diff options
170 files changed, 2538 insertions, 1583 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index e46db1d1b5f..90f061d933d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.HnswIndexParams; import com.yahoo.searchdefinition.document.RankType; import com.yahoo.searchdefinition.document.Stemming; @@ -9,8 +10,11 @@ import java.io.Serializable; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Optional; import java.util.Set; + /** * An index definition in a search definition. * Two indices are equal if they have the same name and the same settings, except @@ -57,6 +61,8 @@ public class Index implements Cloneable, Serializable { /** The boolean index definition, if set */ private BooleanIndexDefinition boolIndex; + private Optional<HnswIndexParams> hnswIndexParams; + /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */ private boolean interleavedFeatures = false; @@ -115,20 +121,25 @@ public class Index implements Cloneable, Serializable { } @Override - public int hashCode() { - return name.hashCode() + ( prefix ? 17 : 0 ); + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Index index = (Index) o; + return prefix == index.prefix && + normalized == index.normalized && + interleavedFeatures == index.interleavedFeatures && + Objects.equals(name, index.name) && + rankType == index.rankType && + Objects.equals(aliases, index.aliases) && + stemming == index.stemming && + type == index.type && + Objects.equals(boolIndex, index.boolIndex) && + Objects.equals(hnswIndexParams, index.hnswIndexParams); } @Override - public boolean equals(Object object) { - if ( ! (object instanceof Index)) return false; - - Index other=(Index)object; - return - this.name.equals(other.name) && - this.prefix==other.prefix && - this.stemming==other.stemming && - this.normalized==other.normalized; + public int hashCode() { + return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, hnswIndexParams, interleavedFeatures); } public String toString() { @@ -176,6 +187,14 @@ public class Index implements Cloneable, Serializable { boolIndex = def; } + public Optional<HnswIndexParams> getHnswIndexParams() { + return hnswIndexParams; + } + + public void setHnswIndexParams(HnswIndexParams params) { + hnswIndexParams = Optional.of(params); + } + public void setInterleavedFeatures(boolean value) { interleavedFeatures = value; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 76dff404568..8de8b239c93 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -240,6 +240,14 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce aaB.tensortype(attribute.tensorType().get().toString()); } aaB.imported(imported); + if (attribute.hnswIndexParams().isPresent()) { + var ib = new AttributesConfig.Attribute.Index.Builder(); + var params = attribute.hnswIndexParams().get(); + ib.hnsw.enabled(true); + ib.hnsw.maxlinkspernode(params.maxLinksPerNode()); + ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert()); + aaB.index(ib); + } return aaB; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java index d8773063053..39a67a69575 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java @@ -5,6 +5,7 @@ import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; @@ -20,7 +21,7 @@ import java.util.List; import java.util.Map; /** - * Deriver of indexschema config containing information of all index fields with name and data type. + * Deriver of indexschema config containing information of all text index fields with name and data type. * * @author geirst */ @@ -44,9 +45,14 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { super.derive(search); } + private boolean isTensorField(ImmutableSDField field) { + return field.getDataType() instanceof TensorDataType; + } + private void deriveIndexFields(ImmutableSDField field, Search search) { - if (!field.doesIndexing() && - !field.isIndexStructureField()) + // Note: Indexes for tensor fields are NOT part of the index schema for text fields. + if ((!field.doesIndexing() && !field.isIndexStructureField()) || + isTensorField(field)) { return; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index fbcaf2a3a80..9ed5e4ca2de 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -66,6 +66,8 @@ public final class Attribute implements Cloneable, Serializable { /** This is set if the type of this is REFERENCE */ private final Optional<StructuredDataType> referenceDocumentType; + private Optional<HnswIndexParams> hnswIndexParams; + private boolean isPosition = false; private final Sorting sorting = new Sorting(); @@ -150,6 +152,7 @@ public final class Attribute implements Cloneable, Serializable { setCollectionType(collectionType); this.tensorType = tensorType; this.referenceDocumentType = referenceDocumentType; + this.hnswIndexParams = Optional.empty(); } public Attribute convertToArray() { @@ -194,6 +197,7 @@ public final class Attribute implements Cloneable, Serializable { public double densePostingListThreshold() { return densePostingListThreshold; } public Optional<TensorType> tensorType() { return tensorType; } public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; } + public Optional<HnswIndexParams> hnswIndexParams() { return hnswIndexParams; } public Sorting getSorting() { return sorting; } @@ -217,6 +221,7 @@ public final class Attribute implements Cloneable, Serializable { public void setUpperBound(long upperBound) { this.upperBound = upperBound; } public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; } public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } + public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } public String getName() { return name; } public Type getType() { return type; } @@ -335,7 +340,7 @@ public final class Attribute implements Cloneable, Serializable { public int hashCode() { return Objects.hash( name, type, collectionType, sorting, isPrefetch(), fastAccess, removeIfZero, createIfNonExistent, - isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType); + isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType, hnswIndexParams); } @Override @@ -362,6 +367,7 @@ public final class Attribute implements Cloneable, Serializable { if ( ! this.sorting.equals(other.sorting)) return false; if (!this.tensorType.equals(other.tensorType)) return false; if (!this.referenceDocumentType.equals(other.referenceDocumentType)) return false; + if (!this.hnswIndexParams.equals(other.hnswIndexParams)) return false; return true; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java new file mode 100644 index 00000000000..70d0df8be7f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java @@ -0,0 +1,60 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import java.util.OptionalInt; + +/** + * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. + * + * @author geirst + */ +public class HnswIndexParams { + + public static final int DEFAULT_MAX_LINKS_PER_NODE = 16; + public static final int DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT = 200; + + private final OptionalInt maxLinksPerNode; + private final OptionalInt neighborsToExploreAtInsert; + + public static class Builder { + private OptionalInt maxLinksPerNode = OptionalInt.empty(); + private OptionalInt neighborsToExploreAtInsert = OptionalInt.empty(); + + public void setMaxLinksPerNode(int value) { + maxLinksPerNode = OptionalInt.of(value); + } + public void setNeighborsToExploreAtInsert(int value) { + neighborsToExploreAtInsert = OptionalInt.of(value); + } + public HnswIndexParams build() { + return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert); + } + } + + public HnswIndexParams() { + this.maxLinksPerNode = OptionalInt.empty(); + this.neighborsToExploreAtInsert = OptionalInt.empty(); + } + + public HnswIndexParams(OptionalInt maxLinksPerNode, OptionalInt neighborsToExploreAtInsert) { + this.maxLinksPerNode = maxLinksPerNode; + this.neighborsToExploreAtInsert = neighborsToExploreAtInsert; + } + + /** + * Creates a new instance where values from the given parameter instance are used where they are present, + * otherwise we use values from this. + */ + public HnswIndexParams overrideFrom(HnswIndexParams rhs) { + return new HnswIndexParams(rhs.maxLinksPerNode.isPresent() ? rhs.maxLinksPerNode : maxLinksPerNode, + rhs.neighborsToExploreAtInsert.isPresent() ? rhs.neighborsToExploreAtInsert : neighborsToExploreAtInsert); + } + + public int maxLinksPerNode() { + return maxLinksPerNode.orElse(DEFAULT_MAX_LINKS_PER_NODE); + } + + public int neighborsToExploreAtInsert() { + return neighborsToExploreAtInsert.orElse(DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java index 39f543c7db3..7f9da28b9ca 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.fieldoperation; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.Index.Type; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.HnswIndexParams; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; @@ -31,6 +32,8 @@ public class IndexOperation implements FieldOperation { private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); private Optional<Boolean> enableBm25 = Optional.empty(); + private Optional<HnswIndexParams.Builder> hnswIndexParams = Optional.empty(); + public String getIndexName() { return indexName; } @@ -91,6 +94,9 @@ public class IndexOperation implements FieldOperation { if (enableBm25.isPresent()) { index.setInterleavedFeatures(enableBm25.get()); } + if (hnswIndexParams.isPresent()) { + index.setHnswIndexParams(hnswIndexParams.get().build()); + } } public Type getType() { @@ -116,8 +122,13 @@ public class IndexOperation implements FieldOperation { public void setDensePostingListThreshold(double densePostingListThreshold) { this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold); } + public void setEnableBm25(boolean value) { enableBm25 = Optional.of(value); } + public void setHnswIndexParams(HnswIndexParams.Builder params) { + this.hnswIndexParams = Optional.of(params); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 8e54d7c00d6..9cd7fb24e42 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -6,7 +6,8 @@ import com.yahoo.document.CollectionDataType; import com.yahoo.document.TensorDataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.HnswIndexParams; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -25,10 +26,11 @@ public class TensorFieldProcessor extends Processor { public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; - for (SDField field : search.allConcreteFields()) { + for (var field : search.allConcreteFields()) { if ( field.getDataType() instanceof TensorDataType ) { validateIndexingScripsForTensorField(field); validateAttributeSettingForTensorField(field); + processIndexSettingsForTensorField(field); } else if (field.getDataType() instanceof CollectionDataType){ validateDataTypeForCollectionField(field); @@ -37,20 +39,53 @@ public class TensorFieldProcessor extends Processor { } private void validateIndexingScripsForTensorField(SDField field) { - if (field.doesIndexing()) { - fail(search, field, "A field of type 'tensor' cannot be specified as an 'index' field."); + if (field.doesIndexing() && !isTensorTypeThatSupportsHnswIndex(field)) { + fail(search, field, "A tensor of type '" + tensorTypeToString(field) + "' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that."); } } + private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) { + var type = ((TensorDataType)field.getDataType()).getTensorType(); + // Tensors with 1 indexed dimension supports a hnsw index (used for approximate nearest neighbor search). + if ((type.dimensions().size() == 1) && + type.dimensions().get(0).isIndexed()) { + return true; + } + return false; + } + + private String tensorTypeToString(ImmutableSDField field) { + return ((TensorDataType)field.getDataType()).getTensorType().toString(); + } + private void validateAttributeSettingForTensorField(SDField field) { if (field.doesAttributing()) { - Attribute attribute = field.getAttributes().get(field.getName()); + var attribute = field.getAttributes().get(field.getName()); if (attribute != null && attribute.isFastSearch()) { fail(search, field, "An attribute of type 'tensor' cannot be 'fast-search'."); } } } + private void processIndexSettingsForTensorField(SDField field) { + if (!field.doesIndexing()) { + return; + } + if (isTensorTypeThatSupportsHnswIndex(field)) { + if (!field.doesAttributing()) { + fail(search, field, "A tensor that has an index must also be an attribute."); + } + var index = field.getIndex(field.getName()); + // TODO: Calculate default params based on tensor dimension size + var params = new HnswIndexParams(); + if (index != null && index.getHnswIndexParams().isPresent()) { + params = params.overrideFrom(index.getHnswIndexParams().get()); + } + field.getAttribute().setHnswIndexParams(params); + } + } + private void validateDataTypeForCollectionField(SDField field) { if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType) fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index fd924eb2a0f..fccacc3210d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metricsproxy.http.metrics.MetricsV2Handler; +import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; @@ -20,6 +22,7 @@ import java.util.Map; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_NAME; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.createMetricsHandler; /** * Container running a metrics proxy. @@ -28,9 +31,11 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus */ public class MetricsProxyContainer extends Container implements NodeDimensionsConfig.Producer, + NodeInfoConfig.Producer, RpcConnectorConfig.Producer, VespaServicesConfig.Producer { + public static final int BASEPORT = 19092; final boolean isHostedVespa; @@ -46,6 +51,7 @@ public class MetricsProxyContainer extends Container implements addMetricsProxyComponent(NodeDimensions.class); addMetricsProxyComponent(RpcConnector.class); addMetricsProxyComponent(VespaServices.class); + addHandler(createMetricsHandler(MetricsV2Handler.class, MetricsV2Handler.V2_PATH)); } @Override @@ -53,8 +59,6 @@ public class MetricsProxyContainer extends Container implements return METRICS_PROXY_CONTAINER; } - static public int BASEPORT = 19092; - @Override public int getWantedPort() { return BASEPORT; @@ -121,7 +125,21 @@ public class MetricsProxyContainer extends Container implements } } - private void addMetricsProxyComponent(Class<?> componentClass) { + @Override + public void getConfig(NodeInfoConfig.Builder builder) { + builder.role(getNodeRole()) + .hostname(getHostName()); + } + + private String getNodeRole() { + String hostConfigId = getHost().getConfigId(); + if (! isHostedVespa) return hostConfigId; + return getHostResource().spec().membership() + .map(ClusterMembership::stringValue) + .orElse(hostConfigId); + } + + private void addMetricsProxyComponent(Class<?> componentClass) { addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index fcc6c3279de..071666b5bc7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -119,11 +119,16 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC } private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { + Handler<AbstractConfigProducer<?>> metricsHandler = createMetricsHandler(clazz, bindingPath); + addComponent(metricsHandler); + } + + static Handler<AbstractConfigProducer<?>> createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { Handler<AbstractConfigProducer<?>> metricsHandler = new Handler<>( new ComponentModel(clazz.getName(), null, METRICS_PROXY_BUNDLE_NAME, null)); metricsHandler.addServerBindings("http://*" + bindingPath, "http://*" + bindingPath + "/*"); - addComponent(metricsHandler); + return metricsHandler; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 38f8cd67601..e308be00faf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -527,8 +527,10 @@ public class VespaMetricSet { metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.sum")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.count")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); + metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.visitor.allthreads.completed.sum.rate")); metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate")); + metrics.add(new Metric("vds.visitor.allthreads.failed.sum.rate")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.max")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.sum")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.count")); @@ -537,19 +539,27 @@ public class VespaMetricSet { metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.sum")); metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.count")); metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.average")); // TODO: Remove in Vespa 8 - + + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.count")); @@ -569,13 +579,13 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.allthreads.splitbuckets.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.joinbuckets.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.filestor.alldisks.allthreads.setbucketstates.count.rate")); - //Distributor metrics.add(new Metric("vds.idealstate.buckets_rechecking.average")); metrics.add(new Metric("vds.idealstate.idealstate_diff.average")); @@ -605,18 +615,24 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.puts.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.puts.sum.ok.rate")); metrics.add(new Metric("vds.distributor.puts.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.puts.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.puts.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.removes.sum.latency.max")); metrics.add(new Metric("vds.distributor.removes.sum.latency.sum")); metrics.add(new Metric("vds.distributor.removes.sum.latency.count")); metrics.add(new Metric("vds.distributor.removes.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.removes.sum.ok.rate")); metrics.add(new Metric("vds.distributor.removes.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.updates.sum.latency.max")); metrics.add(new Metric("vds.distributor.updates.sum.latency.sum")); metrics.add(new Metric("vds.distributor.updates.sum.latency.count")); metrics.add(new Metric("vds.distributor.updates.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.updates.sum.ok.rate")); metrics.add(new Metric("vds.distributor.updates.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.updates.sum.diverging_timestamp_updates.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.ok.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.failures.total.rate")); @@ -626,6 +642,7 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.gets.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.gets.sum.ok.rate")); metrics.add(new Metric("vds.distributor.gets.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.gets.sum.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.max")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.sum")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.count")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index b37caf22216..b492941fd13 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -139,13 +139,9 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> int legalPortInHostedVespa = Container.BASEPORT; if (isHosted && port != legalPortInHostedVespa && ! spec.booleanAttribute("required", false)) { - // TODO: After January 2020: - // - Set required='true' for the http server on port 4443 in the tester services.xml in InternalStepRunner - // - Enable 2 currently ignored tests in this module - // - throw IllegalArgumentException here instead of warning - logger.log(Level.WARNING, "Illegal port " + port + " in http server '" + - spec.stringAttribute("id") + "'" + - ": Port must be set to " + legalPortInHostedVespa); + throw new IllegalArgumentException("Illegal port " + port + " in http server '" + + spec.stringAttribute("id") + "'" + + ": Port must be set to " + legalPortInHostedVespa); } return port; } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index eb7d7763cd1..0cfd13f2a9c 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -332,6 +332,9 @@ TOKEN : | < UPPERBOUND: "upper-bound" > | < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" > | < ENABLE_BM25: "enable-bm25" > +| < HNSW: "hnsw" > +| < MAXLINKSPERNODE: "max-links-per-node" > +| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" > | < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? > | < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > | < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? > @@ -1811,10 +1814,33 @@ Object indexBody(IndexOperation index) : | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); } | <ENABLE_BM25> { index.setEnableBm25(true); } + | hnswIndex(index) { } ) { return null; } } +void hnswIndex(IndexOperation index) : +{ + HnswIndexParams.Builder params = new HnswIndexParams.Builder(); +} +{ + ( LOOKAHEAD(<HNSW> lbrace()) + <HNSW> ( (lbrace() (hnswIndexBody(params) (<NL>)*)* <RBRACE>) ) | + <HNSW> ) + { + index.setHnswIndexParams(params); + } +} + +void hnswIndexBody(HnswIndexParams.Builder params) : +{ + int num; +} +{ + ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); } + | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); } ) +} + /** * Consumes a constant block of a search element. * diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg index 97f480745cf..cf8644ebe83 100644 --- a/config-model/src/test/derived/advanced/attributes.cfg +++ b/config-model/src/test/derived/advanced/attributes.cfg @@ -19,3 +19,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg index 56fd15f9f5d..29d5dd92043 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "elem_array.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -40,3 +43,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg index 022bdbd31a4..773f796ed59 100644 --- a/config-model/src/test/derived/attributeprefetch/attributes.cfg +++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multibyte" attribute[].datatype INT8 attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsbyte" attribute[].datatype INT8 attribute[].collectiontype WEIGHTEDSET @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singleint" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multiint" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsint" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlelong" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multilong" attribute[].datatype INT64 attribute[].collectiontype ARRAY @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wslong" attribute[].datatype INT64 attribute[].collectiontype WEIGHTEDSET @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlefloat" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multifloat" attribute[].datatype FLOAT attribute[].collectiontype ARRAY @@ -229,6 +259,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsfloat" attribute[].datatype FLOAT attribute[].collectiontype WEIGHTEDSET @@ -250,6 +283,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singledouble" attribute[].datatype DOUBLE attribute[].collectiontype SINGLE @@ -271,6 +307,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multidouble" attribute[].datatype DOUBLE attribute[].collectiontype ARRAY @@ -292,6 +331,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsdouble" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET @@ -313,6 +355,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlestring" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -334,6 +379,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multistring" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -355,6 +403,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsstring" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -376,3 +427,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg index 7a21001a9ed..e3faf7662f4 100644 --- a/config-model/src/test/derived/attributes/attributes.cfg +++ b/config-model/src/test/derived/attributes/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a3" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a5" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a6" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b1" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b3" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b4" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b5" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b6" attribute[].datatype INT64 attribute[].collectiontype ARRAY @@ -229,6 +259,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b7" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET @@ -250,6 +283,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a9" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -271,6 +307,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a10" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -292,6 +331,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a11" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -313,6 +355,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a12" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -334,6 +379,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a7_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -355,6 +403,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a8_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -376,3 +427,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg index dda746da5b4..b4971487bd1 100644 --- a/config-model/src/test/derived/complex/attributes.cfg +++ b/config-model/src/test/derived/complex/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "fleeting" attribute[].datatype FLOAT attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "fleeting2" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "foundat" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "collapseby" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "ts" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "combineda" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year_arr" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year_sub" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,3 +211,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg new file mode 100644 index 00000000000..27c9f1e0d13 --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/attributes.cfg @@ -0,0 +1,24 @@ +attribute[].name "t1" +attribute[].datatype TENSOR +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "tensor(x[128])" +attribute[].imported false +attribute[].index.hnsw.enabled true +attribute[].index.hnsw.maxlinkspernode 32 +attribute[].index.hnsw.neighborstoexploreatinsert 300 diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg new file mode 100644 index 00000000000..e9fc265ca67 --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg @@ -0,0 +1,5 @@ +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "test" +ilscript[].docfield[] "t1" +ilscript[].content[] "clear_state | guard { input t1 | attribute t1 | index t1; }" diff --git a/config-model/src/test/derived/hnsw_index/test.sd b/config-model/src/test/derived/hnsw_index/test.sd new file mode 100644 index 00000000000..03ede04208b --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/test.sd @@ -0,0 +1,13 @@ +search test { + document test { + field t1 type tensor(x[128]) { + indexing: attribute | index + index { + hnsw { + max-links-per-node: 32 + neighbors-to-explore-at-insert: 300 + } + } + } + } +} diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg index db2280a7846..5427e856df0 100644 --- a/config-model/src/test/derived/imported_position_field/attributes.cfg +++ b/config-model/src/test/derived/imported_position_field/attributes.cfg @@ -1,42 +1,48 @@ -attribute[0].name "parent_ref" -attribute[0].datatype REFERENCE -attribute[0].collectiontype SINGLE -attribute[0].removeifzero false -attribute[0].createifnonexistent false -attribute[0].fastsearch false -attribute[0].huge false -attribute[0].ismutable false -attribute[0].sortascending true -attribute[0].sortfunction UCA -attribute[0].sortstrength PRIMARY -attribute[0].sortlocale "" -attribute[0].enablebitvectors false -attribute[0].enableonlybitvector false -attribute[0].fastaccess false -attribute[0].arity 8 -attribute[0].lowerbound -9223372036854775808 -attribute[0].upperbound 9223372036854775807 -attribute[0].densepostinglistthreshold 0.4 -attribute[0].tensortype "" -attribute[0].imported false -attribute[1].name "my_pos_zcurve" -attribute[1].datatype INT64 -attribute[1].collectiontype SINGLE -attribute[1].removeifzero false -attribute[1].createifnonexistent false -attribute[1].fastsearch true -attribute[1].huge false -attribute[1].ismutable false -attribute[1].sortascending true -attribute[1].sortfunction UCA -attribute[1].sortstrength PRIMARY -attribute[1].sortlocale "" -attribute[1].enablebitvectors false -attribute[1].enableonlybitvector false -attribute[1].fastaccess false -attribute[1].arity 8 -attribute[1].lowerbound -9223372036854775808 -attribute[1].upperbound 9223372036854775807 -attribute[1].densepostinglistthreshold 0.4 -attribute[1].tensortype "" -attribute[1].imported true
\ No newline at end of file +attribute[].name "parent_ref" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_pos_zcurve" +attribute[].datatype INT64 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg index ce6ff5e54ae..e1969e991dd 100644 --- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg +++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg @@ -1,168 +1,192 @@ -attribute[0].name "parent_ref" -attribute[0].datatype REFERENCE -attribute[0].collectiontype SINGLE -attribute[0].removeifzero false -attribute[0].createifnonexistent false -attribute[0].fastsearch false -attribute[0].huge false -attribute[0].ismutable false -attribute[0].sortascending true -attribute[0].sortfunction UCA -attribute[0].sortstrength PRIMARY -attribute[0].sortlocale "" -attribute[0].enablebitvectors false -attribute[0].enableonlybitvector false -attribute[0].fastaccess false -attribute[0].arity 8 -attribute[0].lowerbound -9223372036854775808 -attribute[0].upperbound 9223372036854775807 -attribute[0].densepostinglistthreshold 0.4 -attribute[0].tensortype "" -attribute[0].imported false -attribute[1].name "my_elem_array.name" -attribute[1].datatype STRING -attribute[1].collectiontype SINGLE -attribute[1].removeifzero false -attribute[1].createifnonexistent false -attribute[1].fastsearch true -attribute[1].huge false -attribute[1].ismutable false -attribute[1].sortascending true -attribute[1].sortfunction UCA -attribute[1].sortstrength PRIMARY -attribute[1].sortlocale "" -attribute[1].enablebitvectors false -attribute[1].enableonlybitvector false -attribute[1].fastaccess false -attribute[1].arity 8 -attribute[1].lowerbound -9223372036854775808 -attribute[1].upperbound 9223372036854775807 -attribute[1].densepostinglistthreshold 0.4 -attribute[1].tensortype "" -attribute[1].imported true -attribute[2].name "my_elem_array.weight" -attribute[2].datatype INT32 -attribute[2].collectiontype SINGLE -attribute[2].removeifzero false -attribute[2].createifnonexistent false -attribute[2].fastsearch false -attribute[2].huge false -attribute[2].ismutable false -attribute[2].sortascending true -attribute[2].sortfunction UCA -attribute[2].sortstrength PRIMARY -attribute[2].sortlocale "" -attribute[2].enablebitvectors false -attribute[2].enableonlybitvector false -attribute[2].fastaccess false -attribute[2].arity 8 -attribute[2].lowerbound -9223372036854775808 -attribute[2].upperbound 9223372036854775807 -attribute[2].densepostinglistthreshold 0.4 -attribute[2].tensortype "" -attribute[2].imported true -attribute[3].name "my_elem_map.key" -attribute[3].datatype STRING -attribute[3].collectiontype SINGLE -attribute[3].removeifzero false -attribute[3].createifnonexistent false -attribute[3].fastsearch true -attribute[3].huge false -attribute[3].ismutable false -attribute[3].sortascending true -attribute[3].sortfunction UCA -attribute[3].sortstrength PRIMARY -attribute[3].sortlocale "" -attribute[3].enablebitvectors false -attribute[3].enableonlybitvector false -attribute[3].fastaccess false -attribute[3].arity 8 -attribute[3].lowerbound -9223372036854775808 -attribute[3].upperbound 9223372036854775807 -attribute[3].densepostinglistthreshold 0.4 -attribute[3].tensortype "" -attribute[3].imported true -attribute[4].name "my_elem_map.value.name" -attribute[4].datatype STRING -attribute[4].collectiontype SINGLE -attribute[4].removeifzero false -attribute[4].createifnonexistent false -attribute[4].fastsearch true -attribute[4].huge false -attribute[4].ismutable false -attribute[4].sortascending true -attribute[4].sortfunction UCA -attribute[4].sortstrength PRIMARY -attribute[4].sortlocale "" -attribute[4].enablebitvectors false -attribute[4].enableonlybitvector false -attribute[4].fastaccess false -attribute[4].arity 8 -attribute[4].lowerbound -9223372036854775808 -attribute[4].upperbound 9223372036854775807 -attribute[4].densepostinglistthreshold 0.4 -attribute[4].tensortype "" -attribute[4].imported true -attribute[5].name "my_elem_map.value.weight" -attribute[5].datatype INT32 -attribute[5].collectiontype SINGLE -attribute[5].removeifzero false -attribute[5].createifnonexistent false -attribute[5].fastsearch false -attribute[5].huge false -attribute[5].ismutable false -attribute[5].sortascending true -attribute[5].sortfunction UCA -attribute[5].sortstrength PRIMARY -attribute[5].sortlocale "" -attribute[5].enablebitvectors false -attribute[5].enableonlybitvector false -attribute[5].fastaccess false -attribute[5].arity 8 -attribute[5].lowerbound -9223372036854775808 -attribute[5].upperbound 9223372036854775807 -attribute[5].densepostinglistthreshold 0.4 -attribute[5].tensortype "" -attribute[5].imported true -attribute[6].name "my_str_int_map.key" -attribute[6].datatype STRING -attribute[6].collectiontype SINGLE -attribute[6].removeifzero false -attribute[6].createifnonexistent false -attribute[6].fastsearch true -attribute[6].huge false -attribute[6].ismutable false -attribute[6].sortascending true -attribute[6].sortfunction UCA -attribute[6].sortstrength PRIMARY -attribute[6].sortlocale "" -attribute[6].enablebitvectors false -attribute[6].enableonlybitvector false -attribute[6].fastaccess false -attribute[6].arity 8 -attribute[6].lowerbound -9223372036854775808 -attribute[6].upperbound 9223372036854775807 -attribute[6].densepostinglistthreshold 0.4 -attribute[6].tensortype "" -attribute[6].imported true -attribute[7].name "my_str_int_map.value" -attribute[7].datatype INT32 -attribute[7].collectiontype SINGLE -attribute[7].removeifzero false -attribute[7].createifnonexistent false -attribute[7].fastsearch false -attribute[7].huge false -attribute[7].ismutable false -attribute[7].sortascending true -attribute[7].sortfunction UCA -attribute[7].sortstrength PRIMARY -attribute[7].sortlocale "" -attribute[7].enablebitvectors false -attribute[7].enableonlybitvector false -attribute[7].fastaccess false -attribute[7].arity 8 -attribute[7].lowerbound -9223372036854775808 -attribute[7].upperbound 9223372036854775807 -attribute[7].densepostinglistthreshold 0.4 -attribute[7].tensortype "" -attribute[7].imported true
\ No newline at end of file +attribute[].name "parent_ref" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_array.name" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_array.weight" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.key" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.value.name" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.value.weight" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_str_int_map.key" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_str_int_map.value" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg index 15b1771a2e8..168c9bd4659 100644 --- a/config-model/src/test/derived/importedfields/attributes.cfg +++ b/config-model/src/test/derived/importedfields/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b_ref_with_summary" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_string_field" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_array_field" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_wset_field" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_ancient_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,3 +187,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg index 4a081edbf54..05a980a8347 100644 --- a/config-model/src/test/derived/inheritance/attributes.cfg +++ b/config-model/src/test/derived/inheritance/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "overridden" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "onlymother" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg index 13f59d4925f..9f01b6c45ce 100644 --- a/config-model/src/test/derived/inheritfromparent/attributes.cfg +++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg @@ -19,3 +19,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg index 8901acf63d1..28cb551cced 100644 --- a/config-model/src/test/derived/map_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_map.value" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg index 665edcdf45d..caae49a0252 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_elem_map.value.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_elem_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -103,3 +115,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg index 7f9592dafc8..a045a532965 100644 --- a/config-model/src/test/derived/music/attributes.cfg +++ b/config-model/src/test/derived/music/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "hiphopvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "metalvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -229,3 +259,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg index b33c2fbdf9b..f728b1b1d33 100644 --- a/config-model/src/test/derived/newrank/attributes.cfg +++ b/config-model/src/test/derived/newrank/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "scorekey" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -208,3 +235,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg index 47e07e2a524..458ee86031d 100644 --- a/config-model/src/test/derived/predicate_attribute/attributes.cfg +++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg @@ -19,3 +19,6 @@ attribute[].upperbound 200 attribute[].densepostinglistthreshold 0.2 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg index d7922a0de69..33fcecb1008 100644 --- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg +++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "attributefield2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,3 +43,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg index 12dbf896edc..58125f73f9c 100644 --- a/config-model/src/test/derived/reference_fields/attributes.cfg +++ b/config-model/src/test/derived/reference_fields/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "other_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "yet_another_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg index e88dfde03bb..3404d6a0384 100644 --- a/config-model/src/test/derived/sorting/attributes.cfg +++ b/config-model/src/test/derived/sorting/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "syntaxcheck2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "infieldonly" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,3 +67,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index 4634e120a3a..a8531f73c1e 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor<float>(x[2],y[1])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f3" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor(x{})" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f4" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor(x[10],y[10])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f5" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor<float>(x[10])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f6" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,3 +115,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index e6ffc37e871..290dd5b9a8b 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -19,6 +19,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "along" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -40,6 +43,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "abool" attribute[].datatype BOOL attribute[].collectiontype SINGLE @@ -61,6 +67,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "ashortfloat" attribute[].datatype FLOAT16 attribute[].collectiontype SINGLE @@ -82,6 +91,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "arrayfield" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -103,6 +115,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -124,6 +139,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield2" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -145,6 +163,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield3" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -166,6 +187,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield4" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -187,6 +211,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "tagfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -208,6 +235,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "juletre" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -229,6 +259,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "album1" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -250,6 +283,9 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "other" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -271,3 +307,6 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 7b4b650295c..c010b23e207 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1315,7 +1315,6 @@ public class ModelProvisioningTest { } @Test - @Ignore // TODO: Enable when turning the port check on public void testThatStandaloneSyntaxOnHostedVespaRequiresDefaultPort() { try { String services = diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 3fc05d789f6..e785792839d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -155,4 +155,9 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectConfigFiles("tensor2"); } + @Test + public void testHnswIndex() throws IOException, ParseException { + assertCorrectDeriving("hnsw_index"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java index b6569357495..b9702c6c4f7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.config.model.test.TestUtil; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; + +import static com.yahoo.searchdefinition.SearchBuilder.createFromString; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -16,7 +21,7 @@ public class TensorFieldTestCase { @Test public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type array<tensor(x{})> {}")); + createFromString(getSd("field f1 type array<tensor(x{})> {}")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -28,11 +33,12 @@ public class TensorFieldTestCase { @Test public void requireThatTensorFieldCannotBeIndexField() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); + createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("For search 'test', field 'f1': A field of type 'tensor' cannot be specified as an 'index' field.", + assertEquals("For search 'test', field 'f1': A tensor of type 'tensor(x{})' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that.", e.getMessage()); } } @@ -40,7 +46,7 @@ public class TensorFieldTestCase { @Test public void requireThatTensorAttributeCannotBeFastSearch() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }")); + createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -51,7 +57,7 @@ public class TensorFieldTestCase { @Test public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); + createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -59,8 +65,67 @@ public class TensorFieldTestCase { } } + @Test + public void hnsw_index_is_default_turned_off() throws ParseException { + var attr = createFromString(getSd("field t1 type tensor(x[64]) { indexing: attribute }")) + .getSearch().getAttribute("t1"); + assertFalse(attr.hnswIndexParams().isPresent()); + } + + @Test + public void hnsw_index_gets_default_parameters_if_not_specified() throws ParseException { + assertHnswIndexParams("", 16, 200); + assertHnswIndexParams("index: hnsw", 16, 200); + } + + @Test + public void hnsw_index_parameters_can_be_specified() throws ParseException { + assertHnswIndexParams("index { hnsw { max-links-per-node: 32 } }", 32, 200); + assertHnswIndexParams("index { hnsw { neighbors-to-explore-at-insert: 300 } }", 16, 300); + assertHnswIndexParams(joinLines("index {", + " hnsw {", + " max-links-per-node: 32", + " neighbors-to-explore-at-insert: 300", + " }", + "}"), + 32, 300); + } + + @Test + public void tensor_with_hnsw_index_must_be_an_attribute() throws ParseException { + try { + createFromString(getSd("field t1 type tensor(x[64]) { indexing: index }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 't1': A tensor that has an index must also be an attribute.", e.getMessage()); + } + } + private static String getSd(String field) { - return "search test {\n document test {\n" + field + "}\n}\n"; + return joinLines("search test {", + " document test {", + " " + field, + " }", + "}"); + } + + private void assertHnswIndexParams(String indexSpec, int maxLinksPerNode, int neighborsToExploreAtInsert) throws ParseException { + var sd = getSdWithIndexSpec(indexSpec); + System.out.println(sd); + var search = createFromString(sd).getSearch(); + var attr = search.getAttribute("t1"); + var params = attr.hnswIndexParams(); + assertTrue(params.isPresent()); + assertEquals(maxLinksPerNode, params.get().maxLinksPerNode()); + assertEquals(neighborsToExploreAtInsert, params.get().neighborsToExploreAtInsert()); + } + + private String getSdWithIndexSpec(String indexSpec) { + return getSd(joinLines("field t1 type tensor(x[64]) {", + " indexing: attribute | index", + " " + indexSpec, + "}")); } private void assertStartsWith(String prefix, String string) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index 621cebd6246..eddad6fce89 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -1,6 +1,7 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; @@ -10,10 +11,10 @@ import com.yahoo.vespa.model.test.VespaModelTester; import org.junit.Test; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CONTAINER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.containerConfigId; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig; @@ -90,13 +91,28 @@ public class MetricsProxyContainerTest { String services = servicesWithContent(); VespaModel hostedModel = getModel(services, hosted); assertEquals(1, hostedModel.getHosts().size()); - String configId = CLUSTER_CONFIG_ID + "/" + hostedModel.getHosts().iterator().next().getHostname(); + String configId = containerConfigId(hostedModel, hosted); NodeDimensionsConfig config = getNodeDimensionsConfig(hostedModel, configId); assertEquals("content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_TYPE)); assertEquals("my-content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_ID)); } + @Test + public void metrics_v2_handler_is_set_up_with_node_info_config() { + String services = servicesWithContent(); + VespaModel hostedModel = getModel(services, hosted); + + var container = (MetricsProxyContainer)hostedModel.id2producer().get(containerConfigId(hostedModel, hosted)); + var handlers = container.getHandlers().getComponents(); + + assertEquals(1, handlers.size()); + var metricsV2Handler = handlers.iterator().next(); + + NodeInfoConfig config = hostedModel.getConfig(NodeInfoConfig.class, metricsV2Handler.getConfigId()); + assertTrue(config.role().startsWith("content/my-content/0/")); + assertTrue(config.hostname().startsWith("node-1-3-9-")); + } @Test public void vespa_services_config_has_all_services() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index f3140aafdaf..7cbc9db5eb2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -49,7 +49,7 @@ class MetricsProxyModelTester { return tester.createModel(servicesXml, true); } - static String configId(VespaModel model, MetricsProxyModelTester.TestMode mode) { + static String containerConfigId(VespaModel model, MetricsProxyModelTester.TestMode mode) { return (mode == hosted) ? CLUSTER_CONFIG_ID + "/" + model.getHosts().iterator().next().getHostname() : CONTAINER_CONFIG_ID; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 75b41ff3667..53c99d1d3dc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -151,7 +151,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - @Ignore // TODO: Enable when turning the port check on public void fail_if_http_port_is_not_default_in_hosted_vespa() throws Exception { try { String servicesXml = diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index f9db9eb2f0d..eb7dad88995 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -30,3 +30,8 @@ attribute[].densepostinglistthreshold double default=0.40 attribute[].tensortype string default="" # Whether this is an imported attribute (from parent document db) or not. attribute[].imported bool default=false + +# Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. +attribute[].index.hnsw.enabled bool default=false +attribute[].index.hnsw.maxlinkspernode int default=16 +attribute[].index.hnsw.neighborstoexploreatinsert int default=200 diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index 8d5e7015947..33c568061fe 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -7,7 +7,6 @@ namespace=cloud.config # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true -tenants{}.applications{}.use4443Upstream bool default=false tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java index 67549672408..2d5cce2d4f1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -34,7 +34,7 @@ public class TesterClient { public HttpResponse getLog(String testerHostname, int port, Long after) { URI testerUri; try { - testerUri = createBuilder(testerHostname, port, "/tester/v1/log2") + testerUri = createBuilder(testerHostname, port, "/tester/v1/log") .addParameter("after", String.valueOf(after)) .build(); } catch (URISyntaxException e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index 5f4895c5ae8..6366576e163 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -9,10 +9,7 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import java.util.Collections; import java.util.Comparator; @@ -35,12 +32,10 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; - private final BooleanFlag use4443Upstream; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; - this.use4443Upstream = Flags.USE_4443_UPSTREAM.bindTo(flagSource); } @Override @@ -72,8 +67,6 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) { LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); ab.activeRotation(getActiveRotation(app)); - ab.use4443Upstream( - use4443Upstream.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value()); app.getModel().getHosts().stream() .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) .forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo))); diff --git a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java index ff301d44798..0188136addb 100644 --- a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java @@ -1,46 +1,41 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; import java.net.URI; +import java.util.List; /** * Returns a response containing an array of links to sub-resources * * @author bratseth */ -public class ResourceResponse extends HttpResponse { +public class ResourceResponse extends SlimeJsonResponse { - private final Slime slime = new Slime(); + public ResourceResponse(URI parentUrl, List<String> subResources) { + super(200, toSlime(parentUrl, subResources)); + } public ResourceResponse(URI parentUrl, String ... subResources) { - super(200); - Cursor resourceArray = slime.setObject().setArray("resources"); - for (String subResource : subResources) { - Cursor resourceEntry = resourceArray.addObject(); - resourceEntry.setString("url", new Uri(parentUrl).append(subResource) - .withTrailingSlash() - .toString()); - } + this(parentUrl, List.of(subResources)); } public ResourceResponse(HttpRequest request, String ... subResources) { this(request.getUri(), subResources); } - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); + private static Slime toSlime(URI parentUrl, List<String> subResources) { + var slime = new Slime(); + var resourceArray = slime.setObject().setArray("resources"); + for (var subResource : subResources) { + var resourceEntry = resourceArray.addObject(); + resourceEntry.setString("url", new Uri(parentUrl).append(subResource) + .withTrailingSlash() + .toString()); + } + return slime; } - @Override - public String getContentType() { return "application/json"; } - } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java index fbd7b62b718..bb9e10fb6b2 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java @@ -50,11 +50,11 @@ public class Deconstructor implements ComponentDeconstructor { } } else if (component instanceof Provider) { // TODO Providers should most likely be deconstructed similarly to AbstractComponent - log.log(LogLevel.DEBUG, "Starting deconstruction of provider " + component); + log.log(LogLevel.DEBUG, () -> "Starting deconstruction of provider " + component); ((Provider<?>) component).deconstruct(); - log.log(LogLevel.DEBUG, "Finished deconstruction of provider " + component); + log.log(LogLevel.DEBUG, () -> "Finished deconstruction of provider " + component); } else if (component instanceof SharedResource) { - log.log(LogLevel.DEBUG, "Releasing container reference to resource " + component); + log.log(LogLevel.DEBUG, () -> "Releasing container reference to resource " + component); // No need to delay release, as jdisc does ref-counting ((SharedResource) component).release(); } @@ -87,10 +87,10 @@ public class Deconstructor implements ComponentDeconstructor { @Override public void run() { for (var component : components) { - log.info("Starting deconstruction of component " + component); + log.log(LogLevel.DEBUG, () -> "Starting deconstruction of component " + component); try { component.deconstruct(); - log.log(LogLevel.INFO, "Finished deconstructing of component " + component); + log.log(LogLevel.DEBUG, () -> "Finished deconstructing of component " + component); } catch (Exception | NoClassDefFoundError e) { // May get class not found due to it being already unloaded log.log(WARNING, "Exception thrown when deconstructing component " + component, e); } catch (Error e) { @@ -110,7 +110,7 @@ public class Deconstructor implements ComponentDeconstructor { // It should now be safe to uninstall the old bundles. for (var bundle : bundles) { try { - log.log(LogLevel.INFO, "Uninstalling bundle " + bundle); + log.log(LogLevel.DEBUG, () -> "Uninstalling bundle " + bundle); bundle.uninstall(); } catch (BundleException e) { log.log(SEVERE, "Could not uninstall bundle " + bundle); diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index d0b3189a79c..421544b5b49 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -78,7 +78,6 @@ public class FederationSearcher extends ForkingSearcher { /** The name of the query property containing the source name added to the query to each source by this */ public final static CompoundName SOURCENAME = new CompoundName("sourceName"); public final static CompoundName PROVIDERNAME = new CompoundName("providerName"); - /** Logging field name constants */ public static final String LOG_COUNT_PREFIX = "count_"; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 998af030b6b..67a6faac606 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -31,7 +31,9 @@ enum PathGroup { "/os/v1/{*}", "/provision/v2/{*}", "/zone/v2/{*}", - "/routing/v1/{*}"), + "/routing/v1/", + "/routing/v1/status/environment/{*}", + "/routing/v1/inactive/environment/{*}"), /** Paths used for creating and reading user resources. */ user(Optional.of("/api"), @@ -53,7 +55,8 @@ enum PathGroup { Optional.of("/api"), "/application/v4/tenant/{tenant}/application/", "/application/v4/tenant/{tenant}/cost", - "/application/v4/tenant/{tenant}/cost/{date}"), + "/application/v4/tenant/{tenant}/cost/{date}", + "/routing/v1/status/tenant/{tenant}/{*}"), tenantKeys(Matcher.tenant, Optional.of("/api"), @@ -97,7 +100,8 @@ enum PathGroup { "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/suspended", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/service/{*}", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/global-rotation/{*}", - "/application/v4/tenant/{tenant}/application/{application}/metering"), + "/application/v4/tenant/{tenant}/application/{application}/metering", + "/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}"), // TODO jonmv: remove /** Path used to restart development nodes. */ diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index da2f64f2893..5348185c276 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.TenantName; import org.junit.Test; import java.net.URI; +import java.util.List; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -30,6 +31,10 @@ public class RoleTest { assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/os/v1/bar"))); assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t2/application/a2"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/status/environment/"))); + assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/status/environment/prod"))); + assertTrue(mainEnforcer.allows(role, Action.create, URI.create("/routing/v1/inactive/environment/prod/region/us-north-1"))); } @Test @@ -165,4 +170,31 @@ public class RoleTest { assertTrue(mainEnforcer.allows(Role.systemFlagsDryrunner(), action, dryrunUri)); assertFalse(mainEnforcer.allows(Role.everyone(), action, dryrunUri)); } + + @Test + public void routing() { + var tenantUrl = URI.create("/routing/v1/status/tenant/t1"); + var applicationUrl = URI.create("/routing/v1/status/tenant/t1/application/a1"); + var instanceUrl = URI.create("/routing/v1/status/tenant/t1/application/a1/instance/i1"); + var deploymentUrl = URI.create("/routing/v1/status/tenant/t1/application/a1/instance/i1/environment/prod/region/us-north-1"); + // Read + for (var url : List.of(tenantUrl, applicationUrl, instanceUrl, deploymentUrl)) { + var allowedRole = Role.reader(TenantName.from("t1")); + var disallowedRole = Role.reader(TenantName.from("t2")); + assertTrue(allowedRole + " can read " + url, mainEnforcer.allows(allowedRole, Action.read, url)); + assertFalse(disallowedRole + " cannot read " + url, mainEnforcer.allows(disallowedRole, Action.read, url)); + } + + // Write + { + var url = URI.create("/routing/v1/inactive/tenant/t1/application/a1/instance/i1/environment/prod/region/us-north-1"); + var allowedRole = Role.applicationAdmin(TenantName.from("t1"), ApplicationName.from("a1")); + var disallowedRole = Role.applicationAdmin(TenantName.from("t2"), ApplicationName.from("a2")); + assertTrue(allowedRole + " can override status at " + url, mainEnforcer.allows(allowedRole, Action.create, url)); + assertTrue(allowedRole + " can clear status at " + url, mainEnforcer.allows(allowedRole, Action.delete, url)); + assertFalse(disallowedRole + " cannot override status at " + url, mainEnforcer.allows(disallowedRole, Action.create, url)); + assertFalse(disallowedRole + " cannot clear status at " + url, mainEnforcer.allows(disallowedRole, Action.delete, url)); + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 3a60c480100..6aebae66bad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import java.time.Duration; import java.time.Instant; @@ -579,8 +578,12 @@ public class DeploymentStatus { Versions versions = Versions.from(change, status.application, status.deploymentFor(job.id()), status.systemVersion); return job.lastSuccess() .filter(run -> versions.targetsMatch(run.versions())) - .filter(run -> status.instanceJobs(instance).get(prodType).lastCompleted() - .map(last -> ! last.end().get().isAfter(run.start())).orElse(false)) + .filter(run -> ! status.jobs() + .instance(instance) + .type(prodType) + .successOn(versions) + .lastCompleted().endedNoLaterThan(run.start()) + .isEmpty()) .map(run -> run.end().get()); } }; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java index c14493a0b72..efa21b71936 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java @@ -3,15 +3,9 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.collections.AbstractFilteringList; import com.yahoo.component.Version; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import java.time.Instant; import java.util.Collection; -import java.util.List; -import java.util.Optional; /** * List for filtering deployment status of applications, for inspection and decision making. @@ -48,14 +42,14 @@ public class DeploymentStatusList extends AbstractFilteringList<DeploymentStatus private static boolean failingUpgradeToVersionSince(JobList jobs, Version version, Instant threshold) { return ! jobs.not().failingApplicationChange() - .firstFailing().endedBefore(threshold) + .firstFailing().endedNoLaterThan(threshold) .lastCompleted().on(version) .isEmpty(); } private static boolean failingApplicationChangeSince(JobList jobs, Instant threshold) { return ! jobs.failingApplicationChange() - .firstFailing().endedBefore(threshold) + .firstFailing().endedNoLaterThan(threshold) .isEmpty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 61dc249feaa..060ffd63fb3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.log.LogLevel; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; @@ -25,7 +26,9 @@ import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; @@ -224,6 +227,13 @@ public class InternalStepRunner implements StepRunner { Instant startTime, DualLogger logger) { try { PrepareResponse prepareResponse = deployment.get().prepareResponse(); + if (prepareResponse.log != null) + logger.logAll(prepareResponse.log.stream() + .map(entry -> new LogEntry(0, // Sequenced by BufferedLogStore. + Instant.ofEpochMilli(entry.time), + LogEntry.typeOf(LogLevel.parse(entry.level)), + entry.message)) + .collect(toList())); if ( ! prepareResponse.configChangeActions.refeedActions.stream().allMatch(action -> action.allowed)) { List<String> messages = new ArrayList<>(); messages.add("Deploy failed due to non-compatible changes that require re-feed."); @@ -237,10 +247,6 @@ public class InternalStepRunner implements StepRunner { .filter(action -> ! action.allowed) .flatMap(action -> action.messages.stream()) .forEach(messages::add); - messages.add("Details:"); - prepareResponse.log.stream() - .map(entry -> entry.message) - .forEach(messages::add); logger.log(messages); return Optional.of(deploymentFailed); } @@ -255,7 +261,7 @@ public class InternalStepRunner implements StepRunner { .map(Hostname::new) .forEach(hostname -> { controller.applications().restart(new DeploymentId(id, type.zone(controller.system())), Optional.of(hostname)); - logger.log("Restarting services on host " + hostname.id() + "."); + logger.log("Schedule service restart on host " + hostname.id() + "."); }); logger.log("Deployment successful."); if (prepareResponse.message != null) @@ -830,6 +836,10 @@ public class InternalStepRunner implements StepRunner { log(List.of(messages)); } + private void logAll(List<LogEntry> messages) { + controller.jobController().log(id, step, messages); + } + private void log(List<String> messages) { controller.jobController().log(id, step, INFO, messages); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index d8d9431dc22..be189004f6d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -73,8 +73,8 @@ import static java.util.stream.Collectors.toUnmodifiableMap; */ public class JobController { - private static final int historyLength = 64; - private static final Duration maxHistoryAge = Duration.ofDays(60); + public static final int historyLength = 64; + public static final Duration maxHistoryAge = Duration.ofDays(60); private final Controller controller; private final CuratorDb curator; @@ -356,7 +356,9 @@ public class JobController { old = oldEntries.next()) { // Make sure we keep the last success and the first failing - if (successes == 1 && old.getValue().status() == RunStatus.success) { + if ( successes == 1 + && old.getValue().status() == RunStatus.success + && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { oldEntries.next(); continue; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java index f353910163f..525eadb6eaf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -117,7 +117,7 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { return new RunFilter(JobStatus::firstFailing); } - /** Allows sub-filters for runs of the given kind */ + /** Allows sub-filters for runs of the indicated kind */ public class RunFilter { private final Function<JobStatus, Optional<Run>> which; @@ -126,47 +126,32 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { this.which = which; } - /** Returns the subset of jobs where the run of the given type exists */ + /** Returns the subset of jobs where the run of the indicated type exists */ public JobList present() { return matching(run -> true); } - /** Returns the runs of the given kind, mapped by the given function, as a list. */ + /** Returns the runs of the indicated kind, mapped by the given function, as a list. */ public <OtherType> List<OtherType> mapToList(Function<? super Run, OtherType> mapper) { return present().mapToList(which.andThen(Optional::get).andThen(mapper)); } - /** Returns the runs of the given kind. */ + /** Returns the runs of the indicated kind. */ public List<Run> asList() { return mapToList(Function.identity()); } - /** Returns the subset of jobs where the run of the given type occurred before the given instant */ - public JobList endedBefore(Instant threshold) { - return matching(run -> run.end().orElse(Instant.MAX).isBefore(threshold)); + /** Returns the subset of jobs where the run of the indicated type ended no later than the given instant */ + public JobList endedNoLaterThan(Instant threshold) { + return matching(run -> ! run.end().orElse(Instant.MAX).isAfter(threshold)); } - /** Returns the subset of jobs where the run of the given type occurred after the given instant */ - public JobList endedAfter(Instant threshold) { - return matching(run -> run.end().orElse(Instant.MIN).isAfter(threshold)); - } - - /** Returns the subset of jobs where the run of the given type occurred before the given instant */ - public JobList startedBefore(Instant threshold) { - return matching(run -> run.start().isBefore(threshold)); - } - - /** Returns the subset of jobs where the run of the given type occurred after the given instant */ - public JobList startedAfter(Instant threshold) { - return matching(run -> run.start().isAfter(threshold)); - } - - /** Returns the subset of jobs where the run of the given type was on the given version */ + /** Returns the subset of jobs where the run of the indicated type was on the given version */ public JobList on(ApplicationVersion version) { return matching(run -> run.versions().targetApplication().equals(version)); } - /** Returns the subset of jobs where the run of the given type was on the given version */ + /** Returns the subset of jobs where the run of the indicated type was on the given version */ public JobList on(Version version) { return matching(run -> run.versions().targetPlatform().equals(version)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java index 1bb449b0a16..d915da21603 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java @@ -58,6 +58,7 @@ public class EndpointCertificateManager { private final Clock clock; private final BooleanFlag useRefreshedEndpointCertificate; private final StringFlag endpointCertificateBackfill; + private final BooleanFlag endpointCertInSharedRouting; public EndpointCertificateManager(ZoneRegistry zoneRegistry, CuratorDb curator, @@ -71,6 +72,7 @@ public class EndpointCertificateManager { this.clock = clock; this.useRefreshedEndpointCertificate = Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.bindTo(flagSource); this.endpointCertificateBackfill = Flags.ENDPOINT_CERTIFICATE_BACKFILL.bindTo(flagSource); + this.endpointCertInSharedRouting = Flags.ENDPOINT_CERT_IN_SHARED_ROUTING.bindTo(flagSource); Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { try { this.backfillCertificateMetadata(); @@ -82,7 +84,8 @@ public class EndpointCertificateManager { public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone) { - if (!zoneRegistry.zones().directlyRouted().ids().contains(zone)) return Optional.empty(); + boolean endpointCertInSharedRouting = this.endpointCertInSharedRouting.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value(); + if (!zoneRegistry.zones().directlyRouted().ids().contains(zone) && !endpointCertInSharedRouting) return Optional.empty(); // Re-use existing certificate if already provisioned var endpointCertificateMetadata = @@ -171,9 +174,9 @@ public class EndpointCertificateManager { } private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance) { - List<ZoneId> directlyRoutedZones = zoneRegistry.zones().directlyRouted().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList()); + List<ZoneId> zones = zoneRegistry.zones().controllerUpgraded().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList()); EndpointCertificateMetadata provisionedCertificateMetadata = endpointCertificateProvider - .requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), directlyRoutedZones)); + .requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones)); curator.writeEndpointCertificateMetadata(instance.id(), provisionedCertificateMetadata); return provisionedCertificateMetadata; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 814be383d65..b43ad28fcec 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -228,6 +228,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -433,13 +434,24 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Slime slime = new Slime(); Cursor array = slime.setArray(); for (Application application : controller.applications().asList(tenant)) { - if (applicationName.map(application.id().application().value()::equals).orElse(true)) - for (InstanceName instance : application.instances().keySet()) + if (applicationName.map(application.id().application().value()::equals).orElse(true)) { + for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet() + : application.instances().keySet()) toSlime(application.id().instance(instance), array.addObject(), request); + } } return new SlimeJsonResponse(slime); } + private HttpResponse devApplicationPackage(ApplicationId id, JobType type) { + if ( ! type.environment().isManuallyDeployed()) + throw new IllegalArgumentException("Only manually deployed zones have dev packages"); + + ZoneId zone = type.zone(controller.system()); + byte[] applicationPackage = controller.applications().applicationStore().getDev(id, zone); + return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", applicationPackage); + } + private HttpResponse applicationPackage(String tenantName, String applicationName, HttpRequest request) { var tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName); var applicationId = ApplicationId.from(tenantName, applicationName, InstanceName.defaultName().value()); @@ -1688,23 +1700,22 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deleteInstance(String tenantName, String applicationName, String instanceName, HttpRequest request) { TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName); - Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user - ? Optional.empty() - : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); controller.applications().deleteInstance(id.instance(instanceName)); - if (controller.applications().requireApplication(id).instances().isEmpty()) + if (controller.applications().requireApplication(id).instances().isEmpty()) { + Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user + ? Optional.empty() + : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); controller.applications().deleteApplication(id, credentials); + } return new MessageResponse("Deleted instance " + id.instance(instanceName).toFullString()); } private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { - Instance instance = controller.applications().requireInstance(ApplicationId.from(tenantName, applicationName, instanceName)); - + DeploymentId id = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), + ZoneId.from(environment, region)); // Attempt to deactivate application even if the deployment is not known by the controller - DeploymentId deploymentId = new DeploymentId(instance.id(), ZoneId.from(environment, region)); - controller.applications().deactivate(deploymentId.applicationId(), deploymentId.zoneId()); - - return new MessageResponse("Deactivated " + deploymentId); + controller.applications().deactivate(id.applicationId(), id.zoneId()); + return new MessageResponse("Deactivated " + id); } /** Returns test config for indicated job, with production deployments of the default instance. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index b28c1fd59e6..32c2d6ec3d1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -192,7 +192,7 @@ class JobControllerApiHandlerHelper { return; Cursor jobObject = jobsArray.addObject(); - jobObject.setString("name", job.type().jobName()); + jobObject.setString("jobName", job.type().jobName()); toSlime(jobObject.setArray("runs"), runs, baseUriForJobs); }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 1815628a1ee..26ccecee3e6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -2,6 +2,9 @@ package com.yahoo.vespa.hosted.controller.restapi.routing; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; @@ -9,21 +12,28 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; +import com.yahoo.restapi.ResourceResponse; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.yolean.Exceptions; +import java.net.URI; import java.time.Instant; +import java.util.Comparator; +import java.util.List; import java.util.Objects; import java.util.logging.Level; +import java.util.stream.Collectors; /** * This implements the /routing/v1 API, which provides operator with global routing control at both zone- and @@ -45,7 +55,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { try { var path = new Path(request.getUri()); switch (request.getMethod()) { - case GET: return get(path); + case GET: return get(path, request); case POST: return post(path); case DELETE: return delete(path); default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); @@ -70,12 +80,92 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse get(Path path) { - if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deploymentStatus(path); - if (path.matches("/routing/v1/status/environment/{environment}/region/{region}")) return zoneStatus(path); + private HttpResponse get(Path path, HttpRequest request) { + if (path.matches("/routing/v1/")) return status(request.getUri()); + if (path.matches("/routing/v1/status/tenant/{tenant}")) return tenant(path, request); + if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}")) return application(path, request); + if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}")) return instance(path, request); + if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path); + if (path.matches("/routing/v1/status/environment")) return environment(request); + if (path.matches("/routing/v1/status/environment/{environment}/region/{region}")) return zone(path); return ErrorResponse.notFoundError("Nothing at " + path); } + private HttpResponse environment(HttpRequest request) { + var zones = controller.zoneRegistry().zones().all().ids(); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + var zonesArray = root.setArray("zones"); + for (var zone : zones) { + toSlime(zone, zonesArray.addObject()); + } + return new SlimeJsonResponse(slime); + } + var resources = controller.zoneRegistry().zones().all().ids().stream() + .map(zone -> zone.environment().value() + + "/region/" + zone.region().value()) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + + private HttpResponse status(URI requestUrl) { + return new ResourceResponse(requestUrl, "status/tenant", "status/environment"); + } + + private HttpResponse tenant(Path path, HttpRequest request) { + var tenantName = tenantFrom(path); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + toSlime(controller.applications().asList(tenantName), null, null, root); + return new SlimeJsonResponse(slime); + } + var resources = controller.applications().asList(tenantName).stream() + .map(Application::id) + .map(TenantAndApplicationId::application) + .map(ApplicationName::value) + .map(application -> "application/" + application) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + + private HttpResponse application(Path path, HttpRequest request) { + var tenantAndApplicationId = tenantAndApplicationIdFrom(path); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + toSlime(List.of(controller.applications().requireApplication(tenantAndApplicationId)), null, + null, root); + return new SlimeJsonResponse(slime); + } + var resources = controller.applications().requireApplication(tenantAndApplicationId).instances().keySet().stream() + .map(InstanceName::value) + .map(instance -> "instance/" + instance) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + + private HttpResponse instance(Path path, HttpRequest request) { + var instanceId = instanceFrom(path); + if (isRecursive(request)) { + var slime = new Slime(); + var root = slime.setObject(); + toSlime(List.of(controller.applications().requireApplication(TenantAndApplicationId.from(instanceId))), + instanceId, null, root); + return new SlimeJsonResponse(slime); + } + var resources = controller.applications().requireInstance(instanceId).deployments().keySet().stream() + .map(zone -> "environment/" + zone.environment().value() + + "/region/" + zone.region().value()) + .sorted() + .collect(Collectors.toList()); + return new ResourceResponse(request.getUri(), resources); + } + private HttpResponse setZoneStatus(Path path, boolean in) { var zone = zoneFrom(path); if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { @@ -88,21 +178,25 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { (in ? "IN" : "OUT")); } - private HttpResponse zoneStatus(Path path) { + private HttpResponse zone(Path path) { var zone = zoneFrom(path); var slime = new Slime(); var root = slime.setObject(); + toSlime(zone, root); + return new SlimeJsonResponse(slime); + } + + private void toSlime(ZoneId zone, Cursor zoneObject) { if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { var zonePolicy = controller.routingController().policies().get(zone); - zoneStatusToSlime(root, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingMethod.exclusive); + zoneStatusToSlime(zoneObject, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingMethod.exclusive); } else { // Rotation status per zone only exposes in/out status, no agent or time of change. var in = controller.serviceRegistry().configServer().getGlobalRotationStatus(zone); var globalRouting = new GlobalRouting(in ? GlobalRouting.Status.in : GlobalRouting.Status.out, GlobalRouting.Agent.operator, Instant.EPOCH); - zoneStatusToSlime(root, zone, globalRouting, RoutingMethod.shared); + zoneStatusToSlime(zoneObject, zone, globalRouting, RoutingMethod.shared); } - return new SlimeJsonResponse(slime); } private HttpResponse setDeploymentStatus(Path path, boolean in) { @@ -124,41 +218,59 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT")); } - private HttpResponse deploymentStatus(Path path) { - var deployment = deploymentFrom(path); - var instance = controller.applications().requireInstance(deployment.applicationId()); + private HttpResponse deployment(Path path) { var slime = new Slime(); - var deploymentsObject = slime.setObject().setArray("deployments"); + var root = slime.setObject(); + var deploymentId = deploymentFrom(path); + var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); + toSlime(List.of(application), deploymentId.applicationId(), deploymentId.zoneId(), root); + return new SlimeJsonResponse(slime); + } - // Include status from rotation - if (rotationCanRouteTo(deployment.zoneId(), instance)) { - var rotationStatus = controller.routingController().globalRotationStatus(deployment); - // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. - var endpointStatus = rotationStatus.values().stream().findFirst(); - if (endpointStatus.isPresent()) { - var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch()); - GlobalRouting.Agent agent; - try { - agent = GlobalRouting.Agent.valueOf(endpointStatus.get().getAgent()); - } catch (IllegalArgumentException e) { - agent = GlobalRouting.Agent.unknown; + private void toSlime(List<Application> applications, ApplicationId instanceId, ZoneId zoneId, Cursor root) { + var deploymentsArray = root.setArray("deployments"); + for (var application : applications) { + var instances = instanceId == null + ? application.instances().values() + : List.of(application.instances().get(instanceId.instance())); + for (var instance : instances) { + var zones = zoneId == null + ? instance.deployments().keySet().stream().sorted(Comparator.comparing(ZoneId::value)) + .collect(Collectors.toList()) + : List.of(zoneId); + for (var zone : zones) { + var deploymentId = new DeploymentId(instance.id(), zone); + // Include status from rotation + if (rotationCanRouteTo(zone, instance)) { + var rotationStatus = controller.routingController().globalRotationStatus(deploymentId); + // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. + var endpointStatus = rotationStatus.values().stream().findFirst(); + if (endpointStatus.isPresent()) { + var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch()); + GlobalRouting.Agent agent; + try { + agent = GlobalRouting.Agent.valueOf(endpointStatus.get().getAgent()); + } catch (IllegalArgumentException e) { + agent = GlobalRouting.Agent.unknown; + } + var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in + ? GlobalRouting.Status.in + : GlobalRouting.Status.out; + deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, + new GlobalRouting(status, agent, changedAt), + RoutingMethod.shared); + } + } + + // Include status from routing policies + var routingPolicies = controller.routingController().policies().get(deploymentId); + for (var policy : routingPolicies.values()) { + deploymentStatusToSlime(deploymentsArray.addObject(), policy); + } } - var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in - ? GlobalRouting.Status.in - : GlobalRouting.Status.out; - deploymentStatusToSlime(deploymentsObject.addObject(), deployment, - new GlobalRouting(status, agent, changedAt), - RoutingMethod.shared); } } - // Include status from routing policies - var routingPolicies = controller.routingController().policies().get(deployment); - for (var policy : routingPolicies.values()) { - deploymentStatusToSlime(deploymentsObject.addObject(), policy); - } - - return new SlimeJsonResponse(slime); } /** Returns whether instance has an assigned rotation and a deployment in given zone */ @@ -190,9 +302,24 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { policy.status().globalRouting(), RoutingMethod.exclusive); } + private TenantName tenantFrom(Path path) { + return TenantName.from(path.get("tenant")); + } + + private ApplicationName applicationFrom(Path path) { + return ApplicationName.from(path.get("application")); + } + + private TenantAndApplicationId tenantAndApplicationIdFrom(Path path) { + return TenantAndApplicationId.from(tenantFrom(path), applicationFrom(path)); + } + + private ApplicationId instanceFrom(Path path) { + return ApplicationId.from(tenantFrom(path), applicationFrom(path), InstanceName.from(path.get("instance"))); + } + private DeploymentId deploymentFrom(Path path) { - return new DeploymentId(ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance")), - zoneFrom(path)); + return new DeploymentId(instanceFrom(path), zoneFrom(path)); } private ZoneId zoneFrom(Path path) { @@ -203,6 +330,10 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { return zone; } + private static boolean isRecursive(HttpRequest request) { + return "true".equals(request.getProperty("recursive")); + } + private static String asString(GlobalRouting.Status status) { switch (status) { case in: return "in"; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 50e567b2024..b082f66b70a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -716,7 +716,7 @@ public class ControllerTest { assertEquals(Stream.concat(Stream.of("vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud", "app1.tenant1.global.vespa.oath.cloud", "*.app1.tenant1.global.vespa.oath.cloud"), - tester.controller().zoneRegistry().zones().directlyRouted().ids().stream() + tester.controller().zoneRegistry().zones().controllerUpgraded().ids().stream() .flatMap(zone -> Stream.of("", "*.") .map(prefix -> prefix + "app1.tenant1." + zone.region().value() + (zone.environment() == Environment.prod ? "" : "." + zone.environment().value()) + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index b3a6ef53d2b..ce4ae7af6b4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -338,6 +338,25 @@ public class JobRunnerTest { } @Test + public void onlySuccessfulRunExpiresThenAnotherFails() { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + var app = tester.newDeploymentContext().submit(); + JobId jobId = new JobId(app.instanceId(), systemTest); + assertFalse(jobs.lastSuccess(jobId).isPresent()); + + app.runJob(systemTest); + assertTrue(jobs.lastSuccess(jobId).isPresent()); + assertEquals(1, jobs.runs(jobId).size()); + + tester.clock().advance(JobController.maxHistoryAge.plusSeconds(1)); + app.submit(); + app.failDeployment(systemTest); + assertFalse(jobs.lastSuccess(jobId).isPresent()); + assertEquals(1, jobs.runs(jobId).size()); + } + + @Test public void timeout() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json deleted file mode 100644 index 31949cce282..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "id": "tenant1:app1:default", - "deploymentSpecField": "<deployment version='1.0'>\n <test />\n <staging />\n <prod>\n <region active=\"true\">us-central-1</region>\n <region active=\"true\">us-west-1</region>\n </prod>\n</deployment>\n", - "validationOverrides": "<deployment version='1.0'/>", - "deployments": [], - "deploymentJobs": { - "jobStatus": [ - { - "jobType": "system-test", - "lastTriggered": { - "version": "6.42.1", - "upgrade": false, - "at": 1506330088050 - } - } - ] - }, - "outstandingChangeField": false -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 9e6d0bc1111..1bdc3a22c2e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -267,6 +267,15 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); app1.runJob(JobType.devUsEast1); + // GET dev application package + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1/package", GET) + .userIdentity(USER_ID), + (response) -> { + assertEquals("attachment; filename=\"tenant1.application1.instance1.dev.us-east-1.zip\"", response.getHeaders().getFirst("Content-Disposition")); + assertArrayEquals(applicationPackageInstance1.zippedContent(), response.getBody()); + }, + 200); + // POST an application package is not generally allowed under user instance tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/otheruser/deploy/dev-us-east-1", POST) .userIdentity(OTHER_USER_ID) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json deleted file mode 100644 index d37e9120837..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.", - "run": 1 -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index 27215b6e2d2..b37d0d41ae4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -32,7 +32,7 @@ }, "deployment": [ { - "name": "dev-us-east-1", + "jobName": "dev-us-east-1", "runs": [ { "id": 1, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json deleted file mode 100644 index 6338306897c..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json +++ /dev/null @@ -1,315 +0,0 @@ -{ - "tenant": "tenant1", - "application": "application1", - "instance": "instance1", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1", - "projectId": 1000, - "deploymentJobs": [ - { - "type": "system-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "staging-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 3, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 3, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-west-1", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-east-3", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - } - ], - "changeBlockers": [], - "compileVersion": "(ignore)", - "globalRotations": [ - "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/" - ], - "rotationId": "rotation-id-1", - "instances": [ - { - "bcpStatus": { - "rotationStatus": "IN" - }, - "endpointStatus": [ - { - "endpointId": "default", - "rotationId": "rotation-id-1", - "clusterId": "foo", - "status": "IN", - "lastUpdated": "(ignore)" - } - ], - "applicationVersion": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "status": "complete", - "environment": "prod", - "region": "us-west-1", - "instance": "instance1", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1" - }, - { - "bcpStatus": { - "rotationStatus": "UNKNOWN" - }, - "endpointStatus": [ - { - "endpointId": "default", - "rotationId": "rotation-id-1", - "clusterId": "foo", - "status": "UNKNOWN", - "lastUpdated": "(ignore)" - } - ], - "applicationVersion": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "status": "complete", - "environment": "prod", - "region": "us-east-3", - "instance": "instance1", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3" - } - ], - "pemDeployKeys": [], - "metrics": { - "queryServiceQuality": 0.5, - "writeServiceQuality": 0.7 - }, - "activity": { - "lastQueried": 1527848130000, - "lastWritten": 1527848130000, - "lastQueriesPerSecond": 1.0, - "lastWritesPerSecond": 2.0 - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 631ec688220..8cd102432d0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -331,7 +331,7 @@ }, "deployment": [ { - "name": "dev-us-east-1", + "jobName": "dev-us-east-1", "runs": [ { "id": 1, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index c0e1f3281dc..aaa9127bdfd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -53,7 +53,7 @@ }, "deployment": [ { - "name": "dev-aws-us-east-2a", + "jobName": "dev-aws-us-east-2a", "runs": [ { "id": 1, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json deleted file mode 100644 index dbaa6623fae..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "versions": [ - { - "version": "0.0.0", - "targetVersion": false, - "cloud": "cloud1", - "nodes": [ - { - "hostname": "node-2-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-2-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-1-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-2-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-1-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - } - ] - }, - { - "version": "0.0.0", - "targetVersion": false, - "cloud": "cloud2", - "nodes": [ - { - "hostname": "node-1-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-1-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-1-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - } - ] - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index 635adc73d1d..d191b460697 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -35,6 +35,88 @@ public class RoutingApiTest extends ControllerContainerTest { } @Test + public void discovery() { + // Deploy + var context = deploymentTester.newDeploymentContext("t1", "a1", "default"); + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context.submit(applicationPackage).deploy(); + + // GET root + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/", "", + Request.Method.GET), + new File("discovery/root.json")); + + // GET tenant + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1", "", + Request.Method.GET), + new File("discovery/tenant.json")); + + // GET application + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/", + "", + Request.Method.GET), + new File("discovery/application.json")); + + // GET instance + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/", + "", + Request.Method.GET), + new File("discovery/instance.json")); + + // GET environment + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/", "", + Request.Method.GET), + new File("discovery/environment.json")); + } + + @Test + public void recursion() { + var context1 = deploymentTester.newDeploymentContext("t1", "a1", "default"); + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var package1 = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context1.submit(package1).deploy(); + + var context2 = deploymentTester.newDeploymentContext("t1", "a2", "default"); + var package2 = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context2.submit(package2).deploy(); + + // GET tenant recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1?recursive=true", "", + Request.Method.GET), + new File("recursion/tenant.json")); + + // GET application recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1?recursive=true", "", + Request.Method.GET), + new File("recursion/application.json")); + + // GET instance recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default?recursive=true", "", + Request.Method.GET), + new File("recursion/application.json")); + + // GET environment recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment?recursive=true", "", + Request.Method.GET), + new File("recursion/environment.json")); + } + + @Test public void exclusive_routing() { var context = deploymentTester.newDeploymentContext(); // Zones support direct routing diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json new file mode 100644 index 00000000000..deda734cbbf --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json @@ -0,0 +1,7 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json new file mode 100644 index 00000000000..1e06b279873 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json @@ -0,0 +1,43 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/environment/dev/region/aws-us-east-2a/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/dev/region/us-east-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/perf/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-northeast-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-northeast-2/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-southeast-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/aws-us-east-1a/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/eu-west-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-central-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/staging/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/test/region/us-east-1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json new file mode 100644 index 00000000000..1a3ad823e14 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json @@ -0,0 +1,10 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-west-1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json new file mode 100644 index 00000000000..9b5630335aa --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json @@ -0,0 +1,10 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json new file mode 100644 index 00000000000..acd05d35c8d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json @@ -0,0 +1,7 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json new file mode 100644 index 00000000000..e0b0e5e9b7a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json @@ -0,0 +1,22 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json new file mode 100644 index 00000000000..f0dd0b7310d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json @@ -0,0 +1,108 @@ +{ + "zones": [ + { + "routingMethod": "shared", + "environment": "test", + "region": "us-east-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "staging", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "dev", + "region": "us-east-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "dev", + "region": "aws-us-east-2a", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "perf", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "aws-us-east-1a", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-northeast-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-northeast-2", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-southeast-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-central-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "eu-west-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json new file mode 100644 index 00000000000..1ee4e1b82ba --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json @@ -0,0 +1,40 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a2:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a2:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + } + ] +} diff --git a/default_build_settings.cmake b/default_build_settings.cmake index d6781f14e75..e29e4c32017 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -77,7 +77,7 @@ endfunction() function(setup_vespa_default_build_settings_fedora_32) message("-- Setting up default build settings for fedora 32") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_ubuntu_18_10) diff --git a/dist/vespa.spec b/dist/vespa.spec index 3a336496f4c..c54e4442167 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -81,7 +81,7 @@ BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc32} -BuildRequires: llvm-devel >= 9.0.0 +BuildRequires: llvm-devel >= 10.0.0 BuildRequires: boost-devel >= 1.69 BuildRequires: gtest-devel BuildRequires: gmock-devel @@ -174,8 +174,8 @@ Requires: llvm-libs >= 9.0.0 %define _vespa_llvm_version 9 %endif %if 0%{?fc32} -Requires: llvm-libs >= 9.0.0 -%define _vespa_llvm_version 9 +Requires: llvm-libs >= 10.0.0 +%define _vespa_llvm_version 10 %endif %define _extra_link_directory %{_vespa_deps_prefix}/lib64 %define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas diff --git a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java index 47599e53ece..8f52c29e84d 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java @@ -144,9 +144,9 @@ public class ComparisonNode implements ExpressionNode { return new ResultList(Result.INVALID); } } else if (oLeft instanceof AttributeNode.VariableValueList) { - return evaluateListAndSingle((AttributeNode.VariableValueList)oLeft, oRight); + return evaluateLhsListAndRhsSingle((AttributeNode.VariableValueList)oLeft, oRight); } else if (oRight instanceof AttributeNode.VariableValueList) { - return evaluateListAndSingle((AttributeNode.VariableValueList)oRight, oLeft); + return evaluateLhsSingleAndRhsList(oLeft, (AttributeNode.VariableValueList)oRight); } return new ResultList(evaluateBool(oLeft, oRight)); } @@ -197,7 +197,7 @@ public class ComparisonNode implements ExpressionNode { } } - private ResultList evaluateListAndSingle(AttributeNode.VariableValueList lhs, Object rhs) { + private ResultList evaluateLhsListAndRhsSingle(AttributeNode.VariableValueList lhs, Object rhs) { if (rhs == null && lhs == null) { return new ResultList(Result.TRUE); } @@ -215,6 +215,24 @@ public class ComparisonNode implements ExpressionNode { return retVal; } + private ResultList evaluateLhsSingleAndRhsList(Object lhs, AttributeNode.VariableValueList rhs) { + if (rhs == null && lhs == null) { + return new ResultList(Result.TRUE); + } + + if (rhs == null || lhs == null) { + return new ResultList(Result.FALSE); + } + + ResultList retVal = new ResultList(); + for (ResultList.VariableValue value : rhs) { + Result result = evaluateBool(lhs, value.getValue()); + retVal.add((FieldPathIteratorHandler.VariableMap)value.getVariables().clone(), result); + } + + return retVal; + } + /** * Evaluate this expression on two operands, given that they are not invalid. * diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 0212be8542e..4e591cdfbd4 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -704,6 +704,23 @@ public class DocumentSelectorTestCase { } @Test + public void using_non_commutative_comparison_operator_with_field_value_is_well_defined() throws ParseException { + var documents = createDocs(); + // Doc 0 contains 24 in `hint` field. + assertEquals(Result.FALSE, evaluate("25 <= test.hint", documents.get(0))); + assertEquals(Result.TRUE, evaluate("24 <= test.hint", documents.get(0))); + assertEquals(Result.TRUE, evaluate("25 > test.hint", documents.get(0))); + assertEquals(Result.FALSE, evaluate("24 > test.hint", documents.get(0))); + assertEquals(Result.TRUE, evaluate("24 >= test.hint", documents.get(0))); + + assertEquals(Result.FALSE, evaluate("test.hint <= 23", documents.get(0))); + assertEquals(Result.TRUE, evaluate("test.hint <= 24", documents.get(0))); + assertEquals(Result.TRUE, evaluate("test.hint > 23", documents.get(0))); + assertEquals(Result.FALSE, evaluate("test.hint > 24", documents.get(0))); + assertEquals(Result.TRUE, evaluate("test.hint >= 24", documents.get(0))); + } + + @Test public void imported_field_references_are_treated_as_valid_field_with_missing_value() throws ParseException { var documents = createDocs(); assertEquals(Result.TRUE, evaluate("test.my_imported_field == null", documents.get(0))); diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index c1e5fbecd14..6d446f6f1d7 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -551,6 +551,22 @@ TEST_F(DocumentSelectParserTest, operators_0) PARSE("30 = 30", *_doc[0], True); } +TEST_F(DocumentSelectParserTest, using_non_commutative_comparison_operator_with_field_value_is_well_defined) { + auto doc = createDoc("testdoctype1", "id:foo:testdoctype1::bar", 24, 0.0, "foo", "bar", 0); + // Document's `headerval` field has value of 24. + PARSE("25 <= testdoctype1.headerval", *doc, False); + PARSE("24 <= testdoctype1.headerval", *doc, True); + PARSE("25 > testdoctype1.headerval", *doc, True); + PARSE("24 > testdoctype1.headerval", *doc, False); + PARSE("24 >= testdoctype1.headerval", *doc, True); + + PARSE("testdoctype1.headerval <= 23", *doc, False); + PARSE("testdoctype1.headerval <= 24", *doc, True); + PARSE("testdoctype1.headerval > 23", *doc, True); + PARSE("testdoctype1.headerval > 24", *doc, False); + PARSE("testdoctype1.headerval >= 24", *doc, True); +} + TEST_F(DocumentSelectParserTest, regex_matching_does_not_bind_anchors_to_newlines) { createDocs(); diff --git a/documentgen-test/etc/complex/book.sd b/documentgen-test/etc/complex/book.sd index 872634bf53b..dd6ccafeab5 100644 --- a/documentgen-test/etc/complex/book.sd +++ b/documentgen-test/etc/complex/book.sd @@ -67,6 +67,10 @@ search book { attribute: tensor(x{}) } } + + import field ref.dummy as my_dummy {} + import field ref.foo as my_foo {} + field sw1 type float { } field didinteger type array<int> { diff --git a/documentgen-test/etc/complex/parent.sd b/documentgen-test/etc/complex/parent.sd index 99a50fdf8ea..50b8e76cf5a 100644 --- a/documentgen-test/etc/complex/parent.sd +++ b/documentgen-test/etc/complex/parent.sd @@ -4,7 +4,10 @@ search parent { document parent { field dummy type string { - + indexing: attribute + } + field foo type string { + indexing: attribute } } } diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 6339416d007..29bee2e9e3e 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -1028,5 +1028,14 @@ public class DocumentGenPluginTest { assertTrue(book.getDataType().fieldSetAll().contains(posZcurve)); assertTrue(book.getDataType().getFields().contains(posZcurve)); } + + @Test + public void imported_fields_are_enumerated_in_document_type() { + var docType = getBook().getDataType(); + assertEquals(2, docType.getImportedFieldNames().size()); + assertTrue(docType.hasImportedField("my_dummy")); + assertTrue(docType.hasImportedField("my_foo")); + assertFalse(docType.hasImportedField("some_field_that_does_not_exist")); + } } diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.h b/eval/src/vespa/eval/eval/llvm/compile_cache.h index 65cec9c0d48..aaadec772a5 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.h +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.h @@ -5,6 +5,7 @@ #include "compiled_function.h" #include <vespa/vespalib/util/executor.h> #include <condition_variable> +#include <atomic> #include <mutex> namespace vespalib { diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index 1d5515d7f4a..cce9838d967 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -12,15 +12,18 @@ #include <llvm/Analysis/Passes.h> #include <llvm/IR/DataLayout.h> #include <llvm/Transforms/Scalar.h> -#if LLVM_VERSION_MAJOR == 9 && defined(__clang__) +#if LLVM_VERSION_MAJOR >= 9 && defined(__clang__) // Avoid reference to undefined symbol llvm::cfg::Update<llvm::BasicBlock*>::dump() const #define NDEBUG #endif #include <llvm/LinkAllPasses.h> -#if LLVM_VERSION_MAJOR == 9 && defined(__clang__) +#if LLVM_VERSION_MAJOR >= 9 && defined(__clang__) #undef NDEBUG #endif #include <llvm/Transforms/IPO/PassManagerBuilder.h> +#if LLVM_VERSION_MAJOR > 9 +#include <llvm/Support/ManagedStatic.h> +#endif #include <vespa/eval/eval/check_type.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/approx.h> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 1a5be921875..e2e85be8b3a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -55,6 +55,12 @@ public class Flags { "Whether to enable Nessus.", "Takes effect on next host admin tick", HOSTNAME); + public static final UnboundBooleanFlag ENABLE_FLEET_SSHD_CONFIG = defineFeatureFlag( + "enable-fleet-sshd-config", false, + "Whether fleet should manage the /etc/ssh/sshd_config file.", + "Takes effect on next host admin tick.", + HOSTNAME); + public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag( "disabled-host-admin-tasks", List.of(), String.class, "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask) that should be skipped", @@ -188,12 +194,6 @@ public class Flags { "Whether to disable CM3.", "Takes effect on next host admin tick", HOSTNAME); - public static final UnboundBooleanFlag USE_4443_UPSTREAM = defineFeatureFlag( - "use-4443-upstream", false, - "Use port 4443 for nginx upstream", - "Takes effect when routing container asks for new config", - APPLICATION_ID); - public static final UnboundBooleanFlag GENERATE_L4_ROUTING_CONFIG = defineFeatureFlag( "generate-l4-routing-config", false, "Whether routing nodes should generate L4 routing config", @@ -229,6 +229,11 @@ public class Flags { "Override the Docker image to use for deployments. This must containing the image name only, without tag", "Takes effect on next host-admin tick", APPLICATION_ID); + public static final UnboundBooleanFlag ENDPOINT_CERT_IN_SHARED_ROUTING = defineFeatureFlag( + "endpoint-cert-in-shared-routing", false, + "Whether to provision and use endpoint certs for apps in shared routing zones", + "Takes effect on next deployment of the application", APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp index 22550a19383..bf98bbd75ef 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp @@ -88,7 +88,7 @@ CfHandler::doConfigure() if (fp != NULL) { fprintf(fp, "[default]\n"); fprintf(fp, "host = %s\n", getenv("VESPA_HOSTNAME")); - fprintf(fp, "_meta = vespa_tenant::%s vespa_application::%s vespa_instance::%s\n", getenv("VESPA_TENANT"), getenv("VESPA_APPLICATION"), getenv("VESPA_INSTANCE")); + fprintf(fp, "_meta = vespa_tenant::%s vespa_app::%s.%s\n", getenv("VESPA_TENANT"), getenv("VESPA_APPLICATION"), getenv("VESPA_INSTANCE")); fclose(fp); rename(tmpPath.c_str(), path.c_str()); } diff --git a/metrics-proxy/CMakeLists.txt b/metrics-proxy/CMakeLists.txt index 41fedb8e8c4..7587159165d 100644 --- a/metrics-proxy/CMakeLists.txt +++ b/metrics-proxy/CMakeLists.txt @@ -6,5 +6,7 @@ install_config_definition(src/main/resources/configdefinitions/consumers.def ai. install_config_definition(src/main/resources/configdefinitions/monitoring.def ai.vespa.metricsproxy.core.monitoring.def) install_config_definition(src/main/resources/configdefinitions/metrics-nodes.def ai.vespa.metricsproxy.http.application.metrics-nodes.def) install_config_definition(src/main/resources/configdefinitions/node-dimensions.def ai.vespa.metricsproxy.metric.dimensions.node-dimensions.def) +install_config_definition(src/main/resources/configdefinitions/node-info.def ai.vespa.metricsproxy.http.metrics.node-info.def) install_config_definition(src/main/resources/configdefinitions/rpc-connector.def ai.vespa.metricsproxy.rpc.rpc-connector.def) install_config_definition(src/main/resources/configdefinitions/vespa-services.def ai.vespa.metricsproxy.service.vespa-services.def) +install_config_definition(src/main/resources/configdefinitions/telegraf.def ai.vespa.metricsproxy.telegraf.telegraf.def) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index 768c1beebef..ae0ef2fa57a 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -46,6 +46,16 @@ public class ValuesFetcher { .collect(Collectors.toList()); } + public List<MetricsPacket.Builder> fetchMetricsAsBuilders(String requestedConsumer) throws JsonRenderingException { + ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); + + return metricsManager.getMetricsAsBuilders(vespaServices.getVespaServices(), Instant.now()) + .stream() + .filter(builder -> builder.hasConsumer(consumer)) + .collect(Collectors.toList()); + } + + public List<MetricsPacket> fetchAllMetrics() throws JsonRenderingException { return metricsManager.getMetrics(vespaServices.getVespaServices(), Instant.now()); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java index c8a8e65be5d..c439a037774 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java @@ -25,7 +25,7 @@ public class Node { } public Node(String role, String hostname, int port, String path) { - Objects.requireNonNull(role, "Null configId is not allowed"); + Objects.requireNonNull(role, "Null role is not allowed"); Objects.requireNonNull(hostname, "Null hostname is not allowed"); Objects.requireNonNull(path, "Null path is not allowed"); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java index 395ec0bea4f..4d1d57644b5 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java @@ -21,7 +21,7 @@ public class PublicDimensionsProcessor implements MetricsProcessor { private final int maxDimensions; private Set<DimensionId> publicDimensions = getPublicDimensions(); - PublicDimensionsProcessor(int maxDimensions) { + public PublicDimensionsProcessor(int maxDimensions) { int numCommonDimensions = PublicDimensions.commonDimensions.size(); if (numCommonDimensions > maxDimensions) { throw new IllegalArgumentException(String.format( diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java new file mode 100644 index 00000000000..71d7857e48a --- /dev/null +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java @@ -0,0 +1,92 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.metrics; + +import ai.vespa.metricsproxy.core.MetricsConsumers; +import ai.vespa.metricsproxy.core.MetricsManager; +import ai.vespa.metricsproxy.http.ValuesFetcher; +import ai.vespa.metricsproxy.http.application.ClusterIdDimensionProcessor; +import ai.vespa.metricsproxy.http.application.Node; +import ai.vespa.metricsproxy.http.application.PublicDimensionsProcessor; +import ai.vespa.metricsproxy.http.application.ServiceIdDimensionProcessor; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor; +import ai.vespa.metricsproxy.service.VespaServices; +import com.google.inject.Inject; +import com.yahoo.container.handler.metrics.ErrorResponse; +import com.yahoo.container.handler.metrics.HttpHandlerBase; +import com.yahoo.container.handler.metrics.JsonResponse; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.Path; + +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; + +import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericApplicationModel; +import static ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor.applyProcessors; +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.OK; +import static java.util.Collections.singletonMap; +import static java.util.stream.Collectors.toList; + +/** + * Http handler for the metrics/v2 rest api. + * + * @author gjoranv + */ +public class MetricsV2Handler extends HttpHandlerBase { + + public static final String V2_PATH = "/metrics/v2"; + public static final String VALUES_PATH = V2_PATH + "/values"; + private static final int MAX_DIMENSIONS = 10; + + private final ValuesFetcher valuesFetcher; + private final NodeInfoConfig nodeInfoConfig; + + @Inject + public MetricsV2Handler(Executor executor, + MetricsManager metricsManager, + VespaServices vespaServices, + MetricsConsumers metricsConsumers, + NodeInfoConfig nodeInfoConfig) { + super(executor); + this.nodeInfoConfig = nodeInfoConfig; + valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers); + } + + @Override + public Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) { + if (apiPath.matches(V2_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); + if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); + return Optional.empty(); + } + + private JsonResponse valuesResponse(String consumer) { + try { + List<MetricsPacket.Builder> builders = valuesFetcher.fetchMetricsAsBuilders(consumer); + List<MetricsPacket> metrics = processAndBuild(builders, + new ServiceIdDimensionProcessor(), + new ClusterIdDimensionProcessor(), + new PublicDimensionsProcessor(MAX_DIMENSIONS)); + + Node localNode = new Node(nodeInfoConfig.role(), nodeInfoConfig.hostname(), 0, ""); + Map<Node, List<MetricsPacket>> metricsByNode = singletonMap(localNode, metrics); + return new JsonResponse(OK, toGenericApplicationModel(metricsByNode).serialize()); + } catch (Exception e) { + log.log(Level.WARNING, "Got exception when rendering metrics:", e); + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + + private static List<MetricsPacket> processAndBuild(List<MetricsPacket.Builder> builders, + MetricsProcessor... processors) { + return builders.stream() + .map(builder -> applyProcessors(builder, processors)) + .map(MetricsPacket.Builder::build) + .collect(toList()); + } + +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java index 8ecf57237ef..8d5a1f50918 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java @@ -179,6 +179,10 @@ public class MetricsPacket { return this; } + public boolean hasConsumer(ConsumerId id) { + return consumers.contains(id); + } + public MetricsPacket build() { return new MetricsPacket(statusCode, statusMessage, timestamp, service, metrics, dimensions, consumers); } diff --git a/metrics-proxy/src/main/resources/configdefinitions/node-info.def b/metrics-proxy/src/main/resources/configdefinitions/node-info.def new file mode 100644 index 00000000000..e66433a96d0 --- /dev/null +++ b/metrics-proxy/src/main/resources/configdefinitions/node-info.def @@ -0,0 +1,5 @@ +# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package=ai.vespa.metricsproxy.http.metrics + +role string +hostname string diff --git a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def new file mode 100644 index 00000000000..f3b5db35d52 --- /dev/null +++ b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def @@ -0,0 +1,20 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package=ai.vespa.metricsproxy.telegraf + +# Metrics pull/push interval +intervalSeconds int default=60 + + +# The consumer to get metrics for +vespa.consumer string default="default" + + +cloudWatch[].region string default="us-east-1" +cloudWatch[].namespace string + +# Only valid and required for hosted Vespa +cloudWatch[].accessKeyName string default="" +cloudWatch[].secretKeyName string default="" + +# Only valid and optional for self-hosted Vespa +cloudWatch[].profile string default="" diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java index 77c3a719cd9..d776368687d 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java @@ -24,6 +24,7 @@ import java.util.List; import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID; import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON; import static ai.vespa.metricsproxy.service.DummyService.METRIC_1; /** @@ -61,11 +62,12 @@ public class HttpHandlerTestBase { } protected static MetricsConsumers getMetricsConsumers() { + // Must use a whitelisted dimension to avoid it being removed for the MetricsV2Handler var defaultConsumerDimension = new ConsumersConfig.Consumer.Metric.Dimension.Builder() - .key("consumer-dim").value("default-val"); + .key(REASON).value("default-val"); var customConsumerDimension = new ConsumersConfig.Consumer.Metric.Dimension.Builder() - .key("consumer-dim").value("custom-val"); + .key(REASON).value("custom-val"); return new MetricsConsumers(new ConsumersConfig.Builder() .consumer(new ConsumersConfig.Consumer.Builder() diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java new file mode 100644 index 00000000000..1c5ce695155 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java @@ -0,0 +1,196 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.metrics; + +import ai.vespa.metricsproxy.http.HttpHandlerTestBase; +import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; +import ai.vespa.metricsproxy.metric.model.json.GenericMetrics; +import ai.vespa.metricsproxy.metric.model.json.GenericService; +import ai.vespa.metricsproxy.service.DownService; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.SERVICE_ID; +import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN; +import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper; +import static ai.vespa.metricsproxy.service.DummyService.METRIC_1; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author gjoranv + */ +public abstract class MetricsHandlerTestBase<MODEL> extends HttpHandlerTestBase { + + static String rootUri; + static String valuesUri; + + Class<MODEL> modelClass; + + abstract GenericJsonModel getGenericJsonModel(MODEL model); + + private MODEL getResponseAsJsonModel(String consumer) { + String response = testDriver.sendRequest(valuesUri + "?consumer=" + consumer).readAll(); + try { + return createObjectMapper().readValue(response, modelClass); + } catch (IOException e) { + fail("Failed to create json model: " + e.getMessage()); + throw new RuntimeException(e); + } + } + + private GenericJsonModel getResponseAsGenericJsonModel(String consumer) { + return getGenericJsonModel(getResponseAsJsonModel(consumer)); + } + + @Test + public void invalid_path_yields_error_response() throws Exception { + String response = testDriver.sendRequest(rootUri + "/invalid").readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("error")); + } + + @Test + public void root_response_contains_values_uri() throws Exception { + String response = testDriver.sendRequest(rootUri).readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("resources")); + + JSONArray resources = root.getJSONArray("resources"); + assertEquals(1, resources.length()); + + JSONObject valuesUrl = resources.getJSONObject(0); + assertEquals(valuesUri, valuesUrl.getString("url")); + } + + @Ignore + @Test + public void visually_inspect_values_response() throws Exception { + String response = testDriver.sendRequest(valuesUri).readAll(); + ObjectMapper mapper = createObjectMapper(); + var jsonModel = mapper.readValue(response, modelClass); + System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel)); + } + + @Test + public void no_explicit_consumer_gives_the_default_consumer() { + String responseDefaultConsumer = testDriver.sendRequest(valuesUri + "?consumer=default").readAll(); + String responseNoConsumer = testDriver.sendRequest(valuesUri).readAll(); + assertEqualsExceptTimestamps(responseDefaultConsumer, responseNoConsumer); + } + + @Test + public void unknown_consumer_gives_the_default_consumer() { + String response = testDriver.sendRequest(valuesUri).readAll(); + String responseUnknownConsumer = testDriver.sendRequest(valuesUri + "?consumer=not_defined").readAll(); + assertEqualsExceptTimestamps(response, responseUnknownConsumer); + } + + private void assertEqualsExceptTimestamps(String s1, String s2) { + assertEquals(replaceTimestamps(s1), replaceTimestamps(s2)); + } + + private String replaceTimestamps(String s) { + return s.replaceAll("timestamp\":\\d+,", "timestamp\":1,"); + } + + @Test + public void response_contains_node_metrics() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER); + + assertNotNull(jsonModel.node); + assertEquals(1, jsonModel.node.metrics.size()); + assertEquals(12.345, jsonModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d); + } + + @Test + public void response_contains_service_metrics() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER); + + assertEquals(2, jsonModel.services.size()); + GenericService dummyService = jsonModel.services.get(0); + assertEquals(2, dummyService.metrics.size()); + + GenericMetrics dummy0Metrics = getMetricsForService("dummy0", dummyService); + assertEquals(1L, dummy0Metrics.values.get(METRIC_1).longValue()); + assertEquals("default-val", dummy0Metrics.dimensions.get(REASON)); + + GenericMetrics dummy1Metrics = getMetricsForService("dummy1", dummyService); + assertEquals(6L, dummy1Metrics.values.get(METRIC_1).longValue()); + assertEquals("default-val", dummy1Metrics.dimensions.get(REASON)); + } + + @Test + public void custom_consumer_gets_only_its_whitelisted_metrics() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(CUSTOM_CONSUMER); + + assertNotNull(jsonModel.node); + // TODO: see comment in ExternalMetrics.setExtraMetrics + // assertEquals(0, jsonModel.node.metrics.size()); + + assertEquals(2, jsonModel.services.size()); + GenericService dummyService = jsonModel.services.get(0); + assertEquals(2, dummyService.metrics.size()); + + GenericMetrics dummy0Metrics = getMetricsForService("dummy0", dummyService); + assertEquals("custom-val", dummy0Metrics.dimensions.get(REASON)); + + GenericMetrics dummy1Metrics = getMetricsForService("dummy1", dummyService); + assertEquals("custom-val", dummy1Metrics.dimensions.get(REASON)); + } + + private static GenericMetrics getMetricsForService(String serviceInstance, GenericService service) { + for (var metrics : service.metrics) { + if (getServiceIdDimension(metrics).equals(serviceInstance)) + return metrics; + } + fail("Could not find metrics for service instance " + serviceInstance); + throw new RuntimeException(); + } + + @Test + public void all_timestamps_are_equal_and_non_zero() { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER); + + Long nodeTimestamp = jsonModel.node.timestamp; + assertNotEquals(0L, (long) nodeTimestamp); + for (var service : jsonModel.services) + assertEquals(nodeTimestamp, service.timestamp); + } + + @Test + public void all_consumers_get_health_from_service_that_is_down() { + assertDownServiceHealth(DEFAULT_CONSUMER); + assertDownServiceHealth(CUSTOM_CONSUMER); + } + + private void assertDownServiceHealth(String consumer) { + GenericJsonModel jsonModel = getResponseAsGenericJsonModel(consumer); + + GenericService downService = jsonModel.services.get(1); + assertEquals(DOWN.status, downService.status.code); + assertEquals("No response", downService.status.description); + + // Service should output metric dimensions, even without metrics, because they contain important info about the service. + assertEquals(1, downService.metrics.size()); + assertEquals(0, downService.metrics.get(0).values.size()); + assertFalse(downService.metrics.get(0).dimensions.isEmpty()); + assertEquals(DownService.NAME, getServiceIdDimension(downService.metrics.get(0))); + } + + private static String getServiceIdDimension(GenericMetrics metrics) { + var instanceDimension = metrics.dimensions.get(INTERNAL_SERVICE_ID); + return instanceDimension != null ? instanceDimension : metrics.dimensions.get(SERVICE_ID); + } + +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java index 22f61114622..fe823466f7b 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java @@ -1,46 +1,30 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.metrics; -import ai.vespa.metricsproxy.http.HttpHandlerTestBase; import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; -import ai.vespa.metricsproxy.metric.model.json.GenericMetrics; -import ai.vespa.metricsproxy.metric.model.json.GenericService; -import ai.vespa.metricsproxy.service.DownService; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.jdisc.RequestHandlerTestDriver; -import org.json.JSONArray; -import org.json.JSONObject; +import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import java.io.IOException; import java.util.concurrent.Executors; -import static ai.vespa.metricsproxy.core.VespaMetrics.INSTANCE_DIMENSION_ID; import static ai.vespa.metricsproxy.http.metrics.MetricsV1Handler.V1_PATH; import static ai.vespa.metricsproxy.http.metrics.MetricsV1Handler.VALUES_PATH; -import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN; -import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper; -import static ai.vespa.metricsproxy.service.DummyService.METRIC_1; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @author gjoranv */ @SuppressWarnings("UnstableApiUsage") -public class MetricsV1HandlerTest extends HttpHandlerTestBase { +public class MetricsV1HandlerTest extends MetricsHandlerTestBase<GenericJsonModel> { private static final String V1_URI = URI_BASE + V1_PATH; private static final String VALUES_URI = URI_BASE + VALUES_PATH; + @BeforeClass public static void setup() { + rootUri = V1_URI; + valuesUri = VALUES_URI; var handler = new MetricsV1Handler(Executors.newSingleThreadExecutor(), getMetricsManager(), vespaServices, @@ -48,149 +32,14 @@ public class MetricsV1HandlerTest extends HttpHandlerTestBase { testDriver = new RequestHandlerTestDriver(handler); } - private GenericJsonModel getResponseAsJsonModel(String consumer) { - String response = testDriver.sendRequest(VALUES_URI + "?consumer=" + consumer).readAll(); - try { - return createObjectMapper().readValue(response, GenericJsonModel.class); - } catch (IOException e) { - fail("Failed to create json model: " + e.getMessage()); - throw new RuntimeException(e); - } - } - - @Test - public void v1_response_contains_values_uri() throws Exception { - String response = testDriver.sendRequest(V1_URI).readAll(); - JSONObject root = new JSONObject(response); - assertTrue(root.has("resources")); - - JSONArray resources = root.getJSONArray("resources"); - assertEquals(1, resources.length()); - - JSONObject valuesUrl = resources.getJSONObject(0); - assertEquals(VALUES_URI, valuesUrl.getString("url")); - } - - @Ignore - @Test - public void visually_inspect_values_response() throws Exception { - String response = testDriver.sendRequest(VALUES_URI).readAll(); - ObjectMapper mapper = createObjectMapper(); - var jsonModel = mapper.readValue(response, GenericJsonModel.class); - System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel)); - } - - @Test - public void no_explicit_consumer_gives_the_default_consumer() { - String responseDefaultConsumer = testDriver.sendRequest(VALUES_URI + "?consumer=default").readAll(); - String responseNoConsumer = testDriver.sendRequest(VALUES_URI).readAll(); - assertEqualsExceptTimestamps(responseDefaultConsumer, responseNoConsumer); - } - - @Test - public void unknown_consumer_gives_the_default_consumer() { - String response = testDriver.sendRequest(VALUES_URI).readAll(); - String responseUnknownConsumer = testDriver.sendRequest(VALUES_URI + "?consumer=not_defined").readAll(); - assertEqualsExceptTimestamps(response, responseUnknownConsumer); - } - - private void assertEqualsExceptTimestamps(String s1, String s2) { - assertEquals(replaceTimestamps(s1), replaceTimestamps(s2)); - } - - @Test - public void response_contains_node_metrics() { - GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER); - - assertNotNull(jsonModel.node); - assertEquals(1, jsonModel.node.metrics.size()); - assertEquals(12.345, jsonModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d); - } - - @Test - public void response_contains_service_metrics() { - GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER); - - assertEquals(2, jsonModel.services.size()); - GenericService dummyService = jsonModel.services.get(0); - assertEquals(2, dummyService.metrics.size()); - - GenericMetrics dummy0Metrics = getMetricsForInstance("dummy0", dummyService); - assertEquals(1L, dummy0Metrics.values.get(METRIC_1).longValue()); - assertEquals("default-val", dummy0Metrics.dimensions.get("consumer-dim")); - - GenericMetrics dummy1Metrics = getMetricsForInstance("dummy1", dummyService); - assertEquals(6L, dummy1Metrics.values.get(METRIC_1).longValue()); - assertEquals("default-val", dummy1Metrics.dimensions.get("consumer-dim")); - } - - @Test - public void all_consumers_get_health_from_service_that_is_down() { - assertDownServiceHealth(DEFAULT_CONSUMER); - assertDownServiceHealth(CUSTOM_CONSUMER); - } - - @Test - public void all_timestamps_are_equal_and_non_zero() { - GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER); - - Long nodeTimestamp = jsonModel.node.timestamp; - assertNotEquals(0L, (long) nodeTimestamp); - for (var service : jsonModel.services) - assertEquals(nodeTimestamp, service.timestamp); - } - - @Test - public void custom_consumer_gets_only_its_whitelisted_metrics() { - GenericJsonModel jsonModel = getResponseAsJsonModel(CUSTOM_CONSUMER); - - assertNotNull(jsonModel.node); - // TODO: see comment in ExternalMetrics.setExtraMetrics - // assertEquals(0, jsonModel.node.metrics.size()); - - assertEquals(2, jsonModel.services.size()); - GenericService dummyService = jsonModel.services.get(0); - assertEquals(2, dummyService.metrics.size()); - - GenericMetrics dummy0Metrics = getMetricsForInstance("dummy0", dummyService); - assertEquals("custom-val", dummy0Metrics.dimensions.get("consumer-dim")); - - GenericMetrics dummy1Metrics = getMetricsForInstance("dummy1", dummyService); - assertEquals("custom-val", dummy1Metrics.dimensions.get("consumer-dim")); - } - - @Test - public void invalid_path_yields_error_response() throws Exception { - String response = testDriver.sendRequest(V1_URI + "/invalid").readAll(); - JSONObject root = new JSONObject(response); - assertTrue(root.has("error")); - } - - private void assertDownServiceHealth(String consumer) { - GenericJsonModel jsonModel = getResponseAsJsonModel(consumer); - - GenericService downService = jsonModel.services.get(1); - assertEquals(DOWN.status, downService.status.code); - assertEquals("No response", downService.status.description); - - // Service should output metric dimensions, even without metrics, because they contain important info about the service. - assertEquals(1, downService.metrics.size()); - assertEquals(0, downService.metrics.get(0).values.size()); - assertFalse(downService.metrics.get(0).dimensions.isEmpty()); - assertEquals(DownService.NAME, downService.metrics.get(0).dimensions.get(INSTANCE_DIMENSION_ID.id)); - } - - private String replaceTimestamps(String s) { - return s.replaceAll("timestamp\":\\d+,", "timestamp\":1,"); + @Before + public void initModelClass() { + modelClass = GenericJsonModel.class; } - private static GenericMetrics getMetricsForInstance(String instance, GenericService service) { - for (var metrics : service.metrics) { - if (metrics.dimensions.get(INSTANCE_DIMENSION_ID.id).equals(instance)) - return metrics; - } - fail("Could not find metrics for service instance " + instance); - throw new RuntimeException(); + @Override + GenericJsonModel getGenericJsonModel(GenericJsonModel genericJsonModel) { + return genericJsonModel; } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java new file mode 100644 index 00000000000..27ee6be4be3 --- /dev/null +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java @@ -0,0 +1,53 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.metricsproxy.http.metrics; + +import ai.vespa.metricsproxy.metric.model.json.GenericApplicationModel; +import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.util.concurrent.Executors; + +import static ai.vespa.metricsproxy.http.metrics.MetricsV2Handler.V2_PATH; +import static ai.vespa.metricsproxy.http.metrics.MetricsV2Handler.VALUES_PATH; + +/** + * @author gjoranv + */ +@SuppressWarnings("UnstableApiUsage") +public class MetricsV2HandlerTest extends MetricsHandlerTestBase<GenericApplicationModel> { + + private static final String V2_URI = URI_BASE + V2_PATH; + private static final String VALUES_URI = URI_BASE + VALUES_PATH; + + + @BeforeClass + public static void setup() { + rootUri = V2_URI; + valuesUri = VALUES_URI; + var handler = new MetricsV2Handler(Executors.newSingleThreadExecutor(), + getMetricsManager(), + vespaServices, + getMetricsConsumers(), + nodeInfoConfig()); + testDriver = new RequestHandlerTestDriver(handler); + } + + @Before + public void initModelClass() { + modelClass = GenericApplicationModel.class; + } + + @Override + GenericJsonModel getGenericJsonModel(GenericApplicationModel genericApplicationModel) { + return genericApplicationModel.nodes.get(0); + } + + private static NodeInfoConfig nodeInfoConfig() { + return new NodeInfoConfig.Builder() + .role("my-role") + .hostname("my-hostname") + .build(); + } +} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java index a85f0425b4b..a224c4090b3 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java @@ -12,6 +12,7 @@ import org.junit.Test; import java.util.concurrent.Executors; +import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -83,7 +84,7 @@ public class PrometheusHandlerTest extends HttpHandlerTestBase { @Test public void service_metrics_have_configured_dimensions() { String dummy0 = getLine(valuesResponse, DummyService.NAME + "0"); - assertTrue(dummy0.contains("consumer_dim=\"default-val\"")); + assertTrue(dummy0.contains(REASON + "=\"default-val\"")); } @Test diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java index 9b37a805245..78c80689299 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java @@ -14,6 +14,7 @@ import java.util.Map; import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; +import static java.util.Collections.singleton; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -45,13 +46,23 @@ public class MetricsPacketTest { MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) .statusCode(0) .statusMessage("") - .addConsumers(Collections.singleton(DUPLICATE_CONSUMER)) - .addConsumers(Collections.singleton(DUPLICATE_CONSUMER)) + .addConsumers(singleton(DUPLICATE_CONSUMER)) + .addConsumers(singleton(DUPLICATE_CONSUMER)) .build(); assertEquals(1, packet.consumers().size()); } @Test + public void builder_allows_inspecting_consumers() { + var consumer = toConsumerId("my-consumer"); + var builder = new MetricsPacket.Builder(toServiceId("foo")) + .statusCode(0) + .statusMessage("") + .addConsumers(singleton(consumer)); + assertTrue(builder.hasConsumer(consumer)); + } + + @Test public void builder_can_retain_subset_of_metrics() { MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) .putMetrics(ImmutableList.of( diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index cb209d710c8..e6d6f9463d3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -114,6 +114,10 @@ public class NodeAdminStateUpdater { // To avoid node agents stalling for too long, we'll force unfrozen ticks now. adjustNodeAgentsToRunFromNodeRepository(); nodeAdmin.setFrozen(false); + + NodeState currentNodeState = nodeRepository.getNode(hostHostname).state(); + if (currentNodeState == NodeState.active) orchestrator.resume(hostHostname); + throw new ConvergenceException("Timed out trying to freeze all nodes: will force an unfrozen tick"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index bb6d53d3304..f75621e18a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer; import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceComparator; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; @@ -44,13 +45,15 @@ public class DynamicProvisioningMaintainer extends Maintainer { private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision"); private final HostProvisioner hostProvisioner; + private final HostResourcesCalculator hostResourcesCalculator; private final BooleanFlag dynamicProvisioningEnabled; private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; - DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, - HostProvisioner hostProvisioner, FlagSource flagSource) { + DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, HostProvisioner hostProvisioner, + HostResourcesCalculator hostResourcesCalculator, FlagSource flagSource) { super(nodeRepository, interval); this.hostProvisioner = hostProvisioner; + this.hostResourcesCalculator = hostResourcesCalculator; this.dynamicProvisioningEnabled = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource); this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); } @@ -68,17 +71,14 @@ public class DynamicProvisioningMaintainer extends Maintainer { } void updateProvisioningNodes(NodeList nodes, Mutex lock) { - Map<String, Node> provisionedHostsByHostname = nodes.state(Node.State.provisioned).nodeType(NodeType.host) - .asList().stream() - .collect(Collectors.toMap(Node::hostname, Function.identity())); - - Map<Node, Set<Node>> nodesByProvisionedParent = nodes.asList().stream() - .filter(node -> node.parentHostname().map(provisionedHostsByHostname::containsKey).orElse(false)) + Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant).asList().stream() + .filter(node -> node.parentHostname().isPresent()) .collect(Collectors.groupingBy( - node -> provisionedHostsByHostname.get(node.parentHostname().get()), + node -> node.parentHostname().get(), Collectors.toSet())); - nodesByProvisionedParent.forEach((host, children) -> { + nodes.state(Node.State.provisioned).nodeType(NodeType.host).forEach(host -> { + Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of()); try { List<Node> updatedNodes = hostProvisioner.provision(host, children); nodeRepository().write(updatedNodes, lock); @@ -112,7 +112,7 @@ public class DynamicProvisioningMaintainer extends Maintainer { NodeResources resources = it.next(); removableHosts.stream() .filter(host -> NodePrioritizer.ALLOCATABLE_HOST_STATES.contains(host.state())) - .filter(host -> host.flavor().resources().satisfies(resources)) + .filter(host -> hostResourcesCalculator.availableCapacityOf(host.flavor().name(), host.flavor().resources()).satisfies(resources)) .min(Comparator.comparingInt(n -> n.flavor().cost())) .ifPresent(host -> { removableHosts.remove(host); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 0f363991310..e2b70608d58 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -119,14 +119,12 @@ public class LoadBalancerExpirer extends Maintainer { /** Apply operation to all load balancers that exist in given state, while holding lock */ private void withLoadBalancersIn(LoadBalancer.State state, Consumer<LoadBalancer> operation) { - try (var legacyLock = db.lockLoadBalancers()) { - for (var id : db.readLoadBalancerIds()) { - try (var lock = db.lockLoadBalancers(id.application())) { - var loadBalancer = db.readLoadBalancer(id); - if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop - if (loadBalancer.get().state() != state) continue; // Wrong state - operation.accept(loadBalancer.get()); - } + for (var id : db.readLoadBalancerIds()) { + try (var lock = db.lockLoadBalancers(id.application())) { + var loadBalancer = db.readLoadBalancer(id); + if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop + if (loadBalancer.get().state() != state) continue; // Wrong state + operation.accept(loadBalancer.get()); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index ae8f8b052db..063b5ad2c2a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -81,7 +81,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService)); dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner -> - new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource)); + new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, provisionServiceProvider.getHostResourcesCalculator(), flagSource)); capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval); osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 0a8575578ce..f211ea9eac5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -521,11 +521,6 @@ public class CuratorDatabaseClient { transaction.commit(); } - // TODO(mpolden): Remove this and all usages once migration to per-application lock is complete - public Lock lockLoadBalancers() { - return lock(lockRoot.append("loadBalancersLock"), defaultLockTimeout); - } - public Lock lockLoadBalancers(ApplicationId application) { return lock(lockRoot.append("loadBalancersLock2").append(application.serializedForm()), defaultLockTimeout); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index a609103ac89..fb76dc54d1a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -26,12 +26,6 @@ public class DockerHostCapacity { this.hostResourcesCalculator = Objects.requireNonNull(hostResourcesCalculator, "hostResourcesCalculator must be non-null"); } - /** Returns the allocation skew of this host */ - public double skew(Node host) { - NodeResources free = freeCapacityOf(host, false); - return Node.skew(host.flavor().resources(), free); - } - int compareWithoutInactive(Node hostA, Node hostB) { int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true)); if (result != 0) return result; @@ -72,7 +66,7 @@ public class DockerHostCapacity { NodeResources freeCapacityOf(Node host, boolean excludeInactive) { // Only hosts have free capacity if (!host.type().canRun(NodeType.tenant)) return new NodeResources(0, 0, 0, 0); - NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().resources()); + NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().name(), host.flavor().resources()); return allNodes.childrenOf(host).asList().stream() .filter(node -> !(excludeInactive && isInactiveOrRetired(node))) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index 05915b82bae..b5c4478cd5a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -30,7 +30,7 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { public static class NoopHostResourcesCalculator implements HostResourcesCalculator { @Override - public NodeResources availableCapacityOf(NodeResources hostResources) { + public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) { return hostResources; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java index c5808a53837..a5570dbf169 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -9,6 +9,6 @@ import com.yahoo.config.provision.NodeResources; public interface HostResourcesCalculator { /** Calculates the resources that are reserved for host level processes and returns the remainder. */ - NodeResources availableCapacityOf(NodeResources hostResources); + NodeResources availableCapacityOf(String flavorName, NodeResources hostResources); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index d4d5b46dfdf..26f8cffa519 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -51,12 +51,10 @@ public class LoadBalancerProvisioner { this.db = nodeRepository.database(); this.service = service; // Read and write all load balancers to make sure they are stored in the latest version of the serialization format - try (var legacyLock = db.lockLoadBalancers()) { - for (var id : db.readLoadBalancerIds()) { - try (var lock = db.lockLoadBalancers(id.application())) { - var loadBalancer = db.readLoadBalancer(id); - loadBalancer.ifPresent(db::writeLoadBalancer); - } + for (var id : db.readLoadBalancerIds()) { + try (var lock = db.lockLoadBalancers(id.application())) { + var loadBalancer = db.readLoadBalancer(id); + loadBalancer.ifPresent(db::writeLoadBalancer); } } } @@ -75,10 +73,8 @@ public class LoadBalancerProvisioner { if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type if (application.instance().isTester()) return; // Do not provision for tester instances - try (var legacyLock = db.lockLoadBalancers()) { - try (var lock = db.lockLoadBalancers(application)) { - provision(application, cluster.id(), false, lock); - } + try (var lock = db.lockLoadBalancers(application)) { + provision(application, cluster.id(), false, lock); } } @@ -94,17 +90,15 @@ public class LoadBalancerProvisioner { */ public void activate(ApplicationId application, Set<ClusterSpec> clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { - try (var legacyLock = db.lockLoadBalancers()) { - try (var lock = db.lockLoadBalancers(application)) { - var containerClusters = containerClusterOf(clusters); - for (var clusterId : containerClusters) { - // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(application, clusterId, true, lock); - } - // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); - deactivate(surplusLoadBalancers, transaction); + try (var lock = db.lockLoadBalancers(application)) { + var containerClusters = containerClusterOf(clusters); + for (var clusterId : containerClusters) { + // Provision again to ensure that load balancer instance is re-configured with correct nodes + provision(application, clusterId, true, lock); } + // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed + var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); + deactivate(surplusLoadBalancers, transaction); } } @@ -114,10 +108,8 @@ public class LoadBalancerProvisioner { */ public void deactivate(ApplicationId application, NestedTransaction transaction) { try (var applicationLock = nodeRepository.lock(application)) { - try (var legacyLock = db.lockLoadBalancers()) { - try (var lock = db.lockLoadBalancers(application)) { - deactivate(nodeRepository.loadBalancers(application).asList(), transaction); - } + try (var lock = db.lockLoadBalancers(application)) { + deactivate(nodeRepository.loadBalancers(application).asList(), transaction); } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 9dd8de6d306..36f876fb6bb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -26,9 +26,11 @@ import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; +import org.junit.Before; import org.junit.Test; import java.time.Duration; @@ -49,6 +51,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -64,18 +67,23 @@ public class DynamicProvisioningMaintainerTest { private final HostProvisionerTester tester = new HostProvisionerTester(); private final HostProvisioner hostProvisioner = mock(HostProvisioner.class); + private final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource() .withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true) .withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(), PreprovisionCapacity.class); private final DynamicProvisioningMaintainer maintainer = new DynamicProvisioningMaintainer( - tester.nodeRepository, Duration.ofDays(1), hostProvisioner, flagSource); + tester.nodeRepository, Duration.ofDays(1), hostProvisioner, hostResourcesCalculator, flagSource); @Test public void delegates_to_host_provisioner_and_writes_back_result() { addNodes(); + Node host3 = tester.nodeRepository.getNode("host3").orElseThrow(); Node host4 = tester.nodeRepository.getNode("host4").orElseThrow(); Node host41 = tester.nodeRepository.getNode("host4-1").orElseThrow(); - assertTrue(Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + assertTrue(Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + + Node host3new = host3.with(host3.ipConfig().with(Set.of("::5"))); + when(hostProvisioner.provision(eq(host3), eq(Set.of()))).thenReturn(List.of(host3new)); Node host4new = host4.with(host4.ipConfig().with(Set.of("::2"))); Node host41new = host41.with(host4.ipConfig().with(Set.of("::4", "10.0.0.1"))); @@ -83,8 +91,10 @@ public class DynamicProvisioningMaintainerTest { maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {}); verify(hostProvisioner).provision(eq(host4), eq(Set.of(host41))); + verify(hostProvisioner).provision(eq(host3), eq(Set.of())); verifyNoMoreInteractions(hostProvisioner); + assertEquals(Optional.of(host3new), tester.nodeRepository.getNode("host3")); assertEquals(Optional.of(host4new), tester.nodeRepository.getNode("host4")); assertEquals(Optional.of(host41new), tester.nodeRepository.getNode("host4-1")); } @@ -127,7 +137,7 @@ public class DynamicProvisioningMaintainerTest { @Test public void provision_deficit_and_deprovision_excess() { - flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(1, 3, 2, 1), new PreprovisionCapacity(2, 3, 2, 2)), PreprovisionCapacity.class); + flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(2, 4, 8, 1), new PreprovisionCapacity(2, 3, 2, 2)), PreprovisionCapacity.class); addNodes(); maintainer.convergeToCapacity(tester.nodeRepository.list()); @@ -150,6 +160,15 @@ public class DynamicProvisioningMaintainerTest { verifyNoMoreInteractions(hostProvisioner); } + @Before + public void setup() { + doAnswer(invocation -> { + String flavorName = invocation.getArgument(0, String.class); + if ("default".equals(flavorName)) return new NodeResources(2, 4, 8, 1); + return invocation.getArguments()[1]; + }).when(hostResourcesCalculator).availableCapacityOf(any(), any()); + } + public void addNodes() { List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)), createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), @@ -157,6 +176,7 @@ public class DynamicProvisioningMaintainerTest { createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)), createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()), + createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index d1a330a3bd6..d0c678bdf45 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -152,7 +152,7 @@ public class RebalancerTest { private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { @Override - public NodeResources availableCapacityOf(NodeResources hostResources) { + public NodeResources availableCapacityOf(String flavorName, NodeResources hostResources) { return hostResources; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java index 7d9ac230771..ba9a04573e1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -37,7 +37,7 @@ public class DockerHostCapacityTest { @Before public void setup() { - doAnswer(invocation -> invocation.getArguments()[0]).when(hostResourcesCalculator).availableCapacityOf(any()); + doAnswer(invocation -> invocation.getArguments()[1]).when(hostResourcesCalculator).availableCapacityOf(any(), any()); // Create flavors NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2"); @@ -95,9 +95,9 @@ public class DockerHostCapacityTest { capacity.freeCapacityOf(host3, false)); doAnswer(invocation -> { - NodeResources totalHostResources = (NodeResources) invocation.getArguments()[0]; + NodeResources totalHostResources = (NodeResources) invocation.getArguments()[1]; return totalHostResources.subtract(new NodeResources(1, 2, 3, 0.5, NodeResources.DiskSpeed.any)); - }).when(hostResourcesCalculator).availableCapacityOf(any()); + }).when(hostResourcesCalculator).availableCapacityOf(any(), any()); assertEquals(new NodeResources(4, 2, 5, 1.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), capacity.freeCapacityOf(host1, false)); diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java index 3f14579dfba..2a582020bb3 100644 --- a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java +++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java @@ -18,6 +18,7 @@ public class GetHostResponse { public static final String FIELD_NAME_HOSTNAME = "hostname"; public static final String FIELD_NAME_STATE = "state"; + public static final String FIELD_NAME_SUSPENDED_SINCE = "suspendedSince"; public static final String FIELD_NAME_APPLICATION_URL = "applicationUrl"; public static final String FIELD_NAME_SERVICES = "services"; @@ -25,15 +26,18 @@ public class GetHostResponse { private final String state; private final String applicationUrl; private final List<HostService> services; + private final String suspendedSince; @JsonCreator public GetHostResponse( @JsonProperty(FIELD_NAME_HOSTNAME) String hostname, @JsonProperty(FIELD_NAME_STATE) String state, + @JsonProperty(FIELD_NAME_SUSPENDED_SINCE) String suspendedSince, @JsonProperty(FIELD_NAME_APPLICATION_URL) String applicationUrl, @JsonProperty(FIELD_NAME_SERVICES) List<HostService> services) { this.hostname = hostname; this.state = state; + this.suspendedSince = suspendedSince; this.applicationUrl = applicationUrl; this.services = services; } @@ -48,6 +52,11 @@ public class GetHostResponse { return state; } + @JsonProperty(FIELD_NAME_SUSPENDED_SINCE) + public String suspendedSince() { + return suspendedSince; + } + @JsonProperty(FIELD_NAME_APPLICATION_URL) public String applicationUrl() { return applicationUrl; @@ -65,12 +74,13 @@ public class GetHostResponse { GetHostResponse that = (GetHostResponse) o; return Objects.equals(hostname, that.hostname) && Objects.equals(state, that.state) && + Objects.equals(suspendedSince, that.suspendedSince) && Objects.equals(applicationUrl, that.applicationUrl) && Objects.equals(services, that.services); } @Override public int hashCode() { - return Objects.hash(hostname, state, applicationUrl, services); + return Objects.hash(hostname, state, suspendedSince, applicationUrl, services); } } diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java new file mode 100644 index 00000000000..39c93291bad --- /dev/null +++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java @@ -0,0 +1,38 @@ +package com.yahoo.vespa.orchestrator.restapi.wire; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.Objects; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WireHostInfo { + private static final String HOST_STATUS_FIELD = "hostStatus"; + private static final String SUSPENDED_SINCE_FIELD = "suspendedSince"; + + private final String hostStatus; + private final String suspendedSinceUtcOrNull; + + /** + * @param hostStatus The host status, e.g. NO_REMARKS. + * @param suspendedSinceUtcOrNull The time the host was suspended in the format + * {@link Instant#toString()}, or null if not suspended + * (NO_REMARKS). + */ + @JsonCreator + public WireHostInfo(@JsonProperty(HOST_STATUS_FIELD) String hostStatus, + @JsonProperty(SUSPENDED_SINCE_FIELD) String suspendedSinceUtcOrNull) { + this.hostStatus = Objects.requireNonNull(hostStatus); + this.suspendedSinceUtcOrNull = suspendedSinceUtcOrNull; + } + + @JsonProperty(HOST_STATUS_FIELD) + public String hostStatus() { return hostStatus; } + + @JsonProperty(SUSPENDED_SINCE_FIELD) + public String getSuspendedSinceUtcOrNull() { return suspendedSinceUtcOrNull; } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java index bda9505d72b..8f5f00af7a0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java @@ -4,23 +4,23 @@ package com.yahoo.vespa.orchestrator; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceInstance; -import com.yahoo.vespa.orchestrator.status.HostStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import java.util.List; public class Host { private final HostName hostName; - private final HostStatus hostStatus; + private final HostInfo hostInfo; private final ApplicationInstanceReference applicationInstanceReference; private final List<ServiceInstance> serviceInstances; public Host(HostName hostName, - HostStatus hostStatus, + HostInfo hostInfo, ApplicationInstanceReference applicationInstanceReference, List<ServiceInstance> serviceInstances) { this.hostName = hostName; - this.hostStatus = hostStatus; + this.hostInfo = hostInfo; this.applicationInstanceReference = applicationInstanceReference; this.serviceInstances = serviceInstances; } @@ -29,8 +29,8 @@ public class Host { return hostName; } - public HostStatus getHostStatus() { - return hostStatus; + public HostInfo getHostInfo() { + return hostInfo; } public ApplicationInstanceReference getApplicationInstanceReference() { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index 414548f8bdc..fbe6864274c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -114,9 +114,10 @@ public class OrchestratorImpl implements Orchestrator { .filter(serviceInstance -> hostName.equals(serviceInstance.hostName())) .collect(Collectors.toList()); + HostInfo hostInfo = statusService.getHostInfo(applicationInstance.reference(), hostName); HostStatus hostStatus = getNodeStatus(applicationInstance.reference(), hostName); - return new Host(hostName, hostStatus, applicationInstance.reference(), serviceInstances); + return new Host(hostName, hostInfo, applicationInstance.reference(), serviceInstances); } @Override diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index 4bb93ffa3cb..fc5c5eb5004 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -31,6 +31,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import java.net.URI; +import java.time.Instant; import java.util.List; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -72,7 +73,8 @@ public class HostResource implements HostApi { return new GetHostResponse( host.getHostName().s(), - host.getHostStatus().name(), + host.getHostInfo().status().name(), + host.getHostInfo().suspendedSince().map(Instant::toString).orElse(null), applicationUri.toString(), hostServices); } catch (UncheckedTimeoutException e) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java index 7b8a74d7fe2..fbb8f445db0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java @@ -14,6 +14,8 @@ import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.InstanceLookupService; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse; +import com.yahoo.vespa.orchestrator.restapi.wire.WireHostInfo; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.StatusService; import com.yahoo.vespa.service.manager.MonitorManager; @@ -29,9 +31,10 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance; @@ -82,13 +85,23 @@ public class InstanceResource { .orElseThrow(() -> new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build())); HostInfos hostInfos = statusService.getHostInfosByApplicationResolver().apply(applicationInstance.reference()); - Map<HostName, String> hostStatusMap = getHostsUsedByApplicationInstance(applicationInstance) - .stream() - .collect(Collectors.toMap(hostName -> hostName, - hostName -> hostInfos.getOrNoRemarks(hostName).status().asString())); + TreeMap<HostName, WireHostInfo> hostStatusMap = + getHostsUsedByApplicationInstance(applicationInstance) + .stream() + .collect(Collectors.toMap( + hostName -> hostName, + hostName -> hostInfoToWire(hostInfos.getOrNoRemarks(hostName)), + (u, v) -> { throw new IllegalStateException(); }, + TreeMap::new)); return InstanceStatusResponse.create(applicationInstance, hostStatusMap); } + private WireHostInfo hostInfoToWire(HostInfo hostInfo) { + String hostStatusString = hostInfo.status().asString(); + String suspendedSinceUtcOrNull = hostInfo.suspendedSince().map(Instant::toString).orElse(null); + return new WireHostInfo(hostStatusString, suspendedSinceUtcOrNull); + } + @GET @Path("/{instanceId}/slobrok") @Produces(MediaType.APPLICATION_JSON) diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java index 068423f7d24..313c73e5c68 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java @@ -4,10 +4,12 @@ package com.yahoo.vespa.orchestrator.resources; import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.orchestrator.status.HostInfo; +import com.yahoo.vespa.orchestrator.restapi.wire.WireHostInfo; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; +import java.util.stream.Collectors; /* * @author andreer @@ -15,16 +17,16 @@ import java.util.Objects; public class InstanceStatusResponse { private final ApplicationInstance applicationInstance; - private final Map<HostName, String> hostStates; + private final TreeMap<HostName, WireHostInfo> hostInfos; - private InstanceStatusResponse(ApplicationInstance applicationInstance, Map<HostName, String> hostStates) { + private InstanceStatusResponse(ApplicationInstance applicationInstance, TreeMap<HostName, WireHostInfo> hostInfos) { this.applicationInstance = applicationInstance; - this.hostStates = hostStates; + this.hostInfos = hostInfos; } public static InstanceStatusResponse create( ApplicationInstance applicationInstance, - Map<HostName, String> hostStates) { + TreeMap<HostName, WireHostInfo> hostStates) { return new InstanceStatusResponse(applicationInstance, hostStates); } @@ -35,14 +37,24 @@ public class InstanceStatusResponse { @JsonProperty("hostStates") public Map<HostName, String> hostStates() { - return hostStates; + // TODO: Remove this once all clients have been moved to hostStatus. + return hostInfos.entrySet().stream() + .collect(Collectors.toMap( + entry -> entry.getKey(), + entry -> entry.getValue().hostStatus() + )); + } + + @JsonProperty("hostInfos") + public TreeMap<HostName, WireHostInfo> hostInfos() { + return hostInfos; } @Override public String toString() { return "InstanceStatusResponse{" + "applicationInstance=" + applicationInstance + - ", hostStates=" + hostStates + + ", hostInfos=" + hostInfos + '}'; } @@ -52,11 +64,11 @@ public class InstanceStatusResponse { if (o == null || getClass() != o.getClass()) return false; InstanceStatusResponse that = (InstanceStatusResponse) o; return Objects.equals(applicationInstance, that.applicationInstance) && - Objects.equals(hostStates, that.hostStates); + Objects.equals(hostInfos, that.hostInfos); } @Override public int hashCode() { - return Objects.hash(applicationInstance, hostStates); + return Objects.hash(applicationInstance, hostInfos); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index a2c99b86ae2..77ec824da54 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -435,7 +435,8 @@ public class OrchestratorImplTest { Host host = orchestrator.getHost(hostName); assertEquals(reference, host.getApplicationInstanceReference()); assertEquals(hostName, host.getHostName()); - assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, host.getHostStatus()); + assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, host.getHostInfo().status()); + assertTrue(host.getHostInfo().suspendedSince().isPresent()); assertEquals(2, host.getServiceInstances().size()); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index b19f96a5867..dc26c1a3770 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -37,6 +37,7 @@ import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; @@ -348,7 +349,7 @@ public class HostResourceTest { Host host = new Host( hostName, - HostStatus.ALLOWED_TO_BE_DOWN, + HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH), new ApplicationInstanceReference( new TenantId("tenantId"), new ApplicationInstanceId("applicationId")), @@ -358,6 +359,7 @@ public class HostResourceTest { assertEquals("https://foo.com/bar", response.applicationUrl()); assertEquals("hostname", response.hostname()); assertEquals("ALLOWED_TO_BE_DOWN", response.state()); + assertEquals("1970-01-01T00:00:00Z", response.suspendedSince()); assertEquals(1, response.services().size()); assertEquals("clusterId", response.services().get(0).clusterId); assertEquals("configId", response.services().get(0).configId); diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp index 7f6bd835cd5..e35a6a74bde 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp @@ -6,9 +6,7 @@ #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/fieldvalue/document.h> -namespace storage { - -namespace spi { +namespace storage::spi { UpdateResult AbstractPersistenceProvider::update(const Bucket& bucket, Timestamp ts, @@ -66,5 +64,3 @@ AbstractPersistenceProvider::move(const Bucket& source, PartitionId target, Cont } } - -} diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h index 27197c2adb8..557a9ec2edd 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/persistence/spi/persistenceprovider.h> +#include "persistenceprovider.h" namespace storage::spi { diff --git a/persistence/src/vespa/persistence/spi/bucket.cpp b/persistence/src/vespa/persistence/spi/bucket.cpp index 9265f995a45..ef94519cdb0 100644 --- a/persistence/src/vespa/persistence/spi/bucket.cpp +++ b/persistence/src/vespa/persistence/spi/bucket.cpp @@ -4,8 +4,7 @@ #include <ostream> #include <vespa/vespalib/stllike/asciistream.h> -namespace storage { -namespace spi { +namespace storage::spi { vespalib::string Bucket::toString() const { @@ -30,5 +29,4 @@ operator<<(std::ostream& os, const Bucket& bucket) { return os << bucket.toString(); } -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/bucket.h b/persistence/src/vespa/persistence/spi/bucket.h index 54760304694..874074d7e24 100644 --- a/persistence/src/vespa/persistence/spi/bucket.h +++ b/persistence/src/vespa/persistence/spi/bucket.h @@ -17,8 +17,7 @@ #include <persistence/spi/types.h> #include <vespa/document/bucket/bucket.h> -namespace storage { -namespace spi { +namespace storage::spi { class Bucket { document::Bucket _bucket; @@ -47,6 +46,4 @@ public: vespalib::asciistream& operator<<(vespalib::asciistream& out, const Bucket& bucket); std::ostream& operator<<(std::ostream& out, const Bucket& bucket); -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/bucketinfo.cpp b/persistence/src/vespa/persistence/spi/bucketinfo.cpp index e60d6152058..5bef59a65f1 100644 --- a/persistence/src/vespa/persistence/spi/bucketinfo.cpp +++ b/persistence/src/vespa/persistence/spi/bucketinfo.cpp @@ -3,8 +3,7 @@ #include "bucketinfo.h" #include <vespa/vespalib/stllike/asciistream.h> -namespace storage { -namespace spi { +namespace storage::spi { BucketInfo::BucketInfo() : _checksum(0), @@ -73,5 +72,4 @@ std::ostream& operator<<(std::ostream& out, const BucketInfo& info) { return out << info.toString(); } -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/bucketinfo.h b/persistence/src/vespa/persistence/spi/bucketinfo.h index fd4605229f8..827aad48d7f 100644 --- a/persistence/src/vespa/persistence/spi/bucketinfo.h +++ b/persistence/src/vespa/persistence/spi/bucketinfo.h @@ -8,9 +8,7 @@ #include <persistence/spi/types.h> -namespace vespalib { - class asciistream; -} +namespace vespalib { class asciistream; } namespace storage::spi { diff --git a/persistence/src/vespa/persistence/spi/clusterstate.cpp b/persistence/src/vespa/persistence/spi/clusterstate.cpp index bc30197978f..567d7ebc1ce 100644 --- a/persistence/src/vespa/persistence/spi/clusterstate.cpp +++ b/persistence/src/vespa/persistence/spi/clusterstate.cpp @@ -12,9 +12,9 @@ namespace storage::spi { ClusterState::ClusterState(const lib::ClusterState& state, uint16_t nodeIndex, const lib::Distribution& distribution) - : _state(new lib::ClusterState(state)), + : _state(std::make_unique<lib::ClusterState>(state)), _nodeIndex(nodeIndex), - _distribution(new lib::Distribution(distribution.serialize())) + _distribution(std::make_unique<lib::Distribution>(distribution.serialize())) { } @@ -26,8 +26,8 @@ void ClusterState::deserialize(vespalib::nbostream& i) { i >> _nodeIndex; i >> distribution; - _state.reset(new lib::ClusterState(clusterState)); - _distribution.reset(new lib::Distribution(distribution)); + _state = std::make_unique<lib::ClusterState>(clusterState); + _distribution = std::make_unique<lib::Distribution>(distribution); } ClusterState::ClusterState(vespalib::nbostream& i) { diff --git a/persistence/src/vespa/persistence/spi/clusterstateimpl.h b/persistence/src/vespa/persistence/spi/clusterstateimpl.h deleted file mode 100644 index 281c37eb9d5..00000000000 --- a/persistence/src/vespa/persistence/spi/clusterstateimpl.h +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/persistence/spi/bucket.h> -#include <vespa/persistence/spi/clusterstate.h> - -namespace storage { - -namespace spi { - -/** - * Used to determine the state of the current node and its buckets. - */ -class ClusterStateImpl : public ClusterState{ -public: - ClusterStateImpl(); - - ClusterStateImpl(const lib::ClusterState& state, - uint16_t nodeIndex, - const lib::Distribution& distribution); - - ClusterStateImpl(vespalib::nbostream& i); - - ClusterStateImpl(const ClusterStateImpl& other); - - ClusterStateImpl& operator=(const ClusterStateImpl& other); - - /** - * Returns true if the given bucket is in the ideal state - * for readiness. - * - * @param b The bucket to check. - */ - bool shouldBeReady(const Bucket& b) const; - - /** - * Returns false if the cluster has been deemed down. This can happen - * if the fleet controller has detected that too many nodes are down - * compared to the complete list of nodes, and deigns the system to be - * unusable. - */ - bool clusterUp() const; - - /** - * Returns false if this node has been set in a state where it should not - * receive external load. - */ - bool nodeUp() const; - - /** - * Returns a serialized form of this object. - */ - void serialize(vespalib::nbostream& o) const; - -private: - std::unique_ptr<lib::ClusterState> _state; - uint16_t _nodeIndex; - std::unique_ptr<lib::Distribution> _distribution; - - void deserialize(vespalib::nbostream&); -}; - -} - -} - diff --git a/persistence/src/vespa/persistence/spi/context.cpp b/persistence/src/vespa/persistence/spi/context.cpp index 5ce34d5d139..429e2fb9d4e 100644 --- a/persistence/src/vespa/persistence/spi/context.cpp +++ b/persistence/src/vespa/persistence/spi/context.cpp @@ -2,8 +2,7 @@ #include "context.h" -namespace storage { -namespace spi { +namespace storage::spi { Context::Context(const LoadType& loadType, Priority pri, int maxTraceLevel) : _loadType(&loadType), @@ -12,7 +11,6 @@ Context::Context(const LoadType& loadType, Priority pri, int maxTraceLevel) _readConsistency(ReadConsistency::STRONG) { } -Context::~Context() { } +Context::~Context() = default; -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/context.h b/persistence/src/vespa/persistence/spi/context.h index ca4c79e3005..8c31439ee75 100644 --- a/persistence/src/vespa/persistence/spi/context.h +++ b/persistence/src/vespa/persistence/spi/context.h @@ -29,13 +29,10 @@ #pragma once -#include <persistence/spi/types.h> -#include <vespa/persistence/spi/read_consistency.h> +#include "read_consistency.h" #include <vespa/vespalib/trace/trace.h> -namespace metrics { - class LoadType; -} +namespace metrics { class LoadType; } namespace storage::spi { @@ -62,10 +59,6 @@ public: const LoadType& getLoadType() const { return *_loadType; } Priority getPriority() const { return _priority; } - int getMaxTraceLevel() const { return _trace.getLevel(); } - void addTrace(const vespalib::TraceNode& traceNode) { - _trace.getRoot().addChild(traceNode); - } /** * A read operation might choose to relax its consistency requirements, diff --git a/persistence/src/vespa/persistence/spi/exceptions.cpp b/persistence/src/vespa/persistence/spi/exceptions.cpp index a1b9d57270c..d17c0f90ca0 100644 --- a/persistence/src/vespa/persistence/spi/exceptions.cpp +++ b/persistence/src/vespa/persistence/spi/exceptions.cpp @@ -2,11 +2,8 @@ #include "exceptions.h" -namespace storage { -namespace spi { +namespace storage::spi { VESPA_IMPLEMENT_EXCEPTION(HandledException, vespalib::Exception); -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/exceptions.h b/persistence/src/vespa/persistence/spi/exceptions.h index 1c434fd8f00..e972e304567 100644 --- a/persistence/src/vespa/persistence/spi/exceptions.h +++ b/persistence/src/vespa/persistence/spi/exceptions.h @@ -3,8 +3,7 @@ #include <vespa/vespalib/util/exceptions.h> -namespace storage { -namespace spi { +namespace storage::spi { /** * Exception used where the cause has already been reported to the user, so @@ -16,6 +15,4 @@ namespace spi { */ VESPA_DEFINE_EXCEPTION(HandledException, vespalib::Exception); -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/matcher.h b/persistence/src/vespa/persistence/spi/matcher.h index 02dc6db2261..bf989530f35 100644 --- a/persistence/src/vespa/persistence/spi/matcher.h +++ b/persistence/src/vespa/persistence/spi/matcher.h @@ -8,12 +8,11 @@ #pragma once -#include <vespa/persistence/spi/docentry.h> +#include "docentry.h" #include <persistence/spi/documentsubset.h> #include <persistence/spi/types.h> -namespace storage { -namespace spi { +namespace storage::spi { class Matcher { DocumentSubset _subset; @@ -37,6 +36,4 @@ struct AllMatcher : public Matcher { bool match(const DocEntry&) const { return true; } }; -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/partitionstate.cpp b/persistence/src/vespa/persistence/spi/partitionstate.cpp index 123a82829ef..7cc42742019 100644 --- a/persistence/src/vespa/persistence/spi/partitionstate.cpp +++ b/persistence/src/vespa/persistence/spi/partitionstate.cpp @@ -4,8 +4,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> -namespace storage { -namespace spi { +namespace storage::spi { PartitionState::PartitionState() : _state(UP), @@ -21,7 +20,7 @@ PartitionStateList::PartitionStateList(PartitionId::Type partitionCount) : _states(partitionCount) { } -PartitionStateList::~PartitionStateList() { } +PartitionStateList::~PartitionStateList() = default; PartitionState& PartitionStateList::operator[](PartitionId::Type index) @@ -34,5 +33,4 @@ PartitionStateList::operator[](PartitionId::Type index) return _states[index]; } -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/partitionstate.h b/persistence/src/vespa/persistence/spi/partitionstate.h index 6296a70b2c1..e5ce24abbe6 100644 --- a/persistence/src/vespa/persistence/spi/partitionstate.h +++ b/persistence/src/vespa/persistence/spi/partitionstate.h @@ -16,8 +16,7 @@ #include <persistence/spi/types.h> -namespace storage { -namespace spi { +namespace storage::spi { struct PartitionState { enum State { UP, DOWN }; @@ -49,6 +48,4 @@ public: PartitionId size() const { return PartitionId(_states.size()); } }; -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp index ec71928baba..61d141c0229 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp @@ -2,11 +2,8 @@ #include "persistenceprovider.h" -namespace storage { -namespace spi { +namespace storage::spi { -PersistenceProvider::~PersistenceProvider() { } - -} // spi -} // storage +PersistenceProvider::~PersistenceProvider() = default; +} diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index 96b3d385b87..857630e2606 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -11,9 +11,7 @@ #include "selection.h" #include "clusterstate.h" -namespace document { - class FieldSet; -} +namespace document { class FieldSet; } namespace storage::spi { diff --git a/persistence/src/vespa/persistence/spi/providerfactory.h b/persistence/src/vespa/persistence/spi/providerfactory.h index 6b4f0b20ae9..8be143851dd 100644 --- a/persistence/src/vespa/persistence/spi/providerfactory.h +++ b/persistence/src/vespa/persistence/spi/providerfactory.h @@ -8,14 +8,11 @@ #pragma once -#include <vespa/persistence/spi/persistenceprovider.h> +#include "persistenceprovider.h" -namespace document { - class DocumentTypeRepo; -} +namespace document { class DocumentTypeRepo; } -namespace storage { -namespace spi { +namespace storage::spi { struct ProviderFactory { virtual ~ProviderFactory() {} diff --git a/persistence/src/vespa/persistence/spi/read_consistency.cpp b/persistence/src/vespa/persistence/spi/read_consistency.cpp index cad15a5263d..c3b55e5a261 100644 --- a/persistence/src/vespa/persistence/spi/read_consistency.cpp +++ b/persistence/src/vespa/persistence/spi/read_consistency.cpp @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "read_consistency.h" -#include <iostream> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".persistence.spi.read_consistency"); -namespace storage { -namespace spi { +namespace storage::spi { std::ostream& operator<<(std::ostream& os, ReadConsistency consistency) @@ -24,6 +23,4 @@ operator<<(std::ostream& os, ReadConsistency consistency) return os; } -} // spi -} // storage - +} diff --git a/persistence/src/vespa/persistence/spi/read_consistency.h b/persistence/src/vespa/persistence/spi/read_consistency.h index 507e05027cc..d90c43af407 100644 --- a/persistence/src/vespa/persistence/spi/read_consistency.h +++ b/persistence/src/vespa/persistence/spi/read_consistency.h @@ -2,10 +2,9 @@ #pragma once #include <iosfwd> -#include <stdint.h> +#include <cstdint> -namespace storage { -namespace spi { +namespace storage::spi { enum class ReadConsistency : uint8_t { /** @@ -28,9 +27,7 @@ enum class ReadConsistency : uint8_t { WEAK }; -std::ostream& -operator<<(std::ostream&, ReadConsistency); +std::ostream& operator<<(std::ostream&, ReadConsistency); -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/result.cpp b/persistence/src/vespa/persistence/spi/result.cpp index 024f5595102..f5131f08b1f 100644 --- a/persistence/src/vespa/persistence/spi/result.cpp +++ b/persistence/src/vespa/persistence/spi/result.cpp @@ -9,7 +9,7 @@ namespace storage::spi { Result::Result(const Result &) = default; Result & Result::operator = (const Result &) = default; -Result::~Result() { } +Result::~Result() = default; vespalib::string Result::toString() const { @@ -33,10 +33,10 @@ GetResult::GetResult(Document::UP doc, Timestamp timestamp) _doc(std::move(doc)) { } -GetResult::~GetResult() { } -BucketIdListResult::~BucketIdListResult() { } +GetResult::~GetResult() = default; +BucketIdListResult::~BucketIdListResult() = default; -IterateResult::~IterateResult() { } +IterateResult::~IterateResult() = default; } diff --git a/persistence/src/vespa/persistence/spi/selection.cpp b/persistence/src/vespa/persistence/spi/selection.cpp index 0cd50446488..c7ed98b9d92 100644 --- a/persistence/src/vespa/persistence/spi/selection.cpp +++ b/persistence/src/vespa/persistence/spi/selection.cpp @@ -2,8 +2,7 @@ #include "selection.h" -namespace storage { -namespace spi { +namespace storage::spi { Selection::Selection(const DocumentSelection& docSel) : _documentSelection(docSel), @@ -12,8 +11,7 @@ Selection::Selection(const DocumentSelection& docSel) _timestampSubset() { } -Selection::~Selection() { } +Selection::~Selection() = default; -} // spi -} // storage +} diff --git a/persistence/src/vespa/persistence/spi/selection.h b/persistence/src/vespa/persistence/spi/selection.h index 5c562cabd34..0f809cf1641 100644 --- a/persistence/src/vespa/persistence/spi/selection.h +++ b/persistence/src/vespa/persistence/spi/selection.h @@ -10,12 +10,7 @@ #include "documentselection.h" -namespace storage { -namespace spi { - -class MetaData { - Timestamp timestamp; -}; +namespace storage::spi { class Selection { public: @@ -67,13 +62,6 @@ public: Timestamp getFromTimestamp() const { return _fromTimestamp; } Timestamp getToTimestamp() const { return _toTimestamp; } - - /** - * Regular usage. - */ - bool match(const Document& doc, const MetaData& metaData) const; }; -} // spi -} // storage - +} diff --git a/persistencetypes/src/persistence/spi/types.cpp b/persistencetypes/src/persistence/spi/types.cpp index 00aa95b9707..e746f02e869 100644 --- a/persistencetypes/src/persistence/spi/types.cpp +++ b/persistencetypes/src/persistence/spi/types.cpp @@ -1,11 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "types.h" - #include <vespa/vespalib/objects/nbostream.h> -namespace storage { - -namespace spi { +namespace storage::spi { DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(NodeIndex); DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(PartitionId); @@ -14,5 +11,3 @@ DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(Timestamp); DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(BucketChecksum); } - -} diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp index da5a8b31671..76dcf53c51d 100644 --- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp @@ -10,6 +10,7 @@ #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/searchlib/attribute/singlenumericattribute.hpp> #include <vespa/log/log.h> LOG_SETUP("attributes_state_explorer_test"); diff --git a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp index ef12b694187..35590cc68f6 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp @@ -46,23 +46,31 @@ DummyPersistenceHandler::SP handler_c(std::make_shared<DummyPersistenceHandler>( DummyPersistenceHandler::SP handler_a_new(std::make_shared<DummyPersistenceHandler>()); + +void +assertHandler(const IPersistenceHandler::SP & lhs, const IPersistenceHandler * rhs) +{ + EXPECT_EQUAL(lhs.get(), rhs); +} + void assertHandler(const IPersistenceHandler::SP &lhs, const IPersistenceHandler::SP &rhs) { EXPECT_EQUAL(lhs.get(), rhs.get()); } +template <typename T> void -assertNullHandler(const IPersistenceHandler::SP &handler) +assertNullHandler(const T & handler) { - EXPECT_TRUE(handler.get() == nullptr); + EXPECT_TRUE(! handler); } void -assertSnapshot(const std::vector<IPersistenceHandler::SP> &exp, const HandlerSnapshot::UP &snapshot) +assertSnapshot(const std::vector<IPersistenceHandler::SP> &exp, HandlerSnapshot snapshot) { - EXPECT_EQUAL(exp.size(), snapshot->size()); - auto &sequence = snapshot->handlers(); + EXPECT_EQUAL(exp.size(), snapshot.size()); + auto &sequence = snapshot.handlers(); for (size_t i = 0; i < exp.size() && sequence.valid(); ++i, sequence.next()) { EXPECT_EQUAL(exp[i].get(), sequence.get()); } diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp index 3ced0f509b5..e80543368d4 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp @@ -119,6 +119,19 @@ BucketDB::cachedGet(const BucketId &bucketId) const return get(bucketId); } +storage::spi::BucketInfo +BucketDB::cachedGetBucketInfo(const BucketId &bucketId) const +{ + if (isCachedBucket(bucketId)) { + return _cachedBucketState; + } + auto itr = _map.find(bucketId); + if (itr != _map.end()) { + return itr->second; + } + return BucketState(); +} + bool BucketDB::hasBucket(const BucketId &bucketId) const { diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h index e64cc53d4a2..05388931e20 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h @@ -51,6 +51,7 @@ public: void cacheBucket(const BucketId &bucketId); void uncacheBucket(); bool isCachedBucket(const BucketId &bucketId) const; + storage::spi::BucketInfo cachedGetBucketInfo(const BucketId &bucketId) const; BucketState cachedGet(const BucketId &bucketId) const; bool hasBucket(const BucketId &bucketId) const; void getBuckets(BucketId::List & buckets) const; diff --git a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp index bddbfed371f..3fb32a9d1e0 100644 --- a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp +++ b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp @@ -5,7 +5,6 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/sequence.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/stllike/hash_map.h> #include <map> #include <vector> @@ -18,9 +17,8 @@ namespace proton { template <typename T> class HandlerMap { private: - typedef typename std::shared_ptr<T> HandlerSP; - typedef std::map<DocTypeName, HandlerSP> StdMap; - typedef typename StdMap::iterator MapIterator; + using HandlerSP = typename std::shared_ptr<T>; + using StdMap = std::map<DocTypeName, HandlerSP>; StdMap _handlers; @@ -40,8 +38,7 @@ public: size_t _offset; public: - typedef std::unique_ptr<Snapshot> UP; - + Snapshot() : _handlers(), _offset(0) { } Snapshot(const StdMap &map) : _handlers(), _offset(0) { _handlers.reserve(map.size()); for (auto itr : map) { @@ -49,6 +46,11 @@ public: } } Snapshot(std::vector<HandlerSP> &&handlers) : _handlers(std::move(handlers)), _offset(0) {} + Snapshot(Snapshot &&) noexcept = default; + Snapshot & operator = (Snapshot &&) noexcept = default; + Snapshot(const Snapshot &) = delete; + Snapshot & operator = (const Snapshot &) = delete; + bool valid() const override { return (_offset < _handlers.size()); } T *get() const override { return _handlers[_offset].get(); } HandlerSP getSP() const { return _handlers[_offset]; } @@ -64,11 +66,7 @@ public: /** * Constructs a new instance of this class. */ - HandlerMap() - : _handlers() - { - // empty - } + HandlerMap() = default; /** * Registers a new handler for the given document type. If another handler @@ -83,7 +81,7 @@ public: putHandler(const DocTypeName &docTypeNameVer, const HandlerSP &handler) { - if (handler.get() == NULL) { + if ( ! handler) { throw vespalib::IllegalArgumentException(vespalib::make_string( "Handler is null for docType '%s'", docTypeNameVer.toString().c_str())); @@ -115,6 +113,20 @@ public: return HandlerSP(); } + /** + * Returns the handler for the given document type. If no handler was + * registered, this method returns a null pointer. + * + * @param docType The document type whose handler to return. + * @return The registered handler, if any. + */ + T * + getHandlerPtr(const DocTypeName &docTypeNameVer) const + { + const_iterator it = _handlers.find(docTypeNameVer); + return (it != _handlers.end()) ? it->second.get() : nullptr; + } + bool hasHandler(const HandlerSP &handler) const { bool found = false; for (const auto &kv : _handlers) { @@ -148,11 +160,7 @@ public: /** * Clear all handlers. */ - void - clear() - { - _handlers.clear(); - } + void clear() { _handlers.clear(); } /** * Create a snapshot of the handlers currently contained in this @@ -161,68 +169,15 @@ public: * * @return handler sequence **/ - std::unique_ptr<Snapshot> - snapshot() const - { - return std::unique_ptr<Snapshot>(new Snapshot(_handlers)); - } + Snapshot snapshot() const { return Snapshot(_handlers); } // we want to use snapshots rather than direct iteration to reduce locking; // the below functions should be deprecated when possible. - /** - * Returns a bidirectional iterator that points at the first element of the - * sequence (or just beyond the end of an empty sequence). - * - * @return The beginning of this map. - */ - iterator - begin() - { - return _handlers.begin(); - } - - /** - * Returns a const bidirectional iterator that points at the first element - * of the sequence (or just beyond the end of an empty sequence). - * - * @return The beginning of this map. - */ - const_iterator - begin() const - { - return _handlers.begin(); - } - - /** - * Returns a bidirectional iterator that points just beyond the end of the - * sequence. - * - * @return The end of this map. - */ - iterator - end() - { - return _handlers.end(); - } - - /** - * Returns a const bidirectional iterator that points just beyond the end of - * the sequence. - * - * @return The end of this map. - */ - const_iterator - end() const - { - return _handlers.end(); - } - - /** - * Returns the number of handlers in this map. - * - * @return the number of handlers. - */ + iterator begin() { return _handlers.begin(); } + const_iterator begin() const { return _handlers.begin(); } + iterator end() { return _handlers.end(); } + const_iterator end() const { return _handlers.end(); } size_t size() const { return _handlers.size(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index 480359f8382..b170f60d71f 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -128,13 +128,13 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req, if (searchHandler) { ret = searchHandler->match(searchHandler, *searchRequest, *threadBundle); } else { - HandlerMap<ISearchHandler>::Snapshot::UP snapshot; + HandlerMap<ISearchHandler>::Snapshot snapshot; { std::lock_guard<std::mutex> guard(_lock); snapshot = _handlers.snapshot(); } - if (snapshot->valid()) { - ISearchHandler::SP handler = snapshot->getSP(); + if (snapshot.valid()) { + ISearchHandler::SP handler = snapshot.getSP(); ret = handler->match(handler, *searchRequest, *threadBundle); // use the first handler } } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp index d34ec8d05a8..bf97bdda29a 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp @@ -106,7 +106,7 @@ void doCleanAttributes(AttributeMetrics &attributes) { auto entries = attributes.release(); - for (const auto entry : entries) { + for (const auto &entry : entries) { attributes.parent()->unregisterMetric(*entry); } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp index 1bcbe4e9683..8175e612bd4 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_iterator.h" +#include <vespa/searchcore/proton/common/cachedselect.h> +#include <vespa/searchcore/proton/common/selectcontext.h> #include <vespa/document/select/gid_filter.h> #include <vespa/document/select/node.h> #include <vespa/document/fieldvalue/document.h> @@ -42,13 +44,6 @@ DocEntry *createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ss } // namespace proton::<unnamed> bool -DocumentIterator::useDocumentSelection() const -{ - return (!_metaOnly && - !_selection.getDocumentSelection().getDocumentSelection().empty()); -} - -bool DocumentIterator::checkMeta(const search::DocumentMetaData &meta) const { if (!meta.valid()) { diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h index 285b3cb2afa..67242e8220f 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h @@ -3,8 +3,6 @@ #pragma once #include "i_document_retriever.h" -#include <vespa/searchcore/proton/common/cachedselect.h> -#include <vespa/searchcore/proton/common/selectcontext.h> #include <vespa/searchlib/common/idocumentmetastore.h> #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/selection.h> @@ -32,10 +30,7 @@ private: storage::spi::IterateResult::List _list; - bool useDocumentSelection() const; bool checkMeta(const search::DocumentMetaData &meta) const; - bool checkDoc(const document::Document &doc) const; - bool checkDoc(const SelectContext &sc) const; void fetchCompleteSource(const IDocumentRetriever & source, storage::spi::IterateResult::List & list); bool isWeakRead() const { return _readConsistency == ReadConsistency::WEAK; } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp index 06969167ab3..5b62a290353 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp @@ -20,15 +20,15 @@ PersistenceHandlerMap::putHandler(document::BucketSpace bucketSpace, return _map[bucketSpace].putHandler(docType, handler); } -IPersistenceHandler::SP +IPersistenceHandler * PersistenceHandlerMap::getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const { auto itr = _map.find(bucketSpace); if (itr != _map.end()) { - return itr->second.getHandler(docType); + return itr->second.getHandlerPtr(docType); } - return IPersistenceHandler::SP(); + return nullptr; } IPersistenceHandler::SP @@ -42,7 +42,7 @@ PersistenceHandlerMap::removeHandler(document::BucketSpace bucketSpace, return IPersistenceHandler::SP(); } -HandlerSnapshot::UP +HandlerSnapshot PersistenceHandlerMap::getHandlerSnapshot() const { std::vector<IPersistenceHandler::SP> handlers; @@ -52,47 +52,17 @@ PersistenceHandlerMap::getHandlerSnapshot() const } } size_t handlersSize = handlers.size(); - return std::make_unique<HandlerSnapshot> - (std::make_unique<DocTypeToHandlerMap::Snapshot>(std::move(handlers)), - handlersSize); -} - -namespace { - -struct EmptySequence : public vespalib::Sequence<IPersistenceHandler *> { - virtual bool valid() const override { return false; } - virtual IPersistenceHandler *get() const override { return nullptr; } - virtual void next() override { } - static EmptySequence::UP make() { return std::make_unique<EmptySequence>(); } -}; - + return HandlerSnapshot(DocTypeToHandlerMap::Snapshot(std::move(handlers)), handlersSize); } -HandlerSnapshot::UP +HandlerSnapshot PersistenceHandlerMap::getHandlerSnapshot(document::BucketSpace bucketSpace) const { auto itr = _map.find(bucketSpace); if (itr != _map.end()) { - return std::make_unique<HandlerSnapshot>(itr->second.snapshot(), itr->second.size()); + return HandlerSnapshot(itr->second.snapshot(), itr->second.size()); } - return std::make_unique<HandlerSnapshot>(EmptySequence::make(), 0); -} - -namespace { - -class SequenceOfOne : public vespalib::Sequence<IPersistenceHandler *> { -private: - bool _done; - IPersistenceHandler *_value; -public: - SequenceOfOne(IPersistenceHandler *value) : _done(false), _value(value) {} - - virtual bool valid() const override { return !_done; } - virtual IPersistenceHandler *get() const override { return _value; } - virtual void next() override { _done = true; } - static SequenceOfOne::UP make(IPersistenceHandler *value) { return std::make_unique<SequenceOfOne>(value); } -}; - + return HandlerSnapshot(); } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h index 003c378d6a7..64241e1ad2b 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h @@ -20,16 +20,17 @@ class IPersistenceHandler; */ class PersistenceHandlerMap { public: - using PersistenceHandlerSequence = vespalib::Sequence<IPersistenceHandler *>; + using DocTypeToHandlerMap = HandlerMap<IPersistenceHandler>; + using PersistenceHandlerSequence = DocTypeToHandlerMap::Snapshot; using PersistenceHandlerSP = std::shared_ptr<IPersistenceHandler>; class HandlerSnapshot { private: - PersistenceHandlerSequence::UP _handlers; - size_t _size; + PersistenceHandlerSequence _handlers; + size_t _size; public: - using UP = std::unique_ptr<HandlerSnapshot>; - HandlerSnapshot(PersistenceHandlerSequence::UP handlers_, size_t size_) + HandlerSnapshot() : _handlers(), _size(0) {} + HandlerSnapshot(DocTypeToHandlerMap::Snapshot handlers_, size_t size_) : _handlers(std::move(handlers_)), _size(size_) {} @@ -37,12 +38,12 @@ public: HandlerSnapshot & operator = (const HandlerSnapshot &) = delete; size_t size() const { return _size; } - PersistenceHandlerSequence &handlers() { return *_handlers; } - static PersistenceHandlerSequence::UP release(HandlerSnapshot &&rhs) { return std::move(rhs._handlers); } + PersistenceHandlerSequence &handlers() { return _handlers; } + static PersistenceHandlerSequence release(HandlerSnapshot &&rhs) { return std::move(rhs._handlers); } }; private: - using DocTypeToHandlerMap = HandlerMap<IPersistenceHandler>; + struct BucketSpaceHash { std::size_t operator() (const document::BucketSpace &bucketSpace) const { return bucketSpace.getId(); } @@ -53,15 +54,11 @@ private: public: PersistenceHandlerMap(); - PersistenceHandlerSP putHandler(document::BucketSpace bucketSpace, - const DocTypeName &docType, - const PersistenceHandlerSP &handler); - PersistenceHandlerSP removeHandler(document::BucketSpace bucketSpace, - const DocTypeName &docType); - PersistenceHandlerSP getHandler(document::BucketSpace bucketSpace, - const DocTypeName &docType) const; - HandlerSnapshot::UP getHandlerSnapshot() const; - HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace) const; + PersistenceHandlerSP putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType, const PersistenceHandlerSP &handler); + PersistenceHandlerSP removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType); + IPersistenceHandler * getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const; + HandlerSnapshot getHandlerSnapshot() const; + HandlerSnapshot getHandlerSnapshot(document::BucketSpace bucketSpace) const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 9d40cbae633..2ede5d45f7e 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -163,19 +163,19 @@ BucketInfoResultHandler::~BucketInfoResultHandler() = default; } -PersistenceEngine::HandlerSnapshot::UP +PersistenceEngine::HandlerSnapshot PersistenceEngine::getHandlerSnapshot(const WriteGuard &) const { return _handlers.getHandlerSnapshot(); } -PersistenceEngine::HandlerSnapshot::UP +PersistenceEngine::HandlerSnapshot PersistenceEngine::getHandlerSnapshot(const ReadGuard &, document::BucketSpace bucketSpace) const { return _handlers.getHandlerSnapshot(bucketSpace); } -PersistenceEngine::HandlerSnapshot::UP +PersistenceEngine::HandlerSnapshot PersistenceEngine::getHandlerSnapshot(const WriteGuard &, document::BucketSpace bucketSpace) const { return _handlers.getHandlerSnapshot(bucketSpace); @@ -212,7 +212,7 @@ PersistenceEngine::putHandler(const WriteGuard &, document::BucketSpace bucketSp } -IPersistenceHandler::SP +IPersistenceHandler * PersistenceEngine::getHandler(const ReadGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType) const { return _handlers.getHandler(bucketSpace, docType); @@ -232,9 +232,9 @@ PersistenceEngine::initialize() { std::unique_lock<std::shared_timed_mutex> wguard(getWLock()); LOG(debug, "Begin initializing persistence handlers"); - HandlerSnapshot::UP snap = getHandlerSnapshot(wguard); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(wguard); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->initialize(); } LOG(debug, "Done initializing persistence handlers"); @@ -260,10 +260,10 @@ PersistenceEngine::listBuckets(BucketSpace bucketSpace, PartitionId id) const BucketIdListResult::List emptyList; return BucketIdListResult(emptyList); } - HandlerSnapshot::UP snap = getHandlerSnapshot(rguard, bucketSpace); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); BucketIdListResultHandler resultHandler; - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleListBuckets(resultHandler); } return resultHandler.getResult(); @@ -275,10 +275,10 @@ PersistenceEngine::setClusterState(BucketSpace bucketSpace, const ClusterState & { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); saveClusterState(bucketSpace, calc); - HandlerSnapshot::UP snap = getHandlerSnapshot(rguard, bucketSpace); - GenericResultHandler resultHandler(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); + GenericResultHandler resultHandler(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleSetClusterState(calc, resultHandler); } resultHandler.await(); @@ -292,10 +292,10 @@ PersistenceEngine::setActiveState(const Bucket& bucket, storage::spi::BucketInfo::ActiveState newState) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snap = getHandlerSnapshot(rguard, bucket.getBucketSpace()); - GenericResultHandler resultHandler(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucket.getBucketSpace()); + GenericResultHandler resultHandler(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleSetActiveState(bucket, newState, resultHandler); } resultHandler.await(); @@ -309,10 +309,10 @@ PersistenceEngine::getBucketInfo(const Bucket& b) const // Runs in SPI thread. // No handover to write threads in persistence handlers. std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snap = getHandlerSnapshot(rguard, b.getBucketSpace()); + HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); BucketInfoResultHandler resultHandler; - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleGetBucketInfo(b, resultHandler); } return resultHandler.getResult(); @@ -338,7 +338,7 @@ PersistenceEngine::put(const Bucket& b, Timestamp t, const document::Document::S return Result(Result::ErrorType::PERMANENT_ERROR, make_string("Old id scheme not supported in elastic mode (%s)", doc->getId().toString().c_str())); } - IPersistenceHandler::SP handler = getHandler(rguard, b.getBucketSpace(), docType); + IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType); if (!handler) { return Result(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str())); @@ -360,7 +360,7 @@ PersistenceEngine::remove(const Bucket& b, Timestamp t, const DocumentId& did, C make_string("Old id scheme not supported in elastic mode (%s)", did.toString().c_str())); } DocTypeName docType(did.getDocType()); - IPersistenceHandler::SP handler = getHandler(rguard, b.getBucketSpace(), docType); + IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType); if (!handler) { return RemoveResult(Result::ErrorType::PERMANENT_ERROR, make_string("No handler for document type '%s'", docType.toString().c_str())); @@ -414,7 +414,7 @@ PersistenceEngine::update(const Bucket& b, Timestamp t, const DocumentUpdate::SP return UpdateResult(Result::ErrorType::PERMANENT_ERROR, make_string("Update operation rejected due to bad id (%s, %s)", upd->getId().toString().c_str(), docType.getName().c_str())); } - IPersistenceHandler::SP handler = getHandler(rguard, b.getBucketSpace(), docType); + IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType); if (handler) { TransportLatch latch(1); @@ -432,9 +432,9 @@ PersistenceEngine::GetResult PersistenceEngine::get(const Bucket& b, const document::FieldSet& fields, const DocumentId& did, Context& context) const { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snapshot = getHandlerSnapshot(rguard, b.getBucketSpace()); + HandlerSnapshot snapshot = getHandlerSnapshot(rguard, b.getBucketSpace()); - for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) { + for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) { BucketGuard::UP bucket_guard = handlers.get()->lockBucket(b); IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency()); for (size_t i = 0; i < retrievers->size(); ++i) { @@ -462,19 +462,19 @@ PersistenceEngine::createIterator(const Bucket &bucket, const document::FieldSet IncludedVersions versions, Context & context) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - HandlerSnapshot::UP snapshot = getHandlerSnapshot(rguard, bucket.getBucketSpace()); + HandlerSnapshot snapshot = getHandlerSnapshot(rguard, bucket.getBucketSpace()); auto entry = std::make_unique<IteratorEntry>(context.getReadConsistency(), bucket, fields, selection, versions, _defaultSerializedSize, _ignoreMaxBytes); - entry->bucket_guards.reserve(snapshot->size()); - for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) { + entry->bucket_guards.reserve(snapshot.size()); + for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) { entry->bucket_guards.push_back(handlers.get()->lockBucket(bucket)); IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency()); for (size_t i = 0; i < retrievers->size(); ++i) { entry->it.add((*retrievers)[i]); } } - entry->handler_sequence = HandlerSnapshot::release(std::move(*snapshot)); + entry->handler_sequence = HandlerSnapshot::release(std::move(snapshot)); std::lock_guard<std::mutex> guard(_iterators_lock); static IteratorId id_counter(0); @@ -541,10 +541,10 @@ PersistenceEngine::createBucket(const Bucket &b, Context &) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); LOG(spam, "createBucket(%s)", b.toString().c_str()); - HandlerSnapshot::UP snap = getHandlerSnapshot(rguard, b.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleCreateBucket(feedtoken::make(latch), b); } latch.await(); @@ -557,10 +557,10 @@ PersistenceEngine::deleteBucket(const Bucket& b, Context&) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); LOG(spam, "deleteBucket(%s)", b.toString().c_str()); - HandlerSnapshot::UP snap = getHandlerSnapshot(rguard, b.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleDeleteBucket(feedtoken::make(latch), b); } latch.await(); @@ -578,10 +578,10 @@ PersistenceEngine::getModifiedBuckets(BucketSpace bucketSpace) const std::lock_guard<std::mutex> guard(_lock); extraModifiedBuckets.swap(_extraModifiedBuckets[bucketSpace]); } - HandlerSnapshot::UP snap = getHandlerSnapshot(rguard, bucketSpace); - SynchronizedBucketIdListResultHandler resultHandler(snap->size() + extraModifiedBuckets.size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); + SynchronizedBucketIdListResultHandler resultHandler(snap.size() + extraModifiedBuckets.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleGetModifiedBuckets(resultHandler); } for (const auto & item : extraModifiedBuckets) { @@ -599,10 +599,10 @@ PersistenceEngine::split(const Bucket& source, const Bucket& target1, const Buck LOG(spam, "split(%s, %s, %s)", source.toString().c_str(), target1.toString().c_str(), target2.toString().c_str()); assert(source.getBucketSpace() == target1.getBucketSpace()); assert(source.getBucketSpace() == target2.getBucketSpace()); - HandlerSnapshot::UP snap = getHandlerSnapshot(rguard, source.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, source.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleSplit(feedtoken::make(latch), source, target1, target2); } latch.await(); @@ -617,10 +617,10 @@ PersistenceEngine::join(const Bucket& source1, const Bucket& source2, const Buck LOG(spam, "join(%s, %s, %s)", source1.toString().c_str(), source2.toString().c_str(), target.toString().c_str()); assert(source1.getBucketSpace() == target.getBucketSpace()); assert(source2.getBucketSpace() == target.getBucketSpace()); - HandlerSnapshot::UP snap = getHandlerSnapshot(rguard, target.getBucketSpace()); - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + HandlerSnapshot snap = getHandlerSnapshot(rguard, target.getBucketSpace()); + TransportLatch latch(snap.size()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleJoin(feedtoken::make(latch), source1, source2, target); } latch.await(); @@ -724,16 +724,16 @@ public: void PersistenceEngine::populateInitialBucketDB(const WriteGuard & guard, BucketSpace bucketSpace, IPersistenceHandler &targetHandler) { - HandlerSnapshot::UP snap = getHandlerSnapshot(guard, bucketSpace); + HandlerSnapshot snap = getHandlerSnapshot(guard, bucketSpace); - size_t snapSize(snap->size()); + size_t snapSize(snap.size()); size_t flawed = 0; // handleListActiveBuckets() runs in SPI thread. // No handover to write threads in persistence handlers. ActiveBucketIdListResultHandler resultHandler; - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler *handler = snap.handlers().get(); handler->handleListActiveBuckets(resultHandler); } typedef std::map<document::BucketId, size_t> BucketIdMap; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index 3f2cf92acb7..5d3be07c532 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -5,12 +5,9 @@ #include "i_resource_write_filter.h" #include "persistence_handler_map.h" #include "ipersistencehandler.h" -#include <vespa/document/bucket/bucketspace.h> #include <vespa/persistence/spi/abstractpersistenceprovider.h> -#include <vespa/searchcore/proton/common/handlermap.hpp> #include <mutex> #include <shared_mutex> -#include <unordered_map> namespace proton { @@ -18,7 +15,7 @@ class IPersistenceEngineOwner; class PersistenceEngine : public storage::spi::AbstractPersistenceProvider { private: - using PersistenceHandlerSequence = vespalib::Sequence<IPersistenceHandler *>; + using PersistenceHandlerSequence = PersistenceHandlerMap::PersistenceHandlerSequence; using HandlerSnapshot = PersistenceHandlerMap::HandlerSnapshot; using DocumentUpdate = document::DocumentUpdate; using Bucket = storage::spi::Bucket; @@ -43,7 +40,7 @@ private: using UpdateResult = storage::spi::UpdateResult; struct IteratorEntry { - PersistenceHandlerSequence::UP handler_sequence; + PersistenceHandlerSequence handler_sequence; DocumentIterator it; bool in_use; std::vector<BucketGuard::UP> bucket_guards; @@ -78,10 +75,10 @@ private: using ReadGuard = std::shared_lock<std::shared_timed_mutex>; using WriteGuard = std::unique_lock<std::shared_timed_mutex>; - IPersistenceHandler::SP getHandler(const ReadGuard & guard, document::BucketSpace bucketSpace, const DocTypeName &docType) const; - HandlerSnapshot::UP getHandlerSnapshot(const WriteGuard & guard) const; - HandlerSnapshot::UP getHandlerSnapshot(const ReadGuard & guard, document::BucketSpace bucketSpace) const; - HandlerSnapshot::UP getHandlerSnapshot(const WriteGuard & guard, document::BucketSpace bucketSpace) const; + IPersistenceHandler * getHandler(const ReadGuard & guard, document::BucketSpace bucketSpace, const DocTypeName &docType) const; + HandlerSnapshot getHandlerSnapshot(const WriteGuard & guard) const; + HandlerSnapshot getHandlerSnapshot(const ReadGuard & guard, document::BucketSpace bucketSpace) const; + HandlerSnapshot getHandlerSnapshot(const WriteGuard & guard, document::BucketSpace bucketSpace) const; void saveClusterState(BucketSpace bucketSpace, const ClusterState &calc); ClusterState::SP savedClusterState(BucketSpace bucketSpace) const; diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp index 1d3b2165c80..57f1ee74e60 100644 --- a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp @@ -116,7 +116,7 @@ BucketHandler::handleGetBucketInfo(const Bucket &bucket, // Called by SPI thread. // BucketDBOwner ensures synchronization between SPI thread and // master write thread in document database. - BucketInfo bucketInfo = _ready->getBucketDB().takeGuard()->cachedGet(bucket); + BucketInfo bucketInfo = _ready->getBucketDB().takeGuard()->cachedGetBucketInfo(bucket); LOG(spam, "handleGetBucketInfo(%s): %s", bucket.toString().c_str(), bucketInfo.toString().c_str()); resultHandler.handle(BucketInfoResult(bucketInfo)); diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp index e154c6761e2..692b05899f4 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp @@ -128,13 +128,13 @@ SummaryEngine::getDocsums(DocsumRequest::UP req) if (searchHandler) { reply = searchHandler->getDocsums(*req); } else { - vespalib::Sequence<ISearchHandler*>::UP snapshot; + HandlerMap<ISearchHandler>::Snapshot snapshot; { std::lock_guard<std::mutex> guard(_lock); snapshot = _handlers.snapshot(); } - if (snapshot->valid()) { - reply = snapshot->get()->getDocsums(*req); // use the first handler + if (snapshot.valid()) { + reply = snapshot.get()->getDocsums(*req); // use the first handler } } updateDocsumMetrics(vespalib::to_s(req->getTimeUsed()), getNumDocs(*reply)); diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index 610fc58e98f..41465f991e3 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -1209,7 +1209,7 @@ TEST("testArithmeticOperations") { testAdd(createScalarInt(I1), createScalarInt(I2), 3469774562ull, 3469774562ull); testAdd(createScalarInt(I1), createScalarFloat(F2), 1793253251ull, 1793253250.767681239); testAdd(createScalarFloat(F1), createScalarFloat(F2), 11, 10.878668839 ); - testMultiply(createScalarInt(I1), createScalarInt(I2), 3006427292488851361ull, 3006427292488851361ull); + testMultiply(createScalarInt(I1), createScalarInt(I2), 3006427292488851361ull, static_cast<double>(3006427292488851361ull)); testMultiply(createScalarInt(I1), createScalarFloat(F2), 17515926039ull, 1793253241.0*9.767681239); testMultiply(createScalarFloat(F1), createScalarFloat(F2), 11, 10.8517727372816364 ); diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp index 503922bdb13..02d459dd0d8 100644 --- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp +++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp @@ -267,7 +267,7 @@ Fixture::assertBools(std::vector<bool> expVals, const vespalib::string &attribut { auto node = makeNode(attributeName, false, preserveAccurateTypes); uint32_t docId = 0; - for (const auto &expDocVal : expVals) { + for (const auto expDocVal : expVals) { ++docId; node->setDocId(docId); node->execute(); diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 7d0ce26b83d..dd44c96555b 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -795,8 +795,8 @@ PersistenceThread::handleCommand(api::StorageCommand& msg) { _context = spi::Context(msg.getLoadType(), msg.getPriority(), msg.getTrace().getLevel()); MessageTracker::UP mtracker(handleCommandSplitByType(msg)); - if (mtracker.get() != 0) { - if (mtracker->getReply().get() != 0) { + if (mtracker) { + if (mtracker->getReply()) { mtracker->getReply()->getTrace().getRoot().addChild(_context.getTrace().getRoot()); } else { msg.getTrace().getRoot().addChild(_context.getTrace().getRoot()); diff --git a/storage/src/vespa/storage/tools/analyzedistribution.cpp b/storage/src/vespa/storage/tools/analyzedistribution.cpp index 52c26c4bb17..472aa63fff6 100644 --- a/storage/src/vespa/storage/tools/analyzedistribution.cpp +++ b/storage/src/vespa/storage/tools/analyzedistribution.cpp @@ -132,7 +132,7 @@ Node::Node(const lib::NodeState& dstate, const lib::NodeState& sstate, uint32_t disks.push_back(Disk(storageState.getDiskState(i))); } } -Node::~Node() {} +Node::~Node() = default; struct Distribution { std::vector<Node> nodes; diff --git a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp index 337709cc591..10691d37b9e 100644 --- a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp +++ b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp @@ -77,7 +77,7 @@ HttpUrlPath::print(std::ostream& out, bool, const std::string&) const if (!_attributes.empty()) { out << "?"; size_t cnt = 0; - for (const auto attr: _attributes) { + for (const auto &attr: _attributes) { if (cnt++ > 0) { out << "&"; } diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index d95eddac57f..95b6528bd77 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -465,7 +465,8 @@ public class DocumentGenMojo extends AbstractMojo { exportStructTypeGetter(docType.getName()+".body", docType.allBody().getFields(), out, 1, "getBodyStructType", "com.yahoo.document.StructDataType"); Collection<Field> allUniqueFields = getAllUniqueFields(multiExtends, docType.getAllFields()); - exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(),out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); + exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(), + docType.getImportedFieldNames(), out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); exportCopyConstructor(className, out, 1, true); exportFieldsAndAccessors(className, "com.yahoo.document.Document".equals(superType) ? allUniqueFields : docType.getFields(), out, 1, true); @@ -627,10 +628,21 @@ public class DocumentGenMojo extends AbstractMojo { } out.write(ind(ind) + "ret.addFieldSets(fieldSets);\n"); } + private static void exportImportedFields(Set<String> importedFieldNames, Writer out, int ind) throws IOException { + out.write(ind(ind) + "java.util.Set<java.lang.String> importedFieldNames = new java.util.HashSet<>();\n"); + for (String importedField : importedFieldNames) { + out.write(ind(ind) + "importedFieldNames.add(\"" + importedField + "\");\n"); + } + } private static void exportExtendedStructTypeGetter(String className, String name, Collection<Field> fields, Set<FieldSet> fieldSets, - Writer out, int ind, String methodName, String retType) throws IOException { + Set<String> importedFieldNames, Writer out, int ind, String methodName, String retType) throws IOException { out.write(ind(ind)+"private static "+retType+" "+methodName+"() {\n"); - out.write(ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n"); + if (importedFieldNames != null) { + exportImportedFields(importedFieldNames, out, ind + 1); + out.write(ind(ind+1)+retType+" ret = new "+retType+"(\"" + name + "\", importedFieldNames);\n"); + } else { + out.write(ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n"); + } for (Field f : fields) { if (f.getDataType().equals(DataType.STRING)) { addExtendedStringField(className, f, out, ind + 1); @@ -783,7 +795,7 @@ public class DocumentGenMojo extends AbstractMojo { ind(ind+2)+"super("+structClassName+".type);\n" + ind(ind+1)+"}\n\n"); exportCopyConstructor(structClassName, out, ind+1, false); - exportExtendedStructTypeGetter(structClassName, structType.getName(), structType.getFields(), null, out, ind+1, "getStructType", "com.yahoo.document.StructDataType"); + exportExtendedStructTypeGetter(structClassName, structType.getName(), structType.getFields(), null, null, out, ind+1, "getStructType", "com.yahoo.document.StructDataType"); exportAssign(structType, structClassName, out, ind+1); exportFieldsAndAccessors(structClassName, structType.getFields(), out, ind+1, true); |