diff options
Diffstat (limited to 'config-model')
40 files changed, 486 insertions, 342 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index e37b0b07746..896c6ea9a7f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -43,7 +43,6 @@ public class TestProperties implements ModelContext.Properties { private double defaultTermwiseLimit = 1.0; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private boolean useNewAthenzFilter = false; - private boolean useDedicatedNodesWhenUnspecified = false; private AthenzDomain athenzDomain; @Override public boolean multitenant() { return multitenant; } @@ -65,7 +64,7 @@ public class TestProperties implements ModelContext.Properties { @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useBucketSpaceMetric() { return true; } @Override public boolean useNewAthenzFilter() { return useNewAthenzFilter; } - @Override public boolean useDedicatedNodesWhenUnspecified() { return useDedicatedNodesWhenUnspecified; } + @Override public boolean useDedicatedNodesWhenUnspecified() { return true; } @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); } public TestProperties setDefaultTermwiseLimit(double limit) { @@ -118,11 +117,6 @@ public class TestProperties implements ModelContext.Properties { return this; } - public TestProperties setUseDedicatedNodesWhenUnspecified(boolean useDedicatedNodesWhenUnspecified) { - this.useDedicatedNodesWhenUnspecified = useDedicatedNodesWhenUnspecified; - return this; - } - public TestProperties setAthenzDomain(AthenzDomain domain) { this.athenzDomain = domain; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java index f909f3864da..201b69c1aae 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java @@ -45,10 +45,16 @@ public class HostsXmlProvisioner implements HostProvisioner { } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, int groups, ProvisionLogger logger) { throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported"); } + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, ProvisionLogger logger) { + throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported"); + } + private HostSpec host2HostSpec(Host host) { return new HostSpec(host.hostname(), host.aliases()); } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 6047b6a9818..298517b85f6 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -6,6 +6,7 @@ import com.yahoo.collections.Pair; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; @@ -57,6 +58,8 @@ public class InMemoryProvisioner implements HostProvisioner { /** Use this index as start index for all clusters */ private final int startIndexForClusters; + private final boolean useMaxResources; + /** Creates this with a number of nodes with resources 1, 3, 9, 1 */ public InMemoryProvisioner(int nodeCount) { this(nodeCount, defaultResources); @@ -64,27 +67,29 @@ public class InMemoryProvisioner implements HostProvisioner { /** Creates this with a number of nodes with given resources */ public InMemoryProvisioner(int nodeCount, NodeResources resources) { - this(Map.of(resources, createHostInstances(nodeCount)), true, 0); + this(Map.of(resources, createHostInstances(nodeCount)), true, false, 0); } /** Creates this with a set of host names of the flavor 'default' */ public InMemoryProvisioner(boolean failOnOutOfCapacity, String... hosts) { - this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, 0); + this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, false, 0); } /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) { - this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, 0, retiredHostNames); + this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, 0, retiredHostNames); } /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { - this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, startIndexForClusters, retiredHostNames); + this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, startIndexForClusters, retiredHostNames); } public InMemoryProvisioner(Map<NodeResources, Collection<Host>> hosts, boolean failOnOutOfCapacity, + boolean useMaxResources, int startIndexForClusters, String ... retiredHostNames) { this.failOnOutOfCapacity = failOnOutOfCapacity; + this.useMaxResources = useMaxResources; for (Map.Entry<NodeResources, Collection<Host>> hostsWithResources : hosts.entrySet()) for (Host host : hostsWithResources.getValue()) freeNodes.put(hostsWithResources.getKey(), host); @@ -113,34 +118,43 @@ public class InMemoryProvisioner implements HostProvisioner { } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) { - if (cluster.group().isPresent() && groups > 1) + return prepare(cluster, requestedCapacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { + if (useMaxResources) + return prepare(cluster, requested.maxResources(), requested.isRequired(), requested.canFail()); + else + return prepare(cluster, requested.minResources(), requested.isRequired(), requested.canFail()); + } + + public List<HostSpec> prepare(ClusterSpec cluster, ClusterResources requested, boolean required, boolean canFail) { + if (cluster.group().isPresent() && requested.groups() > 1) throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created"); - if (requestedCapacity.nodeCount() % groups != 0) - throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " + - groups + " groups, but the node count is not divisible into this number of groups"); - int capacity = failOnOutOfCapacity || requestedCapacity.isRequired() - ? requestedCapacity.nodeCount() - : Math.min(requestedCapacity.nodeCount(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster)); - if (groups > capacity) - groups = capacity; + int capacity = failOnOutOfCapacity || required + ? requested.nodes() + : Math.min(requested.nodes(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster)); + int groups = requested.groups() > capacity ? capacity : requested.groups(); List<HostSpec> allocation = new ArrayList<>(); if (groups == 1) { allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(0))), - requestedCapacity.nodeResources(), + requested.nodeResources(), capacity, startIndexForClusters, - requestedCapacity.canFail())); + canFail)); } else { for (int i = 0; i < groups; i++) { allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(i))), - requestedCapacity.nodeResources(), + requested.nodeResources(), capacity / groups, allocation.size(), - requestedCapacity.canFail())); + canFail)); } } for (ListIterator<HostSpec> i = allocation.listIterator(); i.hasNext(); ) { @@ -162,7 +176,7 @@ public class InMemoryProvisioner implements HostProvisioner { host.dockerImageRepo()); } - private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, Optional<NodeResources> requestedResources, + private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, NodeResources requestedResources, int nodesInGroup, int startIndex, boolean canFail) { List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>()); allocations.put(clusterGroup, allocation); @@ -170,8 +184,8 @@ public class InMemoryProvisioner implements HostProvisioner { // Check if the current allocations are compatible with the new request for (int i = allocation.size() - 1; i >= 0; i--) { Optional<NodeResources> currentResources = allocation.get(0).flavor().map(Flavor::resources); - if (currentResources.isEmpty() || requestedResources.isEmpty()) continue; - if (!currentResources.get().compatibleWith(requestedResources.get())) { + if (currentResources.isEmpty() || requestedResources == NodeResources.unspecified) continue; + if (!currentResources.get().compatibleWith(requestedResources)) { HostSpec removed = allocation.remove(i); freeNodes.put(currentResources.get(), new Host(removed.hostname())); // Return the node back to free pool } @@ -182,7 +196,7 @@ public class InMemoryProvisioner implements HostProvisioner { // Find the smallest host that can fit the requested requested Optional<NodeResources> hostResources = freeNodes.keySet().stream() .sorted(new MemoryDiskCpu()) - .filter(resources -> requestedResources.isEmpty() || resources.satisfies(requestedResources.get())) + .filter(resources -> requestedResources == NodeResources.unspecified || resources.satisfies(requestedResources)) .findFirst(); if (hostResources.isEmpty()) { if (canFail) @@ -195,8 +209,9 @@ public class InMemoryProvisioner implements HostProvisioner { if (freeNodes.get(hostResources.get()).isEmpty()) freeNodes.removeAll(hostResources.get()); ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++); allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(), - hostResources.map(Flavor::new), Optional.of(membership), - newHost.version(), Optional.empty(), requestedResources)); + hostResources.map(Flavor::new), Optional.of(membership), + newHost.version(), Optional.empty(), + requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources))); } nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex); diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index 180a16f3c8f..8945223447f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -30,6 +30,7 @@ public class SingleNodeProvisioner implements HostProvisioner { host = new Host(HostName.getLocalhost()); this.hostSpec = new HostSpec(host.hostname(), host.aliases()); } + public SingleNodeProvisioner(Flavor flavor) { host = new Host(HostName.getLocalhost()); this.hostSpec = new HostSpec(host.hostname(), host.aliases(), flavor); @@ -41,7 +42,14 @@ public class SingleNodeProvisioner implements HostProvisioner { } @Override - public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1 + @Deprecated // TODO: Remove after April 2020 + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + return prepare(cluster, capacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + // TODO: This should fail if capacity requested is more than 1 List<HostSpec> hosts = new ArrayList<>(); hosts.add(new HostSpec(host.hostname(), host.aliases(), ClusterMembership.from(cluster, counter++))); return hosts; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index 90f061d933d..aba6cf9a233 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -10,11 +10,11 @@ import java.io.Serializable; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashSet; +import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.Set; - /** * An index definition in a search definition. * Two indices are equal if they have the same name and the same settings, except @@ -24,6 +24,8 @@ import java.util.Set; */ public class Index implements Cloneable, Serializable { + public static enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES } + public enum Type { VESPA("vespa"); @@ -61,7 +63,9 @@ public class Index implements Cloneable, Serializable { /** The boolean index definition, if set */ private BooleanIndexDefinition boolIndex; - private Optional<HnswIndexParams> hnswIndexParams; + private Optional<HnswIndexParams> hnswIndexParams = Optional.empty(); + + private Optional<DistanceMetric> distanceMetric = Optional.empty(); /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */ private boolean interleavedFeatures = false; @@ -134,12 +138,13 @@ public class Index implements Cloneable, Serializable { stemming == index.stemming && type == index.type && Objects.equals(boolIndex, index.boolIndex) && + Objects.equals(distanceMetric, index.distanceMetric) && Objects.equals(hnswIndexParams, index.hnswIndexParams); } @Override public int hashCode() { - return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, hnswIndexParams, interleavedFeatures); + return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, distanceMetric, hnswIndexParams, interleavedFeatures); } public String toString() { @@ -187,6 +192,16 @@ public class Index implements Cloneable, Serializable { boolIndex = def; } + public Optional<DistanceMetric> getDistanceMetric() { + return distanceMetric; + } + + public void setDistanceMetric(String value) { + String upper = value.toUpperCase(Locale.ENGLISH); + DistanceMetric dm = DistanceMetric.valueOf(upper); + distanceMetric = Optional.of(dm); + } + public Optional<HnswIndexParams> getHnswIndexParams() { return hnswIndexParams; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 5b87fdcf5f6..8b5f7658475 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -240,13 +240,14 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce aaB.tensortype(attribute.tensorType().get().toString()); } aaB.imported(imported); + var dma = attribute.distanceMetric(); if (attribute.hnswIndexParams().isPresent()) { var ib = new AttributesConfig.Attribute.Index.Builder(); var params = attribute.hnswIndexParams().get(); ib.hnsw.enabled(true); ib.hnsw.maxlinkspernode(params.maxLinksPerNode()); ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert()); - var dm = AttributesConfig.Attribute.Index.Hnsw.Distancemetric.Enum.valueOf(params.distanceMetric().toString()); + var dm = AttributesConfig.Attribute.Index.Hnsw.Distancemetric.Enum.valueOf(dma.toString()); ib.hnsw.distancemetric(dm); aaB.index(ib); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index 9ed5e4ca2de..1661a80f238 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -24,6 +24,7 @@ import com.yahoo.document.datatypes.Float16FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.tensor.TensorType; +import static com.yahoo.searchdefinition.Index.DistanceMetric; import java.io.Serializable; import java.util.LinkedHashSet; @@ -66,7 +67,9 @@ public final class Attribute implements Cloneable, Serializable { /** This is set if the type of this is REFERENCE */ private final Optional<StructuredDataType> referenceDocumentType; - private Optional<HnswIndexParams> hnswIndexParams; + private Optional<DistanceMetric> distanceMetric = Optional.empty(); + + private Optional<HnswIndexParams> hnswIndexParams = Optional.empty(); private boolean isPosition = false; private final Sorting sorting = new Sorting(); @@ -152,7 +155,6 @@ public final class Attribute implements Cloneable, Serializable { setCollectionType(collectionType); this.tensorType = tensorType; this.referenceDocumentType = referenceDocumentType; - this.hnswIndexParams = Optional.empty(); } public Attribute convertToArray() { @@ -197,6 +199,11 @@ public final class Attribute implements Cloneable, Serializable { public double densePostingListThreshold() { return densePostingListThreshold; } public Optional<TensorType> tensorType() { return tensorType; } public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; } + + public static final DistanceMetric DEFAULT_DISTANCE_METRIC = DistanceMetric.EUCLIDEAN; + public DistanceMetric distanceMetric() { + return distanceMetric.orElse(DEFAULT_DISTANCE_METRIC); + } public Optional<HnswIndexParams> hnswIndexParams() { return hnswIndexParams; } public Sorting getSorting() { return sorting; } @@ -221,6 +228,7 @@ public final class Attribute implements Cloneable, Serializable { public void setUpperBound(long upperBound) { this.upperBound = upperBound; } public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; } public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } + public void setDistanceMetric(Optional<DistanceMetric> dm) { this.distanceMetric = dm; } public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } public String getName() { return name; } @@ -354,8 +362,8 @@ public final class Attribute implements Cloneable, Serializable { /** Returns whether these attributes describes the same entity, even if they have different names */ public boolean isCompatible(Attribute other) { - if ( ! this.type.equals(other.type)) return false; - if ( ! this.collectionType.equals(other.collectionType)) return false; + if (! this.type.equals(other.type)) return false; + if (! this.collectionType.equals(other.collectionType)) return false; if (this.isPrefetch() != other.isPrefetch()) return false; if (this.removeIfZero != other.removeIfZero) return false; if (this.createIfNonExistent != other.createIfNonExistent) return false; @@ -364,10 +372,11 @@ public final class Attribute implements Cloneable, Serializable { // if (this.noSearch != other.noSearch) return false; No backend consequences so compatible for now if (this.fastSearch != other.fastSearch) return false; if (this.huge != other.huge) return false; - if ( ! this.sorting.equals(other.sorting)) return false; - if (!this.tensorType.equals(other.tensorType)) return false; - if (!this.referenceDocumentType.equals(other.referenceDocumentType)) return false; - if (!this.hnswIndexParams.equals(other.hnswIndexParams)) return false; + if (! this.sorting.equals(other.sorting)) return false; + if (! Objects.equals(tensorType, other.tensorType)) return false; + if (! Objects.equals(referenceDocumentType, other.referenceDocumentType)) return false; + if (! Objects.equals(distanceMetric, other.distanceMetric)) return false; + if (! Objects.equals(hnswIndexParams, other.hnswIndexParams)) return false; return true; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java index 01434be8785..2f084d3e513 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java @@ -13,18 +13,13 @@ public class HnswIndexParams { public static final int DEFAULT_MAX_LINKS_PER_NODE = 16; public static final int DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT = 200; - public static final DistanceMetric DEFAULT_DISTANCE_METRIC = DistanceMetric.EUCLIDEAN; private final Optional<Integer> maxLinksPerNode; private final Optional<Integer> neighborsToExploreAtInsert; - private final Optional<DistanceMetric> distanceMetric; - - public static enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES } public static class Builder { private Optional<Integer> maxLinksPerNode = Optional.empty(); private Optional<Integer> neighborsToExploreAtInsert = Optional.empty(); - private Optional<DistanceMetric> distanceMetric = Optional.empty(); public void setMaxLinksPerNode(int value) { maxLinksPerNode = Optional.of(value); @@ -32,38 +27,31 @@ public class HnswIndexParams { public void setNeighborsToExploreAtInsert(int value) { neighborsToExploreAtInsert = Optional.of(value); } - public void setDistanceMetric(String value) { - String upper = value.toUpperCase(Locale.ENGLISH); - DistanceMetric dm = DistanceMetric.valueOf(upper); - distanceMetric = Optional.of(dm); - } public HnswIndexParams build() { - return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert, distanceMetric); + return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert); } } public HnswIndexParams() { this.maxLinksPerNode = Optional.empty(); this.neighborsToExploreAtInsert = Optional.empty(); - this.distanceMetric = Optional.empty(); } public HnswIndexParams(Optional<Integer> maxLinksPerNode, - Optional<Integer> neighborsToExploreAtInsert, - Optional<DistanceMetric> distanceMetric) { + Optional<Integer> neighborsToExploreAtInsert) { this.maxLinksPerNode = maxLinksPerNode; this.neighborsToExploreAtInsert = neighborsToExploreAtInsert; - this.distanceMetric = distanceMetric; } /** * Creates a new instance where values from the given parameter instance are used where they are present, * otherwise we use values from this. */ - public HnswIndexParams overrideFrom(HnswIndexParams rhs) { + public HnswIndexParams overrideFrom(Optional<HnswIndexParams> other) { + if (! other.isPresent()) return this; + HnswIndexParams rhs = other.get(); return new HnswIndexParams(rhs.maxLinksPerNode.or(() -> maxLinksPerNode), - rhs.neighborsToExploreAtInsert.or(() -> neighborsToExploreAtInsert), - rhs.distanceMetric.or(() -> distanceMetric)); + rhs.neighborsToExploreAtInsert.or(() -> neighborsToExploreAtInsert)); } public int maxLinksPerNode() { @@ -73,8 +61,4 @@ public class HnswIndexParams { public int neighborsToExploreAtInsert() { return neighborsToExploreAtInsert.orElse(DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT); } - - public DistanceMetric distanceMetric() { - return distanceMetric.orElse(DEFAULT_DISTANCE_METRIC); - } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java index 7f9da28b9ca..0c1f443dee3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java @@ -32,6 +32,7 @@ public class IndexOperation implements FieldOperation { private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); private Optional<Boolean> enableBm25 = Optional.empty(); + private Optional<String> distanceMetric = Optional.empty(); private Optional<HnswIndexParams.Builder> hnswIndexParams = Optional.empty(); public String getIndexName() { @@ -94,6 +95,9 @@ public class IndexOperation implements FieldOperation { if (enableBm25.isPresent()) { index.setInterleavedFeatures(enableBm25.get()); } + if (distanceMetric.isPresent()) { + index.setDistanceMetric(distanceMetric.get()); + } if (hnswIndexParams.isPresent()) { index.setHnswIndexParams(hnswIndexParams.get().build()); } @@ -127,6 +131,10 @@ public class IndexOperation implements FieldOperation { enableBm25 = Optional.of(value); } + public void setDistanceMetric(String value) { + this.distanceMetric = Optional.of(value); + } + public void setHnswIndexParams(HnswIndexParams.Builder params) { this.hnswIndexParams = Optional.of(params); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 2790f2ddf6e..c97ee2bd935 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -81,8 +81,9 @@ public class TensorFieldProcessor extends Processor { var index = field.getIndex(field.getName()); // TODO: Calculate default params based on tensor dimension size var params = new HnswIndexParams(); - if (index != null && index.getHnswIndexParams().isPresent()) { - params = params.overrideFrom(index.getHnswIndexParams().get()); + if (index != null) { + params = params.overrideFrom(index.getHnswIndexParams()); + field.getAttribute().setDistanceMetric(index.getDistanceMetric()); } field.getAttribute().setHnswIndexParams(params); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 3765e683b18..557a61ec211 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -111,8 +111,8 @@ public class HostSystem extends AbstractConfigProducer<Host> { } } - public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, int groups, DeployLogger logger) { - List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, groups, new ProvisionDeployLogger(logger)); + public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, DeployLogger logger) { + List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, new ProvisionDeployLogger(logger)); // TODO: Even if HostResource owns a set of memberships, we need to return a map because the caller needs the current membership. Map<HostResource, ClusterMembership> retAllocatedHosts = new LinkedHashMap<>(); for (HostSpec spec : allocatedHosts) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 001e6a9a407..09128e741b9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -301,6 +301,14 @@ public class VespaMetricSet { return metrics; } + private static void addSearchNodeExecutorMetrics(Set<Metric> metrics, String prefix) { + metrics.add(new Metric(prefix + ".queuesize.max")); + metrics.add(new Metric(prefix + ".queuesize.sum")); + metrics.add(new Metric(prefix + ".queuesize.count")); + metrics.add(new Metric(prefix + ".maxpending.last")); // TODO: Remove in Vespa 8 + metrics.add(new Metric(prefix + ".accepted.rate")); + } + private static Set<Metric> getSearchNodeMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); @@ -345,18 +353,12 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.search_protocol.docsum.requested_documents.count")); // Executors shared between all document dbs - metrics.add(new Metric("content.proton.executor.proton.maxpending.last")); - metrics.add(new Metric("content.proton.executor.proton.accepted.rate")); - metrics.add(new Metric("content.proton.executor.flush.maxpending.last")); - metrics.add(new Metric("content.proton.executor.flush.accepted.rate")); - metrics.add(new Metric("content.proton.executor.match.maxpending.last")); - metrics.add(new Metric("content.proton.executor.match.accepted.rate")); - metrics.add(new Metric("content.proton.executor.docsum.maxpending.last")); - metrics.add(new Metric("content.proton.executor.docsum.accepted.rate")); - metrics.add(new Metric("content.proton.executor.shared.maxpending.last")); - metrics.add(new Metric("content.proton.executor.shared.accepted.rate")); - metrics.add(new Metric("content.proton.executor.warmup.maxpending.last")); - metrics.add(new Metric("content.proton.executor.warmup.accepted.rate")); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.proton"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.flush"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.match"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.docsum"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.shared"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.warmup"); // jobs metrics.add(new Metric("content.proton.documentdb.job.total.average")); @@ -370,18 +372,12 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.job.removed_documents_prune.average")); // Threading service (per document db) - metrics.add(new Metric("content.proton.documentdb.threading_service.master.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.master.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.summary.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.summary.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_inverter.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_inverter.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_writer.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_writer.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.attribute_field_writer.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.attribute_field_writer.accepted.rate")); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.master"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.summary"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index_field_inverter"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index_field_writer"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.attribute_field_writer"); // lid space metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_bloat_factor.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index b4246171277..b6f7ab4ff62 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -138,7 +138,7 @@ public class RankSetupValidator extends Validator { private boolean execValidate(String configId, SearchCluster sc, String sdName, DeployLogger deployLogger) { String job = String.format("%s %s", binaryName, configId); - ProcessExecuter executer = new ProcessExecuter(); + ProcessExecuter executer = new ProcessExecuter(true); try { Pair<Integer, String> ret = executer.exec(job); if (ret.getFirst() != 0) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 5629956e8b9..804a5442608 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -65,12 +65,12 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { createSlobroks(deployLogger, admin, allocateHosts(admin.hostSystem(), "slobroks", nodesSpecification)); } else { - createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.count(), 2)); + createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.minResources().nodes(), 2)); } } private void assignLogserver(DeployState deployState, NodesSpecification nodesSpecification, Admin admin) { - if (nodesSpecification.count() > 1) throw new IllegalArgumentException("You can only request a single log server"); + if (nodesSpecification.minResources().nodes() > 1) throw new IllegalArgumentException("You can only request a single log server"); if (deployState.getProperties().applicationId().instance().isTester()) return; // No logserver is needed on tester applications if (nodesSpecification.isDedicated()) { Collection<HostResource> hosts = allocateHosts(admin.hostSystem(), "logserver", nodesSpecification); @@ -79,7 +79,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { Logserver logserver = createLogserver(deployState.getDeployLogger(), admin, hosts); createContainerOnLogserverHost(deployState, admin, logserver.getHostResource()); } else if (containerModels.iterator().hasNext()) { - List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.count(), false); + List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.minResources().nodes(), false); if (hosts.isEmpty()) return; // No log server can be created (and none is needed) createLogserver(deployState.getDeployLogger(), admin, hosts); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java index d34a11abdf4..80c95ad6b59 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java @@ -172,7 +172,12 @@ public class ModelElement { /** Returns the content of the attribute with the given name, or null if none */ public String stringAttribute(String name) { - if ( ! xml.hasAttribute(name)) return null; + return stringAttribute(name, null); + } + + /** Returns the content of the attribute with the given name, or the default value if none */ + public String stringAttribute(String name, String defaultValue) { + if ( ! xml.hasAttribute(name)) return defaultValue; return xml.getAttribute(name); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 384d891003e..6a52ff4f051 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -1,11 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.text.XML; @@ -18,6 +20,8 @@ import org.w3c.dom.Node; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; +import java.util.regex.Pattern; /** * A common utility class to represent a requirement for nodes during model building. @@ -27,11 +31,9 @@ import java.util.Optional; */ public class NodesSpecification { - private final boolean dedicated; - - private final int count; + private final ClusterResources min, max; - private final int groups; + private final boolean dedicated; /** The Vespa version we want the nodes to run */ private Version version; @@ -46,43 +48,51 @@ public class NodesSpecification { private final boolean exclusive; - /** The resources each node should have, or empty to use the default */ - private final Optional<NodeResources> resources; - /** The repo part of a docker image (without tag), optional */ private final Optional<String> dockerImageRepo; /** The ID of the cluster referencing this node specification, if any */ private final Optional<String> combinedId; - private NodesSpecification(boolean dedicated, int count, int groups, Version version, + private NodesSpecification(ClusterResources min, + ClusterResources max, + boolean dedicated, Version version, boolean required, boolean canFail, boolean exclusive, - Optional<NodeResources> resources, Optional<String> dockerImageRepo, + Optional<String> dockerImageRepo, Optional<String> combinedId) { + this.min = min; + this.max = max; this.dedicated = dedicated; - this.count = count; - this.groups = groups; this.version = version; this.required = required; this.canFail = canFail; this.exclusive = exclusive; - this.resources = resources; this.dockerImageRepo = dockerImageRepo; this.combinedId = combinedId; } - private NodesSpecification(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement, - Optional<String> combinedId, Optional<String> dockerImageRepo) { - this(dedicated, - nodesElement.integerAttribute("count", 1), - nodesElement.integerAttribute("groups", 1), - version, - nodesElement.booleanAttribute("required", false), - canFail, - nodesElement.booleanAttribute("exclusive", false), - getResources(nodesElement), - dockerImageToUse(nodesElement, dockerImageRepo), - combinedId); + private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, + ModelElement nodesElement, Optional<String> dockerImageRepo) { + var resolvedElement = resolveElement(nodesElement); + var combinedId = findCombinedId(nodesElement, resolvedElement); + var resources = toResources(resolvedElement); + return new NodesSpecification(resources.getFirst(), + resources.getSecond(), + dedicated, + version, + resolvedElement.booleanAttribute("required", false), + canFail, + resolvedElement.booleanAttribute("exclusive", false), + dockerImageToUse(resolvedElement, dockerImageRepo), + combinedId); + } + + private static Pair<ClusterResources, ClusterResources> toResources(ModelElement nodesElement) { + Pair<Integer, Integer> nodes = toRange(nodesElement.stringAttribute("count"), 1, Integer::parseInt); + Pair<Integer, Integer> groups = toRange(nodesElement.stringAttribute("groups"), 1, Integer::parseInt); + var min = new ClusterResources(nodes.getFirst(), groups.getFirst(), nodeResources(nodesElement).getFirst()); + var max = new ClusterResources(nodes.getSecond(), groups.getSecond(), nodeResources(nodesElement).getSecond()); + return new Pair<>(min, max); } /** Returns the ID of the cluster referencing this node specification, if any */ @@ -95,13 +105,6 @@ public class NodesSpecification { return containerIdReferencing(nodesElement); } - private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, - ModelElement nodesElement, Optional<String> dockerImage) { - var resolvedElement = resolveElement(nodesElement); - var combinedId = findCombinedId(nodesElement, resolvedElement); - return new NodesSpecification(dedicated, canFail, version, resolvedElement, combinedId, dockerImage); - } - /** Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element */ public static NodesSpecification from(ModelElement nodesElement, ConfigModelContext context) { return create(true, @@ -144,32 +147,33 @@ public class NodesSpecification { * Returns a requirement from <code>count</code> non-dedicated nodes in one group */ public static NodesSpecification nonDedicated(int count, ConfigModelContext context) { - return new NodesSpecification(false, - count, - 1, + return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified), + new ClusterResources(count, 1, NodeResources.unspecified), + false, context.getDeployState().getWantedNodeVespaVersion(), false, ! context.getDeployState().getProperties().isBootstrap(), false, - Optional.empty(), context.getDeployState().getWantedDockerImageRepo(), Optional.empty()); } /** Returns a requirement from <code>count</code> dedicated nodes in one group */ public static NodesSpecification dedicated(int count, ConfigModelContext context) { - return new NodesSpecification(true, - count, - 1, + return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified), + new ClusterResources(count, 1, NodeResources.unspecified), + true, context.getDeployState().getWantedNodeVespaVersion(), false, ! context.getDeployState().getProperties().isBootstrap(), false, - Optional.empty(), context.getDeployState().getWantedDockerImageRepo(), Optional.empty()); } + public ClusterResources minResources() { return min; } + public ClusterResources maxResources() { return max; } + /** * Returns whether this requires dedicated nodes. * Otherwise the model encountering this request should reuse nodes requested for other purposes whenever possible. @@ -183,12 +187,6 @@ public class NodesSpecification { */ public boolean isExclusive() { return exclusive; } - /** Returns the number of nodes required */ - public int count() { return count; } - - /** Returns the number of host groups this specifies. Default is 1 */ - public int groups() { return groups; } - public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, @@ -201,29 +199,36 @@ public class NodesSpecification { .combinedId(combinedId.map(ClusterSpec.Id::from)) .dockerImageRepo(dockerImageRepo) .build(); - return hostSystem.allocateHosts(cluster, Capacity.fromCount(count, resources, required, canFail), groups, logger); + return hostSystem.allocateHosts(cluster, Capacity.from(min, max, required, canFail), logger); } - private static Optional<NodeResources> getResources(ModelElement nodesElement) { + private static Pair<NodeResources, NodeResources> nodeResources(ModelElement nodesElement) { ModelElement resources = nodesElement.child("resources"); if (resources != null) { - return Optional.of(new NodeResources(resources.requiredDoubleAttribute("vcpu"), - parseGbAmount(resources.requiredStringAttribute("memory"), "B"), - parseGbAmount(resources.requiredStringAttribute("disk"), "B"), - Optional.ofNullable(resources.stringAttribute("bandwidth")) - .map(b -> parseGbAmount(b, "BPS")) - .orElse(0.3), - parseOptionalDiskSpeed(resources.stringAttribute("disk-speed")), - parseOptionalStorageType(resources.stringAttribute("storage-type")))); + return nodeResourcesFromResorcesElement(resources); } else if (nodesElement.stringAttribute("flavor") != null) { // legacy fallback - return Optional.of(NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor"))); + var flavorResources = NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor")); + return new Pair<>(flavorResources, flavorResources); } - else { // Get the default - return Optional.empty(); + else { + return new Pair<>(NodeResources.unspecified, NodeResources.unspecified); } } + private static Pair<NodeResources, NodeResources> nodeResourcesFromResorcesElement(ModelElement element) { + Pair<Double, Double> vcpu = toRange(element.requiredStringAttribute("vcpu"), .0, Double::parseDouble); + Pair<Double, Double> memory = toRange(element.requiredStringAttribute("memory"), .0, s -> parseGbAmount(s, "B")); + Pair<Double, Double> disk = toRange(element.requiredStringAttribute("disk"), .0, s -> parseGbAmount(s, "B")); + Pair<Double, Double> bandwith = toRange(element.stringAttribute("bandwith"), .3, s -> parseGbAmount(s, "BPS")); + NodeResources.DiskSpeed diskSpeed = parseOptionalDiskSpeed(element.stringAttribute("disk-speed")); + NodeResources.StorageType storageType = parseOptionalStorageType(element.stringAttribute("storage-type")); + + var min = new NodeResources(vcpu.getFirst(), memory.getFirst(), disk.getFirst(), bandwith.getFirst(), diskSpeed, storageType); + var max = new NodeResources(vcpu.getSecond(), memory.getSecond(), disk.getSecond(), bandwith.getSecond(), diskSpeed, storageType); + return new Pair<>(min, max); + } + private static double parseGbAmount(String byteAmount, String unit) { byteAmount = byteAmount.strip(); byteAmount = byteAmount.toUpperCase(); @@ -358,11 +363,28 @@ public class NodesSpecification { return dockerImageFromElement == null ? dockerImage : Optional.of(dockerImageFromElement); } + /** Parses a value ("value") or value range ("[min-value, max-value]") */ + private static <T> Pair<T, T> toRange(String s, T defaultValue, Function<String, T> valueParser) { + try { + if (s == null) return new Pair<>(defaultValue, defaultValue); + s = s.trim(); + if (s.startsWith("[") && s.endsWith("]")) { + String[] numbers = s.substring(1, s.length() - 1).split(","); + if (numbers.length != 2) throw new IllegalArgumentException(); + return new Pair<>(valueParser.apply(numbers[0].trim()), valueParser.apply(numbers[1].trim())); + } else { + return new Pair<>(valueParser.apply(s), valueParser.apply(s)); + } + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Expected a number or range on the form [min, max], but got '" + s + "'", e); + } + } + @Override public String toString() { - return "specification of " + count + (dedicated ? " dedicated " : " ") + "nodes" + - (resources.map(nodeResources -> " with resources " + nodeResources).orElse("")) + - (groups > 1 ? " in " + groups + " groups" : ""); + return "specification of " + (dedicated ? "dedicated " : "") + + (min.equals(max) ? min : "min " + min + " max " + max); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 97b78e1b9b1..c9caca1831f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -141,8 +141,7 @@ public class VespaDomBuilder extends VespaModelBuilder { } private void initializeService(AbstractService t, DeployState deployState, - HostSystem hostSystem, Element producerSpec) - { + HostSystem hostSystem, Element producerSpec) { initializeProducer(t, deployState, producerSpec); if (producerSpec != null) { if (producerSpec.hasAttribute(JVM_OPTIONS)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 7e2d6680827..31c8724d634 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -140,7 +140,7 @@ public abstract class Container extends AbstractService implements if (http == null) { return defaultHttpServer; } else { - return http.getHttpServer(); + return http.getHttpServer().orElse(null); } } @@ -228,10 +228,10 @@ public abstract class Container extends AbstractService implements // XXX unused - remove: from.allocatePort("http/1"); portsMeta.on(offset++).tag("http").tag("external"); - } else if (getHttp().getHttpServer() == null) { + } else if (getHttp().getHttpServer().isEmpty()) { // no http server ports } else { - for (ConnectorFactory connectorFactory : getHttp().getHttpServer().getConnectorFactories()) { + for (ConnectorFactory connectorFactory : getHttp().getHttpServer().get().getConnectorFactories()) { int port = getPort(connectorFactory); String name = "http/" + connectorFactory.getName(); from.requirePort(port, name); @@ -280,7 +280,7 @@ public abstract class Container extends AbstractService implements final Http http = getHttp(); if (http != null) { // TODO: allow the user to specify health port manually - if (http.getHttpServer() == null) { + if (http.getHttpServer().isEmpty()) { return -1; } else { return getRelativePort(0); @@ -303,7 +303,7 @@ public abstract class Container extends AbstractService implements .slobrokId(serviceSlobrokId())). filedistributor(filedistributorConfig()); if (clusterName != null) { - builder.discriminator(clusterName+"."+name); + builder.discriminator(clusterName + "." + name); } else { builder.discriminator(name); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java index ee61b34987a..3d9a1b2e665 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java @@ -10,6 +10,7 @@ import com.yahoo.osgi.provider.model.ComponentModel; * @author Tony Vaagenes */ public class FileStatusHandlerComponent extends Handler implements VipStatusConfig.Producer { + public static final String CLASS = "com.yahoo.container.handler.VipStatusHandler"; private final String fileName; @@ -26,4 +27,5 @@ public class FileStatusHandlerComponent extends Handler implements VipStatusConf builder.accessdisk(true). statusfile(fileName); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index f3758def2b1..470b82496a3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.container.StatisticsConfig; +import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.net.HostName; import com.yahoo.vespa.defaults.Defaults; @@ -29,7 +30,8 @@ public class ConfigserverCluster extends AbstractConfigProducer ZookeeperServerConfig.Producer, ConfigserverConfig.Producer, StatisticsConfig.Producer, - HealthMonitorConfig.Producer { + HealthMonitorConfig.Producer, + VipStatusConfig.Producer { private final CloudConfigOptions options; private ContainerCluster containerCluster; @@ -116,8 +118,8 @@ public class ConfigserverCluster extends AbstractConfigProducer } builder.serverId(HostName.getLocalhost()); - if (!containerCluster.getHttp().getHttpServer().getConnectorFactories().isEmpty()) { - builder.httpport(containerCluster.getHttp().getHttpServer().getConnectorFactories().get(0).getListenPort()); + if (!containerCluster.getHttp().getHttpServer().get().getConnectorFactories().isEmpty()) { + builder.httpport(containerCluster.getHttp().getHttpServer().get().getConnectorFactories().get(0).getListenPort()); } if (options.useVespaVersionInRequest().isPresent()) { builder.useVespaVersionInRequest(options.useVespaVersionInRequest().get()); @@ -178,4 +180,8 @@ public class ConfigserverCluster extends AbstractConfigProducer builder.snapshot_interval(60.0); } + @Override + public void getConfig(VipStatusConfig.Builder builder) { + builder.initiallyInRotation(false); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java index 400ddf80cf9..0fcf7b2d06c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -8,46 +8,39 @@ import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.component.chain.ChainedComponent; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; /** * Represents the http servers and filters of a container cluster. * * @author Tony Vaagenes + * @author bjorncs */ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> implements ServerConfig.Producer { - private FilterChains filterChains; - private JettyHttpServer httpServer; - private List<Binding> bindings; - private final Optional<AccessControl> accessControl; + private final FilterChains filterChains; + private final List<Binding> bindings = new CopyOnWriteArrayList<>(); + private volatile JettyHttpServer httpServer; + private volatile AccessControl accessControl; - public Http(List<Binding> bindings) { - this(bindings, null); + public Http(FilterChains chains) { + super("http"); + this.filterChains = chains; } - public Http(List<Binding> bindings, AccessControl accessControl) { - super( "http"); - this.bindings = Collections.unmodifiableList(bindings); - this.accessControl = Optional.ofNullable(accessControl); - } - - public void setFilterChains(FilterChains filterChains) { - this.filterChains = filterChains; - } - - public void setBindings(List<Binding> bindings) { - this.bindings = Collections.unmodifiableList(bindings); + public void setAccessControl(AccessControl accessControl) { + if (this.accessControl != null) throw new IllegalStateException("Access control already assigned"); + this.accessControl = accessControl; } public FilterChains getFilterChains() { return filterChains; } - public JettyHttpServer getHttpServer() { - return httpServer; + public Optional<JettyHttpServer> getHttpServer() { + return Optional.ofNullable(httpServer); } public void setHttpServer(JettyHttpServer newServer) { @@ -76,25 +69,21 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl } public Optional<AccessControl> getAccessControl() { - return accessControl; + return Optional.ofNullable(accessControl); } @Override public void getConfig(ServerConfig.Builder builder) { for (Binding binding : bindings) { builder.filter(new ServerConfig.Filter.Builder() - .id(binding.filterId().stringValue()) - .binding(binding.binding())); + .id(binding.filterId().stringValue()) + .binding(binding.binding())); } } @Override public void validate() { - validate(bindings); - } - - void validate(Collection<Binding> bindings) { - if (bindings.isEmpty()) return; + if (((Collection<Binding>) bindings).isEmpty()) return; if (filterChains == null) throw new IllegalArgumentException("Null FilterChains are not allowed when there are filter bindings"); @@ -107,5 +96,4 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl throw new RuntimeException("Can't find filter " + binding.filterId() + " for binding " + binding.binding()); } } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index f61618c789b..fb8e9dffbbb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -7,6 +7,7 @@ import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; +import java.time.Duration; import java.util.List; /** @@ -67,10 +68,13 @@ public class HostedSslConnectorFactory extends ConnectorFactory { @Override public void getConfig(ConnectorConfig.Builder connectorBuilder) { super.getConfig(connectorBuilder); - connectorBuilder.tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder() - .pathWhitelist(INSECURE_WHITELISTED_PATHS) - .enable(enforceClientAuth)); - connectorBuilder.proxyProtocol(configureProxyProtocol()); + connectorBuilder + .tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder() + .pathWhitelist(INSECURE_WHITELISTED_PATHS) + .enable(enforceClientAuth)) + .proxyProtocol(configureProxyProtocol()) + .idleTimeout(Duration.ofMinutes(3).toSeconds()) + .maxConnectionLife(Duration.ofMinutes(10).toSeconds()); } private ConnectorConfig.ProxyProtocol.Builder configureProxyProtocol() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index 61a588fb716..bfde9b9add1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -54,11 +54,10 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> filterChains = new FilterChainsBuilder().newChainsInstance(ancestor); } - Http http = new Http(bindings, accessControl); - http.setFilterChains(filterChains); - - buildHttpServers(deployState, ancestor, http, spec); - + Http http = new Http(filterChains); + http.getBindings().addAll(bindings); + http.setAccessControl(accessControl); + http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec)); return http; } @@ -131,10 +130,6 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> return result; } - private void buildHttpServers(DeployState deployState, AbstractConfigProducer ancestor, Http http, Element spec) { - http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec)); - } - static int readPort(ModelElement spec, boolean isHosted, DeployLogger logger) { Integer port = spec.integerAttribute("port"); if (port == null) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index c1f793e255d..06707786136 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -23,9 +23,11 @@ import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; @@ -324,13 +326,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.setHttp(buildHttp(deployState, cluster, httpElement)); } if (isHostedTenantApplication(context)) { - addHostedImplicitHttpIfNotPresent(deployState, cluster); + addHostedImplicitHttpIfNotPresent(cluster); + addHostedImplicitAccessControlIfNotPresent(deployState, cluster); addAdditionalHostedConnector(deployState, cluster); } } private void addAdditionalHostedConnector(DeployState deployState, ApplicationContainerCluster cluster) { - JettyHttpServer server = cluster.getHttp().getHttpServer(); + JettyHttpServer server = cluster.getHttp().getHttpServer().get(); String serverName = server.getComponentId().getName(); String proxyProtocol = deployState.getProperties().proxyProtocol(); @@ -356,39 +359,31 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return deployState.isHosted() && context.getApplicationType() == ApplicationType.DEFAULT && !isTesterApplication; } - private static void addHostedImplicitHttpIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) { + private static void addHostedImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) { if(cluster.getHttp() == null) { - Http http = deployState.getProperties().athenzDomain() - .map(tenantDomain -> createHostedImplicitHttpWithAccessControl(deployState, tenantDomain, cluster)) - .orElseGet(() -> createHostedImplicitHttpWithoutAccessControl(cluster)); - cluster.setHttp(http); + cluster.setHttp(new Http(new FilterChains(cluster))); } - if(cluster.getHttp().getHttpServer() == null) { + if(cluster.getHttp().getHttpServer().isEmpty()) { JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer")); cluster.getHttp().setHttpServer(defaultHttpServer); defaultHttpServer.addConnector(new ConnectorFactory("SearchServer", Defaults.getDefaults().vespaWebServicePort())); } } - private static Http createHostedImplicitHttpWithAccessControl( - DeployState deployState, AthenzDomain tenantDomain, ApplicationContainerCluster cluster) { + private void addHostedImplicitAccessControlIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) { + Http http = cluster.getHttp(); + if (http.getAccessControl().isPresent()) return; // access control added explicitly + AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null); + if (tenantDomain == null) return; // tenant domain not present, cannot add access control. this should eventually be a failure. AccessControl accessControl = new AccessControl.Builder(tenantDomain.value(), deployState.getDeployLogger()) .setHandlers(cluster) .readEnabled(false) .writeEnabled(false) .build(); - Http http = new Http(accessControl.getBindings(), accessControl); - FilterChains filterChains = new FilterChains(cluster); - filterChains.add(new Chain<>(FilterChains.emptyChainSpec(ACCESS_CONTROL_CHAIN_ID))); - http.setFilterChains(filterChains); - return http; - } - - private static Http createHostedImplicitHttpWithoutAccessControl(ApplicationContainerCluster cluster) { - Http http = new Http(Collections.emptyList()); - http.setFilterChains(new FilterChains(cluster)); - return http; + http.getFilterChains().add(new Chain<>(FilterChains.emptyChainSpec(ACCESS_CONTROL_CHAIN_ID))); + http.setAccessControl(accessControl); + http.getBindings().addAll(accessControl.getBindings()); } private Http buildHttp(DeployState deployState, ApplicationContainerCluster cluster, Element httpElement) { @@ -672,11 +667,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .vespaVersion(deployState.getWantedNodeVespaVersion()) .dockerImageRepo(deployState.getWantedDockerImageRepo()) .build(); - Capacity capacity = Capacity.fromCount(1, - Optional.empty(), - false, - ! deployState.getProperties().isBootstrap()); - HostResource host = hostSystem.allocateHosts(clusterSpec, capacity, 1, log).keySet().iterator().next(); + Capacity capacity = Capacity.from(new ClusterResources(1, 1, NodeResources.unspecified), + false, + ! deployState.getProperties().isBootstrap()); + HostResource host = hostSystem.allocateHosts(clusterSpec, capacity, log).keySet().iterator().next(); return singleHostContainerCluster(cluster, host, context); } } @@ -687,11 +681,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .dockerImageRepo(deployState.getWantedDockerImageRepo()) .build(); int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; - Capacity capacity = Capacity.fromCount(nodeCount, - Optional.empty(), - false, - !deployState.getProperties().isBootstrap()); - var hosts = hostSystem.allocateHosts(clusterSpec, capacity, 1, log); + Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified), + false, + !deployState.getProperties().isBootstrap()); + var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); return createNodesFromHosts(log, hosts, cluster); } return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); @@ -721,7 +714,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .build(); Map<HostResource, ClusterMembership> hosts = cluster.getRoot().hostSystem().allocateHosts(clusterSpec, - Capacity.fromRequiredNodeType(type), 1, log); + Capacity.fromRequiredNodeType(type), log); return createNodesFromHosts(context.getDeployLogger(), hosts, cluster); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 066fef727c5..6dd3e619ec2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -286,7 +286,7 @@ public class ContentCluster extends AbstractConfigProducer implements .orElse(NodesSpecification.nonDedicated(3, context)); Collection<HostResource> hosts = nodesSpecification.isDedicated() ? getControllerHosts(nodesSpecification, admin, clusterName, context) : - drawControllerHosts(nodesSpecification.count(), rootGroup, containers); + drawControllerHosts(nodesSpecification.minResources().nodes(), rootGroup, containers); clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone"), hosts, clusterName, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index f5a91297e9e..f93bf6fc872 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -241,7 +241,12 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ public Integer level = null; public void getConfig(ProtonConfig.Summary.Cache.Compression.Builder compression) { - if (type != null) compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name)); + if (type != null) compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name)); + if (level != null) compression.level(level); + } + + public void getConfig(ProtonConfig.Summary.Log.Compact.Compression.Builder compression) { + if (type != null) compression.type(ProtonConfig.Summary.Log.Compact.Compression.Type.Enum.valueOf(type.name)); if (level != null) compression.level(level); } @@ -281,6 +286,12 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ } } + public void getConfig(ProtonConfig.Summary.Log.Compact.Builder compact) { + if (compression != null) { + compression.getConfig(compact.compression); + } + } + public void getConfig(ProtonConfig.Summary.Log.Chunk.Builder chunk) { if (outputInt) { if (maxSize!=null) chunk.maxbytes(maxSize.intValue()); @@ -288,7 +299,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ throw new IllegalStateException("Fix this, chunk does not have long types"); } if (compression != null) { - compression.getConfig(chunk.compression); + compression.getConfig(chunk.compression); } } } @@ -303,6 +314,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ if (minFileSizeFactor!=null) log.minfilesizefactor(minFileSizeFactor); if (chunk != null) { chunk.getConfig(log.chunk); + chunk.getConfig(log.compact); } } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index cca56c209c8..3560cf2cd84 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -1816,6 +1816,7 @@ Object indexBody(IndexOperation index) : | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); } | <ENABLE_BM25> { index.setEnableBm25(true); } + | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { index.setDistanceMetric(str); } | hnswIndex(index) { } ) { return null; } @@ -1841,7 +1842,6 @@ void hnswIndexBody(HnswIndexParams.Builder params) : } { ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); } - | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { params.setDistanceMetric(str); } | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); } ) } diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index c47983adc12..878faabfec1 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -17,14 +17,14 @@ anyElement = element * { JavaId = xsd:string { pattern = "([a-zA-Z_$][a-zA-Z\d_$]*\.)*[a-zA-Z_$][a-zA-Z\d_$]*" } Nodes = element nodes { - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute docker-image { xsd:string }? & Resources? } Resources = element resources { - attribute vcpu { xsd:double { minExclusive = "0.0" } } & + attribute vcpu { xsd:double { minExclusive = "0.0" } | xsd:string } & attribute memory { xsd:string } & attribute disk { xsd:string } & attribute disk-speed { xsd:string }? & @@ -32,7 +32,7 @@ Resources = element resources { } OptionalDedicatedNodes = element nodes { - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute docker-image { xsd:string }? & diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 726fa849c00..3c8b60fb84b 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -239,7 +239,7 @@ NodesOfContainerCluster = element nodes { attribute type { xsd:string } | ( - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute exclusive { xsd:boolean }? & diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index ee451185415..b1821680b14 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -221,11 +221,11 @@ ContentNodes = element nodes { attribute vespamalloc-debug-stacktrace { xsd:string }? & ( ( - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute docker-image { xsd:string }? & - attribute groups { xsd:positiveInteger }? + attribute groups { xsd:positiveInteger | xsd:string }? ) | ContentNode + @@ -266,12 +266,12 @@ Group = element group { | ( element nodes { - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute exclusive { xsd:boolean }? & attribute docker-image { xsd:string }? & - attribute groups { xsd:positiveInteger }? + attribute groups { xsd:positiveInteger | xsd:string }? } ) | diff --git a/config-model/src/test/derived/hnsw_index/test.sd b/config-model/src/test/derived/hnsw_index/test.sd index 3b954e74fc5..207ed764a87 100644 --- a/config-model/src/test/derived/hnsw_index/test.sd +++ b/config-model/src/test/derived/hnsw_index/test.sd @@ -3,9 +3,9 @@ search test { field t1 type tensor(x[128]) { indexing: attribute | index index { + distance-metric: angular hnsw { max-links-per-node: 32 - distance-metric: angular neighbors-to-explore-at-insert: 300 } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 1670ac23ba4..7208d8c5fc1 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1205,6 +1205,62 @@ public class ModelProvisioningTest { } @Test + public void testRequestingRangesMin() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='container'>" + + " <nodes count='[4, 6]'>" + + " <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" + + " </nodes>" + + " </container>" + + " <content version='1.0' id='foo'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='[6, 20]' groups='[3,4]'>" + + " <resources vcpu='8' memory='200Gb' disk='1Pb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + + int totalHosts = 10; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(new NodeResources(11.5, 10, 30, 0.3), 6); + tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20); + VespaModel model = tester.createModel(services, true); + assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); + } + + @Test + public void testRequestingRangesMax() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='container'>" + + " <nodes count='[4, 6]'>" + + " <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" + + " </nodes>" + + " </container>" + + " <content version='1.0' id='foo'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='[6, 20]' groups='[3,4]'>" + + " <resources vcpu='8' memory='200Gb' disk='1Pb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + + int totalHosts = 26; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 6); + tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20); + VespaModel model = tester.createModel(services, true, true); + assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); + } + + @Test public void testContainerOnly() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + @@ -1308,10 +1364,11 @@ public class ModelProvisioningTest { " </http>" + "</container>"; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(1); + tester.addHosts(2); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getHosts().size()); + assertEquals(2, model.getHosts().size()); assertEquals(1, model.getContainerClusters().size()); + assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); } @Test @@ -1374,7 +1431,7 @@ public class ModelProvisioningTest { } @Test - public void testNoNodeTagMeans1Node() { + public void testNoNodeTagMeansTwoNodes() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + @@ -1389,31 +1446,6 @@ public class ModelProvisioningTest { " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(1); - VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().hostSystem().getHosts().size()); - assertEquals(1, model.getAdmin().getSlobroks().size()); - assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); - assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); - } - - @Test - public void testNoNodeTagMeansTwoNodesInContainerClusterWithFeatureFlag() { - String services = - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services>" + - " <container id='foo' version='1.0'>" + - " <search/>" + - " <document-api/>" + - " </container>" + - " <content version='1.0' id='bar'>" + - " <documents>" + - " <document type='type1' mode='index'/>" + - " </documents>" + - " </content>" + - "</services>"; - VespaModelTester tester = new VespaModelTester(); - tester.setUseDedicatedNodesWhenUnspecified(true); tester.addHosts(3); VespaModel model = tester.createModel(services, true); assertEquals(3, model.getRoot().hostSystem().getHosts().size()); @@ -1423,7 +1455,7 @@ public class ModelProvisioningTest { } @Test - public void testNoNodeTagMeans1NodeNoContent() { + public void testNoNodeTagMeansTwoNodesNoContent() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + @@ -1433,11 +1465,11 @@ public class ModelProvisioningTest { " </container>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(1); + tester.addHosts(2); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().hostSystem().getHosts().size()); - assertEquals(1, model.getAdmin().getSlobroks().size()); - assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); + assertEquals(2, model.getRoot().hostSystem().getHosts().size()); + assertEquals(2, model.getAdmin().getSlobroks().size()); + assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java index ead4e586d9f..9f57b22fd58 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java @@ -31,6 +31,9 @@ public class NearestNeighborTestCase extends AbstractExportingTestCase { } catch (QueryException e) { // success assertEquals("Invalid request parameter", e.getMessage()); + } catch (RuntimeException e) { + e.printStackTrace(); + throw e; } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java index d687590faf2..e3dcc925e5e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java @@ -2,13 +2,13 @@ package com.yahoo.searchdefinition.document; +import java.util.Optional; import org.junit.Test; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; -import static com.yahoo.searchdefinition.document.HnswIndexParams.DistanceMetric; public class HnswIndexParamsTestCase { @@ -18,35 +18,27 @@ public class HnswIndexParamsTestCase { var builder = new HnswIndexParams.Builder(); builder.setMaxLinksPerNode(7); var one = builder.build(); - builder.setDistanceMetric("angular"); - var two = builder.build(); builder.setNeighborsToExploreAtInsert(42); var three = builder.build(); builder.setMaxLinksPerNode(17); - builder.setDistanceMetric("geodegrees"); builder.setNeighborsToExploreAtInsert(500); var four = builder.build(); assertThat(empty.maxLinksPerNode(), is(16)); - assertThat(empty.distanceMetric(), is(DistanceMetric.EUCLIDEAN)); assertThat(empty.neighborsToExploreAtInsert(), is(200)); assertThat(one.maxLinksPerNode(), is(7)); - assertThat(two.distanceMetric(), is(DistanceMetric.ANGULAR)); assertThat(three.neighborsToExploreAtInsert(), is(42)); assertThat(four.maxLinksPerNode(), is(17)); - assertThat(four.distanceMetric(), is(DistanceMetric.GEODEGREES)); assertThat(four.neighborsToExploreAtInsert(), is(500)); - var five = four.overrideFrom(empty); + var five = four.overrideFrom(Optional.of(empty)); assertThat(five.maxLinksPerNode(), is(17)); - assertThat(five.distanceMetric(), is(DistanceMetric.GEODEGREES)); assertThat(five.neighborsToExploreAtInsert(), is(500)); - var six = four.overrideFrom(two); + var six = four.overrideFrom(Optional.of(one)); assertThat(six.maxLinksPerNode(), is(7)); - assertThat(six.distanceMetric(), is(DistanceMetric.ANGULAR)); assertThat(six.neighborsToExploreAtInsert(), is(500)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java index 4b7f727ff63..cf1ae637cf9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -109,7 +109,13 @@ public class VespaModelFactoryTest { } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + return prepare(cluster, capacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { return List.of(new HostSpec(hostName, List.of(), ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java index 060fff5bf66..73dd1ca3f3b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.collections.CollectionUtil; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; -import com.yahoo.vespa.model.content.DispatchTuning; import com.yahoo.vespa.model.search.Tuning; import org.junit.Test; import org.w3c.dom.Element; @@ -228,6 +227,8 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest { assertEquals(cfg.summary().log().chunk().maxbytes(), 256); assertEquals(cfg.summary().log().chunk().compression().type(), ProtonConfig.Summary.Log.Chunk.Compression.Type.LZ4); assertEquals(cfg.summary().log().chunk().compression().level(), 5); + assertEquals(cfg.summary().log().compact().compression().type(), ProtonConfig.Summary.Log.Compact.Compression.Type.LZ4); + assertEquals(cfg.summary().log().compact().compression().level(), 5); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index b1e63628852..28e23ce3222 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -3,8 +3,13 @@ package com.yahoo.vespa.model.container.xml; import com.google.common.collect.ImmutableSet; import com.yahoo.collections.CollectionUtil; +import com.yahoo.component.ComponentId; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.container.jdisc.state.StateHandler; +import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.http.AccessControl; import com.yahoo.vespa.model.container.http.Http; @@ -16,14 +21,19 @@ import org.w3c.dom.Element; import java.util.Collection; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -229,6 +239,58 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { } + + @Test + public void access_control_is_implicitly_added_for_hosted_apps() { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + nodesXml, + "</container>" ); + AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); + DeployState state = new DeployState.Builder().properties( + new TestProperties() + .setAthenzDomain(tenantDomain) + .setHostedVespa(true)) + .build(); + createModel(root, state, null, clusterElem); + Optional<AccessControl> maybeAccessControl = + ((ApplicationContainer) root.getProducer("container/container.0")).getHttp().getAccessControl(); + assertThat(maybeAccessControl.isPresent(), is(true)); + AccessControl accessControl = maybeAccessControl.get(); + assertThat(accessControl.writeEnabled, is(false)); + assertThat(accessControl.readEnabled, is(false)); + assertThat(accessControl.domain, equalTo(tenantDomain.value())); + } + + @Test + public void access_control_is_implicitly_added_for_hosted_apps_with_existing_http_element() { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + " <http>", + " <server port='" + getDefaults().vespaWebServicePort() + "' id='main' />", + " <filtering>", + " <filter id='outer' />", + " <request-chain id='myChain'>", + " <filter id='inner' />", + " </request-chain>", + " </filtering>", + " </http>", + nodesXml, + "</container>" ); + AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); + DeployState state = new DeployState.Builder().properties( + new TestProperties() + .setAthenzDomain(tenantDomain) + .setHostedVespa(true)) + .build(); + createModel(root, state, null, clusterElem); + Http http = ((ApplicationContainer) root.getProducer("container/container.0")).getHttp(); + assertThat(http.getAccessControl().isPresent(), is(true)); + assertThat(http.getFilterChains().hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID), is(true)); + assertThat(http.getFilterChains().hasChain(ComponentId.fromString("myChain")), is(true)); + } + + private String httpWithExcludedBinding(String excludedBinding) { return joinLines( " <http>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 73db6e35428..dcd1c46e52f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -14,7 +14,6 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; -import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.RegionName; @@ -44,7 +43,6 @@ import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.http.AccessControl; import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; @@ -626,7 +624,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .setMultitenant(true) .setHostedVespa(true)) .build()); - assertEquals(1, model.hostSystem().getHosts().size()); + assertEquals(2, model.hostSystem().getHosts().size()); } @Test(expected = IllegalArgumentException.class) @@ -811,7 +809,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0"); // Verify that there are two connectors - List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().getConnectorFactories(); + List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories(); assertEquals(2, connectorFactories.size()); List<Integer> ports = connectorFactories.stream() .map(ConnectorFactory::getListenPort) @@ -835,28 +833,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertThat(connectorConfig.ssl().caCertificate(), isEmptyString()); } - @Test - public void access_control_is_implicitly_added_for_hosted_apps() { - Element clusterElem = DomBuilderTest.parse( - "<container version='1.0'>", - nodesXml, - "</container>" ); - AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); - DeployState state = new DeployState.Builder().properties( - new TestProperties() - .setAthenzDomain(tenantDomain) - .setHostedVespa(true)) - .build(); - createModel(root, state, null, clusterElem); - Optional<AccessControl> maybeAccessControl = - ((ApplicationContainer) root.getProducer("container/container.0")).getHttp().getAccessControl(); - assertThat(maybeAccessControl.isPresent(), is(true)); - AccessControl accessControl = maybeAccessControl.get(); - assertThat(accessControl.writeEnabled, is(false)); - assertThat(accessControl.readEnabled, is(false)); - assertThat(accessControl.domain, equalTo(tenantDomain.value())); - } - private Element generateContainerElementWithRenderer(String rendererId) { return DomBuilderTest.parse( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index fd837c6dea3..cdfd9fab194 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -48,7 +48,6 @@ public class VespaModelTester { private Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>(); private ApplicationId applicationId = ApplicationId.defaultId(); private boolean useDedicatedNodeForLogserver = false; - private boolean useDedicatedNodesWhenUnspecified = false; public VespaModelTester() { this(new NullConfigModelRegistry()); @@ -98,10 +97,6 @@ public class VespaModelTester { this.useDedicatedNodeForLogserver = useDedicatedNodeForLogserver; } - public void setUseDedicatedNodesWhenUnspecified(boolean useDedicatedNodesWhenUnspecified) { - this.useDedicatedNodesWhenUnspecified = useDedicatedNodesWhenUnspecified; - } - /** Creates a model which uses 0 as start index and fails on out of capacity */ public VespaModel createModel(String services, String ... retiredHostNames) { return createModel(Zone.defaultZone(), services, true, retiredHostNames); @@ -109,41 +104,48 @@ public class VespaModelTester { /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, 0, retiredHostNames); + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0, retiredHostNames); + } + + /** Creates a model which uses 0 as start index */ + public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, String ... retiredHostNames) { + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, 0, retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, startIndexForClusters, retiredHostNames); + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, startIndexForClusters, retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(zone, services, failOnOutOfCapacity, 0, retiredHostNames); + return createModel(zone, services, failOnOutOfCapacity, false, 0, retiredHostNames); } /** * Creates a model using the hosts already added to this * * @param services the services xml string + * @param useMaxResources false to use the minmal resources (when given a range), true to use max * @param failOnOutOfCapacity whether we should get an exception when not enough hosts of the requested flavor * is available or if we should just silently receive a smaller allocation * @return the resulting model */ - public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { + public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, boolean useMaxResources, + int startIndexForClusters, String ... retiredHostNames) { VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; - HostProvisioner provisioner = hosted ? - new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) : + HostProvisioner provisioner = hosted ? + new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, useMaxResources, + startIndexForClusters, retiredHostNames) : new SingleNodeProvisioner(); TestProperties properties = new TestProperties() .setMultitenant(true) .setHostedVespa(hosted) .setApplicationId(applicationId) - .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver) - .setUseDedicatedNodesWhenUnspecified(useDedicatedNodesWhenUnspecified); + .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver); DeployState deployState = new DeployState.Builder() .applicationPackage(appPkg) diff --git a/config-model/src/test/schema-test-files/services-hosted.xml b/config-model/src/test/schema-test-files/services-hosted.xml index 07839239c81..71a07926240 100644 --- a/config-model/src/test/schema-test-files/services-hosted.xml +++ b/config-model/src/test/schema-test-files/services-hosted.xml @@ -7,7 +7,7 @@ </admin> <container id="container1" version="1.0"> - <nodes count="5" required="true"> + <nodes count="[5,7]" required="true"> <resources vcpu="1.2" memory="10Gb" disk="0.3 TB"/> </nodes> </container> @@ -27,4 +27,11 @@ </nodes> </content> + <content id="ml" version="1.0"> + <redundancy>2</redundancy> + <nodes count="[10,20]" flavor="large" groups="[1,3]"> + <resources vcpu="[3.0, 4]" memory="[32000.0Mb, 33Gb]" disk="[300 Gb, 1Tb]"/> + </nodes> + </content> + </services> |