diff options
255 files changed, 3898 insertions, 3141 deletions
diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java index 3363ddb040f..1d230d6cb47 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java @@ -11,24 +11,28 @@ import java.util.Set; */ public class ApplicationInstance { - private final TenantId tenantId; - private final ApplicationInstanceId applicationInstanceId; + private final ApplicationInstanceReference reference; private final Set<ServiceCluster> serviceClusters; - public ApplicationInstance(TenantId tenantId, ApplicationInstanceId applicationInstanceId, Set<ServiceCluster> serviceClusters) { - this.tenantId = tenantId; - this.applicationInstanceId = applicationInstanceId; + public ApplicationInstance(TenantId tenantId, + ApplicationInstanceId applicationInstanceId, + Set<ServiceCluster> serviceClusters) { + this(new ApplicationInstanceReference(tenantId, applicationInstanceId), serviceClusters); + } + + public ApplicationInstance(ApplicationInstanceReference reference, Set<ServiceCluster> serviceClusters) { + this.reference = reference; this.serviceClusters = serviceClusters; } @JsonProperty("tenantId") public TenantId tenantId() { - return tenantId; + return reference.tenantId(); } @JsonProperty("applicationInstanceId") public ApplicationInstanceId applicationInstanceId() { - return applicationInstanceId; + return reference.applicationInstanceId(); } @JsonProperty("serviceClusters") @@ -38,7 +42,7 @@ public class ApplicationInstance { @JsonProperty("reference") public ApplicationInstanceReference reference() { - return new ApplicationInstanceReference(tenantId, applicationInstanceId); + return reference; } @Override @@ -46,21 +50,19 @@ public class ApplicationInstance { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ApplicationInstance that = (ApplicationInstance) o; - return Objects.equals(tenantId, that.tenantId) && - Objects.equals(applicationInstanceId, that.applicationInstanceId) && + return Objects.equals(reference, that.reference) && Objects.equals(serviceClusters, that.serviceClusters); } @Override public int hashCode() { - return Objects.hash(tenantId, applicationInstanceId, serviceClusters); + return Objects.hash(reference, serviceClusters); } @Override public String toString() { return "ApplicationInstance{" + - "tenantId=" + tenantId + - ", applicationInstanceId=" + applicationInstanceId + + "reference=" + reference + ", serviceClusters=" + serviceClusters + '}'; } diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 90132d6924a..40a62026253 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -880,7 +880,8 @@ "public java.util.Optional endpointCertificateSecrets()", "public abstract double defaultTermwiseLimit()", "public abstract boolean useBucketSpaceMetric()", - "public boolean useNewAthenzFilter()" + "public boolean useNewAthenzFilter()", + "public boolean usePhraseSegmenting()" ], "fields": [] }, diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 2410de55f86..9aad6361b9a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -60,6 +60,7 @@ public interface ModelContext { double defaultTermwiseLimit(); boolean useBucketSpaceMetric(); default boolean useNewAthenzFilter() { return false; } + default boolean usePhraseSegmenting() { return false; } } } diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java index a9677d4b34c..4cd0c1815dd 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java @@ -133,6 +133,11 @@ public final class XmlHelper { return Optional.ofNullable(element.getAttribute(name)).filter(s -> !s.isEmpty()); } + public static Optional<Element> getOptionalChild(Element parent, String childName) { + return Optional.ofNullable(XML.getChild(parent, childName)); + + } + public static Optional<String> getOptionalChildValue(Element parent, String childName) { Element child = XML.getChild(parent, childName); if (child == null) return Optional.empty(); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 0b4562ecd5c..2e4cdd706f7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -42,7 +42,7 @@ public class TestProperties implements ModelContext.Properties { private double defaultTermwiseLimit = 1.0; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private boolean useNewAthenzFilter = false; - + private boolean usePhraseSegmenting = false; @Override public boolean multitenant() { return multitenant; } @Override public ApplicationId applicationId() { return applicationId; } @@ -63,6 +63,7 @@ public class TestProperties implements ModelContext.Properties { @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useBucketSpaceMetric() { return true; } @Override public boolean useNewAthenzFilter() { return useNewAthenzFilter; } + @Override public boolean usePhraseSegmenting() { return usePhraseSegmenting; } public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; @@ -114,6 +115,11 @@ public class TestProperties implements ModelContext.Properties { return this; } + public TestProperties setUsePhraseSegmenting(boolean phraseSegmenting) { + this.usePhraseSegmenting = phraseSegmenting; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index fc8710fa1a1..b16f8c0e5bb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -87,7 +87,7 @@ public class DerivedConfiguration { juniperrc = new Juniperrc(search); rankProfileList = new RankProfileList(search, search.rankingConstants(), attributeFields, rankProfileRegistry, queryProfiles, importedModels, deployProperties); indexingScript = new IndexingScript(search); - indexInfo = new IndexInfo(search); + indexInfo = new IndexInfo(search, deployProperties); indexSchema = new IndexSchema(search); importedFields = new ImportedFields(search); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java index 032f7f58e2a..9ae72badf1c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.document.*; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.Search; @@ -11,6 +12,7 @@ import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.search.config.IndexInfoConfig; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -36,13 +38,16 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { private static final String CMD_FAST_SEARCH = "fast-search"; private static final String CMD_PREDICATE_BOUNDS = "predicate-bounds"; private static final String CMD_NUMERICAL = "numerical"; + private static final String CMD_PHRASE_SEGMENTING = "phrase-segmenting"; private Set<IndexCommand> commands = new java.util.LinkedHashSet<>(); private Map<String, String> aliases = new java.util.LinkedHashMap<>(); private Map<String, FieldSet> fieldSets; private Search search; + private final boolean phraseSegmenting; - public IndexInfo(Search search) { + public IndexInfo(Search search, ModelContext.Properties deployProperties) { this.fieldSets = search.fieldSets().userFieldSets(); + this.phraseSegmenting = deployProperties.usePhraseSegmenting(); addIndexCommand("sddocname", CMD_INDEX); addIndexCommand("sddocname", CMD_WORD); derive(search); @@ -153,6 +158,10 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { addIndexCommand(field, CMD_NUMERICAL); } + if (phraseSegmenting) { + addIndexCommand(field, CMD_PHRASE_SEGMENTING); + } + // Explicit commands for (String command : field.getQueryCommands()) { addIndexCommand(field, command); @@ -293,6 +302,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { boolean anyLowerCasing = false; boolean anyStemming = false; boolean anyNormalizing = false; + String phraseSegmentingCommand = null; String stemmingCommand = null; Matching fieldSetMatching = fieldSet.getMatching(); // null if no explicit matching // First a pass over the fields to read some params to decide field settings implicitly: @@ -313,8 +323,13 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { if (field.getNormalizing().doRemoveAccents()) { anyNormalizing = true; } - if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType) + if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType) { fieldSetMatching = field.getMatching(); + } + Optional<String> explicitPhraseSegmentingCommand = field.getQueryCommands().stream().filter(c -> c.startsWith(CMD_PHRASE_SEGMENTING)).findFirst(); + if (explicitPhraseSegmentingCommand.isPresent()) { + phraseSegmentingCommand = explicitPhraseSegmentingCommand.get(); + } } if (anyIndexing && anyAttributing && fieldSet.getMatching() == null) { // We have both attributes and indexes and no explicit match setting -> @@ -357,6 +372,11 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { new IndexInfoConfig.Indexinfo.Command.Builder() .indexname(fieldSet.getName()) .command(CMD_NORMALIZE)); + if (phraseSegmentingCommand != null) + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(phraseSegmentingCommand)); } } else { // Assume only attribute fields @@ -392,9 +412,15 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } else if (fieldSetMatching.getType().equals(Matching.Type.TEXT)) { } - } - + + if (phraseSegmentingCommand == null + && fieldSet.queryCommands().stream().noneMatch(c -> c.startsWith(CMD_PHRASE_SEGMENTING))) { // use default + if (phraseSegmenting) + iiB.command(new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_PHRASE_SEGMENTING)); + } } private boolean hasMultiValueField(FieldSet fieldSet) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 9c68b3eb28a..a1ec308c808 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -190,7 +190,9 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC cloudWatch.hostedAuth().ifPresent(hostedAuth -> cloudWatchBuilder .accessKeyName(hostedAuth.accessKeyName) .secretKeyName(hostedAuth.secretKeyName)); - cloudWatch.profile().ifPresent(cloudWatchBuilder::profile); + cloudWatch.sharedCredentials().ifPresent(sharedCredentials -> cloudWatchBuilder + .profile(sharedCredentials.profile) + .file(sharedCredentials.file)); builder.cloudWatch(cloudWatchBuilder); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java index fd290409ea5..0f3543d3c36 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java @@ -13,7 +13,7 @@ public class CloudWatch { private final MetricsConsumer consumer; private HostedAuth hostedAuth; - private String profile; + private SharedCredentials sharedCredentials; public CloudWatch(String region, String namespace, MetricsConsumer consumer) { this.region = region; @@ -26,14 +26,14 @@ public class CloudWatch { public String consumer() { return consumer.getId(); } public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); } - public Optional<String> profile() { return Optional.ofNullable(profile); } + public Optional<SharedCredentials> sharedCredentials() {return Optional.ofNullable(sharedCredentials); } - public void setHostedAuth(HostedAuth hostedAuth) { - this.hostedAuth = hostedAuth; + public void setHostedAuth(String accessKeyName, String secretKeyName) { + hostedAuth = new HostedAuth(accessKeyName, secretKeyName); } - public void setProfile(String profile) { - this.profile = profile; + public void setSharedCredentials(String profile, String file) { + sharedCredentials = new SharedCredentials(profile, file); } public static class HostedAuth { @@ -46,4 +46,14 @@ public class CloudWatch { } } + public static class SharedCredentials { + public final String profile; + public final String file; + + public SharedCredentials(String profile, String file) { + this.profile = profile; + this.file = file; + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index e308be00faf..bd16d83d157 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -608,6 +608,8 @@ public class VespaMetricSet { metrics.add(new Metric("vds.idealstate.garbage_collection.done_ok.rate")); metrics.add(new Metric("vds.idealstate.garbage_collection.done_failed.rate")); metrics.add(new Metric("vds.idealstate.garbage_collection.pending.average")); + metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.count")); + metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.rate")); metrics.add(new Metric("vds.distributor.puts.sum.latency.max")); metrics.add(new Metric("vds.distributor.puts.sum.latency.sum")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java index 4b9d5542aa9..5ce941d6638 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java @@ -5,7 +5,7 @@ import com.yahoo.vespa.model.admin.monitoring.CloudWatch.HostedAuth; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import org.w3c.dom.Element; -import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChildValue; +import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChild; /** * @author gjoranv @@ -14,22 +14,26 @@ public class CloudWatchBuilder { private static final String REGION_ATTRIBUTE = "region"; private static final String NAMESPACE_ATTRIBUTE = "namespace"; - private static final String ACCESS_KEY_ELEMENT = "access-key-name"; - private static final String SECRET_KEY_ELEMENT = "secret-key-name"; - private static final String PROFILE_ELEMENT = "profile"; + private static final String CREDENTIALS_ELEMENT = "credentials"; + private static final String ACCESS_KEY_ATTRIBUTE = "access-key-name"; + private static final String SECRET_KEY_ATTRIBUTE = "secret-key-name"; + private static final String SHARED_CREDENTIALS_ELEMENT = "shared-credentials"; + private static final String PROFILE_ATTRIBUTE = "profile"; + private static final String FILE_ATTRIBUTE = "file"; public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) { CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE), cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE), consumer); - getOptionalChildValue(cloudwatchElement, PROFILE_ELEMENT).ifPresent(cloudWatch::setProfile); + getOptionalChild(cloudwatchElement, CREDENTIALS_ELEMENT) + .ifPresent(elem -> cloudWatch.setHostedAuth(elem.getAttribute(ACCESS_KEY_ATTRIBUTE), + elem.getAttribute(SECRET_KEY_ATTRIBUTE))); + + getOptionalChild(cloudwatchElement, SHARED_CREDENTIALS_ELEMENT) + .ifPresent(elem -> cloudWatch.setSharedCredentials(elem.getAttribute(PROFILE_ATTRIBUTE), + elem.getAttribute(FILE_ATTRIBUTE))); - getOptionalChildValue(cloudwatchElement, ACCESS_KEY_ELEMENT) - .ifPresent(accessKey -> cloudWatch.setHostedAuth( - new HostedAuth(accessKey, - getOptionalChildValue(cloudwatchElement, SECRET_KEY_ELEMENT) - .orElseThrow(() -> new IllegalArgumentException("Access key given without a secret key."))))); return cloudWatch; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java new file mode 100644 index 00000000000..462ac39fa84 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java @@ -0,0 +1,37 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; + +import java.util.List; + +import static java.util.stream.Collectors.toList; + +/** + * @author gjoranv + */ +public class CloudWatchValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + if (!deployState.isHosted()) return; + if (deployState.zone().system().isPublic()) return; + if (model.getAdmin().getApplicationType() != ConfigModelContext.ApplicationType.DEFAULT) return; + + var offendingConsumers = model.getAdmin().getUserMetrics().getConsumers().values().stream() + .filter(consumer -> !consumer.cloudWatches().isEmpty()) + .collect(toList()); + + if (! offendingConsumers.isEmpty()) { + throw new IllegalArgumentException("CloudWatch cannot be set up for non-public hosted Vespa and must " + + "be removed for consumers: " + consumerIds(offendingConsumers)); + } + } + + private List<String> consumerIds(List<MetricsConsumer> offendingConsumers) { + return offendingConsumers.stream().map(MetricsConsumer::getId).collect(toList()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 1e4a45428b8..b03917dec3b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -59,6 +59,7 @@ public class Validation { new SecretStoreValidator().validate(model, deployState); new EndpointCertificateSecretsValidator().validate(model, deployState); new AccessControlFilterValidator().validate(model, deployState); + new CloudWatchValidator().validate(model, deployState); List<ConfigChangeAction> result = Collections.emptyList(); if (deployState.getProperties().isFirstTimeDeployment()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 2c88f965f1f..43f4637798a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -15,9 +15,9 @@ import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; import org.w3c.dom.Element; import org.w3c.dom.Node; +import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; /** * A common utility class to represent a requirement for nodes during model building. @@ -46,18 +46,19 @@ public class NodesSpecification { private final boolean exclusive; - /** Whether this requires running container and content processes co-located on the same node. */ - private final boolean combined; - /** The resources each node should have, or empty to use the default */ private final Optional<NodeResources> resources; /** The identifier of the custom docker image layer to use (not supported yet) */ private final Optional<String> dockerImage; + /** The ID of the cluster referencing this node specification, if any */ + private final Optional<String> combinedId; + private NodesSpecification(boolean dedicated, int count, int groups, Version version, - boolean required, boolean canFail, boolean exclusive, boolean combined, - Optional<NodeResources> resources, Optional<String> dockerImage) { + boolean required, boolean canFail, boolean exclusive, + Optional<NodeResources> resources, Optional<String> dockerImage, + Optional<String> combinedId) { this.dedicated = dedicated; this.count = count; this.groups = groups; @@ -67,10 +68,11 @@ public class NodesSpecification { this.exclusive = exclusive; this.resources = resources; this.dockerImage = dockerImage; - this.combined = combined; + this.combinedId = combinedId; } - private NodesSpecification(boolean dedicated, boolean canFail, boolean combined, Version version, ModelElement nodesElement) { + private NodesSpecification(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement, + Optional<String> combinedId) { this(dedicated, nodesElement.integerAttribute("count", 1), nodesElement.integerAttribute("groups", 1), @@ -78,15 +80,25 @@ public class NodesSpecification { nodesElement.booleanAttribute("required", false), canFail, nodesElement.booleanAttribute("exclusive", false), - combined, getResources(nodesElement), - Optional.ofNullable(nodesElement.stringAttribute("docker-image"))); + Optional.ofNullable(nodesElement.stringAttribute("docker-image")), + combinedId); + } + + /** Returns the ID of the cluster referencing this node specification, if any */ + private static Optional<String> findCombinedId(ModelElement nodesElement, ModelElement resolvedElement) { + if (resolvedElement != nodesElement) { + // Specification for a container cluster referencing nodes in a content cluster + return containerIdOf(nodesElement); + } + // Specification for a content cluster that is referenced by a container cluster + return containerIdReferencing(nodesElement); } private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement) { var resolvedElement = resolveElement(nodesElement); - boolean combined = resolvedElement != nodesElement || isReferencedByOtherElement(nodesElement); - return new NodesSpecification(dedicated, canFail, combined, version, resolvedElement); + var combinedId = findCombinedId(nodesElement, resolvedElement); + return new NodesSpecification(dedicated, canFail, version, resolvedElement, combinedId); } /** Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element */ @@ -133,7 +145,7 @@ public class NodesSpecification { false, ! context.getDeployState().getProperties().isBootstrap(), false, - false, + Optional.empty(), Optional.empty(), Optional.empty()); } @@ -147,7 +159,7 @@ public class NodesSpecification { false, ! context.getDeployState().getProperties().isBootstrap(), false, - false, + Optional.empty(), Optional.empty(), Optional.empty()); } @@ -175,9 +187,9 @@ public class NodesSpecification { ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, DeployLogger logger) { - if (combined) + if (combinedId.isPresent()) clusterType = ClusterSpec.Type.combined; - ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId, version, exclusive); + ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId, version, exclusive, combinedId.map(ClusterSpec.Id::from)); return hostSystem.allocateHosts(cluster, Capacity.fromCount(count, resources, required, canFail), groups, logger); } @@ -280,24 +292,35 @@ public class NodesSpecification { return new ModelElement(referencedNodesElement); } - /** Returns whether the given nodesElement is referenced by any other nodes element */ - private static boolean isReferencedByOtherElement(ModelElement nodesElement) { + /** Returns the ID of the parent container element of nodesElement, if any */ + private static Optional<String> containerIdOf(ModelElement nodesElement) { + var element = nodesElement.getXml(); + for (var containerTag : List.of("container", "jdisc")) { + var container = findParentByTag(containerTag, element); + if (container.isEmpty()) continue; + return container.map(el -> el.getAttribute("id")); + } + return Optional.empty(); + } + + /** Returns the ID of the container element referencing nodesElement, if any */ + private static Optional<String> containerIdReferencing(ModelElement nodesElement) { var element = nodesElement.getXml(); var services = findParentByTag("services", element); - if (services.isEmpty()) return false; + if (services.isEmpty()) return Optional.empty(); var content = findParentByTag("content", element); - if (content.isEmpty()) return false; + if (content.isEmpty()) return Optional.empty(); var contentClusterId = content.get().getAttribute("id"); - if (contentClusterId.isEmpty()) return false; + if (contentClusterId.isEmpty()) return Optional.empty(); for (var rootChild : XML.getChildren(services.get())) { if ( ! ContainerModelBuilder.isContainerTag(rootChild)) continue; var nodes = XML.getChild(rootChild, "nodes"); if (nodes == null) continue; if (!contentClusterId.equals(nodes.getAttribute("of"))) continue; - return true; + return Optional.of(rootChild.getAttribute("id")); } - return false; + return Optional.empty(); } private static Optional<Element> findChildById(Element parent, String id) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 9d7274f1bbf..747f8801137 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -629,7 +629,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), deployState.getWantedNodeVespaVersion(), - false); + false, + Optional.empty()); Capacity capacity = Capacity.fromCount(1, Optional.empty(), false, @@ -655,7 +656,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), context.getDeployState().getWantedNodeVespaVersion(), - false); + false, + Optional.empty()); Map<HostResource, ClusterMembership> hosts = cluster.getRoot().hostSystem().allocateHosts(clusterSpec, Capacity.fromRequiredNodeType(type), 1, log); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 1b4c03a2182..445c93ba66e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -202,7 +202,8 @@ public class IndexedSearchCluster extends SearchCluster com.yahoo.searchdefinition.Search search = spec.getSearchDefinition().getSearch(); if ( ! (search instanceof DocumentOnlySearch)) { DocumentDatabase db = new DocumentDatabase(this, search.getName(), - new DerivedConfiguration(search, deployState.getDeployLogger(), + new DerivedConfiguration(search, + deployState.getDeployLogger(), deployState.getProperties(), deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index e3ba7dc500d..f4585b3cf3f 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -91,14 +91,16 @@ Cloudwatch = element cloudwatch { attribute region { xsd:Name } & attribute namespace { xsd:string { pattern = "[\w_\-/#:\.]+" } } & ( - ( - element access-key-name { xsd:Name } & - element secret-key-name { xsd:Name } - ) + element credentials { + attribute access-key-name { xsd:Name } & + attribute secret-key-name { xsd:Name } + } | - element profile { xsd:Name } + element shared-credentials { + attribute profile { xsd:Name } & + attribute file { string }? + } )? - } ClusterControllers = element cluster-controllers { diff --git a/config-model/src/test/derived/fieldset2/index-info.cfg b/config-model/src/test/derived/fieldset2/index-info.cfg index 7c3c1c448db..56cc53b4628 100644 --- a/config-model/src/test/derived/fieldset2/index-info.cfg +++ b/config-model/src/test/derived/fieldset2/index-info.cfg @@ -13,6 +13,8 @@ indexinfo[].command[].indexname "field1" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "field1" indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "field1" +indexinfo[].command[].command "phrase-segmenting" indexinfo[].command[].indexname "field2" indexinfo[].command[].command "index" indexinfo[].command[].indexname "field2" @@ -23,6 +25,8 @@ indexinfo[].command[].indexname "field2" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "field2" indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "field2" +indexinfo[].command[].command "phrase-segmenting" indexinfo[].command[].indexname "default" indexinfo[].command[].command "phrase-segmenting false" indexinfo[].command[].indexname "default" diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg index 65818e088f5..388b212689a 100644 --- a/config-model/src/test/derived/indexschema/index-info.cfg +++ b/config-model/src/test/derived/indexschema/index-info.cfg @@ -334,6 +334,8 @@ indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "gram" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "gram" +indexinfo[].command[].command "phrase-segmenting false" +indexinfo[].command[].indexname "gram" indexinfo[].command[].command "ngram 2" indexinfo[].command[].indexname "nostem1" indexinfo[].command[].command "lowercase" diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index c010b23e207..d215fdbb7a0 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -264,7 +264,7 @@ public class ModelProvisioningTest { assertEquals("Heap size is lowered with combined clusters", 17, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); - assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Type.combined, model); + assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); } } @@ -1805,12 +1805,17 @@ public class ModelProvisioningTest { assertTrue(logdConfig.logserver().use()); } - private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Type type, VespaModel model) { - assertEquals("Nodes in cluster " + id + " with type " + type, nodeCount, + private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Id combinedId, + ClusterSpec.Type type, VespaModel model) { + assertEquals("Nodes in cluster " + id + " with type " + type + (combinedId != null ? ", combinedId " + combinedId : ""), nodeCount, model.hostSystem().getHosts().stream() .map(h -> h.spec().membership().get().cluster()) - .filter(spec -> spec.id().equals(id) && spec.type().equals(type)) + .filter(spec -> spec.id().equals(id) && spec.type().equals(type) && spec.combinedId().equals(Optional.ofNullable(combinedId))) .count()); } + private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Type type, VespaModel model) { + assertProvisioned(nodeCount, id, null, type, model); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index 8ea53172200..7e0eb01f611 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.searchdefinition.Search; @@ -10,8 +12,6 @@ import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import java.io.File; import java.io.IOException; @@ -26,17 +26,22 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase private static final String tempDir = "temp/"; private static final String searchDefRoot = "src/test/derived/"; - private DerivedConfiguration derive(String dirName, String searchDefinitionName) throws IOException, ParseException { + private DerivedConfiguration derive(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException { File toDir = new File(tempDir + dirName); toDir.mkdirs(); deleteContent(toDir); SearchBuilder builder = SearchBuilder.createFromDirectory(searchDefRoot + dirName + "/"); - return derive(dirName, searchDefinitionName, builder); + return derive(dirName, searchDefinitionName, properties, builder); } - private DerivedConfiguration derive(String dirName, String searchDefinitionName, SearchBuilder builder) throws IOException { + private DerivedConfiguration derive(String dirName, + String searchDefinitionName, + TestProperties properties, + SearchBuilder builder) throws IOException { DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName), + new BaseDeployLogger(), + properties, builder.getRankProfileRegistry(), builder.getQueryProfileRegistry(), new ImportedMlModels()); @@ -79,7 +84,11 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase } protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName) throws IOException, ParseException { - DerivedConfiguration derived = derive(dirName, searchDefinitionName); + return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties()); + } + + protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException { + DerivedConfiguration derived = derive(dirName, searchDefinitionName, properties); assertCorrectConfigFiles(dirName); return derived; } @@ -90,7 +99,7 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase */ protected DerivedConfiguration assertCorrectDeriving(SearchBuilder builder, String dirName) throws IOException { builder.build(); - DerivedConfiguration derived = derive(dirName, null, builder); + DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder); assertCorrectConfigFiles(dirName); return derived; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index e785792839d..08d915c35ca 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -121,7 +122,9 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testFieldSet2() throws IOException, ParseException { - assertCorrectDeriving("fieldset2"); + TestProperties properties = new TestProperties(); + properties.setUsePhraseSegmenting(true); + assertCorrectDeriving("fieldset2", null, properties); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java index dbcbad7bf49..9be94e4198e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java @@ -14,6 +14,7 @@ import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author gjoranv @@ -53,6 +54,7 @@ public class TelegrafTest { String services = servicesWithCloudwatch(); VespaModel hostedModel = getModel(services, hosted); TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + assertTrue(config.isHostedVespa()); var cloudWatch0 = config.cloudWatch(0); assertEquals("cloudwatch-consumer", cloudWatch0.consumer()); @@ -72,8 +74,8 @@ public class TelegrafTest { " <consumer id='cloudwatch-consumer'>", " <metric id='my-metric'/>", " <cloudwatch region='us-east-1' namespace='my-namespace' >", - " <access-key-name>my-access-key</access-key-name>", - " <secret-key-name>my-secret-key</secret-key-name>", + " <credentials access-key-name='my-access-key' ", + " secret-key-name='my-secret-key' />", " </cloudwatch>", " </consumer>", " </metrics>", @@ -92,11 +94,11 @@ public class TelegrafTest { " <consumer id='cloudwatch-consumer'>", " <metric id='my-metric'/>", " <cloudwatch region='us-east-1' namespace='namespace-1' >", - " <access-key-name>access-key-1</access-key-name>", - " <secret-key-name>secret-key-1</secret-key-name>", + " <credentials access-key-name='access-key-1' ", + " secret-key-name='secret-key-1' />", " </cloudwatch>", " <cloudwatch region='us-east-1' namespace='namespace-2' >", - " <profile>profile-2</profile>", + " <shared-credentials profile='profile-2' />", " </cloudwatch>", " </consumer>", " </metrics>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java new file mode 100644 index 00000000000..40b8223479d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java @@ -0,0 +1,101 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static com.yahoo.config.provision.Environment.prod; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + + +/** + * @author gjoranv + */ +public class CloudWatchValidatorTest { + + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void cloudwatch_in_public_zones_passes_validation() throws IOException, SAXException { + DeployState deployState = deployState(servicesWithCloudwatch(), true, true); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + new CloudWatchValidator().validate(model, deployState); + } + + @Test + public void cloudwatch_passes_validation_for_self_hosted_vespa() throws IOException, SAXException { + DeployState deployState = deployState(servicesWithCloudwatch(), false, false); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + new CloudWatchValidator().validate(model, deployState); + } + + @Test + public void cloudwatch_in_non_public_zones_fails_validation() throws IOException, SAXException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "CloudWatch cannot be set up for non-public hosted Vespa and must be removed for consumers: [cloudwatch-consumer]"); + + DeployState deployState = deployState(servicesWithCloudwatch(), true, false); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + new CloudWatchValidator().validate(model, deployState); + } + + private static DeployState deployState(String servicesXml, boolean isHosted, boolean isPublic) { + ApplicationPackage app = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build(); + + DeployState.Builder builder = new DeployState.Builder() + .applicationPackage(app) + .properties(new TestProperties().setHostedVespa(isHosted)); + if (isHosted) { + var system = isPublic ? SystemName.Public : SystemName.main; + builder.zone(new Zone(system, Environment.prod, RegionName.from("foo"))); + } + final DeployState deployState = builder.build(); + + if (isHosted) { + assertTrue("Test must emulate a hosted deployment.", deployState.isHosted()); + assertEquals("Test must emulate a prod environment.", prod, deployState.zone().environment()); + } + return deployState; + } + + private String servicesWithCloudwatch() { + return String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='cloudwatch-consumer'>", + " <metric id='my-metric'/>", + " <cloudwatch region='us-east-1' namespace='my-namespace' >", + " <credentials access-key-name='my-access-key' ", + " secret-key-name='my-secret-key' />", + " </cloudwatch>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + } + +} diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 604e2abbbae..0c395fedb96 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -15,26 +15,29 @@ <slobrok hostalias="rtc-1" /> </slobroks> <metrics> + <consumer id="cloudwatch-hosted"> <metric-set id="my-set" /> <metric id="my-metric"/> <metric id="my-metric2" display-name="my-metric3"/> <metric display-name="my-metric4" id="my-metric4.avg"/> <cloudwatch region="us-east1" namespace="my-namespace"> - <access-key-name>my-access-key</access-key-name> - <secret-key-name>my-secret-key</secret-key-name> + <credentials access-key-name="my-access-key" secret-key-name="my-secret-key" /> </cloudwatch> </consumer> + <consumer id="cloudwatch-self-hosted-with-default-auth"> <metric-set id="public" /> <cloudwatch region="us-east1" namespace="namespace_legal.chars:/#1" /> </consumer> + <consumer id="cloudwatch-self-hosted-with-profile"> <metric id="my-custom-metric" /> <cloudwatch region="us-east1" namespace="another-namespace"> - <profile>profile-in-credentials-file</profile> + <shared-credentials profile="profile-in-credentials-file" file="/user/.aws/credentials"/> </cloudwatch> </consumer> + </metrics> <logforwarding> <splunk deployment-server="foo:8989" client-name="foobar" splunk-home="/opt/splunk" phone-home-interval="900"/> diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 9a091f1161c..fa7773a54f1 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -271,11 +271,14 @@ "public com.yahoo.config.provision.ClusterSpec$Id id()", "public com.yahoo.component.Version vespaVersion()", "public java.util.Optional group()", + "public java.util.Optional combinedId()", "public boolean isExclusive()", "public com.yahoo.config.provision.ClusterSpec with(java.util.Optional)", "public com.yahoo.config.provision.ClusterSpec exclusive(boolean)", "public static com.yahoo.config.provision.ClusterSpec request(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.component.Version, boolean)", + "public static com.yahoo.config.provision.ClusterSpec request(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.component.Version, boolean, java.util.Optional)", "public static com.yahoo.config.provision.ClusterSpec from(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.config.provision.ClusterSpec$Group, com.yahoo.component.Version, boolean)", + "public static com.yahoo.config.provision.ClusterSpec from(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.config.provision.ClusterSpec$Group, com.yahoo.component.Version, boolean, java.util.Optional)", "public java.lang.String toString()", "public int hashCode()", "public boolean equals(java.lang.Object)", diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java index f041823bf04..0fb78b59aaf 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java @@ -3,9 +3,11 @@ package com.yahoo.config.provision; import com.yahoo.component.Version; +import java.util.Optional; + /** * A node's membership in a cluster. This is a value object. - * The format is "clusterType/clusterId/groupId/index[/exclusive][/retired]" + * The format is "clusterType/clusterId/groupId/index[/exclusive][/retired][/combinedId]" * * @author bratseth */ @@ -22,21 +24,24 @@ public class ClusterMembership { String[] components = stringValue.split("/"); if (components.length < 4) throw new RuntimeException("Could not parse '" + stringValue + "' to a cluster membership. " + - "Expected 'clusterType/clusterId/groupId/index[/retired][/exclusive]'"); + "Expected 'clusterType/clusterId/groupId/index[/retired][/exclusive][/combinedId]'"); boolean exclusive = false; + var combinedId = Optional.<String>empty(); if (components.length > 4) { for (int i = 4; i < components.length; i++) { String component = components[i]; switch (component) { case "exclusive": exclusive = true; break; case "retired": retired = true; break; + default: combinedId = Optional.of(component); break; } } } this.cluster = ClusterSpec.from(ClusterSpec.Type.valueOf(components[0]), ClusterSpec.Id.from(components[1]), - ClusterSpec.Group.from(Integer.valueOf(components[2])), vespaVersion, exclusive); + ClusterSpec.Group.from(Integer.parseInt(components[2])), vespaVersion, + exclusive, combinedId.map(ClusterSpec.Id::from)); this.index = Integer.parseInt(components[3]); this.stringValue = toStringValue(); } @@ -54,7 +59,8 @@ public class ClusterMembership { (cluster.group().isPresent() ? "/" + cluster.group().get().index() : "") + "/" + index + ( cluster.isExclusive() ? "/exclusive" : "") + - ( retired ? "/retired" : ""); + ( retired ? "/retired" : "") + + ( cluster.combinedId().isPresent() ? "/" + cluster.combinedId().get().value() : ""); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 5aed5d8e2e7..e1a28e3f8d7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -21,13 +21,19 @@ public final class ClusterSpec { private final Optional<Group> groupId; private final Version vespaVersion; private boolean exclusive; + private final Optional<Id> combinedId; - private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive) { + private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive, Optional<Id> combinedId) { this.type = type; this.id = id; this.groupId = groupId; this.vespaVersion = vespaVersion; this.exclusive = exclusive; + // TODO(mpolden): Require combinedId to always be present for type combined after April 2020 + if (type != Type.combined && combinedId.isPresent()) { + throw new IllegalArgumentException("combinedId must be empty for cluster of type " + type); + } + this.combinedId = combinedId; } /** Returns the cluster type */ @@ -42,6 +48,11 @@ public final class ClusterSpec { /** Returns the group within the cluster this specifies, or empty to specify the whole cluster */ public Optional<Group> group() { return groupId; } + /** Returns the ID of the container cluster that is combined with this. This is only present for combined clusters */ + public Optional<Id> combinedId() { + return combinedId; + } + /** * Returns whether the physical hosts running the nodes of this application can * also run nodes of other applications. Using exclusive nodes for containers increases security @@ -50,19 +61,29 @@ public final class ClusterSpec { public boolean isExclusive() { return exclusive; } public ClusterSpec with(Optional<Group> newGroup) { - return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive); + return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive, combinedId); } public ClusterSpec exclusive(boolean exclusive) { - return new ClusterSpec(type, id, groupId, vespaVersion, exclusive); + return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId); } + // TODO(mpolden): Remove after April 2020 public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive) { - return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive); + return request(type, id, vespaVersion, exclusive, Optional.empty()); } + public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive, Optional<Id> combinedId) { + return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive, combinedId); + } + + // TODO(mpolden): Remove after April 2020 public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive) { - return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive); + return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive, Optional.empty()); + } + + public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive, Optional<Id> combinedId) { + return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive, combinedId); } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java index 02da2ab98e2..5c3d9d97bdd 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java @@ -22,4 +22,9 @@ public enum RoutingMethod { return this == exclusive || this == sharedLayer4; } + /** Returns whether this method routes requests through a shared routing layer */ + public boolean isShared() { + return this == shared || this == sharedLayer4; + } + } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java index 3a36afcfdce..aa22747c165 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java @@ -5,6 +5,8 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import org.junit.Test; +import java.util.Optional; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -16,28 +18,41 @@ public class ClusterMembershipTest { @Test public void testContainerServiceInstance() { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false, Optional.empty()); assertContainerService(ClusterMembership.from(cluster, 3)); } @Test - public void testContainerInstanceWithOptionalParts() { + public void testSerializationWithOptionalParts() { { ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive/retired", Vtag.currentVersion); + ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion); + assertEquals(instance, serialized); assertTrue(instance.retired()); assertTrue(instance.cluster().isExclusive()); + assertFalse(instance.cluster().combinedId().isPresent()); } - { ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive", Vtag.currentVersion); + ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion); + assertEquals(instance, serialized); + assertFalse(instance.retired()); + assertTrue(instance.cluster().isExclusive()); + assertFalse(instance.cluster().combinedId().isPresent()); + } + { + ClusterMembership instance = ClusterMembership.from("combined/id1/4/37/exclusive/containerId1", Vtag.currentVersion); + ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion); + assertEquals(instance, serialized); assertFalse(instance.retired()); assertTrue(instance.cluster().isExclusive()); + assertEquals(ClusterSpec.Id.from("containerId1"), instance.cluster().combinedId().get()); } } @Test public void testServiceInstance() { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false, Optional.empty()); assertContentService(ClusterMembership.from(cluster, 37)); } @@ -55,7 +70,7 @@ public class ClusterMembershipTest { @Test public void testServiceInstanceWithRetire() { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false, Optional.empty()); assertContentServiceWithRetire(ClusterMembership.retiredFrom(cluster, 37)); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 829a59a9598..a15b570a55d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -135,6 +135,7 @@ public class ModelContextImpl implements ModelContext { private final double defaultTermwiseLimit; private final boolean useBucketSpaceMetric; private final boolean useNewAthenzFilter; + private final boolean usePhraseSegmenting; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -169,6 +170,8 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useNewAthenzFilter = Flags.USE_NEW_ATHENZ_FILTER.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.usePhraseSegmenting = Flags.PHRASE_SEGMENTING.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); } @Override @@ -223,7 +226,12 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useBucketSpaceMetric() { return useBucketSpaceMetric; } - @Override public boolean useNewAthenzFilter() { return useNewAthenzFilter; } + @Override + public boolean useNewAthenzFilter() { return useNewAthenzFilter; } + + @Override + public boolean usePhraseSegmenting() { return usePhraseSegmenting; } + } } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 970dd49e865..0cd283ca62b 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -63,8 +63,7 @@ <component id="com.yahoo.vespa.service.manager.UnionMonitorManager" bundle="service-monitor" /> <component id="com.yahoo.vespa.service.model.ServiceMonitorImpl" bundle="service-monitor" /> <component id="com.yahoo.vespa.service.duper.DuperModelManager" bundle="service-monitor" /> - <component id="com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService" bundle="orchestrator" /> - <component id="com.yahoo.vespa.orchestrator.status.ZookeeperStatusService" bundle="orchestrator" /> + <component id="com.yahoo.vespa.orchestrator.status.ZkStatusService" bundle="orchestrator" /> <component id="com.yahoo.vespa.orchestrator.controller.RetryingClusterControllerClientFactory" bundle="orchestrator" /> <component id="com.yahoo.vespa.orchestrator.OrchestratorImpl" bundle="orchestrator" /> diff --git a/container-disc/src/main/ssl/jdisc_container.keydb b/container-disc/src/main/ssl/jdisc_container.keydb deleted file mode 100644 index 23479a83442..00000000000 --- a/container-disc/src/main/ssl/jdisc_container.keydb +++ /dev/null @@ -1,11 +0,0 @@ -<keydb> - <keygroup name="jdisc_container" id="0"> - <keyname name="jdisc_key" usage="all" type="transient"> - <key version="0" - value = "2eMsYNKWBOWpsah8d57B65xvmFVZGiPAGv3pbI1mlZU-" current = "true" - timestamp = "20130320234320" - expiry = "20160319234320"> - </key> - </keyname> - </keygroup> -</keydb> diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 82d3223c8fe..51fee99a743 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -858,8 +858,12 @@ "public void <init>(java.lang.String, java.lang.String)", "public int getTargetNumHits()", "public java.lang.String getIndexName()", + "public int getHnswExploreAdditionalHits()", + "public boolean getAllowApproximate()", "public java.lang.String getQueryTensorName()", "public void setTargetNumHits(int)", + "public void setHnswExploreAdditionalHits(int)", + "public void setAllowApproximate(boolean)", "public void setIndexName(java.lang.String)", "public com.yahoo.prelude.query.Item$ItemType getItemType()", "public java.lang.String getName()", diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 365ee299ca4..5e7fddd7fe7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude; - import com.yahoo.language.process.StemMode; import java.util.ArrayList; @@ -10,7 +9,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; - /** * Information about configured settings of a field or field collection (an actual index or not) in a search definition. * There are two types of settings: @@ -74,8 +72,8 @@ public class Index { private boolean isNGram = false; private int gramSize = 2; - /** Whether implicit phrases should lead to a phrase item or an and item */ - private boolean phraseSegmenting = true; + /** Whether implicit phrases should lead to a phrase item or an and item. */ + private Boolean phraseSegmenting = false; /** The string terminating an exact token in this index, or null to use the default (space) */ private String exactTerminator = null; diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java index 062a514056b..00935392683 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java @@ -109,7 +109,6 @@ public final class IndexModel { return searchDefinitions; } - @SuppressWarnings("deprecation") private SearchDefinition unionOf(Collection<SearchDefinition> searchDefinitions) { SearchDefinition union = new SearchDefinition(IndexFacts.unionName); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java index 35b87ec0190..836107138d0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java @@ -20,6 +20,8 @@ import java.nio.ByteBuffer; public class NearestNeighborItem extends SimpleTaggableItem { private int targetNumHits = 0; + private int hnswExploreAdditionalHits = 0; + private boolean approximate = true; private String field; private String queryTensorName; @@ -34,12 +36,24 @@ public class NearestNeighborItem extends SimpleTaggableItem { /** Returns the field name */ public String getIndexName() { return field; } + /** Returns the number of extra hits to explore in HNSW algorithm */ + public int getHnswExploreAdditionalHits() { return hnswExploreAdditionalHits; } + + /** Returns whether approximation is allowed */ + public boolean getAllowApproximate() { return approximate; } + /** Returns the name of the query tensor */ public String getQueryTensorName() { return queryTensorName; } /** Set the K number of hits to produce */ public void setTargetNumHits(int target) { this.targetNumHits = target; } + /** Set the number of extra hits to explore in HNSW algorithm */ + public void setHnswExploreAdditionalHits(int num) { this.hnswExploreAdditionalHits = num; } + + /** Set whether approximation is allowed */ + public void setAllowApproximate(boolean value) { this.approximate = value; } + @Override public void setIndexName(String index) { this.field = index; } @@ -58,6 +72,8 @@ public class NearestNeighborItem extends SimpleTaggableItem { putString(field, buffer); putString(queryTensorName, buffer); IntegerCompressor.putCompressedPositiveNumber(targetNumHits, buffer); + IntegerCompressor.putCompressedPositiveNumber((approximate ? 1 : 0), buffer); + IntegerCompressor.putCompressedPositiveNumber(hnswExploreAdditionalHits, buffer); return 1; // number of encoded stack dump items } @@ -65,6 +81,8 @@ public class NearestNeighborItem extends SimpleTaggableItem { protected void appendBodyString(StringBuilder buffer) { buffer.append("{field=").append(field); buffer.append(",queryTensorName=").append(queryTensorName); + buffer.append(",hnsw.exploreAdditionalHits=").append(hnswExploreAdditionalHits); + buffer.append(",approximate=").append(String.valueOf(approximate)); buffer.append(",targetNumHits=").append(targetNumHits).append("}"); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index d9b969757c2..49bdba2c90f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -30,6 +30,7 @@ public class AllParser extends SimpleParser { super(environment); } + @Override protected Item parseItems() { int position = tokens.getPosition(); try { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java index dd836e9c8e1..b714a1d8b34 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java @@ -35,21 +35,12 @@ public class AnyParser extends SimpleParser { return anyItems(true); } - Item parseFilter(String filter, Language queryLanguage, Set<String> searchDefinitions) { - return parseFilter(filter, queryLanguage, environment.getIndexFacts().newSession(searchDefinitions, Collections.emptySet())); - } - Item parseFilter(String filter, Language queryLanguage, IndexFacts.Session indexFacts) { - Item filterRoot; - setState(queryLanguage, indexFacts); tokenize(filter, null, indexFacts, queryLanguage); - filterRoot = anyItems(true); - - if (filterRoot == null) { - return null; - } + Item filterRoot = anyItems(true); + if (filterRoot == null) return null; markAllTermsAsFilters(filterRoot); return filterRoot; @@ -61,18 +52,10 @@ public class AnyParser extends SimpleParser { try { tokens.skipMultiple(PLUS); + if ( ! tokens.skipMultiple(MINUS)) return null; + if (tokens.currentIsNoIgnore(SPACE)) return null; - if (!tokens.skipMultiple(MINUS)) { - return null; - } - - if (tokens.currentIsNoIgnore(SPACE)) { - return null; - } - - if (item == null) { - item = indexableItem(); - } + item = indexableItem(); if (item == null) { item = compositeItem(); @@ -88,13 +71,13 @@ public class AnyParser extends SimpleParser { } } } - if (item!=null) + if (item != null) item.setProtected(true); + return item; } finally { - if (item == null) { + if (item == null) tokens.setPosition(position); - } } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index 9ddfea6dffb..3d244312b2f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -50,32 +50,28 @@ abstract class SimpleParser extends StructuredParser { private Item anyItemsBody(boolean topLevel) { Item topLevelItem = null; NotItem not = null; - Item item; + Item item = null; do { - item = null; - - if (item == null) { - item = positiveItem(); - if (item != null) { - if (not == null) { - not = new NotItem(); - not.addPositiveItem(item); - topLevelItem = combineItems(topLevelItem, not); - } else { - not.addPositiveItem(item); - } + item = positiveItem(); + if (item != null) { + if (not == null) { + not = new NotItem(); + not.addPositiveItem(item); + topLevelItem = combineItems(topLevelItem, not); + } else { + not.addPositiveItem(item); } } if (item == null) { item = negativeItem(); if (item != null) { - if (not == null && item != null) { + if (not == null) { not = new NotItem(); not.addNegativeItem(item); topLevelItem = combineItems(topLevelItem, not); - } else if (item != null) { + } else { not.addNegativeItem(item); } } @@ -97,9 +93,8 @@ abstract class SimpleParser extends StructuredParser { if (item != null) { if (topLevelItem == null) { topLevelItem = item; - } else if (needNewTopLevel(topLevelItem, item)) { + } else if (needNewORTopLevel(topLevelItem, item)) { CompositeItem newTop = new OrItem(); - newTop.addItem(topLevelItem); newTop.addItem(item); topLevelItem = newTop; @@ -131,6 +126,7 @@ abstract class SimpleParser extends StructuredParser { if (topLevelItem != null && topLevelItem != not) { // => neutral rank items becomes implicit positives + System.out.println("Extracting positive item from " + topLevelItem); not.addPositiveItem(getItemAsPositiveItem(topLevelItem, not)); return not; } else { // Only negatives - ignore them @@ -144,21 +140,13 @@ abstract class SimpleParser extends StructuredParser { } } - - /** Says whether we need a new top level item given the new item */ - private boolean needNewTopLevel(Item topLevelItem, Item item) { - if (item == null) { - return false; - } - if (topLevelItem instanceof TermItem) { - return true; - } - if (topLevelItem instanceof PhraseItem) { - return true; - } - if (topLevelItem instanceof BlockItem) { - return true; - } + /** Says whether we need a new top level OR item given the new item */ + private boolean needNewORTopLevel(Item topLevelItem, Item item) { + if (item == null) return false; + if (topLevelItem instanceof TermItem) return true; + if (topLevelItem instanceof PhraseItem) return true; + if (topLevelItem instanceof BlockItem) return true; + if ( topLevelItem instanceof AndItem) return true; return false; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index 5e292a06b0f..9ba6c1a8101 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -442,9 +442,9 @@ abstract class StructuredParser extends AbstractParser { Item item = null; try { - if (!tokens.currentIs(WORD) - && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS) - && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) { + if ( ! tokens.currentIs(WORD) + && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS) + && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) { return null; } Token word = tokens.next(); @@ -557,6 +557,7 @@ abstract class StructuredParser extends AbstractParser { if (composite != null) { composite.addItem(word); + connectLastTermsIn(composite); } else if (firstWord != null) { if (submodes.site || submodes.url) { UriItem uriItem = new UriItem(); @@ -584,6 +585,7 @@ abstract class StructuredParser extends AbstractParser { } composite.addItem(firstWord); composite.addItem(word); + connectLastTermsIn(composite); } else if (word instanceof PhraseItem) { composite = (PhraseItem)word; } else { @@ -654,6 +656,15 @@ abstract class StructuredParser extends AbstractParser { } } + private void connectLastTermsIn(CompositeItem composite) { + int items = composite.items().size(); + if (items < 2) return; + Item nextToLast = composite.items().get(items - 2); + Item last = composite.items().get(items - 1); + if ( ! (nextToLast instanceof TermItem)) return; + ((TermItem)nextToLast).setConnectivity(last, 1); + } + private boolean addStartMarking() { if (submodes.explicitAnchoring() && tokens.currentIs(HAT)) { tokens.skip(); diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 1e3f11f4f78..dc8e2b70740 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -663,7 +663,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // getQueryTree isn't exception safe try { queryTree = model.getQueryTree().toString(); - } catch (Exception e) { + } catch (Exception | StackOverflowError e) { queryTree = "[Could not parse user input: " + model.getQueryString() + "]"; } return "query '" + queryTree + "'"; @@ -675,7 +675,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // getQueryTree isn't exception safe try { queryTree = model.getQueryTree().toString(); - } catch (Exception e) { + } catch (Exception | StackOverflowError e) { queryTree = "Could not parse user input: " + model.getQueryString(); } return "query=[" + queryTree + "]" + " offset=" + getOffset() + " hits=" + getHits() + "]"; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java index a2821892358..52cb2b4c061 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java @@ -44,13 +44,14 @@ class RpcClient implements Client { // The current shared connection. This will be recycled when it becomes invalid. // All access to this must be synchronized - private Target target = null; + private Target target; public RpcNodeConnection(String hostname, int port, Supervisor supervisor) { this.supervisor = supervisor; this.hostname = hostname; this.port = port; description = "rpc node connection to " + hostname + ":" + port; + target = supervisor.connect(new Spec(hostname, port)); } @Override @@ -79,17 +80,16 @@ class RpcClient implements Client { private void invokeAsync(Request req, double timeout, RequestWaiter waiter) { // TODO: Consider replacing this by a watcher on the target synchronized(this) { // ensure we have exactly 1 valid connection across threads - if (target == null || ! target.isValid()) + if (! target.isValid()) { target = supervisor.connect(new Spec(hostname, port)); + } } target.invokeAsync(req, timeout, waiter); } @Override public void close() { - if (target != null) { - target.close(); - } + target.close(); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java index d39a488626b..e346a766738 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java @@ -97,11 +97,10 @@ public class DocumentSourceSearcher extends Searcher { public Result search(Query query, Execution execution) { queryCount++; Result r = unFilledResults.get(getQueryKeyClone(query)); - if (r == null) { + if (r == null) r = defaultFilledResult.clone(); - } else { + else r = r.clone(); - } r.setQuery(query); r.hits().trim(query.getOffset(), query.getHits()); @@ -182,11 +181,8 @@ public class DocumentSourceSearcher extends Searcher { * reset. For testing - not reliable if multiple threads makes * queries simultaneously */ - public int getQueryCount() { - return queryCount; - } + public int getQueryCount() { return queryCount; } + + public void resetQueryCount() { queryCount = 0; } - public void resetQueryCount() { - queryCount=0; - } } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 6eef1252998..dd52b9e19b8 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -701,7 +701,18 @@ public class VespaSerializer { destination.append(leafAnnotations(item)); comma(destination, initLen); int targetNumHits = item.getTargetNumHits(); - destination.append("\"targetNumHits\": ").append(targetNumHits); + annotationKey(destination, "targetNumHits").append(targetNumHits); + int explore = item.getHnswExploreAdditionalHits(); + if (explore != 0) { + comma(destination, initLen); + String key = YqlParser.HNSW_EXPLORE_ADDITIONAL_HITS; + annotationKey(destination, key).append(explore); + } + boolean allow_approx = item.getAllowApproximate(); + if (! allow_approx) { + comma(destination, initLen); + annotationKey(destination, "approximate").append(allow_approx); + } destination.append("}]"); destination.append(NEAREST_NEIGHBOR).append('('); destination.append(item.getIndexName()).append(", "); @@ -1347,6 +1358,11 @@ public class VespaSerializer { } } + private static StringBuilder annotationKey(StringBuilder annotation, String val) { + annotation.append("\"").append(val).append("\": "); + return annotation; + } + private static void comma(StringBuilder annotation, int initLen) { if (annotation.length() > initLen) { annotation.append(", "); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 8d013e501e8..f4560806dd2 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -137,6 +137,7 @@ public class YqlParser implements Parser { static final String ACCENT_DROP = "accentDrop"; static final String ALTERNATIVES = "alternatives"; static final String AND_SEGMENTING = "andSegmenting"; + static final String APPROXIMATE = "approximate"; static final String BOUNDS = "bounds"; static final String BOUNDS_LEFT_OPEN = "leftOpen"; static final String BOUNDS_OPEN = "open"; @@ -149,6 +150,7 @@ public class YqlParser implements Parser { static final String EQUIV = "equiv"; static final String FILTER = "filter"; static final String HIT_LIMIT = "hitLimit"; + static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; static final String LABEL = "label"; static final String NEAR = "near"; @@ -421,6 +423,14 @@ public class YqlParser implements Parser { if (targetNumHits != null) { item.setTargetNumHits(targetNumHits); } + Integer hnswExploreAdditionalHits = getAnnotation(ast, HNSW_EXPLORE_ADDITIONAL_HITS, + Integer.class, null, "number of extra hits to explore for HNSW algorithm"); + if (hnswExploreAdditionalHits != null) { + item.setHnswExploreAdditionalHits(hnswExploreAdditionalHits); + } + Boolean allowApproximate = getAnnotation(ast, APPROXIMATE, + Boolean.class, Boolean.TRUE, "allow approximate nearest neighbor search"); + item.setAllowApproximate(allowApproximate); String label = getAnnotation(ast, LABEL, String.class, null, "item label"); if (label != null) { item.setLabel(label); diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java index 5cae40bd10d..df35d8dbdea 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java @@ -11,6 +11,7 @@ import org.junit.Test; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Collections; import static org.junit.Assert.assertEquals; @@ -34,7 +35,7 @@ public class ExactMatchAndDefaultIndexTestCase { q.getModel().setExecution(new Execution(new Execution.Context(null, facts, null, null, null))); assertEquals("AND testexact:a/b testexact:foo.com", q.getModel().getQueryTree().getRoot().toString()); q = new Query("?query=" + enc("a/b foo.com")); - assertEquals("AND \"a b\" \"foo com\"", q.getModel().getQueryTree().getRoot().toString()); + assertEquals("AND a b foo com", q.getModel().getQueryTree().getRoot().toString()); } @Test @@ -44,11 +45,7 @@ public class ExactMatchAndDefaultIndexTestCase { } private String enc(String s) { - try { - return URLEncoder.encode(s, "utf-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } + return URLEncoder.encode(s, StandardCharsets.UTF_8); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index 0fdad1a1f9c..c1db7d73561 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -7,6 +7,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; @@ -48,7 +49,9 @@ public class ParseTestCase { @Test public void testTermWithIndexPrefix() { - tester.assertParsed("url:foobar", "url:foobar", Query.Type.ANY); + tester.assertParsed("url:foobar", + "url:foobar", + Query.Type.ANY); } @Test @@ -59,104 +62,98 @@ public class ParseTestCase { @Test public void testMultipleTermsWithUTF8EncodingOred() { tester.assertParsed("OR l\u00e5gen delta M\u00dcNICH M\u00fcnchen", - "l\u00e5gen delta M\u00dcNICH M\u00fcnchen", Query.Type.ANY); + "l\u00e5gen delta M\u00dcNICH M\u00fcnchen", + Query.Type.ANY); } @Test public void testMultipleTermsWithMultiplePrefixes() { tester.assertParsed("RANK (+bar -normal.title:foo -baz) url:foobar", - "url:foobar +bar -normal.title:foo -baz", Query.Type.ANY); + "url:foobar +bar -normal.title:foo -baz", Query.Type.ANY); } @Test public void testSimpleQueryDefaultOr() { - tester.assertParsed("OR foobar foo bar baz", "foobar foo bar baz", - Query.Type.ANY); + tester.assertParsed("OR foobar foo bar baz", "foobar foo bar baz", Query.Type.ANY); } @Test public void testOrAndNot() { tester.assertParsed("RANK (+(AND baz bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy -foobaz +bar", Query.Type.ANY); + "foobar +baz foo -xyzzy -foobaz +bar", Query.Type.ANY); } @Test public void testSimpleOrNestedAnd() { tester.assertParsed("RANK (OR foo bar baz) foobar xyzzy", - "foobar +(foo bar baz) xyzzy", Query.Type.ANY); + "foobar +(foo bar baz) xyzzy", Query.Type.ANY); } @Test public void testSimpleOrNestedNot() { tester.assertParsed("+(OR foobar xyzzy) -(AND foo bar baz)", - "foobar -(foo bar baz) xyzzy", Query.Type.ANY); + "foobar -(foo bar baz) xyzzy", Query.Type.ANY); } @Test public void testOrNotNestedAnd() { - tester.assertParsed( - "RANK (+(AND baz (OR foo bar baz) bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz (OR foo bar baz) bar) -xyzzy -foobaz) foobar foo", + "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +bar", + Query.Type.ANY); } @Test public void testOrAndNotNestedNot() { - tester.assertParsed( - "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz) foobar foo", - "foobar +baz foo -xyzzy -(foo bar baz) -foobaz +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz) foobar foo", + "foobar +baz foo -xyzzy -(foo bar baz) -foobaz +bar", + Query.Type.ANY); } @Test public void testOrMultipleNestedAnd() { - tester.assertParsed( - "RANK (AND (OR fo ba foba) (OR foz baraz)) foobar foo bar baz", - "foobar +(fo ba foba) foo bar +(foz baraz) baz", Query.Type.ANY); + tester.assertParsed("RANK (AND (OR fo ba foba) (OR foz baraz)) foobar foo bar baz", + "foobar +(fo ba foba) foo bar +(foz baraz) baz", + Query.Type.ANY); } @Test public void testOrMultipleNestedNot() { - tester.assertParsed( - "+(OR foobar foo bar baz) -(AND fo ba foba) -(AND foz baraz)", - "foobar -(fo ba foba) foo bar -(foz baraz) baz", Query.Type.ANY); + tester.assertParsed("+(OR foobar foo bar baz) -(AND fo ba foba) -(AND foz baraz)", + "foobar -(fo ba foba) foo bar -(foz baraz) baz", + Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedAnd() { - tester.assertParsed( - "RANK (+(AND baz (OR foo bar baz) (OR foz bazaz) bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +(foz bazaz) +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz (OR foo bar baz) (OR foz bazaz) bar) -xyzzy -foobaz) foobar foo", + "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +(foz bazaz) +bar", + Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedNot() { - tester.assertParsed( - "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz -(AND foz bazaz)) foobar foo", - "foobar +baz foo -xyzzy -(foo bar baz) -foobaz -(foz bazaz) +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz -(AND foz bazaz)) foobar foo", + "foobar +baz foo -xyzzy -(foo bar baz) -foobaz -(foz bazaz) +bar", + Query.Type.ANY); } @Test public void testOrMultipleNestedAndNot() { - tester.assertParsed( - "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof)) -(AND fo ba foba) -(AND foz baraz)) foobar foo bar baz", - "foobar -(fo ba foba) foo +(ffoooo bbaarr) bar +(oof rab raboof) -(foz baraz) baz", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof)) -(AND fo ba foba) -(AND foz baraz)) foobar foo bar baz", + "foobar -(fo ba foba) foo +(ffoooo bbaarr) bar +(oof rab raboof) -(foz baraz) baz", + Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedAndNot() { - tester.assertParsed( - "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof) baz xyxyzzy) -(AND fo ba foba) -foo -bar -(AND foz baraz)) foobar", - "foobar -(fo ba foba) -foo +(ffoooo bbaarr) -bar +(oof rab raboof) -(foz baraz) +baz +xyxyzzy", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof) baz xyxyzzy) -(AND fo ba foba) -foo -bar -(AND foz baraz)) foobar", + "foobar -(fo ba foba) -foo +(ffoooo bbaarr) -bar +(oof rab raboof) -(foz baraz) +baz +xyxyzzy", + Query.Type.ANY); } @Test public void testExplicitPhrase() { - Item root=tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ANY); + Item root = tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ANY); assertTrue(root instanceof PhraseItem); assertTrue(((PhraseItem)root).isExplicit()); } @@ -164,21 +161,20 @@ public class ParseTestCase { @Test public void testPhraseWithIndex() { tester.assertParsed("normal.title:\"foo bar foobar\"", - "normal.title:\"foo bar foobar\"", Query.Type.ANY); + "normal.title:\"foo bar foobar\"", Query.Type.ANY); } @Test public void testPhrasesAndTerms() { tester.assertParsed("OR \"foo bar foobar\" xyzzy \"baz gaz faz\"", - "\"foo bar foobar\" xyzzy \"baz gaz faz\"", Query.Type.ANY); + "\"foo bar foobar\" xyzzy \"baz gaz faz\"", Query.Type.ANY); } @Test public void testPhrasesAndTermsWithOperators() { - tester.assertParsed( - "RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy", - "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy", + "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar", + Query.Type.ANY); } @Test @@ -188,38 +184,40 @@ public class ParseTestCase { @Test public void testTermWithCatalogAndIndexPrefixDefaultAnd() { - tester.assertParsed("normal.title:foobar", "normal.title:foobar", - Query.Type.ALL); + tester.assertParsed("normal.title:foobar", "normal.title:foobar", Query.Type.ALL); } @Test public void testMultipleTermsWithMultiplePrefixesDefaultAnd() { tester.assertParsed("+(AND url:foobar bar) -normal.title:foo -baz", - "url:foobar +bar -normal.title:foo -baz", Query.Type.ALL); + "url:foobar +bar -normal.title:foo -baz", + Query.Type.ALL); } @Test public void testSimpleQueryDefaultAnd() { - tester.assertParsed("AND foobar foo bar baz", "foobar foo bar baz", - Query.Type.ALL); + tester.assertParsed("AND foobar foo bar baz", "foobar foo bar baz", Query.Type.ALL); } @Test public void testNotDefaultAnd() { - tester.assertParsed( - "+(AND foobar (OR foo bar baz) xyzzy) -(AND foz baraz bazar)", - "foobar +(foo bar baz) xyzzy -(foz baraz bazar)", Query.Type.ALL); + tester.assertParsed("+(AND foobar (OR foo bar baz) xyzzy) -(AND foz baraz bazar)", + "foobar +(foo bar baz) xyzzy -(foz baraz bazar)", + Query.Type.ALL); } @Test public void testSimpleTermQueryDefaultPhrase() { - tester.assertParsed("foobar", "foobar", Query.Type.PHRASE); + tester.assertParsed("foobar", + "foobar", + Query.Type.PHRASE); } @Test public void testSimpleQueryDefaultPhrase() { - Item root=tester.assertParsed("\"foobar foo bar baz\"", "foobar foo bar baz", - Query.Type.PHRASE); + Item root = tester.assertParsed("\"foobar foo bar baz\"", + "foobar foo bar baz", + Query.Type.PHRASE); assertTrue(root instanceof PhraseItem); assertFalse(((PhraseItem)root).isExplicit()); } @@ -227,23 +225,25 @@ public class ParseTestCase { @Test public void testMultipleTermsWithMultiplePrefixesDefaultPhrase() { tester.assertParsed("\"url foobar bar normal title foo baz\"", - "url:foobar +bar -normal.title:foo -baz", Query.Type.PHRASE); + "url:foobar +bar -normal.title:foo -baz", + Query.Type.PHRASE); } @Test public void testOdd1() { - tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.ALL); + tester.assertParsed("AND window print error", "+window.print() +error", + Query.Type.ALL); } @Test public void testOdd2() { - tester.assertParsed("normal.title:kaboom", "normal.title:\"kaboom\"",Query.Type.ALL); + tester.assertParsed("normal.title:kaboom", "normal.title:\"kaboom\"", + Query.Type.ALL); } @Test public void testOdd2Uppercase() { - tester.assertParsed("normal.title:KABOOM", "NORMAL.TITLE:\"KABOOM\"", - Query.Type.ALL); + tester.assertParsed("normal.title:KABOOM", "NORMAL.TITLE:\"KABOOM\"", Query.Type.ALL); } @Test @@ -280,19 +280,19 @@ public class ParseTestCase { @Test public void testNestedCompositesDefaultOr() { tester.assertParsed("RANK (OR foobar bar baz) foo xyzzy", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.ANY); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.ANY); } @Test public void testNestedCompositesDefaultAnd() { tester.assertParsed("AND foo (OR foobar bar baz) xyzzy", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.ALL); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.ALL); } @Test public void testNestedCompositesPhraseDefault() { tester.assertParsed("\"foo foobar bar baz xyzzy\"", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.PHRASE); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.PHRASE); } @Test @@ -349,8 +349,7 @@ public class ParseTestCase { @Test public void testNumericWithIndex() { - tester.assertParsed("document.size:[34;454]", "document.size:[34;454]", - Query.Type.ANY); + tester.assertParsed("document.size:[34;454]", "document.size:[34;454]", Query.Type.ANY); } @Test @@ -361,14 +360,14 @@ public class ParseTestCase { @Test public void testMultipleIntegerWithIndex() { tester.assertParsed("OR document.size:[34;454] date:>1234567890", - "document.size:[34;454] date:>1234567890", Query.Type.ANY); + "document.size:[34;454] date:>1234567890", Query.Type.ANY); } @Test public void testMixedNumericAndOtherTerms() { tester.assertParsed("RANK (AND document.size:<1024 xyzzy) foo date:>123456890", - "foo +document.size:<1024 +xyzzy date:>123456890", - Query.Type.ANY); + "foo +document.size:<1024 +xyzzy date:>123456890", + Query.Type.ANY); } @Test @@ -378,20 +377,18 @@ public class ParseTestCase { @Test public void testItemPhraseEmptyPhrase() { - tester.assertParsed("RANK to \"or not to be\"", "+to\"or not to be\"\"\"", - Query.Type.ANY); + tester.assertParsed("RANK to \"or not to be\"", "+to\"or not to be\"\"\"", Query.Type.ANY); } @Test public void testSimpleQuery() { - tester.assertParsed("OR if am \"f g 4 2\" maybe", "if am \" f g 4 2\"\" maybe", - Query.Type.ANY); + tester.assertParsed("OR if am \"f g 4 2\" maybe", "if am \" f g 4 2\"\" maybe", Query.Type.ANY); } @Test public void testExcessivePluses() { tester.assertParsed("+(AND other is nothing) -test", - "++other +++++is ++++++nothing -test", Query.Type.ANY); + "++other +++++is ++++++nothing -test", Query.Type.ANY); } @Test @@ -401,39 +398,38 @@ public class ParseTestCase { @Test public void testPlusesAndMinuses() { - Item root=tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ANY); - assertTrue(root instanceof PhraseItem); - assertFalse(((PhraseItem)root).isExplicit()); + tester.assertParsed("AND a b c d d", "a+b+c+d--d", Query.Type.ANY); } @Test public void testNumbers() { - tester.assertParsed("\"123 2132odfd 934032 32423\"", - "123+2132odfd.934032,,32423", Query.Type.ANY); + tester.assertParsed("AND 123 2132odfd 934032 32423", "123+2132odfd.934032,,32423", Query.Type.ANY); } @Test public void testOtherSignsInQuote() { - tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043", - Query.Type.ANY); + tester.assertParsed("AND 0032 4 320 24329043", "0032+4\\320.24329043", Query.Type.ANY); } @Test public void testGribberish() { tester.assertParsed("1349832840234l3040roer\u00e6lf12", - ",1349832840234l3040roer\u00e6lf12", Query.Type.ANY); + ",1349832840234l3040roer\u00e6lf12", + Query.Type.ANY); } @Test public void testUrl() { - tester.assertParsed("www:\"www hotelaiguablava com\"", - "+www:www.hotelaiguablava:com", Query.Type.ANY); + tester.assertParsed("AND www:www www:hotelaiguablava www:com", + "+www:www.hotelaiguablava:com", + Query.Type.ANY); } @Test public void testUrlGribberish() { - tester.assertParsed("OR \"3 16\" fast.type:lycosoffensive", - "[ 3:16 fast.type:lycosoffensive", Query.Type.ANY); + tester.assertParsed("OR (AND 3 16) fast.type:lycosoffensive", + "[ 3:16 fast.type:lycosoffensive", + Query.Type.ANY); } @Test @@ -475,8 +471,7 @@ public class ParseTestCase { @Test public void testPrefixWithDotAdvanced() { - tester.assertParsed("normal.title:foobar", "normal.title:foobar", - Query.Type.ADVANCED); + tester.assertParsed("normal.title:foobar", "normal.title:foobar", Query.Type.ADVANCED); } @Test @@ -486,20 +481,21 @@ public class ParseTestCase { @Test public void testSimplePhraseAdvanced() { - tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", - Query.Type.ADVANCED); + tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ADVANCED); } @Test public void testSimplePhraseWithIndexAdvanced() { tester.assertParsed("normal.title:\"foo bar foobar\"", - "normal.title:\"foo bar foobar\"", Query.Type.ADVANCED); + "normal.title:\"foo bar foobar\"", + Query.Type.ADVANCED); } @Test public void testMultiplePhrasesAdvanced() { tester.assertParsed("AND \"foo bar foobar\" \"baz gaz faz\"", - "\"foo bar foobar\" and \"baz gaz faz\"", Query.Type.ADVANCED); + "\"foo bar foobar\" and \"baz gaz faz\"", + Query.Type.ADVANCED); } @Test @@ -661,23 +657,23 @@ public class ParseTestCase { @Test public void testImplicitPhrase1Advanced() { - tester.assertParsed("\"test if\"", "--test+-if", Query.Type.ADVANCED); + tester.assertParsed("AND test if", "--test+-if", Query.Type.ADVANCED); } @Test public void testImplicitPhrase2Advanced() { - tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ADVANCED); + tester.assertParsed("AND a b c d d", "a+b+c+d--d", Query.Type.ADVANCED); } @Test public void testImplicitPhrase3Advanced() { - tester.assertParsed("\"123 2132odfd 934032 32423\"", + tester.assertParsed("AND 123 2132odfd 934032 32423", "123+2132odfd.934032,,32423", Query.Type.ADVANCED); } @Test public void testImplicitPhrase4Advanced() { - tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043", Query.Type.ADVANCED); + tester.assertParsed("AND 0032 4 320 24329043", "0032+4\\320.24329043", Query.Type.ADVANCED); } @Test @@ -730,7 +726,7 @@ public class ParseTestCase { @Test public void testSingleHyphen() { - tester.assertParsed("\"a b\"", "a-b", Query.Type.ALL); + tester.assertParsed("AND a b", "a-b", Query.Type.ALL); } @Test @@ -883,27 +879,27 @@ public class ParseTestCase { @Test public void testSimpleDotPhraseAny() { - tester.assertParsed("OR a \"b c\" d", "a b.c d", Query.Type.ANY); + tester.assertParsed("OR a (AND b c) d", "a b.c d", Query.Type.ANY); } @Test public void testSimpleHyphenPhraseAny() { - tester.assertParsed("OR a \"b c\" d", "a b-c d", Query.Type.ANY); + tester.assertParsed("OR a (AND b c) d", "a b-c d", Query.Type.ANY); } @Test public void testAnotherSimpleDotPhraseAny() { - tester.assertParsed("OR \"a b\" c d", "a.b c d", Query.Type.ANY); + tester.assertParsed("OR (AND a b) c d", "a.b c d", Query.Type.ANY); } @Test public void testYetAnotherSimpleDotPhraseAny() { - tester.assertParsed("OR a b \"c d\"", "a b c.d", Query.Type.ANY); + tester.assertParsed("OR a b (AND c d)", "a b c.d", Query.Type.ANY); } @Test public void testVariousSeparatorsPhraseAny() { - tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ANY); + tester.assertParsed("AND a b c d", "a-b.c%d", Query.Type.ANY); } @Test @@ -918,45 +914,44 @@ public class ParseTestCase { @Test public void testIndexedDottedPhraseAny() { - tester.assertParsed("OR a url:\"b c\" d", "a url:b.c d", Query.Type.ANY); + tester.assertParsed("OR a (AND url:b url:c) d", "a url:b.c d", Query.Type.ANY); } @Test public void testIndexedPlusedPhraseAny() { - tester.assertParsed("OR a normal.title:\"b c\" d", "a normal.title:b+c d", - Query.Type.ANY); + tester.assertParsed("OR a (AND normal.title:b normal.title:c) d", "a normal.title:b+c d", Query.Type.ANY); } @Test public void testNestedNotAny() { tester.assertParsed( - "RANK (+(OR normal.title:foobar url:\"www pvv org\") -foo) a", + "RANK (+(OR normal.title:foobar (AND url:www url:pvv url:org)) -foo) a", "a +(normal.title:foobar url:www.pvv.org) -foo", Query.Type.ANY); } @Test public void testDottedPhraseAdvanced() { - tester.assertParsed("OR a \"b c\"", "a or b.c", Query.Type.ADVANCED); + tester.assertParsed("OR a (AND b c)", "a or b.c", Query.Type.ADVANCED); } @Test public void testHyphenPhraseAdvanced() { - tester.assertParsed("OR (AND a \"b c\") d", "a and b-c or d", Query.Type.ADVANCED); + tester.assertParsed("OR (AND a (AND b c)) d", "a and b-c or d", Query.Type.ADVANCED); } @Test public void testAnotherDottedPhraseAdvanced() { - tester.assertParsed("OR \"a b\" c", "a.b or c", Query.Type.ADVANCED); + tester.assertParsed("OR (AND a b) c", "a.b or c", Query.Type.ADVANCED); } @Test public void testNottedDottedPhraseAdvanced() { - tester.assertParsed("+a -\"c d\"", "a andnot c.d", Query.Type.ADVANCED); + tester.assertParsed("+a -(AND c d)", "a andnot c.d", Query.Type.ADVANCED); } @Test public void testVariousSeparatorsPhraseAdvanced() { - tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ADVANCED); + tester.assertParsed("AND a b c d", "a-b.c%d", Query.Type.ADVANCED); } @Test @@ -976,14 +971,14 @@ public class ParseTestCase { @Test public void testNestedPlussedPhraseAdvanced() { - tester.assertParsed("AND (OR a normal.title:\"b c\") d", + tester.assertParsed("AND (OR a (AND normal.title:b normal.title:c)) d", "a or normal.title:b+c and d", Query.Type.ADVANCED); } @Test public void testNottedNestedDottedPhraseAdvanced() { tester.assertParsed( - "+(AND a (OR normal.title:foobar url:\"www pvv org\")) -foo", + "+(AND a (OR normal.title:foobar (AND url:www url:pvv url:org))) -foo", "a and (normal.title:foobar or url:www.pvv.org) andnot foo", Query.Type.ADVANCED); } @@ -995,7 +990,7 @@ public class ParseTestCase { @Test public void testPlusedTwiceThenQuotedPhraseAny() { - tester.assertParsed("\"a b c d\"", "a+b+\"c d\"", Query.Type.ANY); + tester.assertParsed("AND a b c d", "a+b+\"c d\"", Query.Type.ANY); } @Test @@ -1005,7 +1000,7 @@ public class ParseTestCase { @Test public void testPhrasesInBraces() { - tester.assertParsed("url.domain:\"microsoft com\"", + tester.assertParsed("AND url.domain:microsoft url.domain:com", "+(url.domain:microsoft.com)", Query.Type.ALL); } @@ -1053,17 +1048,17 @@ public class ParseTestCase { @Test public void testPhraseNotPrefix() { - tester.assertParsed("OR foo \"prefix bar\"", "foo prefix*bar", Query.Type.ANY); + tester.assertParsed("OR foo (AND prefix bar)", "foo prefix*bar", Query.Type.ANY); } @Test public void testPhraseNotSubstring() { - tester.assertParsed("OR foo \"substring bar\"", "foo *substring*bar", Query.Type.ANY); + tester.assertParsed("OR foo (AND substring bar)", "foo *substring*bar", Query.Type.ANY); } @Test public void testPhraseNotSuffix() { - tester.assertParsed("OR \"foo suffix\" bar", "foo*suffix bar", Query.Type.ANY); + tester.assertParsed("OR (AND foo suffix) bar", "foo*suffix bar", Query.Type.ANY); } @Test @@ -1086,20 +1081,17 @@ public class ParseTestCase { @Test public void testIndexedPhraseNotPrefix() { - tester.assertParsed("foo.bar:\"prefix xyzzy\"", "foo.bar:prefix*xyzzy", - Query.Type.ANY); + tester.assertParsed("AND foo.bar:prefix foo.bar:xyzzy", "foo.bar:prefix*xyzzy", Query.Type.ANY); } @Test public void testIndexedPhraseNotSubstring() { - tester.assertParsed("foo.bar:\"substring xyzzy\"", "foo.bar:*substring*xyzzy", - Query.Type.ANY); + tester.assertParsed("AND foo.bar:substring foo.bar:xyzzy", "foo.bar:*substring*xyzzy", Query.Type.ANY); } @Test public void testIndexedPhraseNotSuffix() { - tester.assertParsed("foo.bar:\"xyzzy suffix\"", "foo.bar:xyzzy*suffix", - Query.Type.ANY); + tester.assertParsed("AND foo.bar:xyzzy foo.bar:suffix", "foo.bar:xyzzy*suffix", Query.Type.ANY); } @Test @@ -1120,20 +1112,20 @@ public class ParseTestCase { assertTrue(root instanceof SuffixItem); } - /** Non existing index → phrase **/ + /** Non existing index → and **/ @Test public void testNonIndexPhraseNotPrefix() { - tester.assertParsed("\"void prefix\"", "void:prefix*", Query.Type.ANY); + tester.assertParsed("AND void prefix", "void:prefix*", Query.Type.ANY); } @Test public void testNonIndexPhraseNotSubstring() { - tester.assertParsed("\"void substring\"", "void:*substring*", Query.Type.ANY); + tester.assertParsed("AND void substring", "void:*substring*", Query.Type.ANY); } @Test public void testNonIndexPhraseNotSuffix() { - tester.assertParsed("\"void suffix\"", "void:*suffix", Query.Type.ANY); + tester.assertParsed("AND void suffix", "void:*suffix", Query.Type.ANY); } /** Explicit phrase → remove '*' **/ @@ -1198,7 +1190,7 @@ public class ParseTestCase { /** Extra spaces with index **/ @Test public void testIndexPrefixExtraSpace() { - tester.assertParsed("\"foo prefix\"", "foo:prefix *", Query.Type.ANY); + tester.assertParsed("AND foo prefix", "foo:prefix *", Query.Type.ANY); } @Test @@ -1419,7 +1411,7 @@ public class ParseTestCase { @Test public void testMultipleDifferentPhraseSeparators() { - tester.assertParsed("\"foo bar\"", "foo.-.bar", Query.Type.ANY); + tester.assertParsed("AND foo bar", "foo.-.bar", Query.Type.ANY); } @Test @@ -1430,19 +1422,17 @@ public class ParseTestCase { @Test public void testReallyNoisyQuery1() { - tester.assertParsed("AND word another", "&word\"()/&#)(/&another!\"", - Query.Type.ALL); + tester.assertParsed("AND word another", "&word\"()/&#)(/&another!\"", Query.Type.ALL); } @Test public void testReallyNoisyQuery2() { - tester.assertParsed("\"\u03bc\u03bc hei\"", "&&&`\u00b5\u00b5=@hei", Query.Type.ALL); + tester.assertParsed("AND \u03bc\u03bc hei", "&&&`\u00b5\u00b5=@hei", Query.Type.ALL); } @Test public void testReallyNoisyQuery3() { - tester.assertParsed("AND \"hei hallo\" du der", "hei-hallo;du;der", - Query.Type.ALL); + tester.assertParsed("AND hei hallo du der", "hei-hallo;du;der", Query.Type.ALL); } @Test @@ -1478,7 +1468,7 @@ public class ParseTestCase { @Test public void testTheStupidSymbolsWhichAreNowWordCharactersInUnicode() { - tester.assertParsed("\"yz a\"", "yz\u00A8\u00AA\u00AF", Query.Type.ANY); + tester.assertParsed("AND yz a", "yz\u00A8\u00AA\u00AF", Query.Type.ANY); } @Test @@ -1498,7 +1488,7 @@ public class ParseTestCase { @Test public void testImplicitPhrasingWithIndex() { - tester.assertParsed("a:\"b c\"", "a:/b/c", Query.Type.ANY); + tester.assertParsed("AND a:b a:c", "a:/b/c", Query.Type.ANY); } @Test @@ -1508,7 +1498,7 @@ public class ParseTestCase { @Test public void testSingleNoisyPhraseWithIndex() { - tester.assertParsed("mail:\"yahoo com\"", "mail:@yahoo.com", Query.Type.ANY); + tester.assertParsed("AND mail:yahoo mail:com", "mail:@yahoo.com", Query.Type.ANY); } @Test @@ -1599,7 +1589,7 @@ public class ParseTestCase { "url.all:http://www.newsadvance.com/servlet/Satellite?pagename=LNA/MGArticle/IMD_BasicArticle&c=MGArticle&cid=1031782787014&path=!mgnetwork!diversions", Query.Type.ALL); tester.assertParsed( - "AND ull:\"http www neue oz de information pub Boulevard index html file a 3 s 4 file\" s:\"37 iptc bdt 20050607 294 dpa 9001170 txt\" s:\"3 dir\" s:\"26 opt DPA parsed boulevard\" s:\"7 bereich\" s:\"9 Boulevard\"", + "AND ull:http ull:www ull:neue ull:oz ull:de ull:information ull:pub ull:Boulevard ull:index ull:html ull:file ull:a ull:3 ull:s ull:4 ull:file s:\"37 iptc bdt 20050607 294 dpa 9001170 txt\" s:\"3 dir\" s:\"26 opt DPA parsed boulevard\" s:\"7 bereich\" s:\"9 Boulevard\"", "ull:http://www.neue-oz.de/information/pub_Boulevard/index.html?file=a:3:{s:4:\"file\";s:37:\"iptc-bdt-20050607-294-dpa_9001170.txt\";s:3:\"dir\";s:26:\"/opt/DPA/parsed/boulevard/\";s:7:\"bereich\";s:9:\"Boulevard\";}", Query.Type.ALL); } @@ -1640,7 +1630,7 @@ public class ParseTestCase { @Test public void testTooLongQueryTerms() { - tester.assertParsed("AND \"545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof filter ew 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"!1000 \"2b 2f 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"", + tester.assertParsed("AND 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof filter ew 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof 2b 2f 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof", "+/545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof&filter=ew:545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof!1000 =.2b..2f.545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof", Query.Type.ALL); } @@ -1648,22 +1638,22 @@ public class ParseTestCase { @Test public void testNonSpecialTokenParsing() { ParsingTester customTester = new ParsingTester(new SpecialTokens("default")); - customTester.assertParsed("OR c or c with \"tcp ip\"", "c# or c++ with tcp/ip", Query.Type.ANY); + customTester.assertParsed("OR c or c with (AND tcp ip)", "c# or c++ with tcp/ip", Query.Type.ANY); } @Test public void testNonIndexWithColons1() { - tester.assertParsed("OR this is \"notan iindex\"", "this is notan:iindex", Query.Type.ANY); + tester.assertParsed("OR this is (AND notan iindex)", "this is notan:iindex", Query.Type.ANY); } @Test public void testNonIndexWithColons2() { - tester.assertParsed("OR this is \"notan iindex either\"", "this is notan:iindex:either", Query.Type.ANY); + tester.assertParsed("OR this is (AND notan iindex either)", "this is notan:iindex:either", Query.Type.ANY); } @Test public void testIndexThenUnderscoreTermBecomesIndex() { - tester.assertParsed("name:\"batch article\"", "name:batch_article", Query.Type.ANY); + tester.assertParsed("AND name:batch name:article", "name:batch_article", Query.Type.ANY); } @Test @@ -1671,17 +1661,17 @@ public class ParseTestCase { // "first" "second" and "third" are segments in the test language Item item = tester.parseQuery("name:firstsecondthird", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertTrue(item instanceof PhraseSegmentItem); - PhraseSegmentItem phrase = (PhraseSegmentItem) item; + assertTrue(item instanceof AndSegmentItem); + AndSegmentItem segment = (AndSegmentItem) item; - assertEquals(3, phrase.getItemCount()); - assertEquals("name:first", phrase.getItem(0).toString()); - assertEquals("name:second", phrase.getItem(1).toString()); - assertEquals("name:third", phrase.getItem(2).toString()); + assertEquals(3, segment.getItemCount()); + assertEquals("name:first", segment.getItem(0).toString()); + assertEquals("name:second", segment.getItem(1).toString()); + assertEquals("name:third", segment.getItem(2).toString()); - assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName()); - assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName()); - assertEquals("name", ((WordItem) phrase.getItem(2)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(0)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(1)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(2)).getIndexName()); } @Test @@ -1689,22 +1679,22 @@ public class ParseTestCase { // "first" "second" and "third" are segments in the test language Item item = tester.parseQuery("name:\"firstsecondthird\"", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertTrue(item instanceof PhraseSegmentItem); - PhraseSegmentItem phrase = (PhraseSegmentItem) item; + assertTrue(item instanceof AndSegmentItem); + AndSegmentItem segment = (AndSegmentItem) item; - assertEquals(3, phrase.getItemCount()); - assertEquals("name:first", phrase.getItem(0).toString()); - assertEquals("name:second", phrase.getItem(1).toString()); - assertEquals("name:third", phrase.getItem(2).toString()); + assertEquals(3, segment.getItemCount()); + assertEquals("name:first", segment.getItem(0).toString()); + assertEquals("name:second", segment.getItem(1).toString()); + assertEquals("name:third", segment.getItem(2).toString()); - assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName()); - assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName()); - assertEquals("name", ((WordItem)phrase.getItem(2)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(0)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(1)).getIndexName()); + assertEquals("name", ((WordItem)segment.getItem(2)).getIndexName()); } @Test public void testAndItemAndImplicitPhrase() { - tester.assertParsed("\"\u00d8 \u00d8 \u00d8 \u00d9\"", + tester.assertParsed("AND \u00d8 \u00d8 \u00d8 \u00d9", "\u00d8\u00b9\u00d8\u00b1\u00d8\u00a8\u00d9", "", Query.Type.ALL, Language.CHINESE_SIMPLIFIED); } @@ -1736,7 +1726,7 @@ public class ParseTestCase { @Test public void testFakeCJKSegmentingOfMultiplePhrases() { Item item = tester.parseQuery("name:firstsecond.s", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertEquals("name:\"'first second' s\"", item.toString()); + assertEquals("AND (SAND name:first name:second) name:s", item.toString()); } @Test @@ -1801,7 +1791,7 @@ public class ParseTestCase { @Test public void testCommaOnlyLeadsToImplicitPhrasing() { - tester.assertParsed("\"A B C\"", "A,B,C", Query.Type.ALL); + tester.assertParsed("AND A B C", "A,B,C", Query.Type.ALL); } @Test @@ -1873,8 +1863,8 @@ public class ParseTestCase { @Test public void testJPMobileExceptionQuery() { - tester.assertParsed("OR concat and \"make string\" 1 47 or", - "(concat \"and\" (make-string 1 47) \"or\")", Query.Type.ALL); + tester.assertParsed("OR concat and (AND make string) 1 47 or", + "(concat \"and\" (make-string 1 47) \"or\")", Query.Type.ALL); } @Test @@ -1882,7 +1872,7 @@ public class ParseTestCase { tester.assertParsed("b", "a: b", Query.Type.ALL); tester.assertParsed("AND a b", "a : b", Query.Type.ALL); tester.assertParsed("AND a b", "a :b", Query.Type.ALL); - tester.assertParsed("\"a b\"", "a.:b", Query.Type.ALL); + tester.assertParsed("AND a b", "a.:b", Query.Type.ALL); tester.assertParsed("a:b", "a:b", Query.Type.ALL); } @@ -1917,8 +1907,7 @@ public class ParseTestCase { tester.assertParsed("AND ringtone AND (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")", "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )", Query.Type.ALL); - // The last one here is a little weird, but it's not a problem, - // so I let it pass for now... + // The last one here is a little weird, but it's not a problem, so let it pass for now... tester.assertParsed("OR (OR ringtone AND) (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")", "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )", Query.Type.ANY); @@ -1926,7 +1915,7 @@ public class ParseTestCase { @Test public void testMixedCaseIndexNames() { - tester.assertParsed("AND mixedCase:a mixedCase:b \"notAnIndex c\" mixedCase:d", + tester.assertParsed("AND mixedCase:a mixedCase:b notAnIndex c mixedCase:d", "mixedcase:a MIXEDCASE:b notAnIndex:c mixedCase:d", Query.Type.ALL); } @@ -1934,7 +1923,7 @@ public class ParseTestCase { /** CJK special tokens should be recognized also on non-boundaries */ @Test public void testChineseSpecialTokens() { - tester.assertParsed("AND \"cat tcp/ip zu\" \"foo dotnet bar dotnet dotnet c# c++ bar dotnet dotnet wiz\"", + tester.assertParsed("AND cat tcp/ip zu foo dotnet bar dotnet dotnet c# c++ bar dotnet dotnet wiz", "cattcp/ipzu foo.netbar.net.netC#c++bar.net.netwiz","", Query.Type.ALL, Language.CHINESE_SIMPLIFIED); } @@ -1945,7 +1934,7 @@ public class ParseTestCase { @Test public void testChineseSpecialTokensWithMultiSegmentReplace() { // special-token-fs is a special token, to be replaced by firstsecond, first and second are segments in test - tester.assertParsed("AND \"tcp/ip firstsecond dotnet\" firstsecond 'first second'","tcp/ipspecial-token-fs.net special-token-fs firstsecond", + tester.assertParsed("AND tcp/ip firstsecond dotnet firstsecond (SAND first second)","tcp/ipspecial-token-fs.net special-token-fs firstsecond", "", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, TestLinguistics.INSTANCE); } @@ -2014,7 +2003,7 @@ public class ParseTestCase { @Test public void testVersionNumbers() { - tester.assertParsed("\"1 0 9\"", "1.0.9", Query.Type.ALL); + tester.assertParsed("AND 1 0 9", "1.0.9", Query.Type.ALL); } @Test @@ -2321,7 +2310,7 @@ public class ParseTestCase { @Test public void testOdd1Web() { - tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.WEB); + tester.assertParsed("AND window print error", "+window.print() +error",Query.Type.WEB); } @Test @@ -2351,13 +2340,13 @@ public class ParseTestCase { @Test public void testDefaultWebIndices() { - tester.assertParsed("\"notanindex b\"","notanindex:b",Query.Type.WEB); - tester.assertParsed("site:\"b $\"","site:b",Query.Type.WEB); - tester.assertParsed("hostname:b","hostname:b",Query.Type.WEB); - tester.assertParsed("link:b","link:b",Query.Type.WEB); - tester.assertParsed("url:b","url:b",Query.Type.WEB); - tester.assertParsed("inurl:b","inurl:b",Query.Type.WEB); - tester.assertParsed("intitle:b","intitle:b",Query.Type.WEB); + tester.assertParsed("AND notanindex b","notanindex:b", Query.Type.WEB); + tester.assertParsed("site:\"b $\"","site:b", Query.Type.WEB); + tester.assertParsed("hostname:b","hostname:b", Query.Type.WEB); + tester.assertParsed("link:b","link:b", Query.Type.WEB); + tester.assertParsed("url:b","url:b", Query.Type.WEB); + tester.assertParsed("inurl:b","inurl:b", Query.Type.WEB); + tester.assertParsed("intitle:b","intitle:b", Query.Type.WEB); } @Test @@ -2527,7 +2516,7 @@ public class ParseTestCase { @Test public void testSiteAndSegmentPhrasesFollowedByText() { - tester.assertParsed("AND host.all:\"www abc com x y-z $\" 'a b'", + tester.assertParsed("AND host.all:\"www abc com x y-z $\" (SAND a b)", "host.all:www.abc.com/x'y-z a'b", "", Query.Type.ALL, Language.ENGLISH); } @@ -2544,7 +2533,7 @@ public class ParseTestCase { @Test public void testNonAsciiNumber() { - tester.assertParsed("title:\"199 119 201 149\"", "title:１９９．１１９．２０１．１４９", Query.Type.ALL); + tester.assertParsed("AND title:199 title:119 title:201 title:149", "title:１９９．１１９．２０１．１４９", Query.Type.ALL); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java index 91cf5015cba..0ca4b8aa615 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java @@ -45,7 +45,7 @@ public class CJKSearcherTestCase { @Test public void testCjkQueryWithOverlappingTokens() { // The test language segmenter will segment "bcd" into the overlapping tokens "bc" "cd" - assertTransformed("bcd", "'bc cd'", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, + assertTransformed("bcd", "SAND bc cd", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, TestLinguistics.INSTANCE); // While "efg" will be segmented into one of the standard options, "e" "fg" diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java index 12e756a07ee..023cd3c2849 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java @@ -71,7 +71,7 @@ public class LiteralBoostSearcherTestCase { @Test public void testQueryWithoutBoost() { - assertEquals("RANK (AND \"nonexistant a\" \"nonexistant b\") default_literal:nonexistant default_literal:a default_literal:nonexistant default_literal:b", + assertEquals("RANK (AND nonexistant a nonexistant b) default_literal:nonexistant default_literal:a default_literal:nonexistant default_literal:b", transformQuery("?query=nonexistant:a nonexistant:b&source=cluster1&restrict=type1")); } diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java index 4875121a501..12619bf0a5e 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java @@ -40,34 +40,8 @@ import static org.junit.Assert.assertTrue; * * @author Steinar Knutsen */ -@SuppressWarnings("deprecation") public class FieldCollapsingSearcherTestCase { - private FastHit createHit(String uri,int relevancy,int mid) { - FastHit hit = new FastHit(uri,relevancy); - hit.setField("amid", String.valueOf(mid)); - return hit; - } - - private void assertHit(String uri,int relevancy,int mid,Hit hit) { - assertEquals(uri,hit.getId().toString()); - assertEquals(relevancy, ((int) hit.getRelevance().getScore())); - assertEquals(mid,Integer.parseInt((String) hit.getField("amid"))); - } - - private static class ZeroHitsControl extends com.yahoo.search.Searcher { - public int queryCount = 0; - public com.yahoo.search.Result search(com.yahoo.search.Query query, - com.yahoo.search.searchchain.Execution execution) { - ++queryCount; - if (query.getHits() == 0) { - return new Result(query); - } else { - return new Result(query, ErrorMessage.createIllegalQuery("Did not request zero hits.")); - } - } - } - @Test public void testFieldCollapsingWithoutHits() { // Set up @@ -116,14 +90,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -133,10 +107,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); } @Test @@ -152,14 +126,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -169,10 +143,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10,0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3, r.hits().get(3)); } @Test @@ -185,14 +159,14 @@ public class FieldCollapsingSearcherTestCase { Query q = new Query("?query=test_collapse"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -220,16 +194,16 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,0)); - r.hits().add(createHit("http://acme.org/d.html", 8,0)); - r.hits().add(createHit("http://acme.org/e.html", 8,0)); - r.hits().add(createHit("http://acme.org/f.html", 7,0)); - r.hits().add(createHit("http://acme.org/g.html", 7,0)); - r.hits().add(createHit("http://acme.org/h.html", 6,0)); - r.hits().add(createHit("http://acme.org/i.html", 5,1)); - r.hits().add(createHit("http://acme.org/j.html", 4,2)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 0)); + r.hits().add(createHit("http://acme.org/d.html", 8, 0)); + r.hits().add(createHit("http://acme.org/e.html", 8, 0)); + r.hits().add(createHit("http://acme.org/f.html", 7, 0)); + r.hits().add(createHit("http://acme.org/g.html", 7, 0)); + r.hits().add(createHit("http://acme.org/h.html", 6, 0)); + r.hits().add(createHit("http://acme.org/i.html", 5, 1)); + r.hits().add(createHit("http://acme.org/j.html", 4, 2)); r.setTotalHitCount(10); docsource.addResult(q, r); @@ -239,15 +213,15 @@ public class FieldCollapsingSearcherTestCase { assertEquals(2, r.getHitCount()); assertEquals(2, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/i.html", 5,1,r.hits().get(1)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/i.html", 5, 1, r.hits().get(1)); // Next results docsource.resetQueryCount(); r = doSearch(collapse, q, 2, 2, chained); assertEquals(1, r.getHitCount()); assertEquals(2, docsource.getQueryCount()); - assertHit("http://acme.org/j.html",4,2,r.hits().get(0)); + assertHit("http://acme.org/j.html",4, 2, r.hits().get(0)); } /** @@ -265,16 +239,16 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,1)); - r.hits().add(createHit("http://acme.org/b.html",10,1)); - r.hits().add(createHit("http://acme.org/c.html",10,0)); - r.hits().add(createHit("http://acme.org/d.html",10,0)); - r.hits().add(createHit("http://acme.org/e.html",10,0)); - r.hits().add(createHit("http://acme.org/f.html",10,0)); - r.hits().add(createHit("http://acme.org/g.html",10,0)); - r.hits().add(createHit("http://acme.org/h.html",10,0)); - r.hits().add(createHit("http://acme.org/i.html",10,0)); - r.hits().add(createHit("http://acme.org/j.html",10,1)); + r.hits().add(createHit("http://acme.org/a.html", 10, 1)); + r.hits().add(createHit("http://acme.org/b.html", 10, 1)); + r.hits().add(createHit("http://acme.org/c.html", 10, 0)); + r.hits().add(createHit("http://acme.org/d.html", 10, 0)); + r.hits().add(createHit("http://acme.org/e.html", 10, 0)); + r.hits().add(createHit("http://acme.org/f.html", 10, 0)); + r.hits().add(createHit("http://acme.org/g.html", 10, 0)); + r.hits().add(createHit("http://acme.org/h.html", 10, 0)); + r.hits().add(createHit("http://acme.org/i.html", 10, 0)); + r.hits().add(createHit("http://acme.org/j.html", 10, 1)); r.setTotalHitCount(10); docsource.addResult(q, r); @@ -287,17 +261,6 @@ public class FieldCollapsingSearcherTestCase { assertHit("http://acme.org/c.html",10,0,r.hits().get(1)); } - public static class QueryMessupSearcher extends Searcher { - public Result search(com.yahoo.search.Query query, Execution execution) { - AndItem a = new AndItem(); - a.addItem(query.getModel().getQueryTree().getRoot()); - a.addItem(new WordItem("b")); - query.getModel().getQueryTree().setRoot(a); - - return execution.search(query); - } - } - @Test public void testQueryTransformAndCollapsing() { // Set up @@ -309,9 +272,9 @@ public class FieldCollapsingSearcherTestCase { chained.put(collapse, messUp); chained.put(messUp, docsource); - // Caveat: Collapse is set to false, because that's what the - // collapser asks for - Query q = new Query("?query=test_collapse+b&collapsefield=amid"); + // Caveat: Collapse is set to false, because that's what the collapser asks for + Query q = new Query("?query=%22test%20collapse%22+b&collapsefield=amid"); + System.out.println(q); // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); @@ -327,13 +290,13 @@ public class FieldCollapsingSearcherTestCase { docsource.addResult(q, r); // Test basic collapsing on mid - q = new Query("?query=test_collapse&collapsefield=amid"); + q = new Query("?query=%22test%20collapse%22&collapsefield=amid"); r = doSearch(collapse, q, 0, 2, chained); assertEquals(2, docsource.getQueryCount()); assertEquals(2, r.getHitCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/h.html", 6,1,r.hits().get(1)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/h.html", 6, 1, r.hits().get(1)); } @Test @@ -367,10 +330,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); assertTrue(r.isFilled("placeholder")); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); docsource.resetQueryCount(); // Test basic collapsing on mid @@ -381,10 +344,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(1, docsource.getQueryCount()); assertFalse(r.isFilled("placeholder")); assertTrue(r.isFilled("short")); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); } @Test @@ -400,14 +363,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -416,29 +379,28 @@ public class FieldCollapsingSearcherTestCase { Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); // Assert that the regular hits are collapsed - assertEquals(4+1, result.getHitCount()); + assertEquals(4 + 1, result.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,result.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,result.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,result.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,result.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, result.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, result.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, result.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, result.hits().get(3)); // Assert that the aggregation group hierarchy is left intact - HitGroup root= getFirstGroupIn(result.hits()); + HitGroup root = getFirstGroupIn(result.hits()); assertNotNull(root); - assertEquals("group:root:",root.getId().stringValue().substring(0,11)); // The id ends by a global counter currently - assertEquals(1,root.size()); - HitGroup groupList= (GroupList)root.get("grouplist:g1"); + assertEquals("group:root:",root.getId().stringValue().substring(0, 11)); // The id ends by a global counter currently + assertEquals(1, root.size()); + HitGroup groupList = (GroupList)root.get("grouplist:g1"); assertNotNull(groupList); - assertEquals(1,groupList.size()); - HitGroup group= (HitGroup)groupList.get("group:long:37"); + assertEquals(1, groupList.size()); + HitGroup group = (HitGroup)groupList.get("group:long:37"); assertNotNull(group); } private Group getFirstGroupIn(HitGroup hits) { - for (Hit h : hits) { + for (Hit h : hits) if (h instanceof Group) return (Group)h; - } return null; } @@ -450,9 +412,8 @@ public class FieldCollapsingSearcherTestCase { private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain, Map<Searcher, Searcher> chained) { List<Searcher> searchers = new ArrayList<>(); - for (Searcher current = topOfChain; current != null; current = chained.get(current)) { + for (Searcher current = topOfChain; current != null; current = chained.get(current)) searchers.add(current); - } return new Chain<>(searchers); } @@ -470,7 +431,7 @@ public class FieldCollapsingSearcherTestCase { @Override public Result search(Query query, Execution execution) { - Result r=execution.search(query); + Result r = execution.search(query); r.hits().add(createAggregationGroup("g1")); return r; } @@ -479,10 +440,51 @@ public class FieldCollapsingSearcherTestCase { Group root = new Group(new RootId(0), new Relevance(1)); GroupList groupList = new GroupList(label); root.add(groupList); - Group value=new Group(new LongId(37l),new Relevance(2.11)); + Group value = new Group(new LongId(37l), new Relevance(2.11)); groupList.add(value); return root; } } + private FastHit createHit(String uri,int relevancy,int mid) { + FastHit hit = new FastHit(uri,relevancy); + hit.setField("amid", String.valueOf(mid)); + return hit; + } + + private void assertHit(String uri,int relevancy,int mid,Hit hit) { + assertEquals(uri,hit.getId().toString()); + assertEquals(relevancy, ((int) hit.getRelevance().getScore())); + assertEquals(mid,Integer.parseInt((String) hit.getField("amid"))); + } + + private static class ZeroHitsControl extends com.yahoo.search.Searcher { + + public int queryCount = 0; + + @Override + public Result search(Query query, Execution execution) { + ++queryCount; + if (query.getHits() == 0) { + return new Result(query); + } else { + return new Result(query, ErrorMessage.createIllegalQuery("Did not request zero hits.")); + } + } + } + + public static class QueryMessupSearcher extends Searcher { + + @Override + public Result search(com.yahoo.search.Query query, Execution execution) { + AndItem a = new AndItem(); + a.addItem(query.getModel().getQueryTree().getRoot()); + a.addItem(new WordItem("b")); + query.getModel().getQueryTree().setRoot(a); + + return execution.search(query); + } + + } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java index b8db5e4d90f..a4cf7d8c380 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java @@ -23,7 +23,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("\"first third\"", q); + assertSemantics("AND first third", q); } @Test @@ -32,7 +32,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("\"bc first third fg\"", q); + assertSemantics("AND bc first third fg", q); } @Test @@ -41,7 +41,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("+bc -\"first third\"", q); + assertSemantics("+bc -(AND first third)", q); } @Test @@ -50,7 +50,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("\"9 2 7 0 bc third 2 3 8 9\"", q); + assertSemantics("AND 9 2 7 0 bc third 2 3 8 9", q); } private static Item parseQuery(String query) { diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java index 82a5a0c7a24..e2ac44316e7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java @@ -73,7 +73,7 @@ public class IndexFactsTestCase { Query q = newQuery("?query=a:b", indexFacts); assertEquals("a:b", q.getModel().getQueryTree().getRoot().toString()); q = newQuery("?query=notarealindex:b", indexFacts); - assertEquals("\"notarealindex b\"", q.getModel().getQueryTree().getRoot().toString()); + assertEquals("AND notarealindex b", q.getModel().getQueryTree().getRoot().toString()); } @Test @@ -302,8 +302,8 @@ public class IndexFactsTestCase { IndexFacts.Session session2 = indexFacts.newSession(query2.getModel().getSources(), query2.getModel().getRestrict()); assertTrue(session1.getIndex("url").isUriIndex()); assertTrue(session2.getIndex("url").isUriIndex()); - assertEquals("url:\"https foo bar\"", query1.getModel().getQueryTree().toString()); - assertEquals("url:\"https foo bar\"", query2.getModel().getQueryTree().toString()); + assertEquals("AND url:https url:foo url:bar", query1.getModel().getQueryTree().toString()); + assertEquals("AND url:https url:foo url:bar", query2.getModel().getQueryTree().toString()); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index 0cbf3a6f92c..c6233ffa50b 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -139,12 +139,24 @@ public class ValidateNearestNeighborTestCase { assertEquals(ErrorMessage.createIllegalQuery(message), r.hits().getError()); } + static String desc(String field, String qt, int th, String errmsg) { + StringBuilder r = new StringBuilder(); + r.append("NEAREST_NEIGHBOR {"); + r.append("field=").append(field); + r.append(",queryTensorName=").append(qt); + r.append(",hnsw.exploreAdditionalHits=0"); + r.append(",approximate=true"); + r.append(",targetNumHits=").append(th); + r.append("} ").append(errmsg); + return r.toString(); + } + @Test public void testMissingTargetNumHits() { String q = "select * from sources * where nearestNeighbor(dvector,qvector);"; Tensor t = makeTensor(tt_dense_dvector_3); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=0} has invalid targetNumHits", r); + assertErrMsg(desc("dvector", "qvector", 0, "has invalid targetNumHits"), r); } @Test @@ -152,16 +164,16 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("dvector", "foo"); Tensor t = makeTensor(tt_dense_dvector_3); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=foo,targetNumHits=1} query tensor not found", r); + assertErrMsg(desc("dvector", "foo", 1, "query tensor not found"), r); } @Test public void testQueryTensorWrongType() { String q = makeQuery("dvector", "qvector"); Result r = doSearch(searcher, q, "tensor string"); - assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=1} query tensor should be a tensor, was: class java.lang.String", r); + assertErrMsg(desc("dvector", "qvector", 1, "query tensor should be a tensor, was: class java.lang.String"), r); r = doSearch(searcher, q, null); - assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=1} query tensor should be a tensor, was: null", r); + assertErrMsg(desc("dvector", "qvector", 1, "query tensor should be a tensor, was: null"), r); } @Test @@ -169,7 +181,7 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("dvector", "qvector"); Tensor t = makeTensor(tt_dense_dvector_2, 2); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=1} field type tensor(x[3]) does not match query tensor type tensor(x[2])", r); + assertErrMsg(desc("dvector", "qvector", 1, "field type tensor(x[3]) does not match query tensor type tensor(x[2])"), r); } @Test @@ -177,7 +189,7 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("foo", "qvector"); Tensor t = makeTensor(tt_dense_dvector_3); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=foo,queryTensorName=qvector,targetNumHits=1} field is not an attribute", r); + assertErrMsg(desc("foo", "qvector", 1, "field is not an attribute"), r); } @Test @@ -185,7 +197,7 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("simple", "qvector"); Tensor t = makeTensor(tt_dense_dvector_3); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=simple,queryTensorName=qvector,targetNumHits=1} field is not a tensor", r); + assertErrMsg(desc("simple", "qvector", 1, "field is not a tensor"), r); } @Test @@ -193,7 +205,7 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("sparse", "qvector"); Tensor t = makeTensor(tt_sparse_vector_x); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=sparse,queryTensorName=qvector,targetNumHits=1} tensor type tensor(x{}) is not a dense vector", r); + assertErrMsg(desc("sparse", "qvector", 1, "tensor type tensor(x{}) is not a dense vector"), r); } @Test @@ -201,7 +213,7 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("matrix", "qvector"); Tensor t = makeMatrix(tt_dense_matrix_xy); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=matrix,queryTensorName=qvector,targetNumHits=1} tensor type tensor(x[3],y[1]) is not a dense vector", r); + assertErrMsg(desc("matrix", "qvector", 1, "tensor type tensor(x[3],y[1]) is not a dense vector"), r); } private static Result doSearch(ValidateNearestNeighborSearcher searcher, String yqlQuery, Object qTensor) { diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java index dbeced57c52..aa507d38be5 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java @@ -5,6 +5,7 @@ import static org.junit.Assert.*; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import org.junit.After; import org.junit.Before; @@ -23,50 +24,50 @@ import com.yahoo.text.Utf8; /** * Functional test for InputCheckingSearcher. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class InputCheckingSearcherTestCase { Execution execution; @Before - public void setUp() throws Exception { + public void setUp() { execution = new Execution(new Chain<Searcher>(new InputCheckingSearcher(MetricReceiver.nullImplementation)), - Execution.Context.createContextStub(new IndexFacts())); + Execution.Context.createContextStub(new IndexFacts())); } @After - public void tearDown() throws Exception { + public void tearDown() { execution = null; } @Test - public final void testCommonCase() { + public void testCommonCase() { Result r = execution.search(new Query("/search/?query=three+blind+mice")); assertNull(r.hits().getErrorHit()); } @Test - public final void candidateButAsciiOnly() { + public void candidateButAsciiOnly() { Result r = execution.search(new Query("/search/?query=a+a+a+a+a+a")); assertNull(r.hits().getErrorHit()); } @Test - public final void candidateButValid() throws UnsupportedEncodingException { + public void candidateButValid() throws UnsupportedEncodingException { Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("å å å å å å", "UTF-8"))); assertNull(r.hits().getErrorHit()); } @Test - public final void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException { + public void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException { Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("œ œ œ œ œ œ", "UTF-8"))); assertNull(r.hits().getErrorHit()); } @Test - public final void testDoubleEncoded() throws UnsupportedEncodingException { + public void testDoubleEncoded() throws UnsupportedEncodingException { String rawQuery = "å å å å å å"; byte[] encodedOnce = Utf8.toBytes(rawQuery); char[] secondEncodingBuffer = new char[encodedOnce.length]; @@ -74,33 +75,42 @@ public class InputCheckingSearcherTestCase { secondEncodingBuffer[i] = (char) (encodedOnce[i] & 0xFF); } String query = new String(secondEncodingBuffer); - Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, "UTF-8"))); + Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, StandardCharsets.UTF_8))); assertEquals(1, r.hits().getErrorHit().errors().size()); } @Test - public final void testRepeatedConsecutiveTermsInPhrase() { - Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); + public void testRepeatedConsecutiveTermsInPhrase() { + Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); + r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22")); assertNotNull(r.hits().getErrorHit()); + assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", + r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); assertNull(r.hits().getErrorHit()); } + @Test - public final void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() { - Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); + public void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() { + Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); + r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22")); assertNotNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); + assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", + r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); + r = execution.search(new Query("/search/?query=%22a.b.0.0.0.1.0.0.0.c%22")); assertNull(r.hits().getErrorHit()); } + @Test - public final void testThatMaxRepeatedTermsInPhraseIs10() { - Result r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a")); + public void testThatMaxRepeatedTermsInPhraseIs10() { + Result r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a%22")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a")); + r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a%22")); assertNotNull(r.hits().getErrorHit()); + assertEquals("Phrase contains more than 10 occurrences of term 'a' in phrase : \"0 a 1 a 2 a 3 a 4 a 5 a 6 a 7 a 8 a 9 a 10 a\"", + r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); } + } diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 34c3da395b7..c831ee29631 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -16,6 +16,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.IndexedItem; @@ -941,12 +942,12 @@ public class QueryTestCase { @Test public void testImplicitPhraseIsDefault() { Query query = new Query(httpEncode("?query=it's fine")); - assertEquals("AND 'it s' fine", query.getModel().getQueryTree().toString()); + assertEquals("AND (SAND it s) fine", query.getModel().getQueryTree().toString()); } @Test public void testImplicitPhrase() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -961,7 +962,7 @@ public class QueryTestCase { @Test public void testImplicitAnd() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:a-b myfield:c")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -972,6 +973,56 @@ public class QueryTestCase { query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); assertEquals("AND (SAND myfield:it myfield:s) myfield:a myfield:b myfield:c", query.getModel().getQueryTree().toString()); + // 'it' and 's' should have connectivity 1 + AndItem root = (AndItem)query.getModel().getQueryTree().getRoot(); + AndSegmentItem sand = (AndSegmentItem)root.getItem(0); + WordItem it = (WordItem)sand.getItem(0); + assertEquals("it", it.getWord()); + WordItem s = (WordItem)sand.getItem(1); + assertEquals("s", s.getWord()); + assertEquals(s, it.getConnectedItem()); + assertEquals(1.0, it.getConnectivity(), 0.00000001); + } + + @Test + public void testImplicitAndConnectivity() { + SearchDefinition test = new SearchDefinition("test"); + Index myField = new Index("myfield"); + myField.addCommand("phrase-segmenting false"); + test.addIndex(myField); + IndexModel indexModel = new IndexModel(test); + + { + Query query = new Query(httpEncode("?query=myfield:b.c.d")); + query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); + assertEquals("AND myfield:b myfield:c myfield:d", query.getModel().getQueryTree().toString()); + AndItem root = (AndItem) query.getModel().getQueryTree().getRoot(); + WordItem b = (WordItem) root.getItem(0); + WordItem c = (WordItem) root.getItem(1); + WordItem d = (WordItem) root.getItem(2); + assertEquals(c, b.getConnectedItem()); + assertEquals(1.0, b.getConnectivity(), 0.00000001); + assertEquals(d, c.getConnectedItem()); + assertEquals(1.0, c.getConnectivity(), 0.00000001); + } + + { + Query query = new Query(httpEncode("?query=myfield:a myfield:b.c.d myfield:e")); + query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); + assertEquals("AND myfield:a myfield:b myfield:c myfield:d myfield:e", query.getModel().getQueryTree().toString()); + AndItem root = (AndItem) query.getModel().getQueryTree().getRoot(); + WordItem a = (WordItem) root.getItem(0); + WordItem b = (WordItem) root.getItem(1); + WordItem c = (WordItem) root.getItem(2); + WordItem d = (WordItem) root.getItem(3); + WordItem e = (WordItem) root.getItem(4); + assertNull(a.getConnectedItem()); + assertEquals(c, b.getConnectedItem()); + assertEquals(1.0, b.getConnectivity(), 0.00000001); + assertEquals(d, c.getConnectedItem()); + assertEquals(1.0, c.getConnectivity(), 0.00000001); + assertNull(d.getConnectedItem()); + } } @Test diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index 1106d8c3999..d770b08d31a 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -128,6 +128,9 @@ public class VespaSerializerTestCase { public void testNearestNeighbor() { parseAndConfirm("[{\"label\": \"foo\", \"targetNumHits\": 1000}]nearestNeighbor(semantic_embedding, my_property)"); parseAndConfirm("[{\"targetNumHits\": 42}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{\"targetNumHits\": 1, \"hnsw.exploreAdditionalHits\": 76}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{\"targetNumHits\": 2, \"approximate\": false}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{\"targetNumHits\": 3, \"hnsw.exploreAdditionalHits\": 67, \"approximate\": false}]nearestNeighbor(semantic_embedding, my_property)"); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 5eb1f3e3de1..e43dbd4e266 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -550,9 +550,11 @@ public class YqlParserTestCase { @Test public void testNearestNeighbor() { assertParse("select foo from bar where nearestNeighbor(semantic_embedding, my_vector);", - "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,targetNumHits=0}"); + "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=0,approximate=true,targetNumHits=0}"); assertParse("select foo from bar where [{\"targetNumHits\": 37}]nearestNeighbor(semantic_embedding, my_vector);", - "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,targetNumHits=37}"); + "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=0,approximate=true,targetNumHits=37}"); + assertParse("select foo from bar where [{\"approximate\": false, \"hnsw.exploreAdditionalHits\": 8, \"targetNumHits\": 3}]nearestNeighbor(semantic_embedding, my_vector);", + "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=8,approximate=false,targetNumHits=3}"); } @Test diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java new file mode 100644 index 00000000000..43d702d108f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -0,0 +1,63 @@ +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; + +import java.util.Optional; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +/** + * Data pertaining to a deployment to be done on a config server. + * + * @author jonmv + */ +public class DeploymentData { + + private final ApplicationId instance; + private final ZoneId zone; + private final byte[] applicationPackage; + private final Version platform; + private final Set<ContainerEndpoint> containerEndpoints; + private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; + + public DeploymentData(ApplicationId instance, ZoneId zone, byte[] applicationPackage, Version platform, + Set<ContainerEndpoint> containerEndpoints, + Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { + this.instance = requireNonNull(instance); + this.zone = requireNonNull(zone); + this.applicationPackage = requireNonNull(applicationPackage); + this.platform = requireNonNull(platform); + this.containerEndpoints = requireNonNull(containerEndpoints); + this.endpointCertificateMetadata = requireNonNull(endpointCertificateMetadata); + } + + public ApplicationId instance() { + return instance; + } + + public ZoneId zone() { + return zone; + } + + public byte[] applicationPackage() { + return applicationPackage; + } + + public Version platform() { + return platform; + } + + public Set<ContainerEndpoint> containerEndpoints() { + return containerEndpoints; + } + + public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() { + return endpointCertificateMetadata; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index 9e5b01a91d7..978a00fbccf 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -31,7 +31,6 @@ import java.time.Clock; * * @author mpolden */ -// TODO(mpolden): Access all services through this public interface ServiceRegistry { ConfigServer configServer(); @@ -42,6 +41,7 @@ public interface ServiceRegistry { GlobalRoutingService globalRoutingService(); + // TODO(mpolden): Remove RoutingGenerator routingGenerator(); Mailer mailer(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java index 171c5caa756..53366c9b922 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java @@ -16,6 +16,7 @@ public class EndpointCertificateMetadata { private final String keyName; private final String certName; private final int version; + // TODO: make these fields required once all certs have them stored private final Optional<String> request_id; private final Optional<List<String>> requestedDnsSans; private final Optional<String> issuer; @@ -24,10 +25,6 @@ public class EndpointCertificateMetadata { this(keyName, certName, version, Optional.empty(), Optional.empty(), Optional.empty()); } - public EndpointCertificateMetadata(String keyName, String certName, int version, String request_id, List<String> requestedDnsSans) { - this(keyName, certName, version, Optional.of(request_id), Optional.of(requestedDnsSans), Optional.empty()); - } - public EndpointCertificateMetadata(String keyName, String certName, int version, Optional<String> request_id, Optional<List<String>> requestedDnsSans, Optional<String> issuer) { this.keyName = keyName; this.certName = certName; @@ -61,6 +58,17 @@ public class EndpointCertificateMetadata { return issuer; } + public EndpointCertificateMetadata withVersion(int version) { + return new EndpointCertificateMetadata( + this.keyName, + this.certName, + version, + this.request_id, + this.requestedDnsSans, + this.issuer + ); + } + @Override public String toString() { return "EndpointCertificateMetadata{" + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 722f9c4e33b..7ea7a350c0c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; @@ -32,9 +33,7 @@ public interface ConfigServer { PrepareResponse prepareResponse(); } - PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, - Set<ContainerEndpoint> containerEndpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, - byte[] content); + PreparedApplication deploy(DeploymentData deployment); void restart(DeploymentId deployment, Optional<Hostname> hostname); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java index 59ad23aaa23..c0ccd0722f3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java @@ -6,6 +6,7 @@ import java.util.Objects; /** * @author smorgrav */ +// TODO(mpolden): Remove together with RoutingGenerator and its implementations public class RoutingEndpoint { private final boolean isGlobal; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java index f5c82018ac6..8150a99979e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java @@ -12,6 +12,7 @@ import java.util.Map; * @author bratseth * @author smorgrav */ +// TODO(mpolden): Remove public interface RoutingGenerator { /** diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java index cd0e6552596..e768a090188 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.routing; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; import java.util.List; @@ -18,31 +16,13 @@ import java.util.stream.Collectors; * @author bratseth * @author jonmv */ +// TODO(mpolden): Remove public class RoutingGeneratorMock implements RoutingGenerator { - public static final List<RoutingEndpoint> TEST_ENDPOINTS = - List.of(new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream3"), - new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"), - new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream2"), - new RoutingEndpoint("http://global-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1"), - new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1")); - private final Map<DeploymentId, List<RoutingEndpoint>> routingTable = new ConcurrentHashMap<>(); - private final List<RoutingEndpoint> defaultEndpoints; - private final ZoneRegistry zoneRegistry; - - public RoutingGeneratorMock(List<RoutingEndpoint> endpoints, ZoneRegistry zoneRegistry) { - this.defaultEndpoints = List.copyOf(endpoints); - this.zoneRegistry = zoneRegistry; - } @Override public List<RoutingEndpoint> endpoints(DeploymentId deployment) { - if (!zoneRegistry.zones().routingMethod(RoutingMethod.shared).ids().contains(deployment.zoneId())) { - throw new IllegalArgumentException(deployment.zoneId() + " does not support routing method " + - RoutingMethod.shared); - } - if (routingTable.isEmpty()) return defaultEndpoints; return routingTable.getOrDefault(deployment, List.of()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 854f72b27fb..bc31a0a02fa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; @@ -36,6 +37,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareRes import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -285,9 +287,65 @@ public class ApplicationController { return deploy(applicationId, zone, applicationPackageFromDeployer, Optional.empty(), options); } - /** Deploys an application. If the application does not exist it is created. */ - // TODO: Get rid of the options arg - // TODO jonmv: Split this, and choose between deployDirectly and deploy in handler, excluding internally built from the latter. + /** Deploys an application package for an existing application instance. */ + public ActivateResult deploy2(JobId job, boolean deploySourceVersions) { // TODO jonmv: make it number one! + if (job.application().instance().isTester()) + throw new IllegalArgumentException("'" + job.application() + "' is a tester application!"); + + TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application()); + ZoneId zone = job.type().zone(controller.system()); + + try (Lock deploymentLock = lockForDeployment(job.application(), zone)) { + Set<ContainerEndpoint> endpoints; + Optional<EndpointCertificateMetadata> endpointCertificateMetadata; + + Run run = controller.jobController().last(job) + .orElseThrow(() -> new IllegalStateException("No known run of '" + job + "'")); + + if (run.hasEnded()) + throw new IllegalStateException("No deployment expected for " + job + " now, as no job is running"); + + Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); + ApplicationVersion revision = run.versions().sourceApplication().filter(__ -> deploySourceVersions).orElse(run.versions().targetApplication()); + ApplicationPackage applicationPackage = getApplicationPackage(job.application(), zone, revision); + + try (Lock lock = lock(applicationId)) { + LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); + Instance instance = application.get().require(job.application().instance()); + + Deployment deployment = instance.deployments().get(zone); + if ( zone.environment().isProduction() && deployment != null + && ( platform.compareTo(deployment.version()) < 0 && ! instance.change().isPinned() + || revision.compareTo(deployment.applicationVersion()) < 0 && ! (revision.isUnknown() && controller.system().isCd()))) + throw new IllegalArgumentException(String.format("Rejecting deployment of application %s to %s, as the requested versions (platform: %s, application: %s)" + + " are older than the currently deployed (platform: %s, application: %s).", + job.application(), zone, platform, revision, deployment.version(), deployment.applicationVersion())); + + if ( ! applicationPackage.trustedCertificates().isEmpty() + && run.testerCertificate().isPresent()) + applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get()); + + endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, zone); + + endpoints = controller.routing().registerEndpointsInDns(applicationPackage.deploymentSpec(), instance, zone); + } // Release application lock while doing the deployment, which is a lengthy task. + + // Carry out deployment without holding the application lock. + ActivateResult result = deploy(job.application(), applicationPackage, zone, platform, endpoints, endpointCertificateMetadata); + + lockApplicationOrThrow(applicationId, application -> + store(application.with(job.application().instance(), + instance -> instance.withNewDeployment(zone, revision, platform, + clock.instant(), warningsFrom(result))))); + return result; + } + } + + private ApplicationPackage getApplicationPackage(ApplicationId application, ZoneId zone, ApplicationVersion revision) { + return new ApplicationPackage(revision.isUnknown() ? applicationStore.getDev(application, zone) + : applicationStore.get(application.tenant(), application.application(), revision)); + } + public ActivateResult deploy(ApplicationId instanceId, ZoneId zone, Optional<ApplicationPackage> applicationPackageFromDeployer, Optional<ApplicationVersion> applicationVersionFromDeployer, @@ -342,13 +400,12 @@ public class ApplicationController { endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(application.get().require(instance), zone); - endpoints = controller.routingController().registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(instanceId.instance()), zone); + endpoints = controller.routing().registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(instanceId.instance()), zone); } // Release application lock while doing the deployment, which is a lengthy task. // Carry out deployment without holding the application lock. - options = withVersion(platformVersion, options); - ActivateResult result = deploy(instanceId, applicationPackage, zone, options, endpoints, - endpointCertificateMetadata); + ActivateResult result = deploy(instanceId, applicationPackage, zone, platformVersion, + endpoints, endpointCertificateMetadata); lockApplicationOrThrow(applicationId, application -> store(application.with(instanceId.instance(), @@ -398,7 +455,7 @@ public class ApplicationController { for (InstanceName instance : declaredInstances) if (applicationPackage.deploymentSpec().requireInstance(instance).concerns(Environment.prod)) - application = controller.routingController().assignRotations(application, instance); + application = controller.routing().assignRotations(application, instance); store(application); return application; @@ -420,31 +477,30 @@ public class ApplicationController { ApplicationPackage applicationPackage = new ApplicationPackage( artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); - DeployOptions options = withVersion(version, DeployOptions.none()); - return deploy(application.id(), applicationPackage, zone, options, Set.of(), /* No application cert */ Optional.empty()); + return deploy(application.id(), applicationPackage, zone, version, Set.of(), /* No application cert */ Optional.empty()); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } } /** Deploys the given tester application to the given zone. */ - public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) { - return deploy(tester.id(), applicationPackage, zone, options, Set.of(), /* No application cert for tester*/ Optional.empty()); + public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, Version platform) { + return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), /* No application cert for tester*/ Optional.empty()); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, - ZoneId zone, DeployOptions deployOptions, Set<ContainerEndpoint> endpoints, + ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { - DeploymentId deploymentId = new DeploymentId(application, zone); try { ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(deploymentId, deployOptions, endpoints, endpointCertificateMetadata, applicationPackage.zippedContent()); + configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, + endpoints, endpointCertificateMetadata)); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { // Even if prepare fails, a load balancer may have been provisioned. Always refresh routing policies so that // any DNS updates can be propagated as early as possible. - controller.routingController().policies().refresh(application, applicationPackage.deploymentSpec(), zone); + controller.routing().policies().refresh(application, applicationPackage.deploymentSpec(), zone); } } @@ -516,7 +572,7 @@ public class ApplicationController { throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments: " + deployments); for (Instance instance : application.get().instances().values()) { - controller.routingController().removeEndpointsInDns(instance); + controller.routing().removeEndpointsInDns(instance); application = application.without(instance.name()); } @@ -551,7 +607,7 @@ public class ApplicationController { && application.get().deploymentSpec().instanceNames().contains(instanceId.instance())) throw new IllegalArgumentException("Can not delete '" + instanceId + "', which is specified in 'deployment.xml'; remove it there first"); - controller.routingController().removeEndpointsInDns(application.get().require(instanceId.instance())); + controller.routing().removeEndpointsInDns(application.get().require(instanceId.instance())); curator.writeApplication(application.without(instanceId.instance()).get()); controller.jobController().collectGarbage(); log.info("Deleted " + instanceId); @@ -633,7 +689,7 @@ public class ApplicationController { } catch (NotFoundException ignored) { // ok; already gone } finally { - controller.routingController().policies().refresh(application.get().id().instance(instanceName), application.get().deploymentSpec(), zone); + controller.routing().policies().refresh(application.get().id().instance(instanceName), application.get().deploymentSpec(), zone); } return application.with(instanceName, instance -> instance.withoutDeploymentIn(zone)); } @@ -776,7 +832,7 @@ public class ApplicationController { } /** Returns the latest known version within the given major. */ - private Optional<Version> lastCompatibleVersion(int targetMajorVersion) { + public Optional<Version> lastCompatibleVersion(int targetMajorVersion) { return controller.versionStatus().versions().stream() .map(VespaVersion::versionNumber) .filter(version -> version.getMajor() == targetMajorVersion) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 3d492bc00d4..5189721df5d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -124,7 +124,7 @@ public class Controller extends AbstractComponent implements ApplicationIdSource public JobController jobController() { return jobController; } /** Returns the instance controlling routing */ - public RoutingController routingController() { + public RoutingController routing() { return routingController; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index a510275b98e..984f6855cb3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -16,8 +15,6 @@ import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; import java.time.Instant; @@ -166,20 +163,6 @@ public class Instance { return rotations; } - /** Returns the default global endpoints for this in given system - for a given endpoint ID */ - public EndpointList endpointsIn(SystemName system, EndpointId endpointId) { - if (rotations.isEmpty()) return EndpointList.EMPTY; - return EndpointList.create(id, endpointId, system); - } - - /** Returns the default global endpoints for this in given system */ - public EndpointList endpointsIn(SystemName system) { - if (rotations.isEmpty()) return EndpointList.EMPTY; - final var endpointStream = rotations.stream() - .flatMap(rotation -> EndpointList.create(id, rotation.endpointId(), system).asList().stream()); - return EndpointList.of(endpointStream); - } - /** Returns the status of the global rotation(s) assigned to this */ public RotationStatus rotationStatus() { return rotationStatus; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index cf272d94dcd..4637de9a32a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -13,12 +14,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerE import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; +import com.yahoo.vespa.hosted.controller.routing.RoutingId; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; @@ -29,13 +31,12 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.TreeMap; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -48,19 +49,15 @@ import java.util.stream.Collectors; */ public class RoutingController { - private static final Logger log = Logger.getLogger(RoutingController.class.getName()); - private final Controller controller; private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; - private final RoutingGenerator routingGenerator; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); this.routingPolicies = new RoutingPolicies(controller); this.rotationRepository = new RotationRepository(rotationsConfig, controller.applications(), controller.curator()); - this.routingGenerator = controller.serviceRegistry().routingGenerator(); } public RoutingPolicies policies() { @@ -71,53 +68,65 @@ public class RoutingController { return rotationRepository; } - /** Returns all legacy endpoint URLs for given deployment, including global, in the shared routing layer */ - public List<URI> legacyEndpointsOf(DeploymentId deployment) { - return routingEndpointsOf(deployment).stream() - .map(RoutingEndpoint::endpoint) - .map(URI::create) - .collect(Collectors.toUnmodifiableList()); + /** Returns zone-scoped endpoints for given deployment */ + public EndpointList endpointsOf(DeploymentId deployment) { + var endpoints = new LinkedHashSet<Endpoint>(); + // TODO(mpolden): Remove this once all applications have deployed once and config server passes correct cluster + // id for combined cluster type + controller.serviceRegistry().routingGenerator().clusterEndpoints(deployment) + .forEach((cluster, url) -> endpoints.add(Endpoint.of(deployment.applicationId()) + .target(cluster, deployment.zoneId()) + .routingMethod(RoutingMethod.shared) + .on(Port.fromRoutingMethod(RoutingMethod.shared)) + .in(controller.system()))); + boolean hasSharedEndpoint = !endpoints.isEmpty(); + for (var policy : routingPolicies.get(deployment).values()) { + if (!policy.status().isActive()) continue; + for (var routingMethod : controller.zoneRegistry().routingMethods(policy.id().zone())) { + if (hasSharedEndpoint && routingMethod == RoutingMethod.shared) continue; + endpoints.add(policy.endpointIn(controller.system(), routingMethod)); + } + } + return EndpointList.copyOf(endpoints); } - /** Returns legacy zone endpoints for given deployment, in the shared routing layer */ - public Map<ClusterSpec.Id, URI> legacyZoneEndpointsOf(DeploymentId deployment) { - if (!supportsRoutingMethod(RoutingMethod.shared, deployment.zoneId())) { - return Map.of(); - } - try { - return routingGenerator.clusterEndpoints(deployment); - } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to get endpoint information for " + deployment, e); - return Map.of(); - } + /** Returns global-scoped endpoints for given instance */ + public EndpointList endpointsOf(ApplicationId instance) { + return endpointsOf(controller.applications().requireInstance(instance)); } - /** - * Returns all non-global endpoint URLs for given deployment, grouped by their cluster ID. If deployment supports - * {@link RoutingMethod#exclusive} endpoints defined through routing polices are returned. - */ - public Map<ClusterSpec.Id, URI> zoneEndpointsOf(DeploymentId deployment) { - if ( ! controller.applications().getInstance(deployment.applicationId()) - .map(application -> application.deployments().containsKey(deployment.zoneId())) - .orElse(deployment.applicationId().instance().isTester())) - throw new NotExistsException("Deployment", deployment.toString()); - - // In exclusively routed zones we create endpoint URLs from routing policies - if (supportsRoutingMethod(RoutingMethod.exclusive, deployment.zoneId())) { - return routingPolicies.get(deployment).values().stream() - .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) - .collect(Collectors.toUnmodifiableMap(policy -> policy.id().cluster(), - policy -> policy.endpointIn(controller.system()) - .url())); + /** Returns global-scoped endpoints for given instance */ + // TODO(mpolden): Add a endpointsOf(Instance, DeploymentId) variant of this that only returns global endpoint of + // which deployment is a member + public EndpointList endpointsOf(Instance instance) { + var endpoints = new LinkedHashSet<Endpoint>(); + // Add global endpoints provided by rotations + for (var rotation : instance.rotations()) { + EndpointList.global(RoutingId.of(instance.id(), rotation.endpointId()), + controller.system(), systemRoutingMethods()) + .requiresRotation() + .forEach(endpoints::add); + } + // Add global endpoints provided by routing policices + for (var policy : routingPolicies.get(instance.id()).values()) { + if (!policy.status().isActive()) continue; + for (var endpointId : policy.endpoints()) { + EndpointList.global(RoutingId.of(instance.id(), endpointId), + controller.system(), systemRoutingMethods()) + .not().requiresRotation() + .forEach(endpoints::add); + } } - return legacyZoneEndpointsOf(deployment); + return EndpointList.copyOf(endpoints); } - /** Returns all non-global endpoint URLs for given deployments, grouped by their cluster ID and zone */ - public Map<ZoneId, Map<ClusterSpec.Id, URI>> zoneEndpointsOf(Collection<DeploymentId> deployments) { - var endpoints = new TreeMap<ZoneId, Map<ClusterSpec.Id, URI>>(Comparator.comparing(ZoneId::value)); + /** Returns all non-global endpoints and corresponding cluster IDs for given deployments, grouped by their zone */ + public Map<ZoneId, Map<URI, ClusterSpec.Id>> zoneEndpointsOf(Collection<DeploymentId> deployments) { + var endpoints = new TreeMap<ZoneId, Map<URI, ClusterSpec.Id>>(Comparator.comparing(ZoneId::value)); for (var deployment : deployments) { - var zoneEndpoints = zoneEndpointsOf(deployment); + var zoneEndpoints = endpointsOf(deployment).scope(Endpoint.Scope.zone).asList().stream() + .collect(Collectors.toUnmodifiableMap(Endpoint::url, + endpoint -> ClusterSpec.Id.from(endpoint.name()))); if (!zoneEndpoints.isEmpty()) { endpoints.put(deployment.zoneId(), zoneEndpoints); } @@ -127,10 +136,9 @@ public class RoutingController { /** Change status of all global endpoints for given deployment */ public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) { - var globalEndpoints = legacyGlobalEndpointsOf(deployment); - globalEndpoints.forEach(endpoint -> { + endpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> { try { - controller.serviceRegistry().configServer().setGlobalRotationStatus(deployment, endpoint.upstreamName(), status); + controller.serviceRegistry().configServer().setGlobalRotationStatus(deployment, endpoint.upstreamIdOf(deployment), status); } catch (Exception e) { throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e); } @@ -138,22 +146,16 @@ public class RoutingController { } /** Get global endpoint status for given deployment */ - public Map<RoutingEndpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) { - var routingEndpoints = new LinkedHashMap<RoutingEndpoint, EndpointStatus>(); - legacyGlobalEndpointsOf(deployment).forEach(endpoint -> { - var status = controller.serviceRegistry().configServer().getGlobalRotationStatus(deployment, endpoint.upstreamName()); + public Map<Endpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) { + var routingEndpoints = new LinkedHashMap<Endpoint, EndpointStatus>(); + endpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> { + var upstreamName = endpoint.upstreamIdOf(deployment); + var status = controller.serviceRegistry().configServer().getGlobalRotationStatus(deployment, upstreamName); routingEndpoints.put(endpoint, status); }); return Collections.unmodifiableMap(routingEndpoints); } - /** Find the global endpoints of given deployment */ - private List<RoutingEndpoint> legacyGlobalEndpointsOf(DeploymentId deployment) { - return routingEndpointsOf(deployment).stream() - .filter(RoutingEndpoint::isGlobal) - .collect(Collectors.toUnmodifiableList()); - } - /** * Assigns one or more global rotations to given application, if eligible. The given application is implicitly * stored, ensuring that the assigned rotation(s) are persisted when this returns. @@ -181,8 +183,7 @@ public class RoutingController { .isPresent(); for (var assignedRotation : instance.rotations()) { var names = new ArrayList<String>(); - var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()) - .scope(Endpoint.Scope.global); + var endpoints = endpointsOf(instance).named(assignedRotation.endpointId()).requiresRotation(); // Skip rotations which do not apply to this zone. Legacy names always point to all zones if (!registerLegacyNames && !assignedRotation.regions().contains(zone.region())) { @@ -191,13 +192,13 @@ public class RoutingController { // Omit legacy DNS names when assigning rotations using <endpoints/> syntax if (!registerLegacyNames) { - endpoints = endpoints.legacy(false); + endpoints = endpoints.not().legacy(); } // Register names in DNS var rotation = rotationRepository.getRotation(assignedRotation.rotationId()); if (rotation.isPresent()) { - endpoints.asList().forEach(endpoint -> { + endpoints.forEach(endpoint -> { controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()), RecordData.fqdn(rotation.get().name()), Priority.normal); @@ -214,32 +215,19 @@ public class RoutingController { /** Remove endpoints in DNS for all rotations assigned to given instance */ public void removeEndpointsInDns(Instance instance) { - instance.rotations().forEach(assignedRotation -> { - var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId()); - endpoints.asList().stream() - .map(Endpoint::dnsName) - .forEach(name -> { - controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), - Priority.normal); - }); - }); - } - - private List<RoutingEndpoint> routingEndpointsOf(DeploymentId deployment) { - if (!supportsRoutingMethod(RoutingMethod.shared, deployment.zoneId())) { - return List.of(); // No rotations/shared routing layer in this zone. - } - try { - return routingGenerator.endpoints(deployment); - } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to get endpoints for " + deployment, e); - return List.of(); - } + endpointsOf(instance).requiresRotation() + .forEach(endpoint -> controller.nameServiceForwarder() + .removeRecords(Record.Type.CNAME, + RecordName.from(endpoint.dnsName()), + Priority.normal)); } - /** Returns whether given routingMethod is supported by zone */ - public boolean supportsRoutingMethod(RoutingMethod routingMethod, ZoneId zone) { - return controller.zoneRegistry().zones().routingMethod(routingMethod).ids().contains(zone); + /** Returns all routing methods supported by this system */ + private List<RoutingMethod> systemRoutingMethods() { + return controller.zoneRegistry().zones().all().ids().stream() + .flatMap(zone -> controller.zoneRegistry().routingMethods(zone).stream()) + .distinct() + .collect(Collectors.toUnmodifiableList()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java deleted file mode 100644 index fb675862320..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import java.util.Objects; - -/** - * Calculate utilization relative to the target utilization, - * tco and waste for one cluster of one deployment. - * - * The target utilization is defined the following assumptions: - * 1. CPU contention starts to cause problems at 0.8 - * 2. Memory management starts to cause problems at 0.7 - * 3. Load is evenly divided between two deployments - each deployments can handle the other. - * 4. Memory and disk are agnostic to query load. - * 5. Peak utilization (daily variations) are twice the size of the average. - * - * With this in mind we get: - * - * CPU: 0.8/2/2 = 0.2 - * Mem: 0.7 - * Disk: 0.7 - * Disk busy: 0.3 - * - * @author smorgrav - */ -public class ClusterCost { - - private final double tco; - private final double waste; - private final ClusterInfo clusterInfo; - private final ClusterUtilization systemUtilization; - private final ClusterUtilization targetUtilization; - private final ClusterUtilization resultUtilization; - - /** - * @param clusterInfo value object with cluster info e.g. the TCO for the hardware used - * @param systemUtilization utilization of system resources (as ratios) - */ - public ClusterCost(ClusterInfo clusterInfo, - ClusterUtilization systemUtilization) { - - Objects.requireNonNull(clusterInfo, "Cluster info cannot be null"); - Objects.requireNonNull(systemUtilization, "Cluster utilization cannot be null"); - - this.clusterInfo = clusterInfo; - this.systemUtilization = systemUtilization; - this.targetUtilization = new ClusterUtilization(0.7,0.2, 0.7, 0.3); - this.resultUtilization = calculateResultUtilization(systemUtilization, targetUtilization); - - this.tco = clusterInfo.getHostnames().size() * clusterInfo.getFlavorCost(); - - double unusedUtilization = 1 - Math.min(1, resultUtilization.getMaxUtilization()); - this.waste = tco * unusedUtilization; - } - - /** @return The TCO in dollars for this cluster (node tco * nodes) */ - public double getTco() { - return tco; - } - - /** @return The amount of dollars spent for unused resources in this cluster */ - public double getWaste() { - return waste; - } - - public ClusterInfo getClusterInfo() { - return clusterInfo; - } - - public ClusterUtilization getSystemUtilization() { - return systemUtilization; - } - - public ClusterUtilization getTargetUtilization() { - return targetUtilization; - } - - public ClusterUtilization getResultUtilization() { - return resultUtilization; - } - - static ClusterUtilization calculateResultUtilization(ClusterUtilization system, ClusterUtilization target) { - double cpu = ratio(system.getCpu(), target.getCpu()); - double mem = ratio(system.getMemory(), target.getMemory()); - double disk = ratio(system.getDisk(), target.getDisk()); - double diskbusy = ratio(system.getDiskBusy(), target.getDiskBusy()); - - return new ClusterUtilization(mem, cpu, disk, diskbusy); - } - - private static double ratio(double a, double b) { - if (b == 0) return 1; - return a/b; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java index 40fc57acdc8..803e88beae2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java @@ -13,6 +13,7 @@ import java.util.List; * * @author smorgrav */ +// TODO(mpolden): Remove when we stop writing these fields. public class ClusterInfo { private final String flavor; private final double flavorCPU; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilization.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilization.java deleted file mode 100644 index ff92ce36d1b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilization.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -/** - * System resources as _ratios_ of available resources. - * - * Can be for actual readings or target numbers. - * - * @author smorgrav - */ -public class ClusterUtilization { - - private final double memory; - private final double cpu; - private final double disk; - private final double diskBusy; - private final double maxUtilization; - - /** - * Resource utilization as ratios. The ratio is normally between 0 and 1 where - * one is fully utilized - but can be higher as it consumes more than it are guaranteed. - * - * @param memory Memory utilization - * @param cpu CPU utilization - * @param disk Disk utilization - * @param diskBusy Disk busy - */ - public ClusterUtilization(double memory, double cpu, double disk, double diskBusy) { - this.memory = memory; - this.cpu = cpu; - this.disk = disk; - this.diskBusy = diskBusy; - - double maxUtil = Math.max(cpu, disk); - maxUtil = Math.max(maxUtil, memory); - this.maxUtilization = Math.max(maxUtil, diskBusy); - } - - /** @return The utilization ratio of the resource that is utilized the most. */ - public double getMaxUtilization() { - return maxUtilization; - } - - /** @return The utilization ratio for memory */ - public double getMemory() { - return memory; - } - - /** @return The utilization ratio for cpu */ - public double getCpu() { - return cpu; - } - - /** @return The utilization ratio for disk */ - public double getDisk() { - return disk; - } - - /** @return The disk busy ratio */ - public double getDiskBusy() { - return diskBusy; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java deleted file mode 100644 index 393c14b35d3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import java.util.HashMap; -import java.util.Map; - -/** - * Calculates cost for for an application deployment. - * - * @author smorgrav - */ -public class DeploymentCost { - - private final double utilization; - private final double waste; - private final double tco; - - private final Map<String, ClusterCost> clusters; - - public DeploymentCost(Map<String, ClusterCost> clusterCosts) { - clusters = new HashMap<>(clusterCosts); - - double tco = 0; - double util = 0; - double waste = 0; - double maxWaste = -1; - - for (ClusterCost costCluster : clusterCosts.values()) { - tco += costCluster.getTco(); - waste += costCluster.getWaste(); - - if (costCluster.getWaste() > maxWaste) { - util = costCluster.getResultUtilization().getMaxUtilization(); - maxWaste = costCluster.getWaste(); - } - } - - this.utilization = util; - this.waste = waste; - this.tco = tco; - } - - public Map<String, ClusterCost> getCluster() { - return clusters; - } - - /** Returns the total monthly cost of ownership for the deployment (sum of all clusters) */ - public double getTco() { - return tco; - } - - /** Returns the utilization of clusters that wastes most money in this deployment */ - public double getUtilization() { - return utilization; - } - - /** Returns the amount of dollars spent and not utilized */ - public double getWaste() { - return waste; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 5fa463233fd..d6b7d3d173e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -6,9 +6,13 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import java.net.URI; import java.util.Objects; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Represents an application's endpoint. The endpoint scope can either be global or a specific zone. This is visible to @@ -23,6 +27,7 @@ public class Endpoint { private static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud"; private static final String PUBLIC_CD_DNS_SUFFIX = ".public-cd.vespa.oath.cloud"; + private final String name; private final URI url; private final Scope scope; private final boolean legacy; @@ -37,6 +42,7 @@ public class Endpoint { Objects.requireNonNull(system, "system must be non-null"); Objects.requireNonNull(port, "port must be non-null"); Objects.requireNonNull(routingMethod, "routingMethod must be non-null"); + this.name = name; this.url = createUrl(name, application, zone, system, port, legacy, routingMethod); this.scope = zone == null ? Scope.global : Scope.zone; this.legacy = legacy; @@ -45,6 +51,17 @@ public class Endpoint { this.wildcard = wildcard; } + /** + * Returns the name of this endpoint (the first component of the DNS name). Depending on the endpoint type, this + * can be one of the following: + * - A wildcard (any scope) + * - A cluster name (only zone scope) + * - An endpoint ID (only global scope) + */ + public String name() { + return name; + } + /** Returns the URL used to access this */ public URI url() { return url; @@ -76,22 +93,35 @@ public class Endpoint { return tls; } + /** Returns whether this requires a rotation to be reachable */ + public boolean requiresRotation() { + return routingMethod.isShared() && scope == Scope.global; + } + /** Returns whether this is a wildcard endpoint (used only in certificates) */ public boolean wildcard() { return wildcard; } + /** Returns the upstream ID of given deployment. This *must* match what the routing layer generates */ + public String upstreamIdOf(DeploymentId deployment) { + if (scope != Scope.global) throw new IllegalArgumentException("Scope " + scope + " does not have upstream name"); + if (!routingMethod.isShared()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not have upstream name"); + return upstreamIdOf(name, deployment.applicationId(), deployment.zoneId()); + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Endpoint endpoint = (Endpoint) o; - return url.equals(endpoint.url); + return url.equals(endpoint.url) && + routingMethod == endpoint.routingMethod; } @Override public int hashCode() { - return Objects.hash(url); + return Objects.hash(url, routingMethod); } @Override @@ -169,6 +199,30 @@ public class Endpoint { } } + private static String upstreamIdOf(String name, ApplicationId application, ZoneId zone) { + return Stream.of(namePart(name, ""), + instancePart(application, ""), + application.application().value(), + application.tenant().value(), + zone.region().value(), + zone.environment().value()) + .filter(Predicate.not(String::isEmpty)) + .map(Endpoint::sanitizeUpstream) + .collect(Collectors.joining(".")); + } + + /** Remove any invalid characters from a upstream part */ + private static String sanitizeUpstream(String part) { + return truncate(part.toLowerCase() + .replace('_', '-') + .replaceAll("[^a-z0-9-]*", "")); + } + + /** Truncate the given part at the front so its length does not exceed 63 characters */ + private static String truncate(String part) { + return part.substring(Math.max(0, part.length() - 63)); + } + /** An endpoint's scope */ public enum Scope { @@ -203,6 +257,12 @@ public class Endpoint { return new Port(443, true); } + /** Returns default port for the given routing method */ + public static Port fromRoutingMethod(RoutingMethod method) { + if (method.isDirect()) return Port.tls(); + return Port.tls(4443); + } + /** Create a HTTPS port */ public static Port tls(int port) { return new Port(port, true); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index e12bb5cda7f..955f8dc755c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -2,16 +2,15 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.collections.AbstractFilteringList; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; +import com.yahoo.vespa.hosted.controller.routing.RoutingId; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** @@ -21,8 +20,6 @@ import java.util.stream.Stream; */ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> { - public static final EndpointList EMPTY = new EndpointList(List.of()); - private EndpointList(Collection<? extends Endpoint> endpoints, boolean negate) { super(endpoints, negate, EndpointList::new); if (endpoints.stream().distinct().count() != endpoints.size()) { @@ -34,14 +31,24 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> this(endpoints, false); } - /** Returns the main endpoint, if any */ - public Optional<Endpoint> main() { - return asList().stream().filter(Predicate.not(Endpoint::legacy)).findFirst(); + /** Returns the primary (non-legacy) endpoint, if any */ + public Optional<Endpoint> primary() { + return not().matching(Endpoint::legacy).asList().stream().findFirst(); + } + + /** Returns the subset of endpoints named according to given ID */ + public EndpointList named(EndpointId id) { + return matching(endpoint -> endpoint.name().equals(id.id())); + } + + /** Returns the subset of endpoints that are considered legacy */ + public EndpointList legacy() { + return matching(Endpoint::legacy); } - /** Returns the subset of endpoints are either legacy or not */ - public EndpointList legacy(boolean legacy) { - return matching(endpoint -> endpoint.legacy() == legacy); + /** Returns the subset of endpoints that require a rotation */ + public EndpointList requiresRotation() { + return matching(Endpoint::requiresRotation); } /** Returns the subset of endpoints with given scope */ @@ -49,22 +56,40 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> return matching(endpoint -> endpoint.scope() == scope); } - public static EndpointList of(Stream<Endpoint> endpoints) { - return new EndpointList(endpoints.collect(Collectors.toUnmodifiableList())); + /** Returns all global endpoints for given routing ID and system provided by given routing methods */ + public static EndpointList global(RoutingId routingId, SystemName system, List<RoutingMethod> routingMethods) { + var endpoints = new ArrayList<Endpoint>(); + for (var method : routingMethods) { + endpoints.add(Endpoint.of(routingId.application()) + .named(routingId.endpointId()) + .on(Port.fromRoutingMethod(method)) + .routingMethod(method) + .in(system)); + // TODO(mpolden): Remove this once all applications have migrated away from legacy endpoints + if (method == RoutingMethod.shared) { + endpoints.add(Endpoint.of(routingId.application()) + .named(routingId.endpointId()) + .on(Port.plain(4080)) + .legacy() + .routingMethod(method) + .in(system)); + endpoints.add(Endpoint.of(routingId.application()) + .named(routingId.endpointId()) + .on(Port.tls(4443)) + .legacy() + .routingMethod(method) + .in(system)); + } + } + return new EndpointList(endpoints); } - /** Returns the default global endpoints in given system. Default endpoints are served by a pre-provisioned routing layer */ - public static EndpointList create(ApplicationId application, EndpointId endpointId, SystemName system) { - switch (system) { - case cd: - case main: - return new EndpointList(List.of( - Endpoint.of(application).named(endpointId).on(Port.plain(4080)).legacy().in(system), - Endpoint.of(application).named(endpointId).on(Port.tls(4443)).legacy().in(system), - Endpoint.of(application).named(endpointId).on(Port.tls(4443)).in(system) - )); - } - return EMPTY; + public static EndpointList global(RoutingId routingId, SystemName system, RoutingMethod routingMethod) { + return global(routingId, system, List.of(routingMethod)); + } + + public static EndpointList copyOf(Collection<Endpoint> endpoints) { + return new EndpointList(endpoints); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index ca359f5953a..d9f7d5f36b4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -122,12 +122,7 @@ public class EndpointCertificateManager { var latestAvailableVersion = latestVersionInSecretStore(currentCertificateMetadata.get()); if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > currentCertificateMetadata.get().version()) { - var refreshedCertificateMetadata = new EndpointCertificateMetadata( - currentCertificateMetadata.get().keyName(), - currentCertificateMetadata.get().certName(), - latestAvailableVersion.getAsInt() - ); - + var refreshedCertificateMetadata = currentCertificateMetadata.get().withVersion(latestAvailableVersion.getAsInt()); validateEndpointCertificate(refreshedCertificateMetadata, instance, zone); curator.writeEndpointCertificateMetadata(instance.id(), refreshedCertificateMetadata); return Optional.of(refreshedCertificateMetadata); @@ -163,7 +158,7 @@ public class EndpointCertificateManager { Map<ApplicationId, EndpointCertificateMetadata> allEndpointCertificateMetadata = curator.readAllEndpointCertificateMetadata(); allEndpointCertificateMetadata.forEach((applicationId, storedMetaData) -> { - if (storedMetaData.requestedDnsSans().isPresent() && storedMetaData.request_id().isPresent()) + if (storedMetaData.requestedDnsSans().isPresent() && storedMetaData.request_id().isPresent() && storedMetaData.issuer().isPresent()) return; var hashedCn = commonNameHashOf(applicationId, zoneRegistry.system()); // use as join key @@ -181,7 +176,7 @@ public class EndpointCertificateManager { storedMetaData.version(), providerMetadata.request_id(), providerMetadata.requestedDnsSans(), - Optional.empty()); + providerMetadata.issuer()); if (mode == BackfillMode.DRYRUN) { log.log(LogLevel.INFO, "Would update stored metadata " + storedMetaData + " with data from provider: " + backfilledMetadata); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index cd1d7796098..dc7fdeb8a98 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -115,6 +115,7 @@ public class InternalStepRunner implements StepRunner { static final NodeResources DEFAULT_TESTER_RESOURCES_AWS = new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any); + static final Duration capacityTimeout = Duration.ofMinutes(5); static final Duration endpointTimeout = Duration.ofMinutes(15); static final Duration endpointCertificateTimeout = Duration.ofMinutes(15); static final Duration testerTimeout = Duration.ofMinutes(30); @@ -159,7 +160,7 @@ public class InternalStepRunner implements StepRunner { } catch (RuntimeException e) { logger.log(WARNING, "Unexpected exception running " + id, e); - if (JobProfile.of(id.type()).alwaysRun().contains(step.get())) { + if (step.get().alwaysRun()) { logger.log("Will keep trying, as this is a cleanup step."); return Optional.empty(); } @@ -173,34 +174,20 @@ public class InternalStepRunner implements StepRunner { versions.sourcePlatform().orElse(versions.targetPlatform()) + " and application version " + versions.sourceApplication().orElse(versions.targetApplication()).id() + " ..."); - return deployReal(id, true, versions, logger); + return deployReal(id, true, logger); } private Optional<RunStatus> deployReal(RunId id, DualLogger logger) { Versions versions = controller.jobController().run(id).get().versions(); logger.log("Deploying platform version " + versions.targetPlatform() + " and application version " + versions.targetApplication().id() + " ..."); - return deployReal(id, false, versions, logger); + return deployReal(id, false, logger); } - private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, Versions versions, DualLogger logger) { - Optional<ApplicationPackage> applicationPackage = id.type().environment().isManuallyDeployed() - ? Optional.of(new ApplicationPackage(controller.applications().applicationStore() - .getDev(id.application(), id.type().zone(controller.system())))) - : Optional.empty(); - - Optional<Version> vespaVersion = id.type().environment().isManuallyDeployed() - ? Optional.of(versions.targetPlatform()) - : Optional.empty(); + private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, DualLogger logger) { return deploy(id.application(), id.type(), - () -> controller.applications().deploy(id.application(), - id.type().zone(controller.system()), - applicationPackage, - new DeployOptions(false, - vespaVersion, - false, - setTheStage)), + () -> controller.applications().deploy2(id.job(), setTheStage), controller.jobController().run(id).get() .stepInfo(setTheStage ? deployInitialReal : deployReal).get() .startTime().get(), @@ -215,10 +202,7 @@ public class InternalStepRunner implements StepRunner { () -> controller.applications().deployTester(id.tester(), testerPackage(id), id.type().zone(controller.system()), - new DeployOptions(true, - Optional.of(platform), - false, - false)), + platform), controller.jobController().run(id).get() .stepInfo(deployTester).get() .startTime().get(), @@ -292,7 +276,9 @@ public class InternalStepRunner implements StepRunner { return result; case OUT_OF_CAPACITY: logger.log(e.getServerMessage()); - return Optional.of(outOfCapacity); + return controller.system().isCd() && startTime.plus(capacityTimeout).isAfter(controller.clock().instant()) + ? Optional.empty() + : Optional.of(outOfCapacity); case INVALID_APPLICATION_PACKAGE: case BAD_REQUEST: logger.log(e.getMessage()); @@ -300,7 +286,8 @@ public class InternalStepRunner implements StepRunner { } throw e; - } catch (EndpointCertificateException e) { + } + catch (EndpointCertificateException e) { switch (e.type()) { case CERT_NOT_AVAILABLE: // Same as CERTIFICATE_NOT_READY above, only from the controller @@ -450,11 +437,11 @@ public class InternalStepRunner implements StepRunner { /** Returns true iff all containers in the deployment give 100 consecutive 200 OK responses on /status.html. */ private boolean containersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { - var endpoints = controller.routingController().zoneEndpointsOf(Set.of(new DeploymentId(id, zoneId))); + var endpoints = controller.routing().zoneEndpointsOf(Set.of(new DeploymentId(id, zoneId))); if ( ! endpoints.containsKey(zoneId)) return false; - for (URI endpoint : endpoints.get(zoneId).values()) { + for (var endpoint : endpoints.get(zoneId).keySet()) { boolean ready = controller.jobController().cloud().ready(endpoint); if (!ready) { logger.log("Failed to get 100 consecutive OKs from " + endpoint); @@ -477,12 +464,12 @@ public class InternalStepRunner implements StepRunner { } private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { - var endpoints = controller.routingController().zoneEndpointsOf(Set.of(new DeploymentId(id, zone))); + var endpoints = controller.routing().zoneEndpointsOf(Set.of(new DeploymentId(id, zone))); if ( ! endpoints.containsKey(zone)) { logger.log("Endpoints not yet ready."); return false; } - for (var endpoint : endpoints.get(zone).values()) + for (var endpoint : endpoints.get(zone).keySet()) if ( ! controller.jobController().cloud().exists(endpoint)) { logger.log(INFO, "DNS lookup yielded no IP address for '" + endpoint + "'."); return false; @@ -492,12 +479,12 @@ public class InternalStepRunner implements StepRunner { return true; } - private void logEndpoints(Map<ZoneId, Map<ClusterSpec.Id, URI>> endpoints, DualLogger logger) { + private void logEndpoints(Map<ZoneId, Map<URI, ClusterSpec.Id>> endpoints, DualLogger logger) { List<String> messages = new ArrayList<>(); messages.add("Found endpoints:"); - endpoints.forEach((zone, uris) -> { + endpoints.forEach((zone, urls) -> { messages.add("- " + zone); - uris.forEach((cluster, uri) -> messages.add(" |-- " + uri + " (" + cluster + ")")); + urls.forEach((url, cluster) -> messages.add(" |-- " + url + " (" + cluster + ")")); }); logger.log(messages); } @@ -550,7 +537,7 @@ public class InternalStepRunner implements StepRunner { deployments.add(new DeploymentId(id.application(), zoneId)); logger.log("Attempting to find endpoints ..."); - var endpoints = controller.routingController().zoneEndpointsOf(deployments); + var endpoints = controller.routing().zoneEndpointsOf(deployments); if ( ! endpoints.containsKey(zoneId)) { logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!"); return Optional.of(error); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 6904bff8548..5d8ad6594df 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -26,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.net.URI; import java.security.cert.X509Certificate; @@ -414,9 +413,11 @@ public class JobController { /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ public void start(ApplicationId id, JobType type, Versions versions) { - if ( ! type.environment().isManuallyDeployed() && versions.targetApplication().isUnknown()) - throw new IllegalArgumentException("Target application must be a valid reference."); + start(id, type, versions, JobProfile.of(type)); + } + /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ + public void start(ApplicationId id, JobType type, Versions versions, JobProfile profile) { controller.applications().lockApplicationIfPresent(TenantAndApplicationId.from(id), application -> { locked(id, type, __ -> { Optional<Run> last = last(id, type); @@ -424,7 +425,7 @@ public class JobController { throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!"); RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); - curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant())); + curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant(), profile)); metric.jobStarted(newId.job()); }); }); @@ -432,9 +433,6 @@ public class JobController { /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment. */ public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage) { - if ( ! type.environment().isManuallyDeployed()) - throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments."); - controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().instances().containsKey(id.instance())) application = controller.applications().withNewInstance(application, id); @@ -445,10 +443,15 @@ public class JobController { last(id, type).filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); locked(id, type, __ -> { controller.applications().applicationStore().putDev(id, type.zone(controller.system()), applicationPackage.zippedContent()); - start(id, type, new Versions(platform.orElse(controller.systemVersion()), - ApplicationVersion.unknown, - Optional.empty(), - Optional.empty())); + start(id, + type, + new Versions(platform.orElse(applicationPackage.deploymentSpec().majorVersion() + .flatMap(controller.applications()::lastCompatibleVersion) + .orElseGet(controller::systemVersion)), + ApplicationVersion.unknown, + Optional.empty(), + Optional.empty()), + JobProfile.development); runner.get().accept(last(id, type).get()); }); @@ -504,7 +507,7 @@ public class JobController { } finally { // Passing an empty DeploymentSpec here is fine as it's used for registering global endpoint names, and // tester instances have none. - controller.routingController().policies().refresh(id.id(), DeploymentSpec.empty, zone); + controller.routing().policies().refresh(id.id(), DeploymentSpec.empty, zone); } } @@ -532,17 +535,12 @@ public class JobController { .collect(toList())); } - /** Returns the tester endpoint URL, if any */ - Optional<URI> testerEndpoint(RunId id) { - var testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); - return controller.routingController().zoneEndpointsOf(testerId).values().stream().findFirst(); - } - private void prunePackages(TenantAndApplicationId id) { controller.applications().lockApplicationIfPresent(id, application -> { application.get().productionDeployments().values().stream() .flatMap(List::stream) .map(Deployment::applicationVersion) + .filter(version -> ! version.isUnknown()) .min(Comparator.comparingLong(applicationVersion -> applicationVersion.buildNumber().getAsLong())) .ifPresent(oldestDeployed -> { controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index 88b3442bd6e..1c1d60a2cf0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -34,8 +34,8 @@ public enum JobProfile { deployTester, installTester, startTests, - endTests), - EnumSet.of(copyVespaLogs, + endTests, + copyVespaLogs, deactivateTester, deactivateReal, report)), @@ -49,37 +49,35 @@ public enum JobProfile { deployReal, installReal, startTests, - endTests), - EnumSet.of(copyVespaLogs, + endTests, + copyVespaLogs, deactivateTester, deactivateReal, report)), production(EnumSet.of(deployReal, - installReal), - EnumSet.of(report)), + installReal, + report)), productionTest(EnumSet.of(deployTester, installTester, startTests, - endTests), - EnumSet.of(deactivateTester, + endTests, + deactivateTester, report)), development(EnumSet.of(deployReal, - installReal), - EnumSet.of(copyVespaLogs)); + installReal, + copyVespaLogs)); private final Set<Step> steps; - private final Set<Step> alwaysRun; - JobProfile(Set<Step> runWhileSuccess, Set<Step> alwaysRun) { - runWhileSuccess.addAll(alwaysRun); - this.steps = Collections.unmodifiableSet(runWhileSuccess); - this.alwaysRun = Collections.unmodifiableSet(alwaysRun); + JobProfile(Set<Step> steps) { + this.steps = Collections.unmodifiableSet(steps); } + // TODO jonmv: Let caller decide profile, and store with run? public static JobProfile of(JobType type) { switch (type.environment()) { case test: return systemTest; @@ -94,7 +92,4 @@ public enum JobProfile { /** Returns all steps in this profile, the default for which is to run only when all prerequisites are successes. */ public Set<Step> steps() { return steps; } - /** Returns the set of steps that should always be run, regardless of outcome. */ - public Set<Step> alwaysRun() { return alwaysRun; } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 8cd57fa7d3a..d2481cd97ad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -56,9 +56,9 @@ public class Run { this.testerCertificate = testerCertificate; } - public static Run initial(RunId id, Versions versions, Instant now) { + public static Run initial(RunId id, Versions versions, Instant now, JobProfile profile) { EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); - JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, StepInfo.initial(step))); + profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step))); return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running, -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty()); } @@ -268,9 +268,9 @@ public class Run { private List<Step> forcedSteps() { return ImmutableList.copyOf(steps.entrySet().stream() .filter(entry -> entry.getValue().status() == unfinished - && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey()) + && entry.getKey().alwaysRun() && entry.getKey().prerequisites().stream() - .filter(JobProfile.of(id.type()).alwaysRun()::contains) + .filter(Step::alwaysRun) .allMatch(step -> steps.get(step) == null || steps.get(step).status() != unfinished)) .map(Map.Entry::getKey) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 44a93a655a8..75d875a29c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -26,54 +26,59 @@ import java.util.List; public enum Step { /** Download test-jar and assemble and deploy tester application. */ - deployTester, + deployTester(false), /** See that tester is done deploying, and is ready to serve. */ - installTester(deployTester), + installTester(false, deployTester), /** Download and deploy the initial real application, for staging tests. */ - deployInitialReal(deployTester), + deployInitialReal(false, deployTester), /** See that the real application has had its nodes converge to the initial state. */ - installInitialReal(deployInitialReal), + installInitialReal(false, deployInitialReal), /** Ask the tester to run its staging setup. */ - startStagingSetup(installInitialReal, installTester), + startStagingSetup(false, installInitialReal, installTester), /** See that the staging setup is done. */ - endStagingSetup(startStagingSetup), + endStagingSetup(false, startStagingSetup), /** Download and deploy real application, restarting services if required. */ - deployReal(endStagingSetup, deployTester), + deployReal(false, endStagingSetup, deployTester), /** See that real application has had its nodes converge to the wanted version and generation. */ - installReal(deployReal), + installReal(false, deployReal), /** Ask the tester to run its tests. */ - startTests(installReal, installTester), + startTests(false, installReal, installTester), /** See that the tests are done running. */ - endTests(startTests), + endTests(false, startTests), /** Fetch and store Vespa logs from the log server cluster of the deployment -- used for test and dev deployments. */ - copyVespaLogs(installReal, endTests), + copyVespaLogs(true, installReal, endTests), /** Delete the real application -- used for test deployments. */ - deactivateReal(deployInitialReal, deployReal, endTests, copyVespaLogs), + deactivateReal(true, deployInitialReal, deployReal, endTests, copyVespaLogs), /** Deactivate the tester. */ - deactivateTester(deployTester, endTests), + deactivateTester(true, deployTester, endTests), /** Report completion to the deployment orchestration machinery. */ - report(deactivateReal, deactivateTester); + report(true, deactivateReal, deactivateTester); + private final boolean alwaysRun; private final List<Step> prerequisites; - Step(Step... prerequisites) { + Step(boolean alwaysRun, Step... prerequisites) { + this.alwaysRun = alwaysRun; this.prerequisites = ImmutableList.copyOf(prerequisites); } + /** Returns whether this is a cleanup-step, and should always run, regardless of job outcome, when specified in a job. */ + public boolean alwaysRun() { return alwaysRun; } + /** Returns the prerequisite steps that must be successfully completed before this, assuming the job contains these steps. */ public List<Step> prerequisites() { return prerequisites; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java index 0dfe945af2b..945ffa529ba 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java @@ -32,7 +32,7 @@ public class TestConfigSerializer { public Slime configSlime(ApplicationId id, JobType type, boolean isCI, - Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments, + Map<ZoneId, Map<URI, ClusterSpec.Id>> deployments, Map<ZoneId, List<String>> clusters) { Slime slime = new Slime(); Cursor root = slime.setObject(); @@ -45,14 +45,14 @@ public class TestConfigSerializer { Cursor endpointsObject = root.setObject("endpoints"); // TODO jvenstad: remove. deployments.forEach((zone, endpoints) -> { Cursor endpointArray = endpointsObject.setArray(zone.value()); - for (URI endpoint : endpoints.values()) + for (URI endpoint : endpoints.keySet()) endpointArray.addString(endpoint.toString()); }); Cursor zoneEndpointsObject = root.setObject("zoneEndpoints"); deployments.forEach((zone, endpoints) -> { Cursor clusterEndpointsObject = zoneEndpointsObject.setObject(zone.value()); - endpoints.forEach((cluster, endpoint) -> { + endpoints.forEach((endpoint, cluster) -> { clusterEndpointsObject.setString(cluster.value(), endpoint.toString()); }); }); @@ -73,7 +73,7 @@ public class TestConfigSerializer { public byte[] configJson(ApplicationId id, JobType type, boolean isCI, - Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments, + Map<ZoneId, Map<URI, ClusterSpec.Id>> deployments, Map<ZoneId, List<String>> clusters) { try { return SlimeUtils.toJsonBytes(configSlime(id, type, isCI, deployments, clusters)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java deleted file mode 100644 index 88c1a48653a..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.application.ClusterInfo; -import com.yahoo.vespa.hosted.controller.application.Deployment; - -import java.time.Duration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Maintains information about hardware, hostnames and cluster specifications. - * - * This is used to calculate cost metrics for the application api. - * - * @author smorgrav - */ -public class ClusterInfoMaintainer extends Maintainer { - - private static final Logger log = Logger.getLogger(ClusterInfoMaintainer.class.getName()); - - private final Controller controller; - private final NodeRepository nodeRepository; - - ClusterInfoMaintainer(Controller controller, Duration duration, JobControl jobControl) { - super(controller, duration, jobControl); - this.controller = controller; - this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - } - - private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(List<Node> nodes) { - Map<ClusterSpec.Id, ClusterInfo> infoMap = new HashMap<>(); - - // Group nodes by clusterid - Map<String, List<Node>> clusters = nodes.stream().collect(Collectors.groupingBy(Node::clusterId)); - - // For each cluster - get info - for (String id : clusters.keySet()) { - List<Node> clusterNodes = clusters.get(id); - - // Assume they are all equal and use first node as a representative for the cluster - Node node = clusterNodes.get(0); - - // Add to map - List<String> hostnames = clusterNodes.stream() - .map(Node::hostname) - .map(HostName::value) - .collect(Collectors.toList()); - ClusterInfo info = new ClusterInfo(node.flavor(), node.cost(), - node.resources().vcpu(), node.resources().memoryGb(), node.resources().diskGb(), - ClusterSpec.Type.from(node.clusterType().name()), hostnames); - infoMap.put(new ClusterSpec.Id(id), info); - } - - return infoMap; - } - - @Override - protected void maintain() { - for (Application application : controller().applications().asList()) { - for (Instance instance : application.instances().values()) { - for (Deployment deployment : instance.deployments().values()) { - DeploymentId deploymentId = new DeploymentId(instance.id(), deployment.zone()); - try { - var nodes = nodeRepository.list(deploymentId.zoneId(), deploymentId.applicationId()); - Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(nodes); - controller().applications().lockApplicationIfPresent(application.id(), lockedApplication -> - controller.applications().store(lockedApplication.with(instance.name(), - locked -> locked.withClusterInfo(deployment.zone(), clusterInfo)))); - } - catch (Exception e) { - log.log(Level.WARNING, "Failing getting cluster information for " + deploymentId, e); - } - } - } - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index ec209c5ed98..18fe96fc9b2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -32,7 +32,6 @@ public class ControllerMaintenance extends AbstractComponent { private final VersionStatusUpdater versionStatusUpdater; private final Upgrader upgrader; private final ReadyJobsTrigger readyJobsTrigger; - private final ClusterInfoMaintainer clusterInfoMaintainer; private final DeploymentMetricsMaintainer deploymentMetricsMaintainer; private final ApplicationOwnershipConfirmer applicationOwnershipConfirmer; private final SystemUpgrader systemUpgrader; @@ -64,7 +63,6 @@ public class ControllerMaintenance extends AbstractComponent { versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(3), jobControl); upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator); readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofMinutes(1), jobControl); - clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl); deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, controller.serviceRegistry().ownershipIssues()); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); @@ -95,7 +93,6 @@ public class ControllerMaintenance extends AbstractComponent { versionStatusUpdater.deconstruct(); upgrader.deconstruct(); readyJobsTrigger.deconstruct(); - clusterInfoMaintainer.deconstruct(); deploymentMetricsMaintainer.deconstruct(); applicationOwnershipConfirmer.deconstruct(); systemUpgrader.deconstruct(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index d88469645b4..a4ace4cf518 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -64,8 +64,8 @@ public class MetricsReporter extends Maintainer { } private void reportRemainingRotations() { - try (RotationLock lock = controller().routingController().rotations().lock()) { - int availableRotations = controller().routingController().rotations().availableRotations(lock).size(); + try (RotationLock lock = controller().routing().rotations().lock()) { + int availableRotations = controller().routing().rotations().availableRotations(lock).size(); metric.set(REMAINING_ROTATIONS, availableRotations, metric.createContext(Map.of())); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java index bdfaa9c098f..1d1d5e93dd8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java @@ -82,7 +82,7 @@ public class RotationStatusUpdater extends Maintainer { private RotationStatus getStatus(Instance instance) { var statusMap = new LinkedHashMap<RotationId, RotationStatus.Targets>(); for (var assignedRotation : instance.rotations()) { - var rotation = controller().routingController().rotations().getRotation(assignedRotation.rotationId()); + var rotation = controller().routing().rotations().getRotation(assignedRotation.rotationId()); if (rotation.isEmpty()) continue; var targets = service.getHealthStatus(rotation.get().name()).entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, (kv) -> from(kv.getValue()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index ad411a895fc..a15bfd07a66 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -92,6 +92,7 @@ public class ApplicationSerializer { private static final String pemDeployKeysField = "pemDeployKeys"; private static final String assignedRotationClusterField = "clusterId"; private static final String assignedRotationRotationField = "rotationId"; + private static final String assignedRotationRegionsField = "regions"; private static final String versionField = "version"; // Instance fields @@ -190,7 +191,7 @@ public class ApplicationSerializer { instanceObject.setString(instanceNameField, instance.name().value()); deploymentsToSlime(instance.deployments().values(), instanceObject.setArray(deploymentsField)); toSlime(instance.jobPauses(), instanceObject.setObject(deploymentJobsField)); - assignedRotationsToSlime(instance.rotations(), instanceObject, assignedRotationsField); + assignedRotationsToSlime(instance.rotations(), instanceObject); toSlime(instance.rotationStatus(), instanceObject.setArray(rotationStatusField)); toSlime(instance.change(), instanceObject, deployingField); } @@ -210,6 +211,7 @@ public class ApplicationSerializer { object.setString(versionField, deployment.version().toString()); object.setLong(deployTimeField, deployment.at().toEpochMilli()); toSlime(deployment.applicationVersion(), object.setObject(applicationPackageRevisionField)); + // TODO(mpolden): Stop writing this after next release. clusterInfoToSlime(deployment.clusterInfo(), object); deploymentMetricsToSlime(deployment.metrics(), object); deployment.activity().lastQueried().ifPresent(instant -> object.setLong(lastQueriedField, instant.toEpochMilli())); @@ -308,13 +310,17 @@ public class ApplicationSerializer { }); } - private void assignedRotationsToSlime(List<AssignedRotation> rotations, Cursor parent, String fieldName) { - var rotationsArray = parent.setArray(fieldName); + private void assignedRotationsToSlime(List<AssignedRotation> rotations, Cursor parent) { + var rotationsArray = parent.setArray(assignedRotationsField); for (var rotation : rotations) { var object = rotationsArray.addObject(); object.setString(assignedRotationEndpointField, rotation.endpointId().id()); object.setString(assignedRotationRotationField, rotation.rotationId().asString()); object.setString(assignedRotationClusterField, rotation.clusterId().value()); + var regionsArray = object.setArray(assignedRotationRegionsField); + for (var region : rotation.regions()) { + regionsArray.addString(region.value()); + } } } @@ -393,7 +399,7 @@ public class ApplicationSerializer { applicationVersionFromSlime(deploymentObject.field(applicationPackageRevisionField)), Version.fromString(deploymentObject.field(versionField).asString()), Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()), - clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)), + Map.of(), deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)), DeploymentActivity.create(Serializers.optionalInstant(deploymentObject.field(lastQueriedField)), Serializers.optionalInstant(deploymentObject.field(lastWrittenField)), @@ -444,25 +450,6 @@ public class ApplicationSerializer { return Collections.unmodifiableMap(rotationStatus); } - private Map<ClusterSpec.Id, ClusterInfo> clusterInfoMapFromSlime (Inspector object) { - Map<ClusterSpec.Id, ClusterInfo> map = new HashMap<>(); - object.traverse((String name, Inspector value) -> map.put(new ClusterSpec.Id(name), clusterInfoFromSlime(value))); - return map; - } - - private ClusterInfo clusterInfoFromSlime(Inspector inspector) { - String flavor = inspector.field(clusterInfoFlavorField).asString(); - int cost = (int)inspector.field(clusterInfoCostField).asLong(); - String type = inspector.field(clusterInfoTypeField).asString(); - double flavorCpu = inspector.field(clusterInfoCpuField).asDouble(); - double flavorMem = inspector.field(clusterInfoMemField).asDouble(); - double flavorDisk = inspector.field(clusterInfoDiskField).asDouble(); - - List<String> hostnames = new ArrayList<>(); - inspector.field(clusterInfoHostnamesField).traverse((ArrayTraverser)(int index, Inspector value) -> hostnames.add(value.asString())); - return new ClusterInfo(flavor, cost, flavorCpu, flavorMem, flavorDisk, ClusterSpec.Type.from(type), hostnames); - } - private ZoneId zoneIdFromSlime(Inspector object) { return ZoneId.from(object.field(environmentField).asString(), object.field(regionField).asString()); } @@ -514,11 +501,11 @@ public class ApplicationSerializer { private List<AssignedRotation> assignedRotationsFromSlime(DeploymentSpec deploymentSpec, InstanceName instance, Inspector root) { var assignedRotations = new LinkedHashMap<EndpointId, AssignedRotation>(); - root.field(assignedRotationsField).traverse((ArrayTraverser) (idx, inspector) -> { var clusterId = new ClusterSpec.Id(inspector.field(assignedRotationClusterField).asString()); var endpointId = EndpointId.of(inspector.field(assignedRotationEndpointField).asString()); var rotationId = new RotationId(inspector.field(assignedRotationRotationField).asString()); + // TODO(mpolden): Read regions from field instead of deployment spec after next release var regions = deploymentSpec.instance(instance) .map(spec -> globalEndpointRegions(spec, endpointId)) .orElse(Set.of()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java index 501d3a06d42..80d8270eaaa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java @@ -48,6 +48,7 @@ public class EndpointCertificateMetadataSerializer { Cursor cursor = object.setArray(requestedDnsSansField); sans.forEach(cursor::addString); }); + metadata.issuer().ifPresent(id -> object.setString(issuerField, id)); return slime; } @@ -77,7 +78,7 @@ public class EndpointCertificateMetadataSerializer { issuer); } - public static EndpointCertificateMetadata fromJsonString(String zkdata) { - return fromSlime(SlimeUtils.jsonToSlime(zkdata).get()); + public static EndpointCertificateMetadata fromJsonString(String zkData) { + return fromSlime(SlimeUtils.jsonToSlime(zkData).get()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 5e234d31322..7b2e1ad7a8e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -29,10 +29,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.hosted.controller.AlreadyExistsException; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -66,10 +63,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.ClusterCost; -import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentCost; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -111,6 +105,7 @@ import java.time.Duration; import java.time.Instant; import java.time.YearMonth; import java.time.format.DateTimeParseException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Comparator; @@ -853,28 +848,20 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } } + // TODO(mpolden): Remove once legacy dashboard and integration tests stop expecting these fields private void globalEndpointsToSlime(Cursor object, Instance instance) { var globalEndpointUrls = new LinkedHashSet<String>(); - // Add default global endpoints. These are backed by rotations. - instance.endpointsIn(controller.system()) - .scope(Endpoint.Scope.global) - .legacy(false) // Hide legacy names - .asList().stream() - .map(Endpoint::url) - .map(URI::toString) - .forEach(globalEndpointUrls::add); - - // Per-cluster endpoints. These are backed by load balancers. - var routingPolicies = controller.routingController().policies().get(instance.id()).values(); - for (var policy : routingPolicies) { - policy.globalEndpointsIn(controller.system()).asList().stream() + // Add global endpoints backed by rotations + controller.routing().endpointsOf(instance.id()) + .requiresRotation() + .not().legacy() // Hide legacy names + .asList().stream() .map(Endpoint::url) .map(URI::toString) .forEach(globalEndpointUrls::add); - } - // TODO(mpolden): Remove once clients stop expecting this field + var globalRotationsArray = object.setArray("globalRotations"); globalEndpointUrls.forEach(globalRotationsArray::addString); @@ -1047,6 +1034,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .ifPresent(version -> toSlime(version, object.setObject("revision"))); } + private void toSlime(Endpoint endpoint, String cluster, Cursor object) { + object.setString("cluster", cluster); + object.setBool("tls", endpoint.tls()); + object.setString("url", endpoint.url().toString()); + object.setString("scope", endpointScopeString(endpoint.scope())); + object.setString("routingMethod", routingMethodString(endpoint.routingMethod())); + } + private void toSlime(Cursor response, DeploymentId deploymentId, Deployment deployment, HttpRequest request) { response.setString("tenant", deploymentId.applicationId().tenant().value()); response.setString("application", deploymentId.applicationId().application().value()); @@ -1054,68 +1049,28 @@ public class ApplicationApiHandler extends LoggingRequestHandler { response.setString("environment", deploymentId.zoneId().environment().value()); response.setString("region", deploymentId.zoneId().region().value()); - // Add zone endpoints defined by routing policies + var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); + var instance = application.instances().get(deploymentId.applicationId().instance()); + + // Add zone endpoints var endpointArray = response.setArray("endpoints"); - for (var policy : controller.routingController().policies().get(deploymentId).values()) { - // TODO(mpolden): Always add endpoints from all policies, independent of routing method. This allows removal - // of RoutingGenerator and eliminates the external call to the routing layer below. - if (!controller.routingController().supportsRoutingMethod(RoutingMethod.exclusive, deployment.zone())) continue; - if (!policy.status().isActive()) continue; - { - var endpointObject = endpointArray.addObject(); - var endpoint = policy.endpointIn(controller.system()); - endpointObject.setString("cluster", policy.id().cluster().value()); - endpointObject.setBool("tls", endpoint.tls()); - endpointObject.setString("url", endpoint.url().toString()); - endpointObject.setString("scope", endpointScopeString(endpoint.scope())); - endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.exclusive)); - } - // Add global endpoints that point to this policy - for (var endpoint : policy.globalEndpointsIn(controller.system()).asList()) { - var endpointObject = endpointArray.addObject(); - endpointObject.setString("cluster", policy.id().cluster().value()); - endpointObject.setBool("tls", endpoint.tls()); - endpointObject.setString("url", endpoint.url().toString()); - endpointObject.setString("scope", endpointScopeString(endpoint.scope())); - endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.exclusive)); + var serviceUrls = new ArrayList<URI>(); + for (var endpoint : controller.routing().endpointsOf(deploymentId)) { + toSlime(endpoint, endpoint.name(), endpointArray.addObject()); + if (endpoint.routingMethod() == RoutingMethod.shared) { + serviceUrls.add(endpoint.url()); } } - // Add zone endpoints served by shared routing layer - for (var clusterAndUrl : controller.routingController().legacyZoneEndpointsOf(deploymentId).entrySet()) { - var endpointObject = endpointArray.addObject(); - endpointObject.setString("cluster", clusterAndUrl.getKey().value()); - endpointObject.setBool("tls", true); - endpointObject.setString("url", clusterAndUrl.getValue().toString()); - endpointObject.setString("scope", endpointScopeString(Endpoint.Scope.zone)); - endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.shared)); - } - // Add global endpoints served by shared routing layer - var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); - var instance = application.instances().get(deploymentId.applicationId().instance()); + // Add global endpoints if (deploymentId.zoneId().environment().isProduction()) { // Global endpoints can only point to production deployments - for (var rotation : instance.rotations()) { - var endpoints = instance.endpointsIn(controller.system(), rotation.endpointId()) - .legacy(false) - .scope(Endpoint.Scope.global) - .asList(); - for (var endpoint : endpoints) { - var endpointObject = endpointArray.addObject(); - endpointObject.setString("cluster", rotation.clusterId().value()); - endpointObject.setBool("tls", true); - endpointObject.setString("url", endpoint.url().toString()); - endpointObject.setString("scope", endpointScopeString(endpoint.scope())); - endpointObject.setString("routingMethod", routingMethodString(RoutingMethod.shared)); - } + for (var endpoint : controller.routing().endpointsOf(instance).not().legacy()) { + // TODO(mpolden): Pass cluster name. Cluster that a global endpoint points to is not available at this level. + toSlime(endpoint, "", endpointArray.addObject()); } } - - // serviceUrls contains all valid endpoints for this deployment, including global. The name of these endpoints - // may contain the cluster name (if non-default). Since the controller has no knowledge of clusters for legacy - // endpoints, we can't generate these URLs on-the-fly and we have to query the routing layer. - // TODO(mpolden): Remove this once all clients stop reading this. + // TODO(mpolden): Remove this once all clients stop reading it Cursor serviceUrlArray = response.setArray("serviceUrls"); - controller.routingController().legacyEndpointsOf(deploymentId) - .forEach(endpoint -> serviceUrlArray.addString(endpoint.toString())); + serviceUrls.forEach(url -> serviceUrlArray.addString(url.toString())); response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); @@ -1155,12 +1110,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { deployment.activity().lastQueriesPerSecond().ifPresent(value -> activity.setDouble("lastQueriesPerSecond", value)); deployment.activity().lastWritesPerSecond().ifPresent(value -> activity.setDouble("lastWritesPerSecond", value)); - // Cost - // TODO(mpolden): Unused, remove this field and related code. - DeploymentCost appCost = new DeploymentCost(Map.of()); - Cursor costObject = response.setObject("cost"); - toSlime(appCost, costObject); - // Metrics DeploymentMetrics metrics = deployment.metrics(); Cursor metricsObject = response.setObject("metrics"); @@ -1257,7 +1206,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private void setGlobalEndpointStatus(DeploymentId deployment, boolean inService, HttpRequest request) { var agent = isOperator(request) ? GlobalRouting.Agent.operator : GlobalRouting.Agent.tenant; var status = inService ? GlobalRouting.Status.in : GlobalRouting.Status.out; - controller.routingController().policies().setGlobalRoutingStatus(deployment, status, agent); + controller.routing().policies().setGlobalRoutingStatus(deployment, status, agent); } /** Set the global rotation status for given deployment. This only applies to global endpoints backed by a rotation */ @@ -1268,7 +1217,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { long timestamp = controller.clock().instant().getEpochSecond(); var status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out; var endpointStatus = new EndpointStatus(status, reason, agent.name(), timestamp); - controller.routingController().setGlobalRotationStatus(deployment, endpointStatus); + controller.routing().setGlobalRotationStatus(deployment, endpointStatus); } private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { @@ -1276,9 +1225,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ZoneId.from(environment, region)); Slime slime = new Slime(); Cursor array = slime.setObject().setArray("globalrotationoverride"); - controller.routingController().globalRotationStatus(deploymentId) + controller.routing().globalRotationStatus(deploymentId) .forEach((endpoint, status) -> { - array.addString(endpoint.upstreamName()); + array.addString(endpoint.upstreamIdOf(deploymentId)); Cursor statusObject = array.addObject(); statusObject.setString("status", status.getStatus().name()); statusObject.setString("reason", status.getReason() == null ? "" : status.getReason()); @@ -1533,6 +1482,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse jobDeploy(ApplicationId id, JobType type, HttpRequest request) { + if ( ! type.environment().isManuallyDeployed() && ! isOperator(request)) + throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments."); + Map<String, byte[]> dataParts = parseDataParts(request); if ( ! dataParts.containsKey("applicationZip")) throw new IllegalArgumentException("Missing required form part 'applicationZip'"); @@ -1722,7 +1674,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(testConfigSerializer.configSlime(id, type, false, - controller.routingController().zoneEndpointsOf(deployments), + controller.routing().zoneEndpointsOf(deployments), controller.applications().contentClustersByZone(deployments))); } @@ -1950,57 +1902,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return controller.versionStatus().versions().stream().anyMatch(v -> v.versionNumber().equals(version)); } - public static void toSlime(DeploymentCost deploymentCost, Cursor object) { - object.setLong("tco", (long)deploymentCost.getTco()); - object.setLong("waste", (long)deploymentCost.getWaste()); - object.setDouble("utilization", deploymentCost.getUtilization()); - Cursor clustersObject = object.setObject("cluster"); - for (Map.Entry<String, ClusterCost> clusterEntry : deploymentCost.getCluster().entrySet()) - toSlime(clusterEntry.getValue(), clustersObject.setObject(clusterEntry.getKey())); - } - - private static void toSlime(ClusterCost clusterCost, Cursor object) { - object.setLong("count", clusterCost.getClusterInfo().getHostnames().size()); - object.setString("resource", getResourceName(clusterCost.getResultUtilization())); - object.setDouble("utilization", clusterCost.getResultUtilization().getMaxUtilization()); - object.setLong("tco", (int)clusterCost.getTco()); - object.setLong("waste", (int)clusterCost.getWaste()); - object.setString("flavor", clusterCost.getClusterInfo().getFlavor()); - object.setDouble("flavorCost", clusterCost.getClusterInfo().getFlavorCost()); - object.setDouble("flavorCpu", clusterCost.getClusterInfo().getFlavorCPU()); - object.setDouble("flavorMem", clusterCost.getClusterInfo().getFlavorMem()); - object.setDouble("flavorDisk", clusterCost.getClusterInfo().getFlavorDisk()); - object.setString("type", clusterCost.getClusterInfo().getClusterType().name()); - Cursor utilObject = object.setObject("util"); - utilObject.setDouble("cpu", clusterCost.getResultUtilization().getCpu()); - utilObject.setDouble("mem", clusterCost.getResultUtilization().getMemory()); - utilObject.setDouble("disk", clusterCost.getResultUtilization().getDisk()); - utilObject.setDouble("diskBusy", clusterCost.getResultUtilization().getDiskBusy()); - Cursor usageObject = object.setObject("usage"); - usageObject.setDouble("cpu", clusterCost.getSystemUtilization().getCpu()); - usageObject.setDouble("mem", clusterCost.getSystemUtilization().getMemory()); - usageObject.setDouble("disk", clusterCost.getSystemUtilization().getDisk()); - usageObject.setDouble("diskBusy", clusterCost.getSystemUtilization().getDiskBusy()); - Cursor hostnamesArray = object.setArray("hostnames"); - for (String hostname : clusterCost.getClusterInfo().getHostnames()) - hostnamesArray.addString(hostname); - } - - private static String getResourceName(ClusterUtilization utilization) { - String name = "cpu"; - double max = utilization.getMaxUtilization(); - - if (utilization.getMemory() == max) { - name = "mem"; - } else if (utilization.getDisk() == max) { - name = "disk"; - } else if (utilization.getDiskBusy() == max) { - name = "diskbusy"; - } - - return name; - } - private static boolean recurseOverTenants(HttpRequest request) { return recurseOverApplications(request) || "tenant".equals(request.getProperty("recursive")); } @@ -2126,6 +2027,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { switch (method) { case exclusive: return "exclusive"; case shared: return "shared"; + case sharedLayer4: return "sharedLayer4"; } throw new IllegalArgumentException("Unknown routing method " + method); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 0d1dca391bd..4d79b7bb24c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -597,7 +597,10 @@ class JobControllerApiHandlerHelper { .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli())); stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli())); stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli())); - stepStatus.blockedUntil(change).ifPresent(until -> stepObject.setLong("blockedUntil", until.toEpochMilli())); + stepStatus.blockedUntil(Change.of(controller.systemVersion())) // Dummy version — just anything with a platform. + .ifPresent(until -> stepObject.setLong("platformBlockedUntil", until.toEpochMilli())); + application.latestVersion().map(Change::of).flatMap(stepStatus::blockedUntil) // Dummy version — just anything with an application. + .ifPresent(until -> stepObject.setLong("applicationBlockedUntil", until.toEpochMilli())); if (stepStatus.type() == DeploymentStatus.StepType.instance) { Cursor deployingObject = stepObject.setObject("deploying"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 26ccecee3e6..ba40f9c2085 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -24,7 +24,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.yolean.Exceptions; import java.net.URI; @@ -170,7 +169,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var zone = zoneFrom(path); if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { var status = in ? GlobalRouting.Status.in : GlobalRouting.Status.out; - controller.routingController().policies().setGlobalRoutingStatus(zone, status); + controller.routing().policies().setGlobalRoutingStatus(zone, status); } else { controller.serviceRegistry().configServer().setGlobalRotationStatus(zone, in); } @@ -188,7 +187,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { private void toSlime(ZoneId zone, Cursor zoneObject) { if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) { - var zonePolicy = controller.routingController().policies().get(zone); + var zonePolicy = controller.routing().policies().get(zone); zoneStatusToSlime(zoneObject, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingMethod.exclusive); } else { // Rotation status per zone only exposes in/out status, no agent or time of change. @@ -210,11 +209,11 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var endpointStatus = new EndpointStatus(in ? EndpointStatus.Status.in : EndpointStatus.Status.out, "", agent.name(), controller.clock().instant().getEpochSecond()); - controller.routingController().setGlobalRotationStatus(deployment, endpointStatus); + controller.routing().setGlobalRotationStatus(deployment, endpointStatus); } // Set policy status - controller.routingController().policies().setGlobalRoutingStatus(deployment, status, agent); + controller.routing().policies().setGlobalRoutingStatus(deployment, status, agent); return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT")); } @@ -242,7 +241,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var deploymentId = new DeploymentId(instance.id(), zone); // Include status from rotation if (rotationCanRouteTo(zone, instance)) { - var rotationStatus = controller.routingController().globalRotationStatus(deploymentId); + var rotationStatus = controller.routing().globalRotationStatus(deploymentId); // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. var endpointStatus = rotationStatus.values().stream().findFirst(); if (endpointStatus.isPresent()) { @@ -263,9 +262,12 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } // Include status from routing policies - var routingPolicies = controller.routingController().policies().get(deploymentId); + var routingPolicies = controller.routing().policies().get(deploymentId); for (var policy : routingPolicies.values()) { - deploymentStatusToSlime(deploymentsArray.addObject(), policy); + if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(RoutingMethod.exclusive)) continue; + deploymentStatusToSlime(deploymentsArray.addObject(), new DeploymentId(policy.id().owner(), + policy.id().zone()), + policy.status().globalRouting(), RoutingMethod.exclusive); } } } @@ -273,9 +275,11 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } - /** Returns whether instance has an assigned rotation and a deployment in given zone */ - private static boolean rotationCanRouteTo(ZoneId zone, Instance instance) { - return !instance.rotations().isEmpty() && instance.deployments().containsKey(zone); + /** Returns whether instance has an assigned rotation that can route to given zone */ + private boolean rotationCanRouteTo(ZoneId zone, Instance instance) { + return !instance.rotations().isEmpty() && + instance.deployments().containsKey(zone) && + controller.zoneRegistry().routingMethods(zone).get(0).isShared(); } private static void zoneStatusToSlime(Cursor object, ZoneId zone, GlobalRouting globalRouting, RoutingMethod method) { @@ -297,11 +301,6 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { object.setLong("changedAt", globalRouting.changedAt().toEpochMilli()); } - private static void deploymentStatusToSlime(Cursor object, RoutingPolicy policy) { - deploymentStatusToSlime(object, new DeploymentId(policy.id().owner(), policy.id().zone()), - policy.status().globalRouting(), RoutingMethod.exclusive); - } - private TenantName tenantFrom(Path path) { return TenantName.from(path.get("tenant")); } @@ -355,7 +354,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { switch (method) { case shared: return "shared"; case exclusive: return "exclusive"; - default: return "unknonwn"; + case sharedLayer4: return "sharedLayer4"; + default: return "unknown"; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 847a6c96a53..66cbf4d17ef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -20,7 +20,6 @@ import com.yahoo.slime.SlimeStream; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedTenant; -import com.yahoo.vespa.hosted.controller.api.integration.ApplicationIdSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; @@ -125,7 +124,6 @@ public class UserApiHandler extends LoggingRequestHandler { User user = getAttribute(request, User.ATTRIBUTE_NAME, User.class); Set<Role> roles = getAttribute(request, SecurityContext.ATTRIBUTE_NAME, SecurityContext.class).roles(); - ApplicationIdSnapshot snapshot = controller.applicationIdSnapshot(); Map<TenantName, List<TenantRole>> tenantRolesByTenantName = roles.stream() .flatMap(role -> filterTenantRoles(role).stream()) .distinct() @@ -156,17 +154,6 @@ public class UserApiHandler extends LoggingRequestHandler { Cursor tenantRolesObject = tenantObject.setArray("roles"); tenantRolesByTenantName.getOrDefault(tenant, List.of()) .forEach(role -> tenantRolesObject.addString(role.definition().name())); - - Cursor tenantApplicationsObject = tenantObject.setObject("applications"); - snapshot.applications(tenant).stream() - .sorted() - .forEach(application -> { - Cursor applicationObject = tenantApplicationsObject.setObject(application.value()); - Cursor applicationInstancesObject = applicationObject.setArray("instances"); - snapshot.instances(tenant, application).stream() - .sorted() - .forEach(instance -> applicationInstancesObject.addString(instance.value())); - }); }); if (!operatorRoles.isEmpty()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java index 5543d0ea0b7..8b012b15cd3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java @@ -48,4 +48,8 @@ public class RoutingId { return "routing id for " + endpointId + " of " + application; } + public static RoutingId of(ApplicationId application, EndpointId endpoint) { + return new RoutingId(application, endpoint); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 8657a601837..033539f64c4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; @@ -13,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -56,14 +58,9 @@ public class RoutingPolicies { /** Read all known routing policies for given deployment */ public Map<RoutingPolicyId, RoutingPolicy> get(DeploymentId deployment) { - return get(deployment.applicationId(), deployment.zoneId()); - } - - /** Read all known routing policies for given deployment */ - public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application, ZoneId zone) { - return db.readRoutingPolicies(application).entrySet() + return db.readRoutingPolicies(deployment.applicationId()).entrySet() .stream() - .filter(kv -> kv.getKey().zone().equals(zone)) + .filter(kv -> kv.getKey().zone().equals(deployment.zoneId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } @@ -77,16 +74,15 @@ public class RoutingPolicies { * load balancers for given application have changed. */ public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { - if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; var loadBalancers = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer() .getLoadBalancers(application, zone), deploymentSpec); var inactiveZones = inactiveZones(application, deploymentSpec); try (var lock = db.lockRoutingPolicies()) { - if (!application.instance().isTester()) removeGlobalDnsUnreferencedBy(loadBalancers, lock); + removeGlobalDnsUnreferencedBy(loadBalancers, lock); storePoliciesOf(loadBalancers, lock); removePoliciesUnreferencedBy(loadBalancers, lock); - if (!application.instance().isTester()) updateGlobalDnsOf(get(loadBalancers.application).values(), inactiveZones, lock); + updateGlobalDnsOf(get(loadBalancers.deployment.applicationId()).values(), inactiveZones, lock); } } @@ -127,6 +123,7 @@ public class RoutingPolicies { var staleTargets = new LinkedHashSet<AliasTarget>(); for (var policy : routeEntry.getValue()) { if (policy.dnsZone().isEmpty()) continue; + if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(RoutingMethod.exclusive)) continue; var target = new AliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone()); var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); // Remove target zone if global routing status is set out at: @@ -146,9 +143,10 @@ public class RoutingPolicies { staleTargets.clear(); } if (!targets.isEmpty()) { - var endpoint = RoutingPolicy.globalEndpointOf(routeEntry.getKey().application(), - routeEntry.getKey().endpointId(), controller.system()); - controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); + var endpoints = controller.routing().endpointsOf(routeEntry.getKey().application()) + .named(routeEntry.getKey().endpointId()) + .not().requiresRotation(); + endpoints.forEach(endpoint -> controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal)); } staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordData.fqdn(t.name().value()), @@ -158,9 +156,9 @@ public class RoutingPolicies { /** Store routing policies for given load balancers */ private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var policies = new LinkedHashMap<>(get(loadBalancers.application)); + var policies = new LinkedHashMap<>(get(loadBalancers.deployment.applicationId())); for (LoadBalancer loadBalancer : loadBalancers.list) { - var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), loadBalancers.zone); + var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), loadBalancers.deployment.zoneId()); var existingPolicy = policies.get(policyId); var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.dnsZone(), loadBalancers.endpointIdsOf(loadBalancer), @@ -172,42 +170,45 @@ public class RoutingPolicies { updateZoneDnsOf(newPolicy); policies.put(newPolicy.id(), newPolicy); } - db.writeRoutingPolicies(loadBalancers.application, policies); + db.writeRoutingPolicies(loadBalancers.deployment.applicationId(), policies); } /** Update zone DNS record for given policy */ private void updateZoneDnsOf(RoutingPolicy policy) { - var name = RecordName.from(policy.endpointIn(controller.system()).dnsName()); + var name = RecordName.from(policy.endpointIn(controller.system(), RoutingMethod.exclusive).dnsName()); var data = RecordData.fqdn(policy.canonicalName().value()); - controller.nameServiceForwarder().createCname(name, data, Priority.normal); + nameUpdaterIn(policy.id().zone()).createCname(name, data); } /** Remove policies and zone DNS records unreferenced by given load balancers */ private void removePoliciesUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var policies = get(loadBalancers.application); + var policies = get(loadBalancers.deployment.applicationId()); var newPolicies = new LinkedHashMap<>(policies); var activeLoadBalancers = loadBalancers.list.stream().map(LoadBalancer::hostname).collect(Collectors.toSet()); for (var policy : policies.values()) { // Leave active load balancers and irrelevant zones alone if (activeLoadBalancers.contains(policy.canonicalName()) || - !policy.id().zone().equals(loadBalancers.zone)) continue; + !policy.id().zone().equals(loadBalancers.deployment.zoneId())) continue; - var dnsName = policy.endpointIn(controller.system()).dnsName(); - controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); + var dnsName = policy.endpointIn(controller.system(), RoutingMethod.exclusive).dnsName(); + nameUpdaterIn(loadBalancers.deployment.zoneId()).removeRecords(Record.Type.CNAME, RecordName.from(dnsName)); newPolicies.remove(policy.id()); } - db.writeRoutingPolicies(loadBalancers.application, newPolicies); + db.writeRoutingPolicies(loadBalancers.deployment.applicationId(), newPolicies); } /** Remove unreferenced global endpoints from DNS */ private void removeGlobalDnsUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var zonePolicies = get(loadBalancers.application, loadBalancers.zone).values(); + var zonePolicies = get(loadBalancers.deployment).values(); var removalCandidates = new HashSet<>(routingTableFrom(zonePolicies).keySet()); var activeRoutingIds = routingIdsFrom(loadBalancers); removalCandidates.removeAll(activeRoutingIds); for (var id : removalCandidates) { - var endpoint = RoutingPolicy.globalEndpointOf(id.application(), id.endpointId(), controller.system()); - controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); + var endpoints = controller.routing().endpointsOf(id.application()) + .not().requiresRotation() + .named(id.endpointId()); + var nameUpdater = nameUpdaterIn(loadBalancers.deployment.zoneId()); + endpoints.forEach(endpoint -> nameUpdater.removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()))); } } @@ -258,22 +259,20 @@ public class RoutingPolicies { /** Load balancers allocated to a deployment */ private static class AllocatedLoadBalancers { - private final ApplicationId application; - private final ZoneId zone; + private final DeploymentId deployment; private final List<LoadBalancer> list; private final DeploymentSpec deploymentSpec; private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, DeploymentSpec deploymentSpec) { - this.application = application; - this.zone = zone; + this.deployment = new DeploymentId(application, zone); this.list = List.copyOf(loadBalancers); this.deploymentSpec = deploymentSpec; } /** Compute all endpoint IDs for given load balancer */ private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { - if (!zone.environment().isProduction()) { // Only production deployments have configurable endpoints + if (!deployment.zoneId().environment().isProduction()) { // Only production deployments have configurable endpoints return Set.of(); } var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance()); @@ -282,7 +281,7 @@ public class RoutingPolicies { } return instanceSpec.get().endpoints().stream() .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) - .filter(endpoint -> endpoint.regions().contains(zone.region())) + .filter(endpoint -> endpoint.regions().contains(deployment.zoneId().region())) .map(com.yahoo.config.application.api.Endpoint::endpointId) .map(EndpointId::of) .collect(Collectors.toSet()); @@ -301,4 +300,53 @@ public class RoutingPolicies { .collect(Collectors.toUnmodifiableSet()); } + /** Returns the name updater to use for given zone */ + private NameUpdater nameUpdaterIn(ZoneId zone) { + if (controller.zoneRegistry().routingMethods(zone).contains(RoutingMethod.exclusive)) { + return new NameUpdater(controller.nameServiceForwarder()); + } + return new DiscardingNameUpdater(); + } + + /** A name updater that passes name service operations to the next handler */ + private static class NameUpdater { + + private final NameServiceForwarder forwarder; + + public NameUpdater(NameServiceForwarder forwarder) { + this.forwarder = forwarder; + } + + public void removeRecords(Record.Type type, RecordName name) { + forwarder.removeRecords(type, name, Priority.normal); + } + + public void createAlias(RecordName name, Set<AliasTarget> targets) { + forwarder.createAlias(name, targets, Priority.normal); + } + + public void createCname(RecordName name, RecordData data) { + forwarder.createCname(name, data, Priority.normal); + } + + } + + /** A name updater that does nothing */ + private static class DiscardingNameUpdater extends NameUpdater { + + private DiscardingNameUpdater() { + super(null); + } + + @Override + public void removeRecords(Record.Type type, RecordName name) {} + + @Override + public void createAlias(RecordName name, Set<AliasTarget> target) {} + + @Override + public void createCname(RecordName name, RecordData data) {} + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 32ec9f359a9..37027e8a8c9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -2,14 +2,12 @@ package com.yahoo.vespa.hosted.controller.routing; import com.google.common.collect.ImmutableSortedSet; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.EndpointList; import java.util.Objects; import java.util.Optional; @@ -70,18 +68,14 @@ public class RoutingPolicy { } /** Returns the endpoint of this */ - public Endpoint endpointIn(SystemName system) { - return Endpoint.of(id.owner()).target(id.cluster(), id.zone()) - .routingMethod(RoutingMethod.exclusive) - .on(Port.tls()) + public Endpoint endpointIn(SystemName system, RoutingMethod routingMethod) { + return Endpoint.of(id.owner()) + .target(id.cluster(), id.zone()) + .on(Port.fromRoutingMethod(routingMethod)) + .routingMethod(routingMethod) .in(system); } - /** Returns global endpoints which this is a member of */ - public EndpointList globalEndpointsIn(SystemName system) { - return EndpointList.of(endpoints.stream().map(endpointId -> globalEndpointOf(id.owner(), endpointId, system))); - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -102,12 +96,4 @@ public class RoutingPolicy { id.zone().value()); } - /** Creates a global endpoint for given application */ - public static Endpoint globalEndpointOf(ApplicationId application, EndpointId endpointId, SystemName system) { - return Endpoint.of(application).named(endpointId) - .on(Port.tls()) - .routingMethod(RoutingMethod.exclusive) - .in(system); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index b082f66b70a..32213a27a83 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -6,7 +6,6 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; @@ -38,11 +37,9 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -176,40 +173,34 @@ public class ControllerTest { @Test public void testGlobalRotations() { - // Setup - ControllerTester tester = this.tester.controllerTester(); - ZoneId zone = ZoneId.from("prod", "us-west-1"); - ApplicationId app = ApplicationId.from("tenant", "app1", "default"); - DeploymentId deployment = new DeploymentId(app, zone); - tester.serviceRegistry().routingGeneratorMock().putEndpoints(deployment, List.of( - new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"), - new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"), - new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3"), - new RoutingEndpoint("http://global-endpoint-2.vespa.yahooapis.com:4080", "host2", true, "upstream4"), - new RoutingEndpoint("http://global-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1"), - new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1") - )); - - Supplier<Map<RoutingEndpoint, EndpointStatus>> globalRotationStatus = () -> tester.controller().routingController().globalRotationStatus(deployment); - Supplier<List<EndpointStatus>> upstreamOneEndpoints = () -> { - return globalRotationStatus.get() - .entrySet().stream() - .filter(kv -> kv.getKey().upstreamName().equals("upstream1")) - .map(Map.Entry::getValue) - .collect(Collectors.toList()); - }; + var context = tester.newDeploymentContext(); + var zone1 = ZoneId.from("prod", "us-west-1"); + var zone2 = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("default", "default", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage).deploy(); // Check initial rotation status - assertEquals(3, globalRotationStatus.get().size()); - assertEquals(2, upstreamOneEndpoints.get().size()); - assertTrue("All upstreams are in", upstreamOneEndpoints.get().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in)); - - // Set the global rotations out of service - EndpointStatus status = new EndpointStatus(EndpointStatus.Status.out, "unit-test", "Test", tester.clock().instant().getEpochSecond()); - tester.controller().routingController().setGlobalRotationStatus(deployment, status); - assertEquals(2, upstreamOneEndpoints.get().size()); - assertTrue("All upstreams are out", upstreamOneEndpoints.get().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out)); - assertTrue("Reason is set", upstreamOneEndpoints.get().stream().allMatch(es -> es.getReason().equals("unit-test"))); + var deployment1 = context.deploymentIdIn(zone1); + var status1 = tester.controller().routing().globalRotationStatus(deployment1); + assertEquals(1, status1.size()); + assertTrue("All upstreams are in", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in)); + + // Set the deployment out of service in the global rotation + var newStatus = new EndpointStatus(EndpointStatus.Status.out, "unit-test", ControllerTest.class.getSimpleName(), tester.clock().instant().getEpochSecond()); + tester.controller().routing().setGlobalRotationStatus(deployment1, newStatus); + status1 = tester.controller().routing().globalRotationStatus(deployment1); + assertEquals(1, status1.size()); + assertTrue("All upstreams are out", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out)); + assertTrue("Reason is set", status1.values().stream().allMatch(es -> es.getReason().equals("unit-test"))); + + // Other deployment remains in + var status2 = tester.controller().routing().globalRotationStatus(context.deploymentIdIn(zone2)); + assertEquals(1, status2.size()); + assertTrue("All upstreams are in", status2.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in)); } @Test @@ -517,9 +508,9 @@ public class ControllerTest { context.submit(applicationPackage); tester.applications().deleteApplication(context.application().id(), tester.controllerTester().credentialsFor(context.application().id().tenant())); - try (RotationLock lock = tester.controller().routingController().rotations().lock()) { + try (RotationLock lock = tester.controller().routing().rotations().lock()) { assertTrue("Rotation is unassigned", - tester.controller().routingController().rotations().availableRotations(lock) + tester.controller().routing().rotations().availableRotations(lock) .containsKey(new RotationId("rotation-id-01"))); } context.flushDnsUpdates(); @@ -788,4 +779,28 @@ public class ControllerTest { assertEquals("Deployed application", 1, context.instance().deployments().size()); } + @Test + public void testDeployWithRoutingGeneratorEndpoints() { + var context = tester.newDeploymentContext(); + var applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-west-1") + .build(); + + var zones = Set.of(systemTest.zone(tester.controller().system()), + stagingTest.zone(tester.controller().system()), + ZoneId.from("prod", "us-west-1")); + for (var zone : zones) { + tester.controllerTester().serviceRegistry().routingGeneratorMock() + .putEndpoints(context.deploymentIdIn(zone), + List.of(new RoutingEndpoint("http://legacy-endpoint", "hostname", + false, "upstreamName"))); + } + // Defer load balancer provisioning in all environments so that routing controller uses routing generator + context.deferLoadBalancerProvisioningIn(zones.stream().map(ZoneId::environment).collect(Collectors.toSet())) + .submit(applicationPackage) + .deploy(); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java deleted file mode 100644 index 6df2d55d52b..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ClusterSpec; -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author smorgrav - */ -public class ClusterCostTest { - - @Test - public void clusterCost() { - List<String> hostnames = new ArrayList<>(); - hostnames.add("host1"); - hostnames.add("host2"); - ClusterInfo info = new ClusterInfo("test", 100, 10, 10, 10, ClusterSpec.Type.container, hostnames); - ClusterUtilization util = new ClusterUtilization(0.3, 0.2, 0.5, 0.1); - ClusterCost cost = new ClusterCost(info, util); - - // CPU is fully utilized - Assert.assertEquals(200, cost.getTco(), Double.MIN_VALUE); - Assert.assertEquals(0, cost.getWaste(), Double.MIN_VALUE); - - // Set Disk as the most utilized resource - util = new ClusterUtilization(0.3, 0.1, 0.5, 0.1); - cost = new ClusterCost(info, util); - Assert.assertEquals(200, cost.getTco(), Double.MIN_NORMAL); // TCO is independent of utilization - Assert.assertEquals(57.1428571429, cost.getWaste(), 0.001); // Waste is not independent - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilizationTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilizationTest.java deleted file mode 100644 index c930978eb1e..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilizationTest.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import org.junit.Assert; -import org.junit.Test; - -/** - * @author smorgrav - */ -public class ClusterUtilizationTest { - - private static final double delta = Double.MIN_NORMAL; - - @Test - public void getMaxUtilization() { - ClusterUtilization resources = new ClusterUtilization(0.3, 0.1, 0.4, 0.5); - Assert.assertEquals(0.5, resources.getMaxUtilization(), delta); - - resources = new ClusterUtilization(0.3, 0.1, 0.4, 0.0); - Assert.assertEquals(0.4, resources.getMaxUtilization(), delta); - - resources = new ClusterUtilization(0.4, 0.3, 0.3, 0.0); - Assert.assertEquals(0.4, resources.getMaxUtilization(), delta); - - resources = new ClusterUtilization(0.1, 0.3, 0.3, 0.0); - Assert.assertEquals(0.3, resources.getMaxUtilization(), delta); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java deleted file mode 100644 index 2e58253d768..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ClusterSpec; -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author smorgrav - */ -public class DeploymentCostTest { - - @Test - public void deploymentCost() { - Map<String, ClusterCost> clusters = new HashMap<>(); - clusters.put("cluster1", createClusterCost(100, 0.2)); - clusters.put("cluster2", createClusterCost(50, 0.1)); - - DeploymentCost cost = new DeploymentCost(clusters); - Assert.assertEquals(300, cost.getTco(), Double.MIN_VALUE); // 2*100 + 2*50 - Assert.assertEquals(28.5714285714, cost.getWaste(), 0.001); // from cluster2 - Assert.assertEquals(0.7142857142857143, cost.getUtilization(), Double.MIN_VALUE); // from cluster2 - } - - private ClusterCost createClusterCost(int flavorCost, double cpuUtil) { - List<String> hostnames = new ArrayList<>(); - hostnames.add("host1"); - hostnames.add("host2"); - ClusterInfo info = new ClusterInfo("test", flavorCost, 10, 10, 10, ClusterSpec.Type.container, hostnames); - ClusterUtilization util = new ClusterUtilization(0.3, cpuUtil, 0.5, 0.1); - return new ClusterCost(info, util); - } -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index 81bbbb479b2..f968b8c76d7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import org.junit.Test; @@ -22,7 +23,7 @@ public class EndpointTest { private static final ApplicationId app2 = ApplicationId.from("t2", "a2", "i2"); @Test - public void test_global_endpoints() { + public void global_endpoints() { EndpointId endpointId = EndpointId.defaultId(); Map<String, Endpoint> tests = Map.of( @@ -70,7 +71,7 @@ public class EndpointTest { } @Test - public void test_global_endpoints_with_endpoint_id() { + public void global_endpoints_with_endpoint_id() { var endpointId = EndpointId.defaultId(); Map<String, Endpoint> tests = Map.of( @@ -118,7 +119,7 @@ public class EndpointTest { } @Test - public void test_zone_endpoints() { + public void zone_endpoints() { var cluster = ClusterSpec.Id.from("default"); // Always default for non-direct routing var prodZone = ZoneId.from("prod", "us-north-1"); var testZone = ZoneId.from("test", "us-north-2"); @@ -168,7 +169,7 @@ public class EndpointTest { } @Test - public void test_wildcard_endpoints() { + public void wildcard_endpoints() { var defaultCluster = ClusterSpec.Id.from("default"); var prodZone = ZoneId.from("prod", "us-north-1"); var testZone = ZoneId.from("test", "us-north-2"); @@ -225,4 +226,30 @@ public class EndpointTest { tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } + + @Test + public void upstream_name() { + var zone = ZoneId.from("prod", "us-north-1"); + var tests1 = Map.of( + // With default cluster + "a1.t1.us-north-1.prod", + Endpoint.of(app1).named(EndpointId.defaultId()).on(Port.tls(4443)).in(SystemName.main), + + // With non-default cluster + "c1.a1.t1.us-north-1.prod", + Endpoint.of(app1).named(EndpointId.of("c1")).on(Port.tls(4443)).in(SystemName.main) + ); + var tests2 = Map.of( + // With non-default instance + "i2.a2.t2.us-north-1.prod", + Endpoint.of(app2).named(EndpointId.defaultId()).on(Port.tls(4443)).in(SystemName.main), + + // With non-default instance and cluster + "c2.i2.a2.t2.us-north-1.prod", + Endpoint.of(app2).named(EndpointId.of("c2")).on(Port.tls(4443)).in(SystemName.main) + ); + tests1.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(new DeploymentId(app1, zone)))); + tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(new DeploymentId(app2, zone)))); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java index f9fed820a5b..d7bc73adf37 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java @@ -112,7 +112,7 @@ public class EndpointCertificateManagerTest { @Test public void reprovisions_certificate_when_necessary() { - mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, "uuid", List.of())); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, Optional.of("uuid"), Optional.of(List.of()), Optional.empty())); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 0); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 3e1f468691c..5eae68adf5a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; @@ -19,7 +18,6 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -27,8 +25,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.EndpointId; @@ -49,7 +45,6 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -102,10 +97,8 @@ public class DeploymentContext { private final ApplicationId instanceId; private final TesterId testerId; private final JobController jobs; - private final RoutingGeneratorMock routing; private final JobRunner runner; private final DeploymentTester tester; - private final Set<Environment> deferLoadBalancerProvisioning = new HashSet<>(); private ApplicationVersion lastSubmission = null; private boolean deferDnsUpdates = false; @@ -118,7 +111,6 @@ public class DeploymentContext { this.jobs = tester.controller().jobController(); this.runner = tester.runner(); this.tester = tester; - this.routing = tester.controllerTester().serviceRegistry().routingGeneratorMock(); createTenantAndApplication(); } @@ -197,15 +189,15 @@ public class DeploymentContext { return this; } - /** - * Defer provisioning of load balancers in zones in given environment. Default behaviour is to automatically - * provision load balancers in all zones that support direct routing. - */ + /** Defer provisioning of load balancers in zones in given environment */ public DeploymentContext deferLoadBalancerProvisioningIn(Environment... environment) { - deferLoadBalancerProvisioning.addAll(List.of(environment)); - return this; + return deferLoadBalancerProvisioningIn(Set.of(environment)); } + public DeploymentContext deferLoadBalancerProvisioningIn(Set<Environment> environments) { + configServer().deferLoadBalancerProvisioningIn(environments); + return this; + } /** Defer DNS updates */ public DeploymentContext deferDnsUpdates() { @@ -229,25 +221,16 @@ public class DeploymentContext { return this; } - /** Add a routing policy for this in given zone, with status set to active */ - public DeploymentContext addRoutingPolicy(ZoneId zone, boolean active) { - return addRoutingPolicy(instanceId, zone, active); - } - - /** Add a routing policy for tester instance of this in given zone, with status set to active */ - public DeploymentContext addTesterRoutingPolicy(ZoneId zone, boolean active) { - return addRoutingPolicy(testerId.id(), zone, active); - } - - private DeploymentContext addRoutingPolicy(ApplicationId instance, ZoneId zone, boolean active) { - var clusterId = "default" + (!active ? "-inactive" : ""); - var id = new RoutingPolicyId(instance, ClusterSpec.Id.from(clusterId), zone); - var policies = new LinkedHashMap<>(tester.controller().curator().readRoutingPolicies(instance)); + /** Add a routing policy for this in given zone, with status set to inactive */ + public DeploymentContext addInactiveRoutingPolicy(ZoneId zone) { + var clusterId = "default-inactive"; + var id = new RoutingPolicyId(instanceId, ClusterSpec.Id.from(clusterId), zone); + var policies = new LinkedHashMap<>(tester.controller().curator().readRoutingPolicies(instanceId)); policies.put(id, new RoutingPolicy(id, HostName.from("lb-host"), Optional.empty(), - Set.of(EndpointId.of("c0")), - new Status(active, GlobalRouting.DEFAULT_STATUS))); - tester.controller().curator().writeRoutingPolicies(instance, policies); + Set.of(EndpointId.of("default")), + new Status(false, GlobalRouting.DEFAULT_STATUS))); + tester.controller().curator().writeRoutingPolicies(instanceId, policies); return this; } @@ -303,7 +286,6 @@ public class DeploymentContext { runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasFailed()); assertTrue(jobs.run(id).get().hasEnded()); - doTeardown(job); return this; } @@ -357,7 +339,6 @@ public class DeploymentContext { } if (job.type().isTest()) doTests(job); - doTeardown(job); return this; } @@ -390,7 +371,6 @@ public class DeploymentContext { runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasFailed()); assertTrue(jobs.run(id).get().hasEnded()); - doTeardown(job); return this; } @@ -405,7 +385,6 @@ public class DeploymentContext { runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasFailed()); assertTrue(jobs.run(id).get().hasEnded()); - doTeardown(job); return this; } @@ -429,11 +408,11 @@ public class DeploymentContext { /** Start tests in system test stage */ public RunId startSystemTestTests() { - RunId id = newRun(JobType.systemTest); + var id = newRun(JobType.systemTest); + var testZone = JobType.systemTest.zone(tester.controller().system()); runner.run(); - configServer().convergeServices(instanceId, JobType.systemTest.zone(tester.controller().system())); - configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.controller().system())); - setEndpoints(JobType.systemTest.zone(tester.controller().system())); + configServer().convergeServices(instanceId, testZone); + configServer().convergeServices(testerId.id(), testZone); runner.run(); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endTests)); assertTrue(jobs.run(id).get().steps().get(Step.endTests).startTime().isPresent()); @@ -456,16 +435,6 @@ public class DeploymentContext { ZoneId zone = zone(job); DeploymentId deployment = new DeploymentId(job.application(), zone); - // Provision load balancers in directly routed zones, unless explicitly deferred - if (provisionLoadBalancerIn(zone)) { - configServer().putLoadBalancers(zone, List.of(new LoadBalancer(deployment.toString(), - deployment.applicationId(), - ClusterSpec.Id.from("default"), - HostName.from("lb-0--" + instanceId.serializedForm() + "--" + zone.toString()), - LoadBalancer.State.active, - Optional.of("dns-zone-1")))); - } - // First step is always a deployment. runner.advance(currentRun(job)); @@ -477,7 +446,6 @@ public class DeploymentContext { Versions versions = currentRun(job).versions(); tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), versions.sourcePlatform().orElse(versions.targetPlatform())); configServer().convergeServices(id.application(), zone); - setEndpoints(zone); runner.advance(currentRun(job)); assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); @@ -514,27 +482,6 @@ public class DeploymentContext { return run; } - /** Sets a single endpoint in the routing layer */ - DeploymentContext setEndpoints(ZoneId zone) { - if (!supportsRoutingMethod(RoutingMethod.shared, zone)) return this; - var id = instanceId; - routing.putEndpoints(new DeploymentId(id, zone), - Collections.singletonList(new RoutingEndpoint(String.format("https://%s--%s--%s.%s.%s.vespa:43", - id.instance().value(), - id.application().value(), - id.tenant().value(), - zone.region().value(), - zone.environment().value()), - "host1", - true, - String.format("cluster1.%s.%s.%s.%s", - id.application().value(), - id.tenant().value(), - zone.region().value(), - zone.environment().value())))); - return this; - } - /** Lets nodes converge on new application version. */ private void doConverge(JobId job) { RunId id = currentRun(job).id(); @@ -542,14 +489,13 @@ public class DeploymentContext { assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installReal)); configServer().convergeServices(id.application(), zone); - setEndpoints(zone); runner.advance(currentRun(job)); if (job.type().environment().isManuallyDeployed()) { assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); assertTrue(jobs.run(id).get().hasEnded()); return; } - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); + assertEquals("Status of " + id, Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); } /** Installs tester and starts tests. */ @@ -587,26 +533,6 @@ public class DeploymentContext { assertTrue(configServer().nodeRepository().list(zone, TesterId.of(id.application()).id()).isEmpty()); } - /** Removes endpoints from routing layer — always call this. */ - private void doTeardown(JobId job) { - ZoneId zone = zone(job); - DeploymentId deployment = new DeploymentId(job.application(), zone); - - if ( ! instance().deployments().containsKey(zone)) - routing.removeEndpoints(deployment); - routing.removeEndpoints(new DeploymentId(TesterId.of(job.application()).id(), zone)); - } - - /** Returns whether a load balancer is expected to be provisioned in given zone */ - private boolean provisionLoadBalancerIn(ZoneId zone) { - return !deferLoadBalancerProvisioning.contains(zone.environment()) && - supportsRoutingMethod(RoutingMethod.exclusive, zone); - } - - private boolean supportsRoutingMethod(RoutingMethod method, ZoneId zone) { - return tester.controller().zoneRegistry().zones().routingMethod(method).ids().contains(zone); - } - private JobId jobId(JobType type) { return new JobId(instanceId, type); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 938b801b88d..7114edcc44e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.log.LogLevel; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; @@ -10,9 +9,7 @@ import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; @@ -28,7 +25,6 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.time.temporal.TemporalAdjusters; -import java.util.Collections; import java.util.logging.Logger; import static org.junit.Assert.assertTrue; @@ -49,7 +45,6 @@ public class DeploymentTester { private final ControllerTester tester; private final JobController jobs; - private final RoutingGeneratorMock routing; private final MockTesterCloud cloud; private final JobRunner runner; private final Upgrader upgrader; @@ -57,7 +52,6 @@ public class DeploymentTester { private final OutstandingChangeDeployer outstandingChangeDeployer; public JobController jobs() { return jobs; } - public RoutingGeneratorMock routing() { return routing; } public MockTesterCloud cloud() { return cloud; } public JobRunner runner() { return runner; } public ConfigServerMock configServer() { return tester.configServer(); } @@ -79,7 +73,6 @@ public class DeploymentTester { public DeploymentTester(ControllerTester controllerTester) { tester = controllerTester; jobs = tester.controller().jobController(); - routing = tester.serviceRegistry().routingGeneratorMock(); cloud = (MockTesterCloud) tester.controller().jobController().cloud(); var jobControl = new JobControl(tester.controller().curator()); runner = new JobRunner(tester.controller(), Duration.ofDays(1), jobControl, @@ -88,7 +81,6 @@ public class DeploymentTester { upgrader.setUpgradesPerMinute(1); // Anything that makes it at least one for any maintenance period is fine. readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval, jobControl); outstandingChangeDeployer = new OutstandingChangeDeployer(tester.controller(), maintenanceInterval, jobControl); - routing.putEndpoints(new DeploymentId(null, null), Collections.emptyList()); // Turn off default behaviour for the mock. // Get deployment job logs to stderr. Logger.getLogger("").setLevel(LogLevel.DEBUG); @@ -138,13 +130,6 @@ public class DeploymentTester { return newDeploymentContext(tenantName, applicationName, instanceName).application(); } - /** - * Sets a single endpoint in the routing mock; this matches that required for the tester. - */ - public void setEndpoints(ApplicationId id, ZoneId zone) { - newDeploymentContext(id).setEndpoints(zone); - } - /** Aborts and finishes all running jobs. */ public void abortAll() { triggerJobs(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index a1c8ccffaf0..e684d479158 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -3,13 +3,19 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; @@ -22,11 +28,14 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.stream.Collectors; +import static com.yahoo.config.provision.SystemName.cd; import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast2; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApSoutheast1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionAwsUsEast1a; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdAwsUsEast1a; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdUsCentral1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionEuWest1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; @@ -57,7 +66,7 @@ import static org.junit.Assert.assertTrue; */ public class DeploymentTriggerTest { - private final DeploymentTester tester = new DeploymentTester(); + private DeploymentTester tester = new DeploymentTester(); @Test public void testTriggerFailing() { @@ -1066,4 +1075,58 @@ public class DeploymentTriggerTest { assertEquals(Change.of(app.lastSubmission().get()), app.instance().change()); } + @Test + public void mixedDirectAndPipelineJobsInProduction() { + ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("cd-us-central-1") + .region("cd-aws-us-east-1a") + .build(); + ServiceRegistryMock services = new ServiceRegistryMock(); + var zones = List.of(ZoneApiMock.fromId("test.cd-us-central-1"), + ZoneApiMock.fromId("staging.cd-us-central-1"), + ZoneApiMock.fromId("prod.cd-us-central-1"), + ZoneApiMock.fromId("prod.cd-aws-us-east-1a")); + services.zoneRegistry() + .setSystemName(SystemName.cd) + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.exclusive); + tester = new DeploymentTester(new ControllerTester(services)); + tester.configServer().bootstrap(services.zoneRegistry().zones().all().ids(), SystemApplication.values()); + tester.controllerTester().upgradeSystem(Version.fromString("6.1")); + tester.controllerTester().computeVersionStatus(); + var app = tester.newDeploymentContext(); + + app.runJob(productionCdUsCentral1, cdPackage); + app.submit(cdPackage); + app.runJob(systemTest); + // Staging test requires unknown initial version, and is broken. + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); + app.runJob(productionCdUsCentral1) + .abortJob(stagingTest) // Complete failing run. + .runJob(stagingTest) + .runJob(productionCdAwsUsEast1a); + + app.runJob(productionCdUsCentral1, cdPackage); + var version = new Version("7.1"); + tester.controllerTester().upgradeSystem(version); + tester.upgrader().maintain(); + // System and staging tests both require unknown versions, and are broken. + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); + app.runJob(productionCdUsCentral1) + .abortJob(systemTest) + .abortJob(stagingTest) + .runJob(systemTest) + .runJob(stagingTest) + .runJob(productionCdAwsUsEast1a); + + app.runJob(productionCdUsCentral1, cdPackage); + app.submit(cdPackage); + app.runJob(systemTest); + // Staging test requires unknown initial version, and is broken. + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); + app.runJob(productionCdUsCentral1) + .jobAborted(stagingTest) + .runJob(stagingTest) + .runJob(productionCdAwsUsEast1a); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index db07aff34e5..a57e302d939 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; @@ -170,9 +169,15 @@ public class InternalStepRunnerTest { public void waitsForEndpointsAndTimesOut() { app.newRun(JobType.systemTest); - // Tester fails to show up for staging tests, and the real deployment for system tests. - tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system())); - tester.setEndpoints(app.instanceId(), JobType.stagingTest.zone(system())); + // Tester endpoint fails to show up for staging tests, and the real deployment for system tests. + var testZone = JobType.systemTest.zone(system()); + var stagingZone = JobType.stagingTest.zone(system()); + tester.newDeploymentContext(app.testerId().id()) + .deferLoadBalancerProvisioningIn(testZone.environment()); + tester.newDeploymentContext(app.instanceId()) + .deferLoadBalancerProvisioningIn(stagingZone.environment()); + tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(new DeploymentId(app.testerId().id(), testZone), List.of()); + tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(new DeploymentId(app.instanceId(), stagingZone), List.of()); tester.runner().run(); tester.configServer().convergeServices(app.instanceId(), JobType.stagingTest.zone(system())); @@ -197,14 +202,10 @@ public class InternalStepRunnerTest { // Node is down too long in system test, and no nodes go down in staging. tester.runner().run(); - tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system())); tester.configServer().setVersion(app.testerId().id(), JobType.systemTest.zone(system()), tester.controller().systemVersion()); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); - tester.setEndpoints(app.instanceId(), JobType.systemTest.zone(system())); - tester.setEndpoints(app.testerId().id(), JobType.stagingTest.zone(system())); tester.configServer().setVersion(app.testerId().id(), JobType.stagingTest.zone(system()), tester.controller().systemVersion()); tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); - tester.setEndpoints(app.instanceId(), JobType.stagingTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester)); @@ -240,13 +241,11 @@ public class InternalStepRunnerTest { app.newRun(JobType.systemTest); tester.runner().run(); tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); - tester.setEndpoints(app.instanceId(), JobType.systemTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); tester.applications().deactivate(app.instanceId(), JobType.systemTest.zone(system())); - tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); @@ -266,8 +265,6 @@ public class InternalStepRunnerTest { tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); - app.addRoutingPolicy(JobType.systemTest.zone(system()), true); - app.addTesterRoutingPolicy(JobType.systemTest.zone(system()), true); tester.runner().run();; assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); @@ -314,13 +311,13 @@ public class InternalStepRunnerTest { RunId id = app.startSystemTestTests(); tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); + var testZone = JobType.systemTest.zone(system()); Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get(); assertEquals(app.instanceId().serializedForm(), configObject.field("application").asString()); - assertEquals(JobType.systemTest.zone(system()).value(), configObject.field("zone").asString()); + assertEquals(testZone.value(), configObject.field("zone").asString()); assertEquals(system().value(), configObject.field("system").asString()); assertEquals(1, configObject.field("endpoints").children()); - assertEquals(1, configObject.field("endpoints").field(JobType.systemTest.zone(system()).value()).entries()); - configObject.field("endpoints").field(JobType.systemTest.zone(system()).value()).traverse((ArrayTraverser) (__, endpoint) -> assertEquals(tester.routing().endpoints(new DeploymentId(instanceId, JobType.systemTest.zone(system()))).get(0).endpoint(), endpoint.asString())); + assertEquals(1, configObject.field("endpoints").field(testZone.value()).entries()); long lastId = tester.jobs().details(id).get().lastId().getAsLong(); tester.cloud().add(new LogEntry(0, Instant.ofEpochMilli(123), info, "Ready!")); @@ -366,7 +363,6 @@ public class InternalStepRunnerTest { tester.runner().run(); // Job run order determined by JobType enum order per application. tester.configServer().convergeServices(app.instanceId(), zone); - tester.setEndpoints(app.instanceId(), zone); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); assertEquals(applicationPackage.hash(), tester.configServer().application(app.instanceId(), zone).get().applicationPackage().hash()); assertEquals(otherPackage.hash(), tester.configServer().application(app.instanceId(), JobType.perfUsEast3.zone(system())).get().applicationPackage().hash()); @@ -375,12 +371,6 @@ public class InternalStepRunnerTest { tester.runner().run(); assertEquals(1, tester.jobs().active().size()); assertEquals(version, tester.instance(app.instanceId()).deployments().get(zone).version()); - - try { - tester.jobs().deploy(app.instanceId(), JobType.productionApNortheast1, Optional.empty(), applicationPackage); - fail("Deployments outside dev should not be allowed."); - } - catch (IllegalArgumentException expected) { } } @Test @@ -422,13 +412,13 @@ public class InternalStepRunnerTest { @Test public void certificateTimeoutAbortsJob() { - tester.controllerTester().zoneRegistry().setSystemName(SystemName.PublicCd); + tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public); var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), - ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c")); + ZoneApiMock.fromId("staging.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-us-east-1c")); tester.controllerTester().zoneRegistry() .setZones(zones) - .setRoutingMethod(zones, RoutingMethod.shared); + .setRoutingMethod(zones, RoutingMethod.exclusive); tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values()); RunId id = app.startSystemTestTests(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java index f31038a7aba..05330d306fe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -29,8 +29,8 @@ public class TestConfigSerializerTest { byte[] json = new TestConfigSerializer(SystemName.PublicCd).configJson(instanceId, JobType.systemTest, true, - Map.of(zone, Map.of(ClusterSpec.Id.from("ai"), - URI.create("https://server/"))), + Map.of(zone, Map.of(URI.create("https://server/"), + ClusterSpec.Id.from("ai"))), Map.of(zone, List.of("facts"))); byte[] expected = Files.readAllBytes(Paths.get("src/test/resources/testConfig.json")); assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlime(expected))), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 5d93881129f..204e0a1c1b8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -5,13 +5,15 @@ import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -19,7 +21,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; @@ -50,6 +51,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -72,6 +74,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Version initialVersion = new Version(6, 1, 0); private final Set<DeploymentId> suspendedApplications = new HashSet<>(); private final Map<ZoneId, Set<LoadBalancer>> loadBalancers = new HashMap<>(); + private final Set<Environment> deferLoadBalancerProvisioning = new HashSet<>(); private final Map<DeploymentId, List<Log>> warnings = new HashMap<>(); private final Map<DeploymentId, Set<String>> rotationNames = new HashMap<>(); private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>(); @@ -245,6 +248,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer this.clusterMetrics.put(deployment, clusterMetrics); } + public void deferLoadBalancerProvisioningIn(Set<Environment> environments) { + deferLoadBalancerProvisioning.addAll(environments); + } + @Override public NodeRepositoryMock nodeRepository() { return nodeRepository; @@ -309,43 +316,51 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, - Set<ContainerEndpoint> containerEndpoints, - Optional<EndpointCertificateMetadata> endpointCertificateMetadata, byte[] content) { - lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null); + public PreparedApplication deploy(DeploymentData deployment) { + lastPrepareVersion = deployment.platform(); if (prepareException != null) { RuntimeException prepareException = this.prepareException; this.prepareException = null; throw prepareException; } - applications.put(deployment, new Application(deployment.applicationId(), lastPrepareVersion, new ApplicationPackage(content))); + DeploymentId id = new DeploymentId(deployment.instance(), deployment.zone()); + applications.put(id, new Application(id.applicationId(), lastPrepareVersion, new ApplicationPackage(deployment.applicationPackage()))); - if (nodeRepository().list(deployment.zoneId(), deployment.applicationId()).isEmpty()) - provision(deployment.zoneId(), deployment.applicationId()); + if (nodeRepository().list(id.zoneId(), id.applicationId()).isEmpty()) + provision(id.zoneId(), id.applicationId()); this.rotationNames.put( - deployment, - containerEndpoints.stream() - .map(ContainerEndpoint::names) - .flatMap(Collection::stream) - .collect(Collectors.toSet()) + id, + deployment.containerEndpoints().stream() + .map(ContainerEndpoint::names) + .flatMap(Collection::stream) + .collect(Collectors.toSet()) ); + if (!deferLoadBalancerProvisioning.contains(id.zoneId().environment())) { + putLoadBalancers(id.zoneId(), List.of(new LoadBalancer(UUID.randomUUID().toString(), + id.applicationId(), + ClusterSpec.Id.from("default"), + HostName.from("lb-0--" + id.applicationId().serializedForm() + "--" + id.zoneId().toString()), + LoadBalancer.State.active, + Optional.of("dns-zone-1")))); + } + return () -> { - Application application = applications.get(deployment); + Application application = applications.get(id); application.activate(); - List<Node> nodes = nodeRepository.list(deployment.zoneId(), deployment.applicationId()); + List<Node> nodes = nodeRepository.list(id.zoneId(), id.applicationId()); for (Node node : nodes) { - nodeRepository.putByHostname(deployment.zoneId(), new Node.Builder(node) + nodeRepository.putByHostname(id.zoneId(), new Node.Builder(node) .state(Node.State.active) .wantedVersion(application.version().get()) .build()); } - serviceStatus.put(deployment, new ServiceConvergence(deployment.applicationId(), - deployment.zoneId(), - false, - 2, - nodes.stream() + serviceStatus.put(id, new ServiceConvergence(id.applicationId(), + id.zoneId(), + false, + 2, + nodes.stream() .map(node -> new ServiceConvergence.Status(node.hostname(), 43, "container", @@ -360,7 +375,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer Collections.emptyList()); setConfigChangeActions(null); prepareResponse.tenant = new TenantId("tenant"); - prepareResponse.log = warnings.getOrDefault(deployment, Collections.emptyList()); + prepareResponse.log = warnings.getOrDefault(id, Collections.emptyList()); return prepareResponse; }; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 323b86be1d3..cbb93a51bfd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -64,7 +64,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); this.configServerMock = new ConfigServerMock(zoneRegistryMock); - this.routingGeneratorMock = new RoutingGeneratorMock(RoutingGeneratorMock.TEST_ENDPOINTS, zoneRegistryMock); + this.routingGeneratorMock = new RoutingGeneratorMock(); } @Inject diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 2092c5ec9a3..efc875b06f5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -116,6 +116,9 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } public ZoneRegistryMock setRoutingMethod(ZoneApi zone, List<RoutingMethod> routingMethods) { + if (routingMethods.stream().distinct().count() != routingMethods.size()) { + throw new IllegalArgumentException("Routing methods must be distinct"); + } this.zoneRoutingMethods.put(zone, List.copyOf(routingMethods)); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java deleted file mode 100644 index ff72a2f7231..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import org.junit.Test; - -import java.time.Duration; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author smorgrav - */ -public class ClusterInfoMaintainerTest { - - private final ControllerTester tester = new ControllerTester(); - - @Test - public void maintain() { - tester.createTenant("tenant1", "domain123", 321L); - ApplicationId app = tester.createApplication("tenant1", "app1", "default").id().defaultInstance(); - ZoneId zone = ZoneId.from("dev", "us-east-1"); - tester.deploy(app, zone); - - // Precondition: no cluster info attached to the deployments - Deployment deployment = tester.controller().applications().getInstance(app).get().deployments().values().stream() - .findFirst() - .get(); - assertEquals(0, deployment.clusterInfo().size()); - - addNodes(zone); - ClusterInfoMaintainer maintainer = new ClusterInfoMaintainer(tester.controller(), Duration.ofHours(1), - new JobControl(new MockCuratorDb())); - maintainer.maintain(); - - deployment = tester.controller().applications().getInstance(app).get().deployments().values().stream() - .findFirst() - .get(); - assertEquals(2, deployment.clusterInfo().size()); - assertEquals(10, deployment.clusterInfo().get(ClusterSpec.Id.from("clusterA")).getFlavorCost()); - } - - private void addNodes(ZoneId zone) { - var nodeA = new Node.Builder() - .hostname(HostName.from("hostA")) - .parentHostname(HostName.from("parentHostA")) - .state(Node.State.active) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(1, 1, 1, 1)) - .cost(10) - .clusterId("clusterA") - .clusterType(Node.ClusterType.container) - .build(); - var nodeB = new Node.Builder() - .hostname(HostName.from("hostB")) - .parentHostname(HostName.from("parentHostB")) - .state(Node.State.active) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(1, 1, 1, 1)) - .cost(20) - .clusterId("clusterB") - .clusterType(Node.ClusterType.container) - .build(); - tester.configServer().nodeRepository().addNodes(zone, List.of(nodeA, nodeB)); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index dde1333eb5f..008052d1b5d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -99,7 +99,7 @@ public class ApplicationSerializerTest { Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), - createClusterInfo(3, 4), + Map.of(), new DeploymentMetrics(2, 3, 4, 5, 6, Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)), Map.of(DeploymentMetrics.Warning.all, 3)), @@ -188,14 +188,7 @@ public class ApplicationSerializerTest { assertEquals(original.require(id3.instance()).change(), serialized.require(id3.instance()).change()); // Test cluster info - assertEquals(3, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().size()); - assertEquals(10, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCost()); - assertEquals(ClusterSpec.Type.content, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getClusterType()); - assertEquals("flavor2", serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavor()); - assertEquals(4, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getHostnames().size()); - assertEquals(2, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCPU(), Double.MIN_VALUE); - assertEquals(4, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorMem(), Double.MIN_VALUE); - assertEquals(50, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorDisk(), Double.MIN_VALUE); + assertEquals(0, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().size()); // Test metrics assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java index 5f8a3eaa98a..58957e50d2d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java @@ -4,6 +4,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCe import org.junit.Test; import java.util.List; +import java.util.Optional; import static org.junit.Assert.*; @@ -12,7 +13,7 @@ public class EndpointCertificateMetadataSerializerTest { private EndpointCertificateMetadata sample = new EndpointCertificateMetadata("keyName", "certName", 1); private EndpointCertificateMetadata sampleWithRequestMetadata = - new EndpointCertificateMetadata("keyName", "certName", 1, "requestId", List.of("SAN1", "SAN2")); + new EndpointCertificateMetadata("keyName", "certName", 1, Optional.of("requestId"), Optional.of(List.of("SAN1", "SAN2")), Optional.of("issuer")); @Test public void serialize() { @@ -24,7 +25,7 @@ public class EndpointCertificateMetadataSerializerTest { @Test public void serializeWithRequestMetadata() { assertEquals( - "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"]}", + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}", EndpointCertificateMetadataSerializer.toSlime(sampleWithRequestMetadata).toString()); } @@ -41,6 +42,6 @@ public class EndpointCertificateMetadataSerializerTest { assertEquals( sampleWithRequestMetadata, EndpointCertificateMetadataSerializer.fromJsonString( - "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"]}")); + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}")); } }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 0df94598935..11bcb983fe2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; +import com.yahoo.vespa.hosted.controller.deployment.JobProfile; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; @@ -148,7 +149,7 @@ public class RunSerializerTest { assertEquals(run.versions(), phoenix.versions()); assertEquals(run.steps(), phoenix.steps()); - Run initial = Run.initial(id, run.versions(), run.start()); + Run initial = Run.initial(id, run.versions(), run.start(), JobProfile.production); assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 019231321d6..810b9c2550c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -45,7 +45,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; import com.yahoo.vespa.hosted.controller.api.integration.resource.MockTenantCost; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; @@ -258,8 +257,22 @@ public class ApplicationApiTest extends ControllerContainerTest { ApplicationId id = ApplicationId.from("tenant1", "application1", "instance1"); var app1 = deploymentTester.newDeploymentContext(id); - // POST (deploy) an application to start a manual deployment to dev + // POST (deploy) an application to start a manual deployment in prod is not allowed MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST) + .data(entity) + .userIdentity(USER_ID), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Direct deployments are only allowed to manually deployed environments.\"}", 400); + + // POST (deploy) an application to start a manual deployment in prod is allowed for operators + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST) + .data(entity) + .userIdentity(HOSTED_VESPA_OPERATOR), + "{\"message\":\"Deployment started in run 1 of production-us-east-3 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); + app1.runJob(JobType.productionUsEast3); + tester.controller().applications().deactivate(app1.instanceId(), ZoneId.from("prod", "us-east-3")); + + // POST (deploy) an application to start a manual deployment to dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1/", POST) .data(entity) .userIdentity(USER_ID), @@ -634,10 +647,7 @@ public class ApplicationApiTest extends ControllerContainerTest { ZoneId.from("dev", "us-east-1"), Optional.of(applicationPackageDefault), new DeployOptions(false, Optional.empty(), false, false)); - tester.serviceRegistry().routingGeneratorMock().putEndpoints(new DeploymentId(ApplicationId.from("tenant1", "application1", "default"), ZoneId.from("prod", "us-central-1")), - List.of(new RoutingEndpoint("https://us-central-1.prod.default", "host", false, "upstream"))); - tester.serviceRegistry().routingGeneratorMock().putEndpoints(new DeploymentId(ApplicationId.from("tenant1", "application1", "my-user"), ZoneId.from("dev", "us-east-1")), - List.of(new RoutingEndpoint("https://us-east-1.dev.my-user", "host", false, "upstream"))); + // GET test-config for local tests against a dev deployment tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/my-user/job/dev-us-east-1/test-config", GET) .userIdentity(USER_ID), @@ -798,8 +808,6 @@ public class ApplicationApiTest extends ControllerContainerTest { // Create tenant and deploy var app = deploymentTester.newDeploymentContext(createTenantAndApplication()); app.submit(applicationPackage).deploy(); - app.addRoutingPolicy(westZone, true); - app.addRoutingPolicy(eastZone, true); // Invalid application fails tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation", GET) @@ -1467,8 +1475,6 @@ public class ApplicationApiTest extends ControllerContainerTest { } - - @Test public void applicationWithRoutingPolicy() { var app = deploymentTester.newDeploymentContext(createTenantAndApplication()); @@ -1481,8 +1487,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .region(zone.region().value()) .build(); app.submit(applicationPackage).deploy(); - app.addRoutingPolicy(zone, true); - app.addRoutingPolicy(zone, false); + app.addInactiveRoutingPolicy(zone); // GET application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) @@ -1643,7 +1648,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private void assertGlobalRouting(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) { var changedAt = tester.controller().clock().instant(); - var westPolicies = tester.controller().routingController().policies().get(deployment); + var westPolicies = tester.controller().routing().policies().get(deployment); assertEquals(1, westPolicies.size()); var westPolicy = westPolicies.values().iterator().next(); assertEquals(status, westPolicy.status().globalRouting().status()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 1c96f46dd31..6b71968822f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -23,7 +23,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; -import java.util.Date; import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE; @@ -153,7 +152,6 @@ public class JobControllerApiHandlerHelperTest { tester.configServer().setLogStream("Nope, this won't be logged"); tester.configServer().convergeServices(app.instanceId(), zone); - tester.setEndpoints(app.instanceId(), zone); tester.runner().run(); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root")), "dev-overview.json"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 282c18046d3..fe9d0573ed2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -464,8 +464,8 @@ ], "runs": [ { - "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1", + "id": 2, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2", "start": "(ignore)", "status": "aborted", "versions": { @@ -491,6 +491,31 @@ "status": "unfinished" } ] + }, + { + "id": 1, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1", + "start": "(ignore)", + "end": "(ignore)", + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": {} + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] } ] }, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index eb8bf523474..cd47859c7cc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -15,20 +15,13 @@ { "cluster": "default", "tls": true, - "url": "https://c0.instance1.application1.tenant1.global.vespa.oath.cloud/", - "scope": "global", - "routingMethod": "exclusive" - }, - { - "cluster": "default", - "tls": true, - "url": "https://instance1--application1--tenant1.us-west-1.prod.vespa:43", + "url": "https://instance1--application1--tenant1.us-west-1.vespa.oath.cloud:4443/", "scope": "zone", "routingMethod": "shared" } ], "serviceUrls": [ - "https://instance1--application1--tenant1.us-west-1.prod.vespa:43" + "https://instance1--application1--tenant1.us-west-1.vespa.oath.cloud:4443/" ], "nodes": "http://localhost:8080/zone/v2/prod/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-west-1&application=tenant1.application1.instance1", @@ -52,12 +45,6 @@ }, "status": "complete", "activity": {}, - "cost": { - "tco": 0, - "waste": 0, - "utilization": 0.0, - "cluster": {} - }, "metrics": { "queriesPerSecond": 0.0, "writesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index ef8899c0860..726df575028 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -8,12 +8,12 @@ { "cluster": "default", "tls": true, - "url": "https://instance1--application1--tenant1.us-central-1.prod.vespa:43", + "url": "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/", "scope": "zone", "routingMethod": "shared" }, { - "cluster": "foo", + "cluster": "", "tls": true, "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", "scope": "global", @@ -21,7 +21,7 @@ } ], "serviceUrls": [ - "https://instance1--application1--tenant1.us-central-1.prod.vespa:43" + "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/" ], "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", @@ -59,12 +59,6 @@ "lastQueriesPerSecond": 1.0, "lastWritesPerSecond": 2.0 }, - "cost": { - "tco": 0, - "waste": 0, - "utilization": 0.0, - "cluster": {} - }, "metrics": { "queriesPerSecond": 1.0, "writesPerSecond": 2.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json index b63031fab4f..d151e615f17 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json @@ -6,7 +6,7 @@ { "at": 0, "type": "info", - "message": " |-- https://default--application--tenant.us-east-1.dev.vespa:43 (cluster 'default')" + "message": " |-- https://application--tenant.us-east-1.dev.vespa.oath.cloud:4443/ (cluster 'default')" }, { "at": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index cc930c94051..7c231beb5ed 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -8,13 +8,13 @@ { "cluster": "default", "tls": true, - "url": "https://instance1--application1--tenant1.us-east-1.dev.vespa:43", + "url": "https://instance1--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/", "scope": "zone", "routingMethod": "shared" } ], "serviceUrls": [ - "https://instance1--application1--tenant1.us-east-1.dev.vespa:43" + "https://instance1--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/" ], "nodes": "http://localhost:8080/zone/v2/dev/us-east-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-east-1&application=tenant1.application1.instance1", @@ -28,12 +28,6 @@ "lastQueriesPerSecond": 1.0, "lastWritesPerSecond": 2.0 }, - "cost": { - "tco": 0, - "waste": 0, - "utilization": 0.0, - "cluster": {} - }, "metrics": { "queriesPerSecond": 1.0, "writesPerSecond": 2.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json index a21b1558aee..f7c512842fd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json @@ -1 +1,11 @@ -{"globalrotationoverride":["cluster1.application1.tenant1.us-west-1.prod",{"status":"in","reason":"","agent":"","timestamp":1497618757}]} +{ + "globalrotationoverride": [ + "instance1.application1.tenant1.us-west-1.prod", + { + "status": "in", + "reason": "", + "agent": "", + "timestamp": 1497618757 + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json index 18f5127718f..16061053b98 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json @@ -180,9 +180,7 @@ ], "changeBlockers": [], "compileVersion": "(ignore)", - "globalRotations": [ - "https://c0.instance1.application1.tenant1.global.vespa.oath.cloud/" - ], + "globalRotations": [], "instances": [ { "environment": "prod", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json index fd8bc256ac5..3d1c5ade300 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json @@ -13,8 +13,8 @@ "projectId": 123, "deploying": { "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -30,10 +30,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -47,10 +47,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -64,10 +64,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -85,10 +85,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -102,10 +102,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -119,10 +119,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -140,10 +140,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -157,10 +157,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -174,10 +174,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -192,13 +192,13 @@ }, { "type": "production-us-east-3", - "success": false, + "success": true, "lastTriggered": { - "id": 1, - "version": "(ignore)", + "id": 2, + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -209,6 +209,18 @@ }, "reason": "unknown reason", "at": "(ignore)" + }, + "lastCompleted": { + "id": 1, + "version": "6.1.0", + "reason": "unknown reason", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 1, + "version": "6.1.0", + "reason": "unknown reason", + "at": "(ignore)" } }, { @@ -263,7 +275,7 @@ "rotationId": "rotation-id-1", "clusterId": "foo", "status": "IN", - "lastUpdated":"(ignore)" + "lastUpdated": "(ignore)" } ], "environment": "prod", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json index ee75d129241..1b7e7893222 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json @@ -13,8 +13,8 @@ "projectId": 123, "deploying": { "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -30,10 +30,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -47,10 +47,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -64,10 +64,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -85,10 +85,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -102,10 +102,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -119,10 +119,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -140,10 +140,10 @@ "success": true, "lastTriggered": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -157,10 +157,10 @@ }, "lastCompleted": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -174,10 +174,10 @@ }, "lastSuccess": { "id": 1, - "version": "(ignore)", + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -192,13 +192,13 @@ }, { "type": "production-us-east-3", - "success": false, + "success": true, "lastTriggered": { - "id": 1, - "version": "(ignore)", + "id": 2, + "version": "6.1.0", "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -209,6 +209,18 @@ }, "reason": "unknown reason", "at": "(ignore)" + }, + "lastCompleted": { + "id": 1, + "version": "6.1.0", + "reason": "unknown reason", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 1, + "version": "6.1.0", + "reason": "unknown reason", + "at": "(ignore)" } }, { @@ -241,7 +253,7 @@ ] } ], - "compileVersion": "(ignore)", + "compileVersion": "6.0.0", "globalRotations": [ "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 8cd102432d0..b16ca4cc67c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -275,7 +275,7 @@ "us-east-3": { "runs": [ { - "id": 1, + "id": 2, "status": "aborted", "start": "(ignore)", "wantedPlatform": "6.1", @@ -296,6 +296,26 @@ "report": "unfinished" }, "tasks": {}, + "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2" + }, + { + "id": 1, + "status": "success", + "start": "(ignore)", + "end": "(ignore)", + "wantedPlatform": "6.1", + "wantedApplication": { + "hash": "unknown" + }, + "steps": { + "deployReal": "succeeded", + "installReal": "succeeded", + "copyVespaLogs": "succeeded" + }, + "tasks": { + "deploy": "succeeded", + "install": "succeeded" + }, "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1" } ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 436c2767b3e..41f3908f12f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -11,12 +11,12 @@ { "cluster": "default", "tls": true, - "url": "https://instance1--application1--tenant1.us-central-1.prod.vespa:43", + "url": "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/", "scope": "zone", "routingMethod": "shared" }, { - "cluster": "foo", + "cluster": "", "tls": true, "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", "scope": "global", @@ -24,7 +24,7 @@ } ], "serviceUrls": [ - "https://instance1--application1--tenant1.us-central-1.prod.vespa:43" + "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/" ], "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", @@ -62,12 +62,6 @@ "lastQueriesPerSecond": 1.0, "lastWritesPerSecond": 2.0 }, - "cost": { - "tco": 0, - "waste": 0, - "utilization": 0.0, - "cluster": {} - }, "metrics": { "queriesPerSecond": 1.0, "writesPerSecond": 2.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index 6db83522c26..c01ab2f61b3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -212,7 +212,7 @@ { "at": "(ignore)", "type": "info", - "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa:43 (cluster 'default')" + "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')" }, { "at": "(ignore)", @@ -239,7 +239,7 @@ { "at": "(ignore)", "type": "info", - "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa:43 (cluster 'default')" + "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')" }, { "at": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json index 818dc72a9d9..0632ab7a67b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json @@ -5,18 +5,18 @@ "isCI": false, "endpoints": { "dev.us-east-1": [ - "https://us-east-1.dev.my-user" + "https://my-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/" ], "prod.us-central-1": [ - "https://us-central-1.prod.default" + "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" ] }, "zoneEndpoints": { "dev.us-east-1": { - "default": "https://us-east-1.dev.my-user" + "default": "https://my-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/" }, "prod.us-central-1": { - "default": "https://us-central-1.prod.default" + "default": "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" } }, "clusters": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json index c1ccbc7100f..c81ed767239 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json @@ -5,12 +5,12 @@ "isCI": false, "endpoints": { "prod.us-central-1": [ - "https://us-central-1.prod.default" + "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" ] }, "zoneEndpoints": { "prod.us-central-1": { - "default": "https://us-central-1.prod.default" + "default": "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" } }, "clusters": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 3371c5563c9..fbdf8caaed7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -10,9 +10,6 @@ "name": "CloudEventReporter" }, { - "name": "ClusterInfoMaintainer" - }, - { "name": "ContactInformationMaintainer" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index d191b460697..f079bd1d7e6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -131,8 +131,6 @@ public class RoutingApiTest extends ControllerContainerTest { .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) .build(); context.submit(applicationPackage).deploy(); - context.addRoutingPolicy(westZone, true); - context.addRoutingPolicy(eastZone, true); // GET initial deployment status tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", @@ -255,9 +253,6 @@ public class RoutingApiTest extends ControllerContainerTest { .build(); context.submit(applicationPackage).deploy(); - // Assign policy in one zone - context.addRoutingPolicy(westZone, true); - // GET status with both policy and rotation assigned tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json index 079e2c9c388..56108dce94f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json @@ -11,57 +11,21 @@ "administrator", "developer", "reader" - ], - "applications": { - "app1": { - "instances": [ - "default" - ] - }, - "app2": { - "instances": [ - "default", - "dev" - ] - } - } + ] }, "tenant1": { "roles": [ "administrator", "developer", "reader" - ], - "applications": { - "app1": { - "instances": [ - "default" - ] - }, - "app2": { - "instances": [ - "default", - "myinstance" - ] - }, - "app3": { - "instances": [] - } - } + ] }, "tenant2": { "roles": [ "administrator", "developer", "reader" - ], - "applications": { - "app2": { - "instances": [ - "test" - ] - } - } + ] } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json index f0ea10ed888..ea76aa977ce 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json @@ -10,53 +10,17 @@ "roles": [ "developer", "reader" - ], - "applications": { - "app1": { - "instances": [ - "default" - ] - }, - "app2": { - "instances": [ - "default", - "dev" - ] - } - } + ] }, "tenant1": { "roles": [ "administrator" - ], - "applications": { - "app1": { - "instances": [ - "default" - ] - }, - "app2": { - "instances": [ - "default", - "myinstance" - ] - }, - "app3": { - "instances": [] - } - } + ] }, "tenant2": { "roles": [ "developer" - ], - "applications": { - "app2": { - "instances": [ - "test" - ] - } - } + ] } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index 1bb531edbc5..b67422bc06b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -2,12 +2,16 @@ package com.yahoo.vespa.hosted.controller.rotation; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.routing.RoutingId; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Before; import org.junit.Rule; @@ -55,7 +59,7 @@ public class RotationRepositoryTest { @Before public void before() { tester = new DeploymentTester(new ControllerTester(rotationsConfig)); - repository = tester.controller().routingController().rotations(); + repository = tester.controller().routing().rotations(); application = tester.newDeploymentContext("tenant1", "app1", "default"); } @@ -67,7 +71,8 @@ public class RotationRepositoryTest { assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), - application.instance().endpointsIn(SystemName.main).main().get().url()); + EndpointList.global(RoutingId.of(application.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared) + .primary().get().url()); try (RotationLock lock = repository.lock()) { List<AssignedRotation> rotations = repository.getOrAssignRotations(application.application().deploymentSpec(), application.instance(), @@ -83,7 +88,7 @@ public class RotationRepositoryTest { @Test public void strips_whitespace_in_rotation_fqdn() { tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces)); - RotationRepository repository = tester.controller().routingController().rotations(); + RotationRepository repository = tester.controller().routing().rotations(); var application2 = tester.newDeploymentContext("tenant1", "app2", "default"); application2.submit(applicationPackage); @@ -143,7 +148,8 @@ public class RotationRepositoryTest { application2.submit(applicationPackage); assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations())); assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", - application2.instance().endpointsIn(SystemName.cd).main().get().url().toString()); + EndpointList.global(RoutingId.of(application2.instanceId(), EndpointId.defaultId()), SystemName.cd, RoutingMethod.shared) + .primary().get().url().toString()); } @Test @@ -159,9 +165,11 @@ public class RotationRepositoryTest { assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - instance1.instance().endpointsIn(SystemName.main).main().get().url()); + EndpointList.global(RoutingId.of(instance1.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared) + .primary().get().url()); assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - instance2.instance().endpointsIn(SystemName.main).main().get().url()); + EndpointList.global(RoutingId.of(instance2.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared) + .primary().get().url()); } @Test @@ -179,9 +187,11 @@ public class RotationRepositoryTest { assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - instance1.instance().endpointsIn(SystemName.main).main().get().url()); + EndpointList.global(RoutingId.of(instance1.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared) + .primary().get().url()); assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - instance2.instance().endpointsIn(SystemName.main).main().get().url()); + EndpointList.global(RoutingId.of(instance2.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared) + .primary().get().url()); } private void assertSingleRotation(Rotation expected, List<AssignedRotation> assignedRotations, RotationRepository repository) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index a27b2e1084a..a48554e2af4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; @@ -31,7 +32,6 @@ import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; -import java.net.URI; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -39,7 +39,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -248,22 +247,6 @@ public class RoutingPoliciesTest { } @Test - public void cluster_endpoints_resolve_from_policies() { - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.provisionLoadBalancers(3, context.instanceId(), zone1, zone2); - context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(context.deploymentIdIn(zone1), List.of()); - assertEquals(Map.of(ClusterSpec.Id.from("c0"), - URI.create("https://c0.app1.tenant1.us-west-1.vespa.oath.cloud/"), - ClusterSpec.Id.from("c1"), - URI.create("https://c1.app1.tenant1.us-west-1.vespa.oath.cloud/"), - ClusterSpec.Id.from("c2"), - URI.create("https://c2.app1.tenant1.us-west-1.vespa.oath.cloud/")), - tester.controllerTester().controller().routingController().zoneEndpointsOf(context.deploymentIdIn(zone1))); - } - - @Test public void manual_deployment_creates_routing_policy() { // Empty application package is valid in manually deployed environments var tester = new RoutingPoliciesTester(); @@ -274,7 +257,6 @@ public class RoutingPoliciesTest { tester.controllerTester().serviceRegistry().zoneRegistry() .setZones(zoneApi) .exclusiveRoutingIn(zoneApi); - tester.provisionLoadBalancers(1, context.instanceId(), zone); // Deploy to dev tester.controllerTester().controller().applications().deploy(context.instanceId(), zone, Optional.of(emptyApplicationPackage), DeployOptions.none()); @@ -283,7 +265,7 @@ public class RoutingPoliciesTest { // Routing policy is created and DNS is updated assertEquals(1, tester.policiesOf(context.instanceId()).size()); - assertEquals(Set.of("c0.app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); + assertEquals(Set.of("app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); } @Test @@ -302,14 +284,13 @@ public class RoutingPoliciesTest { // Deploy to dev under different instance var devInstance = context.application().id().instance("user"); - tester.provisionLoadBalancers(1, devInstance, zone); tester.controllerTester().controller().applications().deploy(devInstance, zone, Optional.of(applicationPackage), DeployOptions.none()); assertEquals("DeploymentSpec is persisted", applicationPackage.deploymentSpec(), context.application().deploymentSpec()); context.flushDnsUpdates(); // Routing policy is created and DNS is updated assertEquals(1, tester.policiesOf(devInstance).size()); - assertEquals(Sets.union(prodRecords, Set.of("c0.user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames()); + assertEquals(Sets.union(prodRecords, Set.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames()); } @Test @@ -618,7 +599,7 @@ public class RoutingPoliciesTest { } public RoutingPolicies routingPolicies() { - return tester.controllerTester().controller().routingController().policies(); + return tester.controllerTester().controller().routing().policies(); } public DeploymentContext newDeploymentContext(String tenant, String application, String instance) { @@ -668,7 +649,8 @@ public class RoutingPoliciesTest { } private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) { - var endpoint = RoutingPolicy.globalEndpointOf(application, endpointId, tester.controller().system()).dnsName(); + var endpoint = EndpointList.global(RoutingId.of(application, endpointId), tester.controller().system(), RoutingMethod.exclusive) + .primary().get().dnsName(); var zoneTargets = Arrays.stream(zone) .map(z -> "lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + z.value() + "/dns-zone-1/" + z.value()) diff --git a/dist/vespa.spec b/dist/vespa.spec index 1c1f5071684..4281ce243fa 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -108,6 +108,7 @@ BuildRequires: openblas-devel BuildRequires: lz4-devel BuildRequires: libzstd-devel BuildRequires: zlib-devel +BuildRequires: re2-devel %if ! 0%{?el7} BuildRequires: libicu-devel %endif @@ -151,6 +152,7 @@ Requires: openblas-serial Requires: lz4 Requires: libzstd Requires: zlib +Requires: re2 %if ! 0%{?el7} Requires: libicu %endif @@ -162,6 +164,7 @@ Requires: llvm5.0 Requires: vespa-openssl >= 1.1.1c-1 Requires: vespa-icu >= 65.1.0-1 Requires: vespa-protobuf >= 3.7.0-4 +Requires: vespa-telegraf >= 1.1.1-1 %define _vespa_llvm_version 5.0 %define _extra_link_directory /usr/lib64/llvm5.0/lib;%{_vespa_deps_prefix}/lib64 %define _extra_include_directory /usr/include/llvm5.0;%{_vespa_deps_prefix}/include;/usr/include/openblas diff --git a/docker-api/CMakeLists.txt b/docker-api/CMakeLists.txt index 1288c624890..edcdcfb5bfc 100644 --- a/docker-api/CMakeLists.txt +++ b/docker-api/CMakeLists.txt @@ -1,4 +1,2 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(docker-api) -install(DIRECTORY DESTINATION conf/node-admin-app/components) -install_symlink(lib/jars/docker-api-jar-with-dependencies.jar conf/node-admin-app/components/docker-api.jar) +install(FILES target/docker-api-jar-with-dependencies.jar DESTINATION conf/node-admin-app/components) diff --git a/eval/src/tests/ann/bruteforce-nns.h b/eval/src/tests/ann/bruteforce-nns.h index 0c7c48654f7..ecac73c0d10 100644 --- a/eval/src/tests/ann/bruteforce-nns.h +++ b/eval/src/tests/ann/bruteforce-nns.h @@ -47,7 +47,7 @@ public: TopK bruteforce_nns(const PointVector &query) { TopK result; BfHitHeap heap(result.K); - for (uint32_t docid = 0; docid < NUM_DOCS; ++docid) { + for (uint32_t docid = 0; docid < EFFECTIVE_DOCS; ++docid) { const PointVector &docvector = generatedDocs[docid]; double d = l2distCalc.l2sq_dist(query, docvector); Hit h(docid, d); @@ -64,7 +64,7 @@ void verifyBF(uint32_t qid) { const PointVector &query = generatedQueries[qid]; TopK &result = bruteforceResults[qid]; double min_distance = result.hits[0].distance; - for (uint32_t i = 0; i < NUM_DOCS; ++i) { + for (uint32_t i = 0; i < EFFECTIVE_DOCS; ++i) { double dist = computeDistance(query, i); if (dist < min_distance) { fprintf(stderr, "WARN dist %.9g < mindist %.9g\n", dist, min_distance); diff --git a/eval/src/tests/ann/gist_benchmark.cpp b/eval/src/tests/ann/gist_benchmark.cpp index de8bff877e6..5a317e77e72 100644 --- a/eval/src/tests/ann/gist_benchmark.cpp +++ b/eval/src/tests/ann/gist_benchmark.cpp @@ -11,6 +11,7 @@ #define NUM_DIMS 960 #define NUM_DOCS 200000 +#define EFFECTIVE_DOCS NUM_DOCS #define NUM_REACH 10000 #define NUM_Q 1000 diff --git a/eval/src/tests/ann/sift_benchmark.cpp b/eval/src/tests/ann/sift_benchmark.cpp index b2fa66cd0f1..4bbe8f61ef1 100644 --- a/eval/src/tests/ann/sift_benchmark.cpp +++ b/eval/src/tests/ann/sift_benchmark.cpp @@ -12,6 +12,7 @@ #define NUM_DIMS 128 #define NUM_DOCS 1000000 +#define EFFECTIVE_DOCS NUM_DOCS #define NUM_Q 1000 #define NUM_REACH 10000 diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 57228dfe49d..c97edf9ef3d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -192,7 +192,19 @@ public class Flags { "Regularly issue a small write to disk and fail the host if it is not successful", "Takes effect on next node agent tick (but does not clear existing failure reports)", HOSTNAME); - + + public static final UnboundBooleanFlag RESTRICT_ACQUIRING_NEW_PRIVILEGES = defineFeatureFlag( + "restrict-acquiring-new-privileges", false, + "Whether docker daemon should restrict containers from acquiring new privileges", + "Takes effect on next host admin tick", + HOSTNAME); + + public static final UnboundListFlag<String> AUDITED_PATHS = defineListFlag( + "audited-paths", List.of(), String.class, + "List of paths that should audited", + "Takes effect on next host admin tick", + HOSTNAME); + public static final UnboundBooleanFlag GENERATE_L4_ROUTING_CONFIG = defineFeatureFlag( "generate-l4-routing-config", false, "Whether routing nodes should generate L4 routing config", @@ -213,8 +225,7 @@ public class Flags { public static final UnboundStringFlag ENDPOINT_CERTIFICATE_BACKFILL = defineStringFlag( "endpoint-certificate-backfill", "disable", "Whether the endpoint certificate maintainer should backfill missing certificate data from cameo", - "Takes effect on next scheduled run of maintainer - set to \"disable\", \"dryrun\" or \"enable\"" - ); + "Takes effect on next scheduled run of maintainer - set to \"disable\", \"dryrun\" or \"enable\""); public static final UnboundBooleanFlag USE_NEW_ATHENZ_FILTER = defineFeatureFlag( "use-new-athenz-filter", false, @@ -238,6 +249,12 @@ public class Flags { "Whether to provision and use endpoint certs for apps in shared routing zones", "Takes effect on next deployment of the application", APPLICATION_ID); + public static final UnboundBooleanFlag PHRASE_SEGMENTING = defineFeatureFlag( + "phrase-segmenting", true, + "Should 'implicit phrases' in queries we parsed to a phrase or and?", + "Takes effect on redeploy", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index f1486ae7117..ec3baf30eae 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -17,6 +17,7 @@ import com.yahoo.slime.JsonFormat; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; +import javax.net.ssl.SSLContext; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -37,6 +38,7 @@ import java.util.ArrayList; import java.util.List; import java.util.OptionalLong; import java.util.concurrent.Callable; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; @@ -76,6 +78,11 @@ public abstract class ControllerHttpClient { return new SigningControllerHttpClient(endpoint, privateKeyFile, id); } + /** Creates an HTTP client against the given endpoint, which uses the given SSL context for authentication. */ + public static ControllerHttpClient withSSLContext(URI endpoint, SSLContext sslContext) { + return new MutualTlsControllerHttpClient(endpoint, sslContext); + } + /** Creates an HTTP client against the given endpoint, which uses the given private key and certificate identity. */ public static ControllerHttpClient withKeyAndCertificate(URI endpoint, Path privateKeyFile, Path certificateFile) { var privateKey = unchecked(() -> KeyUtils.fromPemEncodedPrivateKey(Files.readString(privateKeyFile, UTF_8))); @@ -141,10 +148,36 @@ public abstract class ControllerHttpClient { /** Returns the sorted list of log entries after the given after from the deployment job of the given ids. */ public DeploymentLog deploymentLog(ApplicationId id, ZoneId zone, long run, long after) { return toDeploymentLog(send(request(HttpRequest.newBuilder(runPath(id, zone, run, after)) - .timeout(Duration.ofSeconds(10)), + .timeout(Duration.ofMinutes(2)), GET))); } + /** Follows the given deployment job until it is done, or this thread is interrupted, at which point the current status is returned. */ + public DeploymentLog followDeploymentUntilDone(ApplicationId id, ZoneId zone, long run, + Consumer<DeploymentLog.Entry> out) { + long last = -1; + DeploymentLog log = null; + while (true) { + DeploymentLog update = deploymentLog(id, zone, run, last); + for (DeploymentLog.Entry entry : update.entries()) + out.accept(entry); + log = (log == null ? update : log.updatedWith(update)); + last = log.last().orElse(last); + + if ( ! log.isActive()) + break; + + try { + Thread.sleep(1000); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + return log; + } + /** Returns the sorted list of log entries from the deployment job of the given ids. */ public DeploymentLog deploymentLog(ApplicationId id, ZoneId zone, long run) { return deploymentLog(id, zone, run, -1); @@ -384,14 +417,17 @@ public abstract class ControllerHttpClient { /** Client that uses a given key / certificate identity to authenticate to the remote controller. */ private static class MutualTlsControllerHttpClient extends ControllerHttpClient { + private MutualTlsControllerHttpClient(URI endpoint, SSLContext sslContext) { + super(endpoint, HttpClient.newBuilder().sslContext(sslContext)); + } + private MutualTlsControllerHttpClient(URI endpoint, PrivateKey privateKey, List<X509Certificate> certs) { - super(endpoint, - HttpClient.newBuilder() - .sslContext(new SslContextBuilder().withKeyStore(privateKey, certs).build())); + this(endpoint, new SslContextBuilder().withKeyStore(privateKey, certs).build()); } } + private static DeploymentLog.Status valueOf(String status) { switch (status) { case "running": return DeploymentLog.Status.running; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java index 177c72107e0..9eae9a33cff 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java @@ -4,6 +4,7 @@ package ai.vespa.hosted.api; import java.time.Instant; import java.util.List; import java.util.OptionalLong; +import java.util.stream.Stream; import static java.util.Comparator.comparing; import static java.util.stream.Collectors.toUnmodifiableList; @@ -27,6 +28,14 @@ public class DeploymentLog { this.last = last; } + /** Returns this log updated with the content of the other. */ + public DeploymentLog updatedWith(DeploymentLog other) { + return new DeploymentLog(Stream.concat(entries.stream(), other.entries.stream()).collect(toUnmodifiableList()), + other.active, + other.status, + other.last); + } + public List<Entry> entries() { return entries; } diff --git a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def index cb03ce6f1f6..8a87b92cd60 100644 --- a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def +++ b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def @@ -19,3 +19,4 @@ cloudWatch[].secretKeyName string default="" # Only valid and optional for self-hosted Vespa cloudWatch[].profile string default="" +cloudWatch[].file string default="" diff --git a/node-admin/CMakeLists.txt b/node-admin/CMakeLists.txt index 03bf09121c3..8dbb32c7de1 100644 --- a/node-admin/CMakeLists.txt +++ b/node-admin/CMakeLists.txt @@ -1,2 +1,6 @@ # Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install(DIRECTORY DESTINATION logs/vespa/node-admin) +install(FILES target/node-admin-jar-with-dependencies.jar DESTINATION conf/node-admin-app/components) +install_symlink(lib/jars/flags-jar-with-dependencies.jar conf/node-admin-app/components/flags-jar-with-dependencies.jar) +install(FILES src/main/application/services.xml DESTINATION conf/node-admin-app) +install(PROGRAMS src/main/sh/node-admin.sh DESTINATION libexec/vespa) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 71f7dc3701e..31f3ed5fc95 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; import java.time.Duration; import java.util.List; import java.util.Optional; +import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -24,6 +25,8 @@ import java.util.stream.Collectors; */ public class Autoscaler { + private Logger log = Logger.getLogger(Autoscaler.class.getName()); + /* TODO: - Scale group size @@ -69,20 +72,28 @@ public class Autoscaler { Optional<Double> cpuLoad = averageLoad(Resource.cpu, cluster, clusterNodes); Optional<Double> memoryLoad = averageLoad(Resource.memory, cluster, clusterNodes); Optional<Double> diskLoad = averageLoad(Resource.disk, cluster, clusterNodes); - if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) return Optional.empty(); + if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) { + log.fine("Autoscaling " + applicationId + " " + cluster + ": Insufficient metrics to decide"); + return Optional.empty(); + } Optional<ClusterResourcesWithCost> bestAllocation = findBestAllocation(cpuLoad.get(), memoryLoad.get(), diskLoad.get(), currentAllocation, cluster); - if (bestAllocation.isEmpty()) return Optional.empty(); + if (bestAllocation.isEmpty()) { + log.fine("Autoscaling " + applicationId + " " + cluster + ": Could not find a better allocation"); + return Optional.empty(); + } if (closeToIdeal(Resource.cpu, cpuLoad.get()) && closeToIdeal(Resource.memory, memoryLoad.get()) && closeToIdeal(Resource.disk, diskLoad.get()) && - similarCost(bestAllocation.get().cost(), currentAllocation.nodes() * costOf(currentAllocation.nodeResources()))) + similarCost(bestAllocation.get().cost(), currentAllocation.nodes() * costOf(currentAllocation.nodeResources()))) { + log.fine("Autoscaling " + applicationId + " " + cluster + ": Resources are almost ideal and price difference is small"); return Optional.empty(); // Avoid small, unnecessary changes + } return bestAllocation.map(a -> a.clusterResources()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index ed6e7cc71ef..e65b7273b9e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -60,8 +60,7 @@ public class MetricsReporter extends Maintainer { @Override public void maintain() { NodeList nodes = nodeRepository().list(); - Map<HostName, List<ServiceInstance>> servicesByHost = - serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName(); + Map<HostName, List<ServiceInstance>> servicesByHost = serviceMonitor.getServicesByHostname(); nodes.forEach(node -> updateNodeMetrics(node, servicesByHost)); updateStateMetrics(nodes); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index ca53b215237..8a5a119f6a7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; -import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.service.monitor.ServiceMonitor; import com.yahoo.yolean.Exceptions; @@ -187,7 +186,7 @@ public class NodeFailer extends Maintainer { Map<String, Node> activeNodesByHostname = nodeRepository().getNodes(Node.State.active).stream() .collect(Collectors.toMap(Node::hostname, node -> node)); - serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName() + serviceMonitor.getServicesByHostname() .forEach((hostName, serviceInstances) -> { Node node = activeNodesByHostname.get(hostName.s()); if (node == null) return; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index 178e8385008..6ae086146c4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -39,6 +39,7 @@ public class NodeMetricsDbMaintainer extends Maintainer { nodeMetricsDb.add(nodeMetrics.fetchMetrics(application)); } catch (Exception e) { + // TODO: Don't warn if this only happens occasionally if (warnings++ < maxWarningsPerInvocation) log.log(Level.WARNING, "Could not update metrics for " + application, e); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 26f8cffa519..1d3fe114e23 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -74,7 +74,7 @@ public class LoadBalancerProvisioner { if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type if (application.instance().isTester()) return; // Do not provision for tester instances try (var lock = db.lockLoadBalancers(application)) { - provision(application, cluster.id(), false, lock); + provision(application, effectiveId(cluster), false, lock); } } @@ -91,7 +91,7 @@ public class LoadBalancerProvisioner { public void activate(ApplicationId application, Set<ClusterSpec> clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { try (var lock = db.lockLoadBalancers(application)) { - var containerClusters = containerClusterOf(clusters); + var containerClusters = containerClustersOf(clusters); for (var clusterId : containerClusters) { // Provision again to ensure that load balancer instance is re-configured with correct nodes provision(application, clusterId, true, lock); @@ -183,7 +183,7 @@ public class LoadBalancerProvisioner { .owner(application) .filter(node -> node.state().isAllocated()) .container() - .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterId)) + .filter(node -> effectiveId(node.allocation().get().membership().cluster()).equals(clusterId)) .asList(); } @@ -202,11 +202,16 @@ public class LoadBalancerProvisioner { return reachable; } - private static Set<ClusterSpec.Id> containerClusterOf(Set<ClusterSpec> clusters) { + /** Returns the container cluster IDs of the given clusters */ + private static Set<ClusterSpec.Id> containerClustersOf(Set<ClusterSpec> clusters) { return clusters.stream() .filter(c -> c.type().isContainer()) - .map(ClusterSpec::id) + .map(LoadBalancerProvisioner::effectiveId) .collect(Collectors.toUnmodifiableSet()); } + private static ClusterSpec.Id effectiveId(ClusterSpec cluster) { + return cluster.combinedId().orElse(cluster.id()); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index a2579bee0a1..43d4c731759 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -141,28 +141,28 @@ public class MockNodeRepository extends NodeRepository { ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), - false); + false, Optional.empty()); activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), 1, null), zoneApp, provisioner); ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), - false); + false, Optional.empty()); provisioner.prepare(app1, cluster1, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null); ApplicationId app2 = ApplicationId.from(TenantName.from("tenant2"), ApplicationName.from("application2"), InstanceName.from("instance2")); ClusterSpec cluster2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id2"), Version.fromString("6.42"), - false); + false, Optional.empty()); activate(provisioner.prepare(app2, cluster2, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null), app2, provisioner); ApplicationId app3 = ApplicationId.from(TenantName.from("tenant3"), ApplicationName.from("application3"), InstanceName.from("instance3")); ClusterSpec cluster3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id3"), Version.fromString("6.42"), - false); + false, Optional.empty()); activate(provisioner.prepare(app3, cluster3, Capacity.fromCount(2, new NodeResources(1, 4, 100, 1), false, true), 1, null), app3, provisioner); List<Node> largeNodes = new ArrayList<>(); @@ -176,7 +176,7 @@ public class MockNodeRepository extends NodeRepository { ClusterSpec cluster4 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id4"), Version.fromString("6.42"), - false); + false, Optional.empty()); activate(provisioner.prepare(app4, cluster4, Capacity.fromCount(2, new NodeResources(10, 48, 500, 1), false, true), 1, null), app4, provisioner); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index f15b7e4220b..66a883aec2e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -78,7 +78,8 @@ class AutoscalingTester { return ClusterSpec.request(type, ClusterSpec.Id.from(clusterId), Version.fromString("7"), - false); + false, + Optional.empty()); } public void deploy(ApplicationId application, ClusterSpec cluster, ClusterResources resources) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index c293a3436b8..d0179cdda00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -57,7 +57,7 @@ public class FailedExpirerTest { ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), - false); + false, Optional.empty()); private static final Capacity tenantHostApplicationCapacity = Capacity.fromRequiredNodeType(NodeType.host); @@ -144,7 +144,7 @@ public class FailedExpirerTest { .withNode(NodeType.proxy, FailureScenario.defaultFlavor, "proxy3") .setReady("proxy1", "proxy2", "proxy3") .allocate( ApplicationId.from("vespa", "zone-app", "default"), - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("routing"), Version.fromString("6.42"), false), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("routing"), Version.fromString("6.42"), false, Optional.empty()), Capacity.fromRequiredNodeType(NodeType.proxy)) .failNode(1, "proxy1"); @@ -326,8 +326,8 @@ public class FailedExpirerTest { ClusterSpec clusterSpec = ClusterSpec.request(clusterType, ClusterSpec.Id.from("test"), Version.fromString("6.42"), - false - ); + false, + Optional.empty()); Capacity capacity = Capacity.fromCount(hostname.length, Optional.of(flavor), false, true); return allocate(applicationId, clusterSpec, capacity); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index f9c2ef90bbd..b617a2dcf85 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -54,7 +55,7 @@ public class InactiveAndFailedExpirerTest { List<Node> nodes = tester.makeReadyNodes(2, nodeResources); // Allocate then deallocate 2 nodes - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1); tester.activate(applicationId, new HashSet<>(preparedNodes)); assertEquals(2, tester.getNodes(applicationId, Node.State.active).size()); @@ -95,7 +96,7 @@ public class InactiveAndFailedExpirerTest { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), - false); + false, Optional.empty()); List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1); tester.activate(applicationId, new HashSet<>(preparedNodes)); assertEquals(2, tester.getNodes(applicationId, Node.State.active).size()); @@ -122,7 +123,7 @@ public class InactiveAndFailedExpirerTest { public void node_that_wants_to_retire_is_moved_to_parked() throws OrchestrationException { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), - Version.fromString("6.42"), false); + Version.fromString("6.42"), false, Optional.empty()); tester.makeReadyNodes(5, nodeResources); // Allocate two nodes @@ -178,7 +179,7 @@ public class InactiveAndFailedExpirerTest { // Allocate then deallocate a node ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); tester.makeReadyNodes(1, nodeResources); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); List<HostSpec> preparedNodes = tester.prepare(testerId, cluster, Capacity.fromCount(2, nodeResources), 1); tester.activate(testerId, new HashSet<>(preparedNodes)); assertEquals(1, tester.getNodes(testerId, Node.State.active).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index 7e8fcddb1ae..a3c95e1a825 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -17,6 +17,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -159,7 +160,7 @@ public class LoadBalancerExpirerTest { List<HostSpec> hosts = new ArrayList<>(); for (var cluster : clusters) { hosts.addAll(tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster, - Vtag.currentVersion, false), + Vtag.currentVersion, false, Optional.empty()), 2, 1, new NodeResources(1, 4, 10, 0.3))); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 321812497bd..8daec1d641e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -26,7 +26,6 @@ import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; -import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; import org.junit.Test; @@ -97,9 +96,7 @@ public class MetricsReporterTest { when(orchestrator.getHostResolver()).thenReturn(hostName -> Optional.of(HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.ofEpochSecond(1))) ); - ServiceModel serviceModel = mock(ServiceModel.class); - when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); - when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); + when(serviceMonitor.getServicesByHostname()).thenReturn(Map.of()); TestMetric metric = new TestMetric(); MetricsReporter metricsReporter = new MetricsReporter( @@ -146,9 +143,7 @@ public class MetricsReporterTest { Orchestrator orchestrator = mock(Orchestrator.class); ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); when(orchestrator.getHostResolver()).thenReturn(hostName -> Optional.of(HostInfo.createNoRemarks())); - ServiceModel serviceModel = mock(ServiceModel.class); - when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); - when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); + when(serviceMonitor.getServicesByHostname()).thenReturn(Map.of()); TestMetric metric = new TestMetric(); ManualClock clock = new ManualClock(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 033ddcd827e..e0e0a320763 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -89,8 +89,8 @@ public class NodeFailTester { tester.createHostNodes(3); // Create applications - ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); - ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); + ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); Capacity capacity1 = Capacity.fromCount(5, nodeResources, false, true); Capacity capacity2 = Capacity.fromCount(7, nodeResources, false, true); @@ -120,9 +120,9 @@ public class NodeFailTester { } // Create applications - ClusterSpec clusterNodeAdminApp = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), false); - ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false); - ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false); + ClusterSpec clusterNodeAdminApp = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), false, Optional.empty()); + ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false, Optional.empty()); + ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false, Optional.empty()); Capacity allHosts = Capacity.fromRequiredNodeType(NodeType.host); Capacity capacity1 = Capacity.fromCount(3, new NodeResources(1, 4, 10, 0.3), false, true); Capacity capacity2 = Capacity.fromCount(5, new NodeResources(1, 4, 10, 0.3), false, true); @@ -154,7 +154,7 @@ public class NodeFailTester { ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), - false); + false, Optional.empty()); tester.activate(app1, clusterApp1, allNodes); assertEquals(count, tester.nodeRepository.getNodes(nodeType, Node.State.active).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index 22d7f03c449..e2745760637 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -134,9 +134,9 @@ public class OperatorChangeApplicationMaintainerTest { final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz")); final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz")); final ApplicationId app3 = ApplicationId.from(TenantName.from("vespa-hosted"), ApplicationName.from("routing"), InstanceName.from("default")); - final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); - final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); - final ClusterSpec clusterApp3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("routing"), Version.fromString("6.42"), false); + final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + final ClusterSpec clusterApp3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("routing"), Version.fromString("6.42"), false, Optional.empty()); final int wantedNodesApp1 = 5; final int wantedNodesApp2 = 7; final int wantedNodesApp3 = 2; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index 913b8b53c46..5c87684da76 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -244,8 +244,8 @@ public class PeriodicApplicationMaintainerTest { final NodeResources nodeResources = new NodeResources(2, 8, 50, 1); final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz")); final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz")); - final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); - final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); + final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); + final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); final int wantedNodesApp1 = 5; final int wantedNodesApp2 = 7; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index d0c678bdf45..d80aad7cc25 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -129,7 +129,7 @@ public class RebalancerTest { } private ClusterSpec clusterSpec(String clusterId) { - return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false); + return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false, Optional.empty()); } private ApplicationId makeApplicationId(String tenant, String appName) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 96c3cc09b6b..83e41211da9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -61,7 +61,7 @@ public class ReservationExpirerTest { assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.dirty).size()); nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("bar").instanceName("fuz").build(); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); provisioner.prepare(applicationId, cluster, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null); assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.reserved).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index bdbe046fbdf..c82cbafaaa1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -86,7 +86,7 @@ public class RetiredExpirerTest { // Allocate content cluster of sizes 7 -> 2 -> 3: // Should end up with 3 nodes in the cluster (one previously retired), and 4 retired - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); int wantedNodes; activate(applicationId, cluster, wantedNodes=7, 1, provisioner); activate(applicationId, cluster, wantedNodes=2, 1, provisioner); @@ -117,7 +117,7 @@ public class RetiredExpirerTest { ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); activate(applicationId, cluster, 8, 8, provisioner); activate(applicationId, cluster, 2, 2, provisioner); assertEquals(8, nodeRepository.getNodes(applicationId, Node.State.active).size()); @@ -148,7 +148,7 @@ public class RetiredExpirerTest { // Allocate content cluster of sizes 7 -> 2 -> 3: // Should end up with 3 nodes in the cluster (one previously retired), and 4 retired - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); int wantedNodes; activate(applicationId, cluster, wantedNodes=7, 1, provisioner); activate(applicationId, cluster, wantedNodes=2, 1, provisioner); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index 1e59add8304..cb2e3f65c40 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -52,7 +52,7 @@ public class DockerProvisioningTest { Version wantedVespaVersion = Version.fromString("6.39"); int nodeCount = 7; List<HostSpec> hosts = tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), nodeCount, 1, dockerFlavor); tester.activate(application1, new HashSet<>(hosts)); @@ -63,7 +63,7 @@ public class DockerProvisioningTest { // Upgrade Vespa version on nodes Version upgradedWantedVespaVersion = Version.fromString("6.40"); List<HostSpec> upgradedHosts = tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), upgradedWantedVespaVersion, false), + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), upgradedWantedVespaVersion, false, Optional.empty()), nodeCount, 1, dockerFlavor); tester.activate(application1, new HashSet<>(upgradedHosts)); NodeList upgradedNodes = tester.getNodes(application1, Node.State.active); @@ -85,7 +85,7 @@ public class DockerProvisioningTest { Version wantedVespaVersion = Version.fromString("6.39"); int nodeCount = 7; List<HostSpec> nodes = tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), nodeCount, 1, dockerFlavor); try { tester.activate(application1, new HashSet<>(nodes)); @@ -94,13 +94,13 @@ public class DockerProvisioningTest { // Activate the zone-app, thereby allocating the parents List<HostSpec> hosts = tester.prepare(zoneApplication, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("zone-app"), wantedVespaVersion, false), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("zone-app"), wantedVespaVersion, false, Optional.empty()), Capacity.fromRequiredNodeType(NodeType.host), 1); tester.activate(zoneApplication, hosts); // Try allocating tenants again nodes = tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), nodeCount, 1, dockerFlavor); tester.activate(application1, new HashSet<>(nodes)); @@ -124,14 +124,14 @@ public class DockerProvisioningTest { Version wantedVespaVersion = Version.fromString("6.39"); List<HostSpec> nodes = tester.prepare(application2_1, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), 6, 1, resources); assertHostSpecParentReservation(nodes, Optional.empty(), tester); // We do not get nodes on hosts reserved to tenant1 tester.activate(application2_1, nodes); try { tester.prepare(application2_2, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), 5, 1, resources); fail("Expected exception"); } @@ -140,7 +140,7 @@ public class DockerProvisioningTest { } nodes = tester.prepare(application1_1, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false, Optional.empty()), 10, 1, resources); assertHostSpecParentReservation(nodes, Optional.of(tenant1), tester); tester.activate(application1_1, nodes); @@ -262,7 +262,7 @@ public class DockerProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost"); - List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false), 1, 1, dockerFlavor); + List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false, Optional.empty()), 1, 1, dockerFlavor); tester.activate(application1, new HashSet<>(hosts)); NodeList nodes = tester.getNodes(application1, Node.State.active); @@ -280,7 +280,7 @@ public class DockerProvisioningTest { tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost2"); List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), - Version.fromString("6.42"), false), 2, 1, + Version.fromString("6.42"), false, Optional.empty()), 2, 1, dockerFlavor.with(NodeResources.StorageType.remote)); } catch (OutOfCapacityException e) { @@ -294,7 +294,7 @@ public class DockerProvisioningTest { private void prepareAndActivate(ApplicationId application, int nodeCount, boolean exclusive, ProvisioningTester tester) { Set<HostSpec> hosts = new HashSet<>(tester.prepare(application, - ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.39"), exclusive), + ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.39"), exclusive, Optional.empty()), Capacity.fromCount(nodeCount, Optional.of(dockerFlavor), false, true), 1)); tester.activate(application, hosts); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 53fecbf6095..7eb379aa404 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -343,7 +343,7 @@ public class DynamicDockerAllocationTest { tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty()); NodeResources resources = new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any); List<HostSpec> hosts = tester.prepare(application, cluster, 2, 1, resources); @@ -360,7 +360,7 @@ public class DynamicDockerAllocationTest { tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty()); NodeResources resources = new NodeResources(1, 4, 10, 1, requestDiskSpeed); try { @@ -382,7 +382,7 @@ public class DynamicDockerAllocationTest { tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty()); NodeResources resources = new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.fast); List<HostSpec> hosts = tester.prepare(application, cluster, 4, 1, resources); @@ -401,7 +401,7 @@ public class DynamicDockerAllocationTest { tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false, Optional.empty()); List<HostSpec> hosts1 = tester.prepare(application, cluster, Capacity.fromCount(2, Optional.of(NodeResources.fromLegacyName("d-2-8-50")), false, true), 1); tester.activate(application, hosts1); @@ -466,7 +466,7 @@ public class DynamicDockerAllocationTest { } private ClusterSpec clusterSpec(String clusterId) { - return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false); + return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false, Optional.empty()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 8706661f261..2868edeaa58 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -1,10 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -23,6 +21,7 @@ import org.junit.Test; import java.time.Instant; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -146,7 +145,7 @@ public class DynamicDockerProvisionTest { } private static ClusterSpec clusterSpec(String clusterId) { - return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false); + return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false, Optional.empty()); } @SuppressWarnings("unchecked") diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java index 0ad7d37d13b..d83ed0c12dc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java @@ -18,6 +18,7 @@ import org.junit.Test; import java.util.Collection; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -51,9 +52,9 @@ public class InPlaceResizeProvisionTest { private static final NodeResources mediumResources = new NodeResources(4, 8, 16, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); private static final NodeResources largeResources = new NodeResources(8, 16, 32, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); - private static final ClusterSpec container1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), Version.fromString("7.157.9"), false); - private static final ClusterSpec container2 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container2"), Version.fromString("7.157.9"), false); - private static final ClusterSpec content1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("content1"), Version.fromString("7.157.9"), false); + private static final ClusterSpec container1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), Version.fromString("7.157.9"), false, Optional.empty()); + private static final ClusterSpec container2 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container2"), Version.fromString("7.157.9"), false, Optional.empty()); + private static final ClusterSpec content1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("content1"), Version.fromString("7.157.9"), false, Optional.empty()); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final ProvisioningTester tester = new ProvisioningTester.Builder() diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index ee9a582c4db..d4037c94a45 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -199,8 +200,9 @@ public class LoadBalancerProvisionerTest { assertEquals(List.of(), tester.nodeRepository().loadBalancers(app1).asList()); } + // TODO(mpolden): Remove when ClusterSpec with combined type rejects empty combinedId @Test - public void provision_load_balancer_combined_cluster() { + public void provision_load_balancer_combined_cluster_without_id() { Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(app1).asList(); ClusterSpec.Id cluster = ClusterSpec.Id.from("foo"); @@ -211,6 +213,18 @@ public class LoadBalancerProvisionerTest { assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); } + @Test + public void provision_load_balancer_combined_cluster() { + Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(app1).asList(); + var combinedId = ClusterSpec.Id.from("container1"); + var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, ClusterSpec.Id.from("content1"), Optional.of(combinedId))); + assertEquals(1, lbs.get().size()); + assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().reals().size()); + tester.activate(app1, nodes); + assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); + assertEquals(combinedId, lbs.get().get(0).id().cluster()); + } + private void dirtyNodesOf(ApplicationId application) { tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, this.getClass().getSimpleName()); } @@ -254,7 +268,11 @@ public class LoadBalancerProvisionerTest { } private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id) { - return ClusterSpec.request(type, id, Version.fromString("6.42"), false); + return clusterRequest(type, id, Optional.empty()); + } + + private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id, Optional<ClusterSpec.Id> combinedId) { + return ClusterSpec.request(type, id, Version.fromString("6.42"), false, combinedId); } private static <T> T get(Set<T> set, int position) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index 6ab027cd143..549441eb9e6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -204,7 +204,7 @@ public class MultigroupProvisioningTest { assertEquals("No additional groups are retained containing retired nodes", wantedGroups, allGroups.size()); } - private ClusterSpec cluster() { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); } + private ClusterSpec cluster() { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false, Optional.empty()); } private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, int groupCount, ProvisioningTester tester) { return new HashSet<>(tester.prepare(application, cluster(), capacity, groupCount)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java index f0bd43b6bae..5cd31f00895 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java @@ -18,6 +18,7 @@ import java.time.Duration; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -38,7 +39,7 @@ public class NodeTypeProvisioningTest { private final ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), - false); + false, Optional.empty()); @Before public void setup() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 166d5a1f654..1ab94e852ba 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -3,20 +3,17 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; @@ -33,7 +30,6 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -477,7 +473,7 @@ public class ProvisioningTest { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music"), new com.yahoo.component.Version(4, 5, 6), - false); + false, Optional.empty()); tester.prepare(application, cluster, Capacity.fromCount(5, Optional.empty(), false, false), 1); // No exception; Success } @@ -514,7 +510,7 @@ public class ProvisioningTest { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music"), new com.yahoo.component.Version(4, 5, 6), - false); + false, Optional.empty()); tester.activate(application, tester.prepare(application, cluster, capacity, 1)); assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size()); assertEquals(0, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); @@ -691,10 +687,10 @@ public class ProvisioningTest { int content1Size, boolean required, NodeResources nodeResources, Version wantedVersion, ProvisioningTester tester) { // "deploy prepare" with a two container clusters and a storage cluster having of two groups - ClusterSpec containerCluster0 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container0"), wantedVersion, false); - ClusterSpec containerCluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), wantedVersion, false); - ClusterSpec contentCluster0 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content0"), wantedVersion, false); - ClusterSpec contentCluster1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1"), wantedVersion, false); + ClusterSpec containerCluster0 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container0"), wantedVersion, false, Optional.empty()); + ClusterSpec containerCluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), wantedVersion, false, Optional.empty()); + ClusterSpec contentCluster0 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content0"), wantedVersion, false, Optional.empty()); + ClusterSpec contentCluster1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1"), wantedVersion, false, Optional.empty()); Set<HostSpec> container0 = prepare(application, containerCluster0, container0Size, 1, required, nodeResources, tester); Set<HostSpec> container1 = prepare(application, containerCluster1, container1Size, 1, required, nodeResources, tester); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 85a6ed31073..03d0298900a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -158,7 +158,7 @@ public class ProvisioningTester { } public void prepareAndActivateInfraApplication(ApplicationId application, NodeType nodeType, Version version) { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString()), version, false); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString()), version, false, Optional.empty()); Capacity capacity = Capacity.fromRequiredNodeType(nodeType); List<HostSpec> hostSpecs = prepare(application, cluster, capacity, 1, true); activate(application, hostSpecs); @@ -412,7 +412,7 @@ public class ProvisioningTester { ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), - false), + false, Optional.empty()), Capacity.fromRequiredNodeType(NodeType.host), 1); activate(applicationId, Set.copyOf(list)); @@ -420,7 +420,7 @@ public class ProvisioningTester { public List<Node> deploy(ApplicationId application, Capacity capacity) { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), - Version.fromString("6.42"), false); + Version.fromString("6.42"), false, Optional.empty()); List<HostSpec> prepared = prepare(application, cluster, capacity, 1); activate(application, Set.copyOf(prepared)); return getNodes(application, Node.State.active).asList(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index f8c6e31cfd7..60616481665 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -36,8 +36,8 @@ import static org.junit.Assert.assertNotNull; public class VirtualNodeProvisioningTest { private static final NodeResources flavor = new NodeResources(4, 8, 100, 1); - private static final ClusterSpec contentClusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false); - private static final ClusterSpec containerClusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.42"), false); + private static final ClusterSpec contentClusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false, Optional.empty()); + private static final ClusterSpec containerClusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.42"), false, Optional.empty()); private ProvisioningTester tester = new ProvisioningTester.Builder().build(); private ApplicationId applicationId = tester.makeApplicationId(); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/InstanceLookupService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/InstanceLookupService.java deleted file mode 100644 index 554b6e11501..00000000000 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/InstanceLookupService.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.orchestrator; - -import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; -import com.yahoo.vespa.applicationmodel.ApplicationInstance; -import com.yahoo.vespa.applicationmodel.HostName; - -import java.util.Optional; -import java.util.Set; - -/** - * @author oyving - */ -public interface InstanceLookupService { - - Optional<ApplicationInstance> findInstanceById(ApplicationInstanceReference applicationInstanceReference); - Optional<ApplicationInstance> findInstanceByHost(HostName hostName); - Set<ApplicationInstanceReference> knownInstances(); - -} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index d0062966a6d..d4b55e92271 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -31,11 +31,12 @@ import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; import com.yahoo.vespa.orchestrator.policy.Policy; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.ApplicationLock; import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; -import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import java.io.IOException; import java.time.Clock; @@ -59,7 +60,7 @@ public class OrchestratorImpl implements Orchestrator { private final Policy policy; private final StatusService statusService; - private final InstanceLookupService instanceLookupService; + private final ServiceMonitor serviceMonitor; private final int serviceMonitorConvergenceLatencySeconds; private final ClusterControllerClientFactory clusterControllerClientFactory; private final Clock clock; @@ -70,24 +71,26 @@ public class OrchestratorImpl implements Orchestrator { public OrchestratorImpl(ClusterControllerClientFactory clusterControllerClientFactory, StatusService statusService, OrchestratorConfig orchestratorConfig, - InstanceLookupService instanceLookupService, + ServiceMonitor serviceMonitor, ConfigserverConfig configServerConfig, FlagSource flagSource) { - this(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, new ApplicationApiFactory(configServerConfig.zookeeperserver().size())), - clusterControllerClientFactory, - statusService, - instanceLookupService, - orchestratorConfig.serviceMonitorConvergenceLatencySeconds(), - Clock.systemUTC(), - new ApplicationApiFactory(configServerConfig.zookeeperserver().size()), - flagSource); + this(new HostedVespaPolicy(new HostedVespaClusterPolicy(), + clusterControllerClientFactory, + new ApplicationApiFactory(configServerConfig.zookeeperserver().size())), + clusterControllerClientFactory, + statusService, + serviceMonitor, + orchestratorConfig.serviceMonitorConvergenceLatencySeconds(), + Clock.systemUTC(), + new ApplicationApiFactory(configServerConfig.zookeeperserver().size()), + flagSource); } public OrchestratorImpl(Policy policy, ClusterControllerClientFactory clusterControllerClientFactory, StatusService statusService, - InstanceLookupService instanceLookupService, + ServiceMonitor serviceMonitor, int serviceMonitorConvergenceLatencySeconds, Clock clock, ApplicationApiFactory applicationApiFactory, @@ -97,7 +100,7 @@ public class OrchestratorImpl implements Orchestrator { this.clusterControllerClientFactory = clusterControllerClientFactory; this.statusService = statusService; this.serviceMonitorConvergenceLatencySeconds = serviceMonitorConvergenceLatencySeconds; - this.instanceLookupService = instanceLookupService; + this.serviceMonitor = serviceMonitor; this.clock = clock; this.applicationApiFactory = applicationApiFactory; this.retireWithPermanentlyDownFlag = Flags.RETIRE_WITH_PERMANENTLY_DOWN.bindTo(flagSource); @@ -105,7 +108,10 @@ public class OrchestratorImpl implements Orchestrator { @Override public Host getHost(HostName hostName) throws HostNameNotFoundException { - ApplicationInstance applicationInstance = getApplicationInstance(hostName); + ApplicationInstance applicationInstance = serviceMonitor + .getApplicationNarrowedTo(hostName) + .orElseThrow(() -> new HostNameNotFoundException(hostName)); + List<ServiceInstance> serviceInstances = applicationInstance .serviceClusters().stream() .flatMap(cluster -> cluster.serviceInstances().stream()) @@ -113,7 +119,6 @@ public class OrchestratorImpl implements Orchestrator { .collect(Collectors.toList()); HostInfo hostInfo = statusService.getHostInfo(applicationInstance.reference(), hostName); - HostStatus hostStatus = getNodeStatus(applicationInstance.reference(), hostName); return new Host(hostName, hostInfo, applicationInstance.reference(), serviceInstances); } @@ -125,17 +130,17 @@ public class OrchestratorImpl implements Orchestrator { @Override public Function<HostName, Optional<HostInfo>> getHostResolver() { - return hostName -> instanceLookupService.findInstanceByHost(hostName) - .map(application -> statusService.getHostInfo(application.reference(), hostName)); + return hostName -> serviceMonitor + .getApplication(hostName) + .map(application -> statusService.getHostInfo(application.reference(), hostName)); } @Override public void setNodeStatus(HostName hostName, HostStatus status) throws OrchestrationException { ApplicationInstanceReference reference = getApplicationInstance(hostName).reference(); OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock); - try (MutableStatusRegistry statusRegistry = statusService - .lockApplicationInstance_forCurrentThreadOnly(context, reference)) { - statusRegistry.setHostState(hostName, status); + try (ApplicationLock lock = statusService.lockApplication(context, reference)) { + lock.setHostState(hostName, status); } } @@ -163,9 +168,8 @@ public class OrchestratorImpl implements Orchestrator { ApplicationInstance appInstance = getApplicationInstance(hostName); OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock); - try (MutableStatusRegistry statusRegistry = statusService - .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) { - HostStatus currentHostState = statusRegistry.getHostInfos().getOrNoRemarks(hostName).status(); + try (ApplicationLock lock = statusService.lockApplication(context, appInstance.reference())) { + HostStatus currentHostState = lock.getHostInfos().getOrNoRemarks(hostName).status(); if (currentHostState == HostStatus.NO_REMARKS) { return; } @@ -176,11 +180,11 @@ public class OrchestratorImpl implements Orchestrator { // 2. The whole application is down: the content cluster states are set to maintenance, // and the host may be taken down manually at any moment. if (currentHostState == HostStatus.PERMANENTLY_DOWN || - statusRegistry.getStatus() == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) { + lock.getApplicationInstanceStatus() == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) { return; } - policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, statusRegistry); + policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, lock); } } @@ -200,10 +204,8 @@ public class OrchestratorImpl implements Orchestrator { .with(FetchVector.Dimension.HOSTNAME, hostName.s()) .value(); OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock, usePermanentlyDownStatus); - try (MutableStatusRegistry statusRegistry = statusService - .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) { - ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, statusRegistry, - clusterControllerClientFactory); + try (ApplicationLock lock = statusService.lockApplication(context, appInstance.reference())) { + ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, lock, clusterControllerClientFactory); policy.acquirePermissionToRemove(context.createSubcontextWithinLock(), applicationApi); } @@ -218,23 +220,22 @@ public class OrchestratorImpl implements Orchestrator { void suspendGroup(OrchestratorContext context, NodeGroup nodeGroup) throws HostStateChangeDeniedException { ApplicationInstanceReference applicationReference = nodeGroup.getApplicationReference(); - try (MutableStatusRegistry hostStatusRegistry = - statusService.lockApplicationInstance_forCurrentThreadOnly(context, applicationReference)) { - ApplicationInstanceStatus appStatus = hostStatusRegistry.getStatus(); + try (ApplicationLock lock = statusService.lockApplication(context, applicationReference)) { + ApplicationInstanceStatus appStatus = lock.getApplicationInstanceStatus(); if (appStatus == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) { return; } - ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, hostStatusRegistry, - clusterControllerClientFactory); + ApplicationApi applicationApi = applicationApiFactory.create( + nodeGroup, lock, clusterControllerClientFactory); policy.grantSuspensionRequest(context.createSubcontextWithinLock(), applicationApi); } } @Override public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationId appId) throws ApplicationIdNotFoundException { - ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(appId, instanceLookupService); - return statusService.getApplicationInstanceStatus(appRef); + ApplicationInstanceReference reference = OrchestratorUtil.toApplicationInstanceReference(appId, serviceMonitor); + return statusService.getApplicationInstanceStatus(reference); } @Override @@ -351,18 +352,17 @@ public class OrchestratorImpl implements Orchestrator { private void setApplicationStatus(ApplicationId appId, ApplicationInstanceStatus status) throws ApplicationStateChangeDeniedException, ApplicationIdNotFoundException{ OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock); - ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(appId, instanceLookupService); - try (MutableStatusRegistry statusRegistry = - statusService.lockApplicationInstance_forCurrentThreadOnly(context, appRef)) { + ApplicationInstanceReference reference = OrchestratorUtil.toApplicationInstanceReference(appId, serviceMonitor); + try (ApplicationLock lock = statusService.lockApplication(context, reference)) { // Short-circuit if already in wanted state - if (status == statusRegistry.getStatus()) return; + if (status == lock.getApplicationInstanceStatus()) return; // Set content clusters for this application in maintenance on suspend if (status == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) { - ApplicationInstance application = getApplicationInstance(appRef); + ApplicationInstance application = getApplicationInstance(reference); - HostInfos hostInfosSnapshot = statusRegistry.getHostInfos(); + HostInfos hostInfosSnapshot = lock.getHostInfos(); // Mark it allowed to be down before we manipulate the clustercontroller OrchestratorUtil.getHostsUsedByApplicationInstance(application) @@ -370,14 +370,14 @@ public class OrchestratorImpl implements Orchestrator { // This filter also ensures host status is not modified if a suspended host // has status != ALLOWED_TO_BE_DOWN. .filter(hostname -> !hostInfosSnapshot.getOrNoRemarks(hostname).status().isSuspended()) - .forEach(hostname -> statusRegistry.setHostState(hostname, HostStatus.ALLOWED_TO_BE_DOWN)); + .forEach(hostname -> lock.setHostState(hostname, HostStatus.ALLOWED_TO_BE_DOWN)); // If the clustercontroller throws an error the nodes will be marked as allowed to be down // and be set back up on next resume invocation. setClusterStateInController(context.createSubcontextWithinLock(), application, ClusterControllerNodeState.MAINTENANCE); } - statusRegistry.setApplicationInstanceStatus(status); + lock.setApplicationInstanceStatus(status); } } @@ -417,12 +417,13 @@ public class OrchestratorImpl implements Orchestrator { } private ApplicationInstance getApplicationInstance(HostName hostName) throws HostNameNotFoundException{ - return instanceLookupService.findInstanceByHost(hostName).orElseThrow( + return serviceMonitor.getApplication(hostName).orElseThrow( () -> new HostNameNotFoundException(hostName)); } - private ApplicationInstance getApplicationInstance(ApplicationInstanceReference appRef) throws ApplicationIdNotFoundException { - return instanceLookupService.findInstanceById(appRef).orElseThrow(ApplicationIdNotFoundException::new); + private ApplicationInstance getApplicationInstance(ApplicationInstanceReference reference) + throws ApplicationIdNotFoundException { + return serviceMonitor.getApplication(reference).orElseThrow(ApplicationIdNotFoundException::new); } private static void sleep(long time, TimeUnit timeUnit) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java index 91da046840d..40a45627f4d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import java.util.Collection; import java.util.List; @@ -65,7 +66,7 @@ public class OrchestratorUtil { private static final Pattern APPLICATION_INSTANCE_REFERENCE_REST_FORMAT_PATTERN = Pattern.compile("^([^:]+):(.+)$"); /** Returns an ApplicationInstanceReference constructed from the serialized format used in the REST API. */ - public static ApplicationInstanceReference parseAppInstanceReference(String restFormat) { + public static ApplicationInstanceReference parseApplicationInstanceReference(String restFormat) { if (restFormat == null) { throw new IllegalArgumentException("Could not construct instance id from null string"); } @@ -84,26 +85,26 @@ public class OrchestratorUtil { return applicationInstanceReference.tenantId() + ":" + applicationInstanceReference.applicationInstanceId(); } - - public static ApplicationInstanceReference toApplicationInstanceReference(ApplicationId appId, - InstanceLookupService instanceLookupService) + public static ApplicationInstanceReference toApplicationInstanceReference( + ApplicationId applicationid, + ServiceMonitor serviceMonitor) throws ApplicationIdNotFoundException { - Set<ApplicationInstanceReference> appRefs = instanceLookupService.knownInstances(); - List<ApplicationInstanceReference> appRefList = appRefs.stream() - .filter(a -> OrchestratorUtil.toApplicationId(a).equals(appId)) + Set<ApplicationInstanceReference> references = serviceMonitor.getAllApplicationInstanceReferences(); + List<ApplicationInstanceReference> referencesWithId = references.stream() + .filter(a -> OrchestratorUtil.toApplicationId(a).equals(applicationid)) .collect(Collectors.toList()); - if (appRefList.size() > 1) { - String msg = String.format("ApplicationId '%s' was not unique but mapped to '%s'", appId, appRefList); + if (referencesWithId.size() > 1) { + String msg = String.format("ApplicationId '%s' was not unique but mapped to '%s'", applicationid, referencesWithId); throw new ApplicationIdNotFoundException(msg); } - if (appRefList.size() == 0) { + if (referencesWithId.size() == 0) { throw new ApplicationIdNotFoundException(); } - return appRefList.get(0); + return referencesWithId.get(0); } public static ApplicationId toApplicationId(ApplicationInstanceReference appRef) { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java deleted file mode 100644 index 1a859cfacc5..00000000000 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.orchestrator; - -import com.google.inject.Inject; -import com.yahoo.vespa.applicationmodel.ApplicationInstance; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; -import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.service.monitor.ServiceMonitor; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Uses slobrok data (a.k.a. heartbeat) to implement {@link InstanceLookupService}. - * - * @author bakksjo - */ -public class ServiceMonitorInstanceLookupService implements InstanceLookupService { - - private final ServiceMonitor serviceMonitor; - - @Inject - public ServiceMonitorInstanceLookupService(ServiceMonitor serviceMonitor) { - this.serviceMonitor = serviceMonitor; - } - - @Override - public Optional<ApplicationInstance> findInstanceById(ApplicationInstanceReference applicationInstanceReference) { - return serviceMonitor.getServiceModelSnapshot().getApplicationInstance(applicationInstanceReference); - } - - @Override - public Optional<ApplicationInstance> findInstanceByHost(HostName hostName) { - return Optional.ofNullable(serviceMonitor.getServiceModelSnapshot().getApplicationsByHostName().get(hostName)); - } - - @Override - public Set<ApplicationInstanceReference> knownInstances() { - return serviceMonitor.getServiceModelSnapshot().getAllApplicationInstances().keySet(); - } - -} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java index 5e6ec59c28d..fb52eed2048 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.orchestrator.model; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; -import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; +import com.yahoo.vespa.orchestrator.status.ApplicationLock; /** * @author mpolden @@ -16,9 +16,9 @@ public class ApplicationApiFactory { } public ApplicationApi create(NodeGroup nodeGroup, - MutableStatusRegistry hostStatusService, + ApplicationLock lock, ClusterControllerClientFactory clusterControllerClientFactory) { - return new ApplicationApiImpl(nodeGroup, hostStatusService, clusterControllerClientFactory, numberOfConfigServers); + return new ApplicationApiImpl(nodeGroup, lock, clusterControllerClientFactory, numberOfConfigServers); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java index cf6946fa7f8..ccc89fa9191 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; -import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; +import com.yahoo.vespa.orchestrator.status.ApplicationLock; import java.util.Collection; import java.util.Comparator; @@ -32,19 +32,19 @@ public class ApplicationApiImpl implements ApplicationApi { private final ApplicationInstance applicationInstance; private final NodeGroup nodeGroup; - private final MutableStatusRegistry hostStatusService; + private final ApplicationLock lock; private final List<ClusterApi> clusterInOrder; private final HostInfos hostInfos; public ApplicationApiImpl(NodeGroup nodeGroup, - MutableStatusRegistry hostStatusService, + ApplicationLock lock, ClusterControllerClientFactory clusterControllerClientFactory, int numberOfConfigServers) { this.applicationInstance = nodeGroup.getApplication(); this.nodeGroup = nodeGroup; - this.hostStatusService = hostStatusService; + this.lock = lock; Collection<HostName> hosts = getHostsUsedByApplicationInstance(applicationInstance); - this.hostInfos = hostStatusService.getHostInfos(); + this.hostInfos = lock.getHostInfos(); this.clusterInOrder = makeClustersInOrder(nodeGroup, hostInfos, clusterControllerClientFactory, numberOfConfigServers); } @@ -95,12 +95,12 @@ public class ApplicationApiImpl implements ApplicationApi { @Override public ApplicationInstanceStatus getApplicationStatus() { - return hostStatusService.getStatus(); + return lock.getApplicationInstanceStatus(); } @Override public void setHostState(OrchestratorContext context, HostName hostName, HostStatus status) { - hostStatusService.setHostState(hostName, status); + lock.setHostState(hostName, status); } @Override diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java index f6a1e4f91f0..8b74d8a40ef 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.orchestrator.model.NodeGroup; import com.yahoo.vespa.orchestrator.model.StorageNode; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.HostStatus; -import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; +import com.yahoo.vespa.orchestrator.status.ApplicationLock; /** * @author oyving @@ -113,9 +113,9 @@ public class HostedVespaPolicy implements Policy { OrchestratorContext context, ApplicationInstance applicationInstance, HostName hostName, - MutableStatusRegistry hostStatusService) throws HostStateChangeDeniedException { + ApplicationLock lock) throws HostStateChangeDeniedException { NodeGroup nodeGroup = new NodeGroup(applicationInstance, hostName); - ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, hostStatusService, clusterControllerClientFactory); + ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, lock, clusterControllerClientFactory); releaseSuspensionGrant(context, applicationApi); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java index aa7636227c8..c410cda23a8 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java @@ -5,8 +5,7 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.model.ApplicationApi; -import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; -import com.yahoo.vespa.orchestrator.status.StatusService; +import com.yahoo.vespa.orchestrator.status.ApplicationLock; /** * @author oyving @@ -33,6 +32,6 @@ public interface Policy { void releaseSuspensionGrant( OrchestratorContext context, ApplicationInstance applicationInstance, HostName hostName, - MutableStatusRegistry hostStatusService) throws HostStateChangeDeniedException; + ApplicationLock hostStatusService) throws HostStateChangeDeniedException; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java index fbb8f445db0..1cf2a2a4965 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.orchestrator.InstanceLookupService; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse; import com.yahoo.vespa.orchestrator.restapi.wire.WireHostInfo; @@ -20,6 +19,7 @@ import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.StatusService; import com.yahoo.vespa.service.manager.MonitorManager; import com.yahoo.vespa.service.manager.UnionMonitorManager; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import com.yahoo.vespa.service.monitor.SlobrokApi; import javax.inject.Inject; @@ -33,12 +33,11 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.time.Instant; import java.util.List; -import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance; -import static com.yahoo.vespa.orchestrator.OrchestratorUtil.parseAppInstanceReference; +import static com.yahoo.vespa.orchestrator.OrchestratorUtil.parseApplicationInstanceReference; /** * Provides a read-only API for looking into the current state as seen by the Orchestrator. @@ -55,14 +54,14 @@ public class InstanceResource { private final StatusService statusService; private final SlobrokApi slobrokApi; private final MonitorManager rootManager; - private final InstanceLookupService instanceLookupService; + private final ServiceMonitor serviceMonitor; @Inject - public InstanceResource(@Component InstanceLookupService instanceLookupService, + public InstanceResource(@Component ServiceMonitor serviceMonitor, @Component StatusService statusService, @Component SlobrokApi slobrokApi, @Component UnionMonitorManager rootManager) { - this.instanceLookupService = instanceLookupService; + this.serviceMonitor = serviceMonitor; this.statusService = statusService; this.slobrokApi = slobrokApi; this.rootManager = rootManager; @@ -70,8 +69,8 @@ public class InstanceResource { @GET @Produces(MediaType.APPLICATION_JSON) - public Set<ApplicationInstanceReference> getAllInstances() { - return instanceLookupService.knownInstances(); + public List<ApplicationInstanceReference> getAllInstances() { + return serviceMonitor.getAllApplicationInstanceReferences().stream().sorted().collect(Collectors.toList()); } @GET @@ -81,7 +80,7 @@ public class InstanceResource { ApplicationInstanceReference instanceId = parseInstanceId(instanceIdString); ApplicationInstance applicationInstance - = instanceLookupService.findInstanceById(instanceId) + = serviceMonitor.getApplication(instanceId) .orElseThrow(() -> new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build())); HostInfos hostInfos = statusService.getHostInfosByApplicationResolver().apply(applicationInstance.reference()); @@ -153,7 +152,7 @@ public class InstanceResource { static ApplicationInstanceReference parseInstanceId(String instanceIdString) { try { - return parseAppInstanceReference(instanceIdString); + return parseApplicationInstanceReference(instanceIdString); } catch (IllegalArgumentException e) { throwBadRequest(e.getMessage()); return null; // Necessary for compiler diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ApplicationLock.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ApplicationLock.java new file mode 100644 index 00000000000..8883f78b693 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ApplicationLock.java @@ -0,0 +1,36 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; + +/** + * The exclusive lock of an application in the status service, and methods to mutate host status + * and data structures guarded by such a lock. + * + * @author oyving + * @author Tony Vaagenes + * @author bakksjo + */ +public interface ApplicationLock extends AutoCloseable { + + /** The reference of the locked application. */ + ApplicationInstanceReference getApplicationInstanceReference(); + + /** Returns all host infos for this application. */ + HostInfos getHostInfos(); + + /** Sets the state for the given host. */ + void setHostState(HostName hostName, HostStatus status); + + /** Returns the application status. */ + ApplicationInstanceStatus getApplicationInstanceStatus(); + + /** Sets the orchestration status for the application instance. */ + void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus); + + /** WARNING: Must not throw an exception. */ + @Override + void close(); + +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java new file mode 100644 index 00000000000..2d6c3eb82a1 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java @@ -0,0 +1,128 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.orchestrator.status; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.jdisc.Timer; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.orchestrator.OrchestratorUtil; +import com.yahoo.vespa.orchestrator.status.json.WireHostInfo; +import org.apache.zookeeper.KeeperException.NoNodeException; + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Handles all ZooKeeper data structures related to each active application, including HostInfo. + * Cache (if any) is above and not visible here. + * + * @author hakonhall + */ +public class HostInfosServiceImpl implements HostInfosService { + private static final Logger log = Logger.getLogger(HostInfosServiceImpl.class.getName()); + + private final Curator curator; + private final Timer timer; + + HostInfosServiceImpl(Curator curator, Timer timer) { + this.curator = curator; + this.timer = timer; + } + + @Override + public HostInfos getHostInfos(ApplicationInstanceReference reference) { + ApplicationId application = OrchestratorUtil.toApplicationId(reference); + String hostsRootPath = hostsPath(application); + if (uncheck(() -> curator.framework().checkExists().forPath(hostsRootPath)) == null) { + return new HostInfos(); + } else { + List<String> hostnames = uncheck(() -> curator.framework().getChildren().forPath(hostsRootPath)); + Map<HostName, HostInfo> hostInfos = hostnames.stream().collect(Collectors.toMap( + hostname -> new HostName(hostname), + hostname -> { + byte[] bytes = uncheck(() -> curator.framework().getData().forPath(hostsRootPath + "/" + hostname)); + return WireHostInfo.deserialize(bytes); + })); + return new HostInfos(hostInfos); + } + } + + @Override + public boolean setHostStatus(ApplicationInstanceReference reference, HostName hostname, HostStatus status) { + ApplicationId application = OrchestratorUtil.toApplicationId(reference); + String path = hostPath(application, hostname); + + if (status == HostStatus.NO_REMARKS) { + return deleteNode_ignoreNoNodeException(path, "Host already has state NO_REMARKS, path = " + path); + } + + Optional<HostInfo> currentHostInfo = uncheck(() -> readBytesFromZk(path)).map(WireHostInfo::deserialize); + if (currentHostInfo.isEmpty()) { + Instant suspendedSince = timer.currentTime(); + HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince); + byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo); + uncheck(() -> curator.framework().create().creatingParentsIfNeeded().forPath(path, hostInfoBytes)); + } else if (currentHostInfo.get().status() == status) { + return false; + } else { + Instant suspendedSince = currentHostInfo.get().suspendedSince().orElseGet(timer::currentTime); + HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince); + byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo); + uncheck(() -> curator.framework().setData().forPath(path, hostInfoBytes)); + } + + return true; + } + + private Optional<byte[]> readBytesFromZk(String path) throws Exception { + try { + return Optional.of(curator.framework().getData().forPath(path)); + } catch (NoNodeException e) { + return Optional.empty(); + } + } + + private boolean deleteNode_ignoreNoNodeException(String path, String debugLogMessageIfNotExists) { + try { + curator.framework().delete().forPath(path); + return true; + } catch (NoNodeException e) { + log.log(LogLevel.DEBUG, debugLogMessageIfNotExists, e); + return false; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static String applicationPath(ApplicationId application) { + return "/vespa/host-status/" + application.serializedForm(); + } + + private static String hostsPath(ApplicationId application) { + return applicationPath(application) + "/hosts"; + } + + private static String hostPath(ApplicationId application, HostName hostname) { + return hostsPath(application) + "/" + hostname.s(); + } + + private <T> T uncheck(SupplierThrowingException<T> supplier) { + try { + return supplier.get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @FunctionalInterface + private interface SupplierThrowingException<T> { + T get() throws Exception; + } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java deleted file mode 100644 index 24da83364aa..00000000000 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.orchestrator.status; - -import com.yahoo.vespa.applicationmodel.HostName; - -import java.util.Set; - -/** - * Registry of the suspension and host statuses for an application instance. - * - * @author oyving - * @author Tony Vaagenes - * @author bakksjo - */ -public interface MutableStatusRegistry extends AutoCloseable { - - /** Returns the status of this application. */ - ApplicationInstanceStatus getStatus(); - - /** Returns a snapshot of all host infos for this application. */ - HostInfos getHostInfos(); - - /** Sets the state for the given host. */ - void setHostState(HostName hostName, HostStatus status); - - /** - * Sets the orchestration status for the application instance. - */ - void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus); - - /** - * We don't want {@link AutoCloseable#close()} to throw an exception (what to do about it anyway?), - * so we override it here to strip the exception from the signature. - */ - @Override - void close(); - -} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java index e2be5ec7eb6..0d4076f3d59 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java @@ -24,29 +24,9 @@ public interface StatusService { * Returns a mutable host status registry for a locked application instance. All operations performed on * the returned registry are executed in the context of a lock, including read operations. Hence, multi-step * operations (e.g. read-then-write) are guaranteed to be consistent. - * - * Some limitations/caveats apply for certain implementations, and since clients of this API must be aware of - * these limitations/caveats when using those implementations, they are expressed here, at interface level - * rather than at implementation level, because the interface represents the lowest common denominator - * of guarantees offered by implementations. Specifically, it is the zookeeper-based implementation's semantics - * that "leak through" in this spec. Now, to the specific caveats: - * - * Locking this application instance only guarantees that the holder is the only one that can mutate host statuses - * for the application instance. - * It is _not_ safe to assume that there is only one entity holding the lock for a given application instance - * reference at any given time. - * - * You cannot have multiple locks in a single thread, even if they are for different application instances, - * (i.e. different HostStatusRegistry instances). (This is due to a limitation in SessionFailRetryLoop.) - * - * While read-then-write-operations are consistent (i.e. the current value doesn't change between the read - * and the write), it is possible that the lock is lost before it is explicitly released by the code. In - * this case, subsequent mutating operations will fail, but previous mutating operations are NOT rolled back. - * This may leave the registry in an inconsistent state (as judged by the client code). */ - MutableStatusRegistry lockApplicationInstance_forCurrentThreadOnly( - OrchestratorContext context, - ApplicationInstanceReference applicationInstanceReference) throws UncheckedTimeoutException; + ApplicationLock lockApplication(OrchestratorContext context, ApplicationInstanceReference reference) + throws UncheckedTimeoutException; /** * Returns all application instances that are allowed to be down. The intention is to use this @@ -68,5 +48,5 @@ public interface StatusService { ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference application); /** Get host info for hostname in application. This is consistent if its lock is held. */ - HostInfo getHostInfo(ApplicationInstanceReference applicationInstanceReference, HostName hostName); + HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostName); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java new file mode 100644 index 00000000000..479dc5062a8 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java @@ -0,0 +1,114 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.orchestrator.status; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.curator.Curator; +import org.apache.zookeeper.KeeperException; + +import java.util.logging.Logger; + +/** + * ZooKeeper implementation of {@link ApplicationLock}. + * + * @author hakonhall + */ +class ZkApplicationLock implements ApplicationLock { + + private static final Logger log = Logger.getLogger(ZkApplicationLock.class.getName()); + + private final ZkStatusService statusService; + private final Curator curator; + private final Runnable onClose; + private final ApplicationInstanceReference reference; + private final boolean probe; + private final HostInfosCache hostInfosCache; + + ZkApplicationLock(ZkStatusService statusService, + Curator curator, + Runnable onClose, + ApplicationInstanceReference reference, + boolean probe, + HostInfosCache hostInfosCache) { + this.statusService = statusService; + this.curator = curator; + this.onClose = onClose; + this.reference = reference; + this.probe = probe; + this.hostInfosCache = hostInfosCache; + } + + @Override + public ApplicationInstanceReference getApplicationInstanceReference() { + return reference; + } + + @Override + public ApplicationInstanceStatus getApplicationInstanceStatus() { + return statusService.getApplicationInstanceStatus(reference); + } + + @Override + public HostInfos getHostInfos() { + return hostInfosCache.getHostInfos(reference); + } + + @Override + public void setHostState(final HostName hostName, final HostStatus status) { + if (probe) return; + log.log(LogLevel.INFO, "Setting host " + hostName + " to status " + status); + hostInfosCache.setHostStatus(reference, hostName, status); + } + + @Override + public void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus) { + if (probe) return; + + log.log(LogLevel.INFO, "Setting app " + reference.asString() + " to status " + applicationInstanceStatus); + + String path = statusService.applicationInstanceSuspendedPath(reference); + switch (applicationInstanceStatus) { + case NO_REMARKS: + deleteNode_ignoreNoNodeException(path); + break; + case ALLOWED_TO_BE_DOWN: + createNode_ignoreNodeExistsException(path); + break; + } + } + + @Override + public void close() { + try { + onClose.run(); + } catch (RuntimeException e) { + // We may want to avoid logging some exceptions that may be expected, like when session expires. + log.log(LogLevel.WARNING, + "Failed close application lock in " + + ZkApplicationLock.class.getSimpleName() + ", will ignore and continue", + e); + } + } + + void deleteNode_ignoreNoNodeException(String path) { + try { + curator.framework().delete().forPath(path); + } catch (KeeperException.NoNodeException e) { + // ok + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + void createNode_ignoreNodeExistsException(String path) { + try { + curator.framework().create().creatingParentsIfNeeded().forPath(path); + } catch (KeeperException.NodeExistsException e) { + // ok + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java new file mode 100644 index 00000000000..c1ca04aea75 --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java @@ -0,0 +1,237 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.status; + +import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jaxrs.annotation.Component; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.Timer; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.orchestrator.OrchestratorContext; +import com.yahoo.vespa.orchestrator.OrchestratorUtil; +import org.apache.zookeeper.data.Stat; + +import javax.inject.Inject; +import java.time.Duration; +import java.time.Instant; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.logging.Logger; + +/** + * Stores instance suspension status and which hosts are allowed to go down in zookeeper. + * + * TODO: expiry of old application instances + * @author Tony Vaagenes + */ +public class ZkStatusService implements StatusService { + + private static final Logger log = Logger.getLogger(ZkStatusService.class.getName()); + + final static String HOST_STATUS_BASE_PATH = "/vespa/host-status-service"; + final static String APPLICATION_STATUS_BASE_PATH = "/vespa/application-status-service"; + + private final Curator curator; + private final HostInfosCache hostInfosCache; + private final Metric metric; + private final Timer timer; + + /** + * A cache of metric contexts for each possible dimension map. In practice, there is one dimension map + * for each application, so up to hundreds of elements. + */ + private final ConcurrentHashMap<Map<String, String>, Metric.Context> cachedContexts = new ConcurrentHashMap<>(); + + @Inject + public ZkStatusService(@Component Curator curator, @Component Metric metric, @Component Timer timer) { + this.curator = curator; + this.metric = metric; + this.timer = timer; + this.hostInfosCache = new HostInfosCache(curator, new HostInfosServiceImpl(curator, timer)); + } + + /** Non-private for testing only. */ + ZkStatusService(Curator curator, Metric metric, Timer timer, HostInfosCache hostInfosCache) { + this.curator = curator; + this.metric = metric; + this.timer = timer; + this.hostInfosCache = hostInfosCache; + } + + @Override + public Set<ApplicationInstanceReference> getAllSuspendedApplications() { + try { + Set<ApplicationInstanceReference> resultSet = new HashSet<>(); + + // Return empty set if the base path does not exist + Stat stat = curator.framework().checkExists().forPath(APPLICATION_STATUS_BASE_PATH); + if (stat == null) return resultSet; + + // The path exist and we may have children + for (String referenceString : curator.framework().getChildren().forPath(APPLICATION_STATUS_BASE_PATH)) { + ApplicationInstanceReference reference = OrchestratorUtil.parseApplicationInstanceReference(referenceString); + resultSet.add(reference); + } + + return resultSet; + } catch (Exception e) { + log.log(LogLevel.DEBUG, "Something went wrong while listing out applications in suspend.", e); + throw new RuntimeException(e); + } + } + + /** + * Cache is checked for freshness when this mapping is created, and may be invalidated again later + * by other users of the cache. Since this function is backed by the cache, any such invalidation + * will be reflected in the returned mapping; all users of the cache collaborate in repopulating it. + */ + @Override + public Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver() { + hostInfosCache.refreshCache(); + return hostInfosCache::getCachedHostInfos; + } + + + /** + * 1) locks the status service for an application instance. + * 2) fails all operations in this thread when the session is lost, + * since session loss might cause the lock to be lost. + * Since it only fails operations in this thread, + * all operations depending on a lock, including the locking itself, must be done in this thread. + * Note that since it is the thread that fails, all status operations in this thread will fail + * even if they're not supposed to be guarded by this lock + * (i.e. the request is for another applicationInstanceReference) + */ + @Override + public ApplicationLock lockApplication(OrchestratorContext context, ApplicationInstanceReference reference) + throws UncheckedTimeoutException { + + Runnable onRegistryClose; + + // A multi-application operation, aka batch suspension, will first issue a probe + // then a non-probe. With "large locks", the lock is not release in between - + // no lock is taken on the non-probe. Instead, the release is done on the multi-application + // context close. + if (context.hasLock(reference)) { + onRegistryClose = () -> {}; + } else { + Runnable unlock = acquireLock(context, reference); + if (context.registerLockAcquisition(reference, unlock)) { + onRegistryClose = () -> {}; + } else { + onRegistryClose = unlock; + } + } + + try { + return new ZkApplicationLock( + this, + curator, + onRegistryClose, + reference, + context.isProbe(), + hostInfosCache); + } catch (Throwable t) { + // In case the constructor throws an exception. + onRegistryClose.run(); + throw t; + } + } + + private Runnable acquireLock(OrchestratorContext context, ApplicationInstanceReference reference) + throws UncheckedTimeoutException { + ApplicationId applicationId = OrchestratorUtil.toApplicationId(reference); + String app = applicationId.application().value() + "." + applicationId.instance().value(); + Map<String, String> dimensions = Map.of( + "tenantName", applicationId.tenant().value(), + "applicationId", applicationId.toFullString(), + "app", app); + Metric.Context metricContext = cachedContexts.computeIfAbsent(dimensions, metric::createContext); + + Duration duration = context.getTimeLeft(); + String lockPath = applicationInstanceLock2Path(reference); + Lock lock = new Lock(lockPath, curator); + + Instant startTime = timer.currentTime(); + Instant acquireEndTime; + boolean lockAcquired = false; + try { + lock.acquire(duration); + lockAcquired = true; + } finally { + acquireEndTime = timer.currentTime(); + double seconds = durationInSeconds(startTime, acquireEndTime); + metric.set("orchestrator.lock.acquire-latency", seconds, metricContext); + metric.set("orchestrator.lock.acquired", lockAcquired ? 1 : 0, metricContext); + + metric.add("orchestrator.lock.acquire", 1, metricContext); + String acquireResultMetricName = lockAcquired ? "orchestrator.lock.acquire-success" : "orchestrator.lock.acquire-timedout"; + metric.add(acquireResultMetricName, 1, metricContext); + } + + return () -> { + try { + lock.close(); + } catch (RuntimeException e) { + // We may want to avoid logging some exceptions that may be expected, like when session expires. + log.log(LogLevel.WARNING, + "Failed to close application lock for " + + ZkStatusService.class.getSimpleName() + ", will ignore and continue", + e); + } + + Instant lockReleasedTime = timer.currentTime(); + double seconds = durationInSeconds(acquireEndTime, lockReleasedTime); + metric.set("orchestrator.lock.hold-latency", seconds, metricContext); + }; + } + + private double durationInSeconds(Instant startInstant, Instant endInstant) { + return Duration.between(startInstant, endInstant).toMillis() / 1000.0; + } + + @Override + public HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostName) { + return hostInfosCache.getHostInfos(reference).getOrNoRemarks(hostName); + } + + @Override + public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference reference) { + try { + Stat statOrNull = curator.framework().checkExists().forPath( + applicationInstanceSuspendedPath(reference)); + + return (statOrNull == null) ? ApplicationInstanceStatus.NO_REMARKS : ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + static String applicationInstanceReferencePath(ApplicationInstanceReference reference) { + return HOST_STATUS_BASE_PATH + '/' + reference.tenantId() + ":" + reference.applicationInstanceId(); + } + + private static String hostsAllowedDownPath(ApplicationInstanceReference reference) { + return applicationInstanceReferencePath(reference) + "/hosts-allowed-down"; + } + + private static String applicationInstanceLock2Path(ApplicationInstanceReference reference) { + return applicationInstanceReferencePath(reference) + "/lock2"; + } + + String applicationInstanceSuspendedPath(ApplicationInstanceReference reference) { + return APPLICATION_STATUS_BASE_PATH + "/" + OrchestratorUtil.toRestApiFormat(reference); + } + + private static String hostAllowedDownPath(ApplicationInstanceReference reference, HostName hostname) { + return hostsAllowedDownPath(reference) + '/' + hostname.s(); + } + +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java deleted file mode 100644 index 2cdce350377..00000000000 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java +++ /dev/null @@ -1,418 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.orchestrator.status; - -import com.google.common.util.concurrent.UncheckedTimeoutException; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.Timer; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; -import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.orchestrator.OrchestratorContext; -import com.yahoo.vespa.orchestrator.OrchestratorUtil; -import com.yahoo.vespa.orchestrator.status.json.WireHostInfo; -import org.apache.zookeeper.KeeperException.NoNodeException; -import org.apache.zookeeper.KeeperException.NodeExistsException; -import org.apache.zookeeper.data.Stat; - -import javax.inject.Inject; -import java.time.Duration; -import java.time.Instant; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Stores instance suspension status and which hosts are allowed to go down in zookeeper. - * - * TODO: expiry of old application instances - * @author Tony Vaagenes - */ -public class ZookeeperStatusService implements StatusService { - - private static final Logger log = Logger.getLogger(ZookeeperStatusService.class.getName()); - - final static String HOST_STATUS_BASE_PATH = "/vespa/host-status-service"; - final static String APPLICATION_STATUS_BASE_PATH = "/vespa/application-status-service"; - - private final Curator curator; - private final HostInfosCache hostInfosCache; - private final Metric metric; - private final Timer timer; - - /** - * A cache of metric contexts for each possible dimension map. In practice, there is one dimension map - * for each application, so up to hundreds of elements. - */ - private final ConcurrentHashMap<Map<String, String>, Metric.Context> cachedContexts = new ConcurrentHashMap<>(); - - @Inject - public ZookeeperStatusService(@Component Curator curator, @Component Metric metric, @Component Timer timer) { - this.curator = curator; - this.metric = metric; - this.timer = timer; - - // Insert a cache above some ZooKeeper accesses - this.hostInfosCache = new HostInfosCache(curator, new HostInfosService() { - @Override - public HostInfos getHostInfos(ApplicationInstanceReference application) { - return ZookeeperStatusService.this.getHostInfosFromZk(application); - } - - @Override - public boolean setHostStatus(ApplicationInstanceReference application, HostName hostName, HostStatus hostStatus) { - return ZookeeperStatusService.this.setHostInfoInZk(application, hostName, hostStatus); - } - }); - } - - /** Non-private for testing only. */ - ZookeeperStatusService(Curator curator, Metric metric, Timer timer, HostInfosCache hostInfosCache) { - this.curator = curator; - this.metric = metric; - this.timer = timer; - this.hostInfosCache = hostInfosCache; - } - - @Override - public Set<ApplicationInstanceReference> getAllSuspendedApplications() { - try { - Set<ApplicationInstanceReference> resultSet = new HashSet<>(); - - // Return empty set if the base path does not exist - Stat stat = curator.framework().checkExists().forPath(APPLICATION_STATUS_BASE_PATH); - if (stat == null) return resultSet; - - // The path exist and we may have children - for (String appRefStr : curator.framework().getChildren().forPath(APPLICATION_STATUS_BASE_PATH)) { - ApplicationInstanceReference appRef = OrchestratorUtil.parseAppInstanceReference(appRefStr); - resultSet.add(appRef); - } - - return resultSet; - } catch (Exception e) { - log.log(LogLevel.DEBUG, "Something went wrong while listing out applications in suspend.", e); - throw new RuntimeException(e); - } - } - - /** - * Cache is checked for freshness when this mapping is created, and may be invalidated again later - * by other users of the cache. Since this function is backed by the cache, any such invalidation - * will be reflected in the returned mapping; all users of the cache collaborate in repopulating it. - */ - @Override - public Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver() { - hostInfosCache.refreshCache(); - return hostInfosCache::getCachedHostInfos; - } - - - /** - * 1) locks the status service for an application instance. - * 2) fails all operations in this thread when the session is lost, - * since session loss might cause the lock to be lost. - * Since it only fails operations in this thread, - * all operations depending on a lock, including the locking itself, must be done in this thread. - * Note that since it is the thread that fails, all status operations in this thread will fail - * even if they're not supposed to be guarded by this lock - * (i.e. the request is for another applicationInstanceReference) - */ - @Override - public MutableStatusRegistry lockApplicationInstance_forCurrentThreadOnly( - OrchestratorContext context, - ApplicationInstanceReference applicationInstanceReference) throws UncheckedTimeoutException { - Runnable onRegistryClose; - - // A multi-application operation, aka batch suspension, will first issue a probe - // then a non-probe. With "large locks", the lock is not release in between - - // no lock is taken on the non-probe. Instead, the release is done on the multi-application - // context close. - if (context.hasLock(applicationInstanceReference)) { - onRegistryClose = () -> {}; - } else { - Runnable unlock = acquireLock(context, applicationInstanceReference); - if (context.registerLockAcquisition(applicationInstanceReference, unlock)) { - onRegistryClose = () -> {}; - } else { - onRegistryClose = unlock; - } - } - - try { - return new ZkMutableStatusRegistry(onRegistryClose, applicationInstanceReference, context.isProbe()); - } catch (Throwable t) { - // In case the constructor throws an exception. - onRegistryClose.run(); - throw t; - } - } - - private Runnable acquireLock(OrchestratorContext context, - ApplicationInstanceReference applicationInstanceReference) - throws UncheckedTimeoutException { - ApplicationId applicationId = OrchestratorUtil.toApplicationId(applicationInstanceReference); - String app = applicationId.application().value() + "." + applicationId.instance().value(); - Map<String, String> dimensions = Map.of( - "tenantName", applicationId.tenant().value(), - "applicationId", applicationId.toFullString(), - "app", app); - Metric.Context metricContext = cachedContexts.computeIfAbsent(dimensions, metric::createContext); - - Duration duration = context.getTimeLeft(); - String lockPath = applicationInstanceLock2Path(applicationInstanceReference); - Lock lock = new Lock(lockPath, curator); - - Instant startTime = timer.currentTime(); - Instant acquireEndTime; - boolean lockAcquired = false; - try { - lock.acquire(duration); - lockAcquired = true; - } finally { - acquireEndTime = timer.currentTime(); - double seconds = durationInSeconds(startTime, acquireEndTime); - metric.set("orchestrator.lock.acquire-latency", seconds, metricContext); - metric.set("orchestrator.lock.acquired", lockAcquired ? 1 : 0, metricContext); - - metric.add("orchestrator.lock.acquire", 1, metricContext); - String acquireResultMetricName = lockAcquired ? "orchestrator.lock.acquire-success" : "orchestrator.lock.acquire-timedout"; - metric.add(acquireResultMetricName, 1, metricContext); - } - - return () -> { - try { - lock.close(); - } catch (RuntimeException e) { - // We may want to avoid logging some exceptions that may be expected, like when session expires. - log.log(LogLevel.WARNING, - "Failed to close application lock for " + - ZookeeperStatusService.class.getSimpleName() + ", will ignore and continue", - e); - } - - Instant lockReleasedTime = timer.currentTime(); - double seconds = durationInSeconds(acquireEndTime, lockReleasedTime); - metric.set("orchestrator.lock.hold-latency", seconds, metricContext); - }; - } - - private double durationInSeconds(Instant startInstant, Instant endInstant) { - return Duration.between(startInstant, endInstant).toMillis() / 1000.0; - } - - /** Returns false if no changes were made. */ - private boolean setHostInfoInZk(ApplicationInstanceReference application, HostName hostname, HostStatus status) { - String path = hostPath(application, hostname); - - if (status == HostStatus.NO_REMARKS) { - return deleteNode_ignoreNoNodeException(path, "Host already has state NO_REMARKS, path = " + path); - } - - Optional<HostInfo> currentHostInfo = uncheck(() -> readBytesFromZk(path)).map(WireHostInfo::deserialize); - if (currentHostInfo.isEmpty()) { - Instant suspendedSince = timer.currentTime(); - HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince); - byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo); - uncheck(() -> curator.framework().create().creatingParentsIfNeeded().forPath(path, hostInfoBytes)); - } else if (currentHostInfo.get().status() == status) { - return false; - } else { - Instant suspendedSince = currentHostInfo.get().suspendedSince().orElseGet(timer::currentTime); - HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince); - byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo); - uncheck(() -> curator.framework().setData().forPath(path, hostInfoBytes)); - } - - return true; - } - - private boolean deleteNode_ignoreNoNodeException(String path, String debugLogMessageIfNotExists) { - try { - curator.framework().delete().forPath(path); - return true; - } catch (NoNodeException e) { - log.log(LogLevel.DEBUG, debugLogMessageIfNotExists, e); - return false; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private boolean createNode_ignoreNodeExistsException(String path, String debugLogMessageIfExists) { - try { - curator.framework().create() - .creatingParentsIfNeeded() - .forPath(path); - return true; - } catch (NodeExistsException e) { - log.log(LogLevel.DEBUG, debugLogMessageIfExists, e); - return false; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private Optional<byte[]> readBytesFromZk(String path) throws Exception { - try { - return Optional.of(curator.framework().getData().forPath(path)); - } catch (NoNodeException e) { - return Optional.empty(); - } - } - - @Override - public HostInfo getHostInfo(ApplicationInstanceReference applicationInstanceReference, HostName hostName) { - return hostInfosCache.getHostInfos(applicationInstanceReference).getOrNoRemarks(hostName); - } - - /** Do not call this directly: should be called behind a cache. */ - private HostInfos getHostInfosFromZk(ApplicationInstanceReference application) { - String hostsRootPath = hostsPath(application); - if (uncheck(() -> curator.framework().checkExists().forPath(hostsRootPath)) == null) { - return new HostInfos(); - } else { - List<String> hostnames = uncheck(() -> curator.framework().getChildren().forPath(hostsRootPath)); - Map<HostName, HostInfo> hostInfos = hostnames.stream().collect(Collectors.toMap( - hostname -> new HostName(hostname), - hostname -> { - byte[] bytes = uncheck(() -> curator.framework().getData().forPath(hostsRootPath + "/" + hostname)); - return WireHostInfo.deserialize(bytes); - })); - return new HostInfos(hostInfos); - } - } - - private <T> T uncheck(SupplierThrowingException<T> supplier) { - try { - return supplier.get(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @FunctionalInterface - interface SupplierThrowingException<T> { - T get() throws Exception; - } - - @Override - public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference applicationInstanceReference) { - try { - Stat statOrNull = curator.framework().checkExists().forPath( - applicationInstanceSuspendedPath(applicationInstanceReference)); - - return (statOrNull == null) ? ApplicationInstanceStatus.NO_REMARKS : ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - static String applicationInstanceReferencePath(ApplicationInstanceReference applicationInstanceReference) { - return HOST_STATUS_BASE_PATH + '/' + - applicationInstanceReference.tenantId() + ":" + applicationInstanceReference.applicationInstanceId(); - } - - private static String applicationPath(ApplicationInstanceReference applicationInstanceReference) { - ApplicationId applicationId = OrchestratorUtil.toApplicationId(applicationInstanceReference); - return "/vespa/host-status/" + applicationId.serializedForm(); - } - - private static String hostsPath(ApplicationInstanceReference applicationInstanceReference) { - return applicationPath(applicationInstanceReference) + "/hosts"; - } - - private static String hostsAllowedDownPath(ApplicationInstanceReference applicationInstanceReference) { - return applicationInstanceReferencePath(applicationInstanceReference) + "/hosts-allowed-down"; - } - - private static String applicationInstanceLock2Path(ApplicationInstanceReference applicationInstanceReference) { - return applicationInstanceReferencePath(applicationInstanceReference) + "/lock2"; - } - - private String applicationInstanceSuspendedPath(ApplicationInstanceReference applicationInstanceReference) { - return APPLICATION_STATUS_BASE_PATH + "/" + OrchestratorUtil.toRestApiFormat(applicationInstanceReference); - } - - private static String hostAllowedDownPath(ApplicationInstanceReference applicationInstanceReference, HostName hostname) { - return hostsAllowedDownPath(applicationInstanceReference) + '/' + hostname.s(); - } - - private static String hostPath(ApplicationInstanceReference application, HostName hostname) { - return hostsPath(application) + "/" + hostname.s(); - } - - private class ZkMutableStatusRegistry implements MutableStatusRegistry { - - private final Runnable onClose; - private final ApplicationInstanceReference applicationInstanceReference; - private final boolean probe; - - public ZkMutableStatusRegistry(Runnable onClose, - ApplicationInstanceReference applicationInstanceReference, - boolean probe) { - this.onClose = onClose; - this.applicationInstanceReference = applicationInstanceReference; - this.probe = probe; - } - - @Override - public ApplicationInstanceStatus getStatus() { - return getApplicationInstanceStatus(applicationInstanceReference); - } - - @Override - public HostInfos getHostInfos() { - return hostInfosCache.getHostInfos(applicationInstanceReference); - } - - @Override - public void setHostState(final HostName hostName, final HostStatus status) { - if (probe) return; - log.log(LogLevel.INFO, "Setting host " + hostName + " to status " + status); - hostInfosCache.setHostStatus(applicationInstanceReference, hostName, status); - } - - @Override - public void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus) { - if (probe) return; - - log.log(LogLevel.INFO, "Setting app " + applicationInstanceReference.asString() + " to status " + applicationInstanceStatus); - - String path = applicationInstanceSuspendedPath(applicationInstanceReference); - switch (applicationInstanceStatus) { - case NO_REMARKS: - deleteNode_ignoreNoNodeException(path, - "Instance is already in state NO_REMARKS, path = " + path); - break; - case ALLOWED_TO_BE_DOWN: - createNode_ignoreNodeExistsException(path, - "Instance is already in state ALLOWED_TO_BE_DOWN, path = " + path); - break; - } - } - - @Override - public void close() { - try { - onClose.run(); - } catch (RuntimeException e) { - // We may want to avoid logging some exceptions that may be expected, like when session expires. - log.log(LogLevel.WARNING, - "Failed close application lock in " + - ZookeeperStatusService.class.getSimpleName() + ", will ignore and continue", - e); - } - } - } - -} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyServiceMonitor.java index a54f5284ee0..09fb6296866 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyServiceMonitor.java @@ -15,8 +15,12 @@ import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.orchestrator.model.NodeGroup; import com.yahoo.vespa.orchestrator.model.VespaModelUtil; +import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -27,7 +31,7 @@ import java.util.stream.Collectors; * @author oyving * @author smorgrav */ -public class DummyInstanceLookupService implements InstanceLookupService { +public class DummyServiceMonitor implements ServiceMonitor { public static final HostName TEST1_HOST_NAME = new HostName("test1.hostname.tld"); public static final HostName TEST3_HOST_NAME = new HostName("test3.hostname.tld"); @@ -121,26 +125,27 @@ public class DummyInstanceLookupService implements InstanceLookupService { } // A node group is tied to an application, so we need to define them after we have populated the above applications. - public final static NodeGroup TEST1_NODE_GROUP = new NodeGroup(new DummyInstanceLookupService().findInstanceByHost(TEST1_HOST_NAME).get(), TEST1_HOST_NAME); - public final static NodeGroup TEST3_NODE_GROUP = new NodeGroup(new DummyInstanceLookupService().findInstanceByHost(TEST3_HOST_NAME).get(), TEST3_HOST_NAME); - public final static NodeGroup TEST6_NODE_GROUP = new NodeGroup(new DummyInstanceLookupService().findInstanceByHost(TEST6_HOST_NAME).get(), TEST6_HOST_NAME); + public final static NodeGroup TEST1_NODE_GROUP = new NodeGroup(new DummyServiceMonitor().getApplication(TEST1_HOST_NAME).get(), TEST1_HOST_NAME); + public final static NodeGroup TEST3_NODE_GROUP = new NodeGroup(new DummyServiceMonitor().getApplication(TEST3_HOST_NAME).get(), TEST3_HOST_NAME); + public final static NodeGroup TEST6_NODE_GROUP = new NodeGroup(new DummyServiceMonitor().getApplication(TEST6_HOST_NAME).get(), TEST6_HOST_NAME); + @Override + public ServiceModel getServiceModelSnapshot() { + throw new UnsupportedOperationException(); + } @Override - public Optional<ApplicationInstance> findInstanceById( - final ApplicationInstanceReference applicationInstanceReference) { - for (ApplicationInstance app : apps) { - if (app.reference().equals(applicationInstanceReference)) return Optional.of(app); - } - return Optional.empty(); + public Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() { + return apps.stream().map(a -> + new ApplicationInstanceReference(a.tenantId(),a.applicationInstanceId())).collect(Collectors.toSet()); } @Override - public Optional<ApplicationInstance> findInstanceByHost(HostName hostName) { + public Optional<ApplicationInstance> getApplication(HostName hostname) { for (ApplicationInstance app : apps) { for (ServiceCluster cluster : app.serviceClusters()) { for (ServiceInstance service : cluster.serviceInstances()) { - if (hostName.equals(service.hostName())) return Optional.of(app); + if (hostname.equals(service.hostName())) return Optional.of(app); } } } @@ -148,10 +153,16 @@ public class DummyInstanceLookupService implements InstanceLookupService { } @Override - public Set<ApplicationInstanceReference> knownInstances() { - return apps.stream().map(a -> - new ApplicationInstanceReference(a.tenantId(),a.applicationInstanceId())).collect(Collectors.toSet()); + public Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) { + for (ApplicationInstance app : apps) { + if (app.reference().equals(reference)) return Optional.of(app); + } + return Optional.empty(); + } + @Override + public Map<HostName, List<ServiceInstance>> getServicesByHostname() { + throw new UnsupportedOperationException(); } public static Set<HostName> getContentHosts(ApplicationInstanceReference appRef) { diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index bfe4b523e4a..9bcbdba074e 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -26,11 +26,13 @@ import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; +import com.yahoo.vespa.orchestrator.status.ApplicationLock; import com.yahoo.vespa.orchestrator.status.HostStatus; -import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; -import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService; +import com.yahoo.vespa.orchestrator.status.ZkStatusService; +import com.yahoo.vespa.service.model.ServiceModelCache; import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -85,17 +87,17 @@ public class OrchestratorImplTest { @Before public void setUp() { // Extract applications and hosts from dummy instance lookup service - Iterator<ApplicationInstance> iterator = DummyInstanceLookupService.getApplications().iterator(); + Iterator<ApplicationInstance> iterator = DummyServiceMonitor.getApplications().iterator(); ApplicationInstanceReference app1_ref = iterator.next().reference(); app1 = OrchestratorUtil.toApplicationId(app1_ref); - app1_host1 = DummyInstanceLookupService.getContentHosts(app1_ref).iterator().next(); + app1_host1 = DummyServiceMonitor.getContentHosts(app1_ref).iterator().next(); app2 = OrchestratorUtil.toApplicationId(iterator.next().reference()); clustercontroller = new ClusterControllerClientFactoryMock(); orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clustercontroller, applicationApiFactory), clustercontroller, - new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer()), - new DummyInstanceLookupService(), + new ZkStatusService(new MockCurator(), mock(Metric.class), new TestTimer()), + new DummyServiceMonitor(), 0, new ManualClock(), applicationApiFactory, @@ -241,8 +243,8 @@ public class OrchestratorImplTest { @Test public void applicationReferenceHasTenantAndAppInstance() { - InstanceLookupService service = new DummyInstanceLookupService(); - String applicationInstanceId = service.findInstanceByHost(DummyInstanceLookupService.TEST1_HOST_NAME).get() + ServiceMonitor service = new DummyServiceMonitor(); + String applicationInstanceId = service.getApplication(DummyServiceMonitor.TEST1_HOST_NAME).get() .reference().toString(); assertEquals("test-tenant-id:application:prod:utopia-1:instance", applicationInstanceId); } @@ -264,21 +266,21 @@ public class OrchestratorImplTest { orchestrator.suspendAll( new HostName("parentHostname"), Arrays.asList( - DummyInstanceLookupService.TEST1_HOST_NAME, - DummyInstanceLookupService.TEST3_HOST_NAME, - DummyInstanceLookupService.TEST6_HOST_NAME)); + DummyServiceMonitor.TEST1_HOST_NAME, + DummyServiceMonitor.TEST3_HOST_NAME, + DummyServiceMonitor.TEST6_HOST_NAME)); // As of 2016-06-07 the order of the node groups are as follows: // TEST3: mediasearch:imagesearch:default // TEST6: tenant-id-3:application-instance-3:default // TEST1: test-tenant-id:application:instance InOrder order = inOrder(orchestrator); - verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST3_NODE_GROUP, true); - verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST6_NODE_GROUP, true); - verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST1_NODE_GROUP, true); - verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST3_NODE_GROUP, false); - verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST6_NODE_GROUP, false); - verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST1_NODE_GROUP, false); + verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST3_NODE_GROUP, true); + verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST6_NODE_GROUP, true); + verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST1_NODE_GROUP, true); + verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST3_NODE_GROUP, false); + verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST6_NODE_GROUP, false); + verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST1_NODE_GROUP, false); order.verifyNoMoreInteractions(); } @@ -295,18 +297,18 @@ public class OrchestratorImplTest { OrchestratorImpl orchestrator = spy(this.orchestrator); Throwable supensionFailure = new HostStateChangeDeniedException( - DummyInstanceLookupService.TEST6_HOST_NAME, + DummyServiceMonitor.TEST6_HOST_NAME, "some-constraint", "error message"); - doThrow(supensionFailure).when(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST6_NODE_GROUP)); + doThrow(supensionFailure).when(orchestrator).suspendGroup(any(), eq(DummyServiceMonitor.TEST6_NODE_GROUP)); try { orchestrator.suspendAll( new HostName("parentHostname"), Arrays.asList( - DummyInstanceLookupService.TEST1_HOST_NAME, - DummyInstanceLookupService.TEST3_HOST_NAME, - DummyInstanceLookupService.TEST6_HOST_NAME)); + DummyServiceMonitor.TEST1_HOST_NAME, + DummyServiceMonitor.TEST3_HOST_NAME, + DummyServiceMonitor.TEST6_HOST_NAME)); fail(); } catch (BatchHostStateChangeDeniedException e) { assertEquals("Failed to suspend NodeGroup{application=tenant-id-3:application-instance-3:prod:utopia-1:default, " + @@ -317,8 +319,8 @@ public class OrchestratorImplTest { } InOrder order = inOrder(orchestrator); - order.verify(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST3_NODE_GROUP)); - order.verify(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST6_NODE_GROUP)); + order.verify(orchestrator).suspendGroup(any(), eq(DummyServiceMonitor.TEST3_NODE_GROUP)); + order.verify(orchestrator).suspendGroup(any(), eq(DummyServiceMonitor.TEST6_NODE_GROUP)); order.verifyNoMoreInteractions(); } @@ -329,25 +331,24 @@ public class OrchestratorImplTest { var applicationInstanceReference = new ApplicationInstanceReference(tenantId, applicationInstanceId); var policy = mock(HostedVespaPolicy.class); - var zookeeperStatusService = mock(ZookeeperStatusService.class); - var instanceLookupService = mock(InstanceLookupService.class); + var zookeeperStatusService = mock(ZkStatusService.class); + var serviceMonitor = mock(ServiceMonitor.class); var applicationInstance = mock(ApplicationInstance.class); var clusterControllerClientFactory = mock(ClusterControllerClientFactory.class); var clock = new ManualClock(); var applicationApiFactory = mock(ApplicationApiFactory.class); - var hostStatusRegistry = mock(MutableStatusRegistry.class); + var lock = mock(ApplicationLock.class); - when(instanceLookupService.findInstanceByHost(any())).thenReturn(Optional.of(applicationInstance)); + when(serviceMonitor.getApplication(any(HostName.class))).thenReturn(Optional.of(applicationInstance)); when(applicationInstance.reference()).thenReturn(applicationInstanceReference); - when(zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(any(), any())) - .thenReturn(hostStatusRegistry); - when(hostStatusRegistry.getStatus()).thenReturn(NO_REMARKS); + when(zookeeperStatusService.lockApplication(any(), any())).thenReturn(lock); + when(lock.getApplicationInstanceStatus()).thenReturn(NO_REMARKS); var orchestrator = new OrchestratorImpl( policy, clusterControllerClientFactory, zookeeperStatusService, - instanceLookupService, + serviceMonitor, 20, clock, applicationApiFactory, @@ -358,7 +359,7 @@ public class OrchestratorImplTest { orchestrator.suspendAll(parentHostname, List.of(parentHostname)); ArgumentCaptor<OrchestratorContext> contextCaptor = ArgumentCaptor.forClass(OrchestratorContext.class); - verify(zookeeperStatusService, times(2)).lockApplicationInstance_forCurrentThreadOnly(contextCaptor.capture(), any()); + verify(zookeeperStatusService, times(2)).lockApplication(contextCaptor.capture(), any()); List<OrchestratorContext> contexts = contextCaptor.getAllValues(); // First invocation is probe, second is not. @@ -370,26 +371,26 @@ public class OrchestratorImplTest { verify(applicationApiFactory, times(2)).create(any(), any(), any()); verify(policy, times(2)).grantSuspensionRequest(any(), any()); - verify(instanceLookupService, atLeastOnce()).findInstanceByHost(any()); - verify(hostStatusRegistry, times(2)).getStatus(); + verify(serviceMonitor, atLeastOnce()).getApplication(any(HostName.class)); + verify(lock, times(2)).getApplicationInstanceStatus(); // Each zookeeperStatusService that is created, is closed. - verify(zookeeperStatusService, times(2)).lockApplicationInstance_forCurrentThreadOnly(any(), any()); - verify(hostStatusRegistry, times(2)).close(); + verify(zookeeperStatusService, times(2)).lockApplication(any(), any()); + verify(lock, times(2)).close(); verifyNoMoreInteractions( policy, clusterControllerClientFactory, zookeeperStatusService, - hostStatusRegistry, - instanceLookupService, + lock, + serviceMonitor, applicationApiFactory); } @Test public void testGetHost() throws Exception { ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock(); - StatusService statusService = new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer()); + StatusService statusService = new ZkStatusService(new MockCurator(), mock(Metric.class), new TestTimer()); HostName hostName = new HostName("host.yahoo.com"); TenantId tenantId = new TenantId("tenant"); @@ -415,13 +416,14 @@ public class OrchestratorImplTest { hostName, ServiceStatus.NOT_CHECKED))))); - InstanceLookupService lookupService = new ServiceMonitorInstanceLookupService( - () -> new ServiceModel(Map.of(reference, applicationInstance))); + ServiceMonitor serviceMonitor = new ServiceModelCache( + () -> new ServiceModel(Map.of(reference, applicationInstance)), + new TestTimer()); orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, applicationApiFactory), clusterControllerClientFactory, statusService, - lookupService, + serviceMonitor, 0, new ManualClock(), applicationApiFactory, @@ -438,8 +440,8 @@ public class OrchestratorImplTest { } private boolean isInMaintenance(ApplicationId appId, HostName hostName) throws ApplicationIdNotFoundException { - for (ApplicationInstance app : DummyInstanceLookupService.getApplications()) { - if (app.reference().equals(OrchestratorUtil.toApplicationInstanceReference(appId, new DummyInstanceLookupService()))) { + for (ApplicationInstance app : DummyServiceMonitor.getApplications()) { + if (app.reference().equals(OrchestratorUtil.toApplicationInstanceReference(appId, new DummyServiceMonitor()))) { return clustercontroller.isInMaintenance(app, hostName); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java index 230048f505d..76fbb5c9fe2 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java @@ -39,14 +39,14 @@ public class OrchestratorUtilTest { public void applicationid_conversion_are_symmetric() throws Exception { // From appId to appRef and back - ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(APPID_1, new DummyInstanceLookupService()); + ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(APPID_1, new DummyServiceMonitor()); ApplicationId appIdRoundTrip = OrchestratorUtil.toApplicationId(appRef); Assert.assertEquals(APPID_1, appIdRoundTrip); // From appRef to appId and back ApplicationId appId = OrchestratorUtil.toApplicationId(APPREF_1); - ApplicationInstanceReference appRefRoundTrip = OrchestratorUtil.toApplicationInstanceReference(appId, new DummyInstanceLookupService()); + ApplicationInstanceReference appRefRoundTrip = OrchestratorUtil.toApplicationInstanceReference(appId, new DummyServiceMonitor()); Assert.assertEquals(APPREF_1, appRefRoundTrip); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java index 5c5ee7d2260..502e42481aa 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.orchestrator.controller; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.orchestrator.DummyInstanceLookupService; +import com.yahoo.vespa.orchestrator.DummyServiceMonitor; import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.model.VespaModelUtil; @@ -37,8 +37,8 @@ public class ClusterControllerClientFactoryMock implements ClusterControllerClie } public void setAllDummyNodesAsUp() { - for (ApplicationInstance app : DummyInstanceLookupService.getApplications()) { - Set<HostName> hosts = DummyInstanceLookupService.getContentHosts(app.reference()); + for (ApplicationInstance app : DummyServiceMonitor.getApplications()) { + Set<HostName> hosts = DummyServiceMonitor.getContentHosts(app.reference()); for (HostName host : hosts) { ClusterId clusterName = VespaModelUtil.getContentClusterName(app, host); int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(app, host); diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java index eff222bc074..3d0746e5a36 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java @@ -21,18 +21,19 @@ import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.OrchestratorImpl; -import com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; +import com.yahoo.vespa.orchestrator.status.ApplicationLock; import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostInfos; import com.yahoo.vespa.orchestrator.status.HostStatus; -import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; -import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService; +import com.yahoo.vespa.orchestrator.status.ZkStatusService; +import com.yahoo.vespa.service.model.ServiceModelCache; import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import com.yahoo.yolean.Exceptions; import java.time.Clock; @@ -57,11 +58,13 @@ class ModelTestUtils { private final Map<ApplicationInstanceReference, ApplicationInstance> applications = new HashMap<>(); private final ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock(); private final Map<HostName, HostStatus> hostStatusMap = new HashMap<>(); - private final StatusService statusService = new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer()); + private final StatusService statusService = new ZkStatusService(new MockCurator(), mock(Metric.class), new TestTimer()); + private final TestTimer timer = new TestTimer(); + private final ServiceMonitor serviceMonitor = new ServiceModelCache(() -> new ServiceModel(applications), timer); private final Orchestrator orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, applicationApiFactory()), clusterControllerClientFactory, statusService, - new ServiceMonitorInstanceLookupService(() -> new ServiceModel(applications)), + serviceMonitor, 0, new ManualClock(), applicationApiFactory(), @@ -110,10 +113,10 @@ class ModelTestUtils { ApplicationInstance applicationInstance, HostName... hostnames) { NodeGroup nodeGroup = new NodeGroup(applicationInstance, hostnames); - MutableStatusRegistry registry = statusService.lockApplicationInstance_forCurrentThreadOnly( + ApplicationLock lock = statusService.lockApplication( OrchestratorContext.createContextForSingleAppOp(Clock.systemUTC()), applicationInstance.reference()); - return applicationApiFactory().create(nodeGroup, registry, clusterControllerClientFactory); + return applicationApiFactory().create(nodeGroup, lock, clusterControllerClientFactory); } ApplicationInstance createApplicationInstance( diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java index 80d0af09792..1fff8f976bb 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java @@ -161,8 +161,8 @@ public class ApplicationSuspensionResourceTest { " </config>\n" + " <component id=\"com.yahoo.vespa.flags.InMemoryFlagSource\" bundle=\"flags\" />\n" + " <component id=\"com.yahoo.vespa.curator.mock.MockCurator\" bundle=\"zkfacade\" />\n" + - " <component id=\"com.yahoo.vespa.orchestrator.status.ZookeeperStatusService\" bundle=\"orchestrator\" />\n" + - " <component id=\"com.yahoo.vespa.orchestrator.DummyInstanceLookupService\" bundle=\"orchestrator\" />\n" + + " <component id=\"com.yahoo.vespa.orchestrator.status.ZkStatusService\" bundle=\"orchestrator\" />\n" + + " <component id=\"com.yahoo.vespa.orchestrator.DummyServiceMonitor\" bundle=\"orchestrator\" />\n" + " <component id=\"com.yahoo.vespa.orchestrator.OrchestratorImpl\" bundle=\"orchestrator\" />\n" + " <component id=\"com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock\" bundle=\"orchestrator\" />\n" + "\n" + diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index bfa68145828..b620b0798be 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException; import com.yahoo.vespa.orchestrator.BatchInternalErrorException; import com.yahoo.vespa.orchestrator.Host; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; -import com.yahoo.vespa.orchestrator.InstanceLookupService; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.OrchestratorContext; @@ -37,11 +36,13 @@ import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; +import com.yahoo.vespa.orchestrator.status.ApplicationLock; import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; -import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; -import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService; +import com.yahoo.vespa.orchestrator.status.ZkStatusService; +import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import org.junit.Before; import org.junit.Test; @@ -56,8 +57,8 @@ import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; -import java.util.Set; import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet; import static org.junit.Assert.assertEquals; @@ -78,13 +79,13 @@ public class HostResourceTest { private static final int SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS = 0; private static final TenantId TENANT_ID = new TenantId("tenantId"); private static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("applicationId"); - private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = new ZookeeperStatusService( + private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = new ZkStatusService( new MockCurator(), mock(Metric.class), new TestTimer()); private static final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3); - private static final InstanceLookupService mockInstanceLookupService = mock(InstanceLookupService.class); + private static final ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); static { - when(mockInstanceLookupService.findInstanceByHost(any())) + when(serviceMonitor.getApplication(any(HostName.class))) .thenReturn(Optional.of( new ApplicationInstance( TENANT_ID, @@ -94,21 +95,12 @@ public class HostResourceTest { private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - private static final InstanceLookupService alwaysEmptyInstanceLookUpService = new InstanceLookupService() { - @Override - public Optional<ApplicationInstance> findInstanceById( - final ApplicationInstanceReference applicationInstanceReference) { - return Optional.empty(); - } - - @Override - public Optional<ApplicationInstance> findInstanceByHost(final HostName hostName) { - return Optional.empty(); - } + private static final ServiceMonitor alwaysEmptyServiceMonitor = new ServiceMonitor() { + private final ServiceModel emptyServiceModel = new ServiceModel(Map.of()); @Override - public Set<ApplicationInstanceReference> knownInstances() { - return Collections.emptySet(); + public ServiceModel getServiceModelSnapshot() { + return emptyServiceModel; } }; @@ -129,14 +121,14 @@ public class HostResourceTest { public void releaseSuspensionGrant( OrchestratorContext context, ApplicationInstance applicationInstance, HostName hostName, - MutableStatusRegistry hostStatusRegistry) { + ApplicationLock hostStatusRegistry) { } } private final OrchestratorImpl alwaysAllowOrchestrator = new OrchestratorImpl( new AlwaysAllowPolicy(), new ClusterControllerClientFactoryMock(), - EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, mockInstanceLookupService, + EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, serviceMonitor, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, applicationApiFactory, @@ -145,7 +137,7 @@ public class HostResourceTest { private final OrchestratorImpl hostNotFoundOrchestrator = new OrchestratorImpl( new AlwaysAllowPolicy(), new ClusterControllerClientFactoryMock(), - EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, alwaysEmptyInstanceLookUpService, + EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, alwaysEmptyServiceMonitor, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, applicationApiFactory, @@ -231,7 +223,7 @@ public class HostResourceTest { public void releaseSuspensionGrant( OrchestratorContext context, ApplicationInstance applicationInstance, HostName hostName, - MutableStatusRegistry hostStatusRegistry) throws HostStateChangeDeniedException { + ApplicationLock hostStatusRegistry) throws HostStateChangeDeniedException { doThrow(); } @@ -248,7 +240,7 @@ public class HostResourceTest { final OrchestratorImpl alwaysRejectResolver = new OrchestratorImpl( new AlwaysFailPolicy(), new ClusterControllerClientFactoryMock(), - EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,mockInstanceLookupService, + EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, serviceMonitor, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, applicationApiFactory, @@ -269,7 +261,7 @@ public class HostResourceTest { new AlwaysFailPolicy(), new ClusterControllerClientFactoryMock(), EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, - mockInstanceLookupService, + serviceMonitor, SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, applicationApiFactory, diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService2Test.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusService2Test.java index 8f530f4abf3..c5d390050ee 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService2Test.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusService2Test.java @@ -27,12 +27,12 @@ import static org.mockito.Mockito.when; /** * @author hakonhall */ -public class ZookeeperStatusService2Test { +public class ZkStatusService2Test { private final Curator curator = mock(Curator.class); private final Timer timer = new TestTimer(); private final Metric metric = mock(Metric.class); private final HostInfosCache cache = mock(HostInfosCache.class); - private final ZookeeperStatusService zookeeperStatusService = new ZookeeperStatusService(curator, metric, timer, cache); + private final ZkStatusService zkStatusService = new ZkStatusService(curator, metric, timer, cache); private final OrchestratorContext context = mock(OrchestratorContext.class); private final InterProcessMutex mutex = mock(InterProcessMutex.class); @@ -50,7 +50,7 @@ public class ZookeeperStatusService2Test { when(context.getTimeLeft()).thenReturn(Duration.ofSeconds(12)); - try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + try (ApplicationLock lock = zkStatusService.lockApplication(context, reference)) { // nothing } @@ -65,7 +65,7 @@ public class ZookeeperStatusService2Test { when(context.isProbe()).thenReturn(false); - try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + try (ApplicationLock lock = zkStatusService.lockApplication(context, reference)) { // nothing } @@ -88,7 +88,7 @@ public class ZookeeperStatusService2Test { when(context.getTimeLeft()).thenReturn(Duration.ofSeconds(12)); - try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + try (ApplicationLock lock = zkStatusService.lockApplication(context, reference)) { // nothing } @@ -105,7 +105,7 @@ public class ZookeeperStatusService2Test { when(context.hasLock(any())).thenReturn(true); when(context.registerLockAcquisition(any(), any())).thenReturn(false); - try (MutableStatusRegistry registry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly(context, reference)) { + try (ApplicationLock lock = zkStatusService.lockApplication(context, reference)) { // nothing } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java index 687ea951f88..a6d7f09d69b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java @@ -53,9 +53,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) -public class ZookeeperStatusServiceTest { +public class ZkStatusServiceTest { private TestingServer testingServer; - private ZookeeperStatusService zookeeperStatusService; + private ZkStatusService zkStatusService; private Curator curator; private final Timer timer = mock(Timer.class); private final Metric metric = mock(Metric.class); @@ -70,7 +70,7 @@ public class ZookeeperStatusServiceTest { testingServer = new TestingServer(); curator = createConnectedCurator(testingServer); - zookeeperStatusService = new ZookeeperStatusService(curator, metric, timer); + zkStatusService = new ZkStatusService(curator, metric, timer); when(context.getTimeLeft()).thenReturn(Duration.ofSeconds(10)); when(context.isProbe()).thenReturn(false); when(timer.currentTime()).thenReturn(Instant.ofEpochMilli(1)); @@ -96,7 +96,7 @@ public class ZookeeperStatusServiceTest { @Test public void host_state_for_unknown_hosts_is_no_remarks() { assertThat( - zookeeperStatusService.getHostInfo(TestIds.APPLICATION_INSTANCE_REFERENCE, TestIds.HOST_NAME1).status(), + zkStatusService.getHostInfo(TestIds.APPLICATION_INSTANCE_REFERENCE, TestIds.HOST_NAME1).status(), is(HostStatus.NO_REMARKS)); } @@ -107,17 +107,14 @@ public class ZookeeperStatusServiceTest { Instant.ofEpochMilli((3)), Instant.ofEpochMilli(6)); - try (MutableStatusRegistry statusRegistry = zookeeperStatusService - .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { + try (ApplicationLock lock = zkStatusService.lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { //shuffling to catch "clean database" failures for all cases. for (HostStatus hostStatus: shuffledList(HostStatus.NO_REMARKS, HostStatus.ALLOWED_TO_BE_DOWN)) { for (int i = 0; i < 2; i++) { - statusRegistry.setHostState( - TestIds.HOST_NAME1, - hostStatus); + lock.setHostState(TestIds.HOST_NAME1, hostStatus); - assertThat(statusRegistry.getHostInfos().getOrNoRemarks(TestIds.HOST_NAME1).status(), + assertThat(lock.getHostInfos().getOrNoRemarks(TestIds.HOST_NAME1).status(), is(hostStatus)); } } @@ -141,15 +138,15 @@ public class ZookeeperStatusServiceTest { @Test public void locks_are_exclusive() throws Exception { - ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curator, mock(Metric.class), new TestTimer()); + ZkStatusService zkStatusService2 = new ZkStatusService(curator, mock(Metric.class), new TestTimer()); final CompletableFuture<Void> lockedSuccessfullyFuture; - try (MutableStatusRegistry statusRegistry = zookeeperStatusService - .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { + try (ApplicationLock lock = zkStatusService + .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { lockedSuccessfullyFuture = CompletableFuture.runAsync(() -> { - try (MutableStatusRegistry statusRegistry2 = zookeeperStatusService2 - .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) + try (ApplicationLock lock2 = zkStatusService2 + .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { } }); @@ -166,15 +163,15 @@ public class ZookeeperStatusServiceTest { @Test public void failing_to_get_lock_closes_SessionFailRetryLoop() throws Exception { - ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curator, mock(Metric.class), new TestTimer()); + ZkStatusService zkStatusService2 = new ZkStatusService(curator, mock(Metric.class), new TestTimer()); - try (MutableStatusRegistry statusRegistry = zookeeperStatusService - .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { + try (ApplicationLock lock = zkStatusService + .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { //must run in separate thread, since having 2 locks in the same thread fails CompletableFuture<Void> resultOfZkOperationAfterLockFailure = CompletableFuture.runAsync(() -> { try { - zookeeperStatusService2.lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE); + zkStatusService2.lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE); fail("Both zookeeper host status services locked simultaneously for the same application instance"); } catch (RuntimeException e) { } @@ -182,7 +179,7 @@ public class ZookeeperStatusServiceTest { killSession(curator.framework(), testingServer); //Throws SessionFailedException if the SessionFailRetryLoop has not been closed. - statusRegistry.getHostInfos().getOrNoRemarks(TestIds.HOST_NAME1); + lock.getHostInfos().getOrNoRemarks(TestIds.HOST_NAME1); }); assertThat(resultOfZkOperationAfterLockFailure, notHoldsException()); @@ -239,29 +236,29 @@ public class ZookeeperStatusServiceTest { // Initial state is NO_REMARK assertThat( - zookeeperStatusService + zkStatusService .getApplicationInstanceStatus(TestIds.APPLICATION_INSTANCE_REFERENCE), is(ApplicationInstanceStatus.NO_REMARKS)); // Suspend - try (MutableStatusRegistry statusRegistry = zookeeperStatusService - .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { - statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN); + try (ApplicationLock lock = zkStatusService + .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { + lock.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN); } assertThat( - zookeeperStatusService + zkStatusService .getApplicationInstanceStatus(TestIds.APPLICATION_INSTANCE_REFERENCE), is(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN)); // Resume - try (MutableStatusRegistry statusRegistry = zookeeperStatusService - .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { - statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.NO_REMARKS); + try (ApplicationLock lock = zkStatusService + .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { + lock.setApplicationInstanceStatus(ApplicationInstanceStatus.NO_REMARKS); } assertThat( - zookeeperStatusService + zkStatusService .getApplicationInstanceStatus(TestIds.APPLICATION_INSTANCE_REFERENCE), is(ApplicationInstanceStatus.NO_REMARKS)); } @@ -269,20 +266,20 @@ public class ZookeeperStatusServiceTest { @Test public void suspending_two_applications_returns_two_applications() { Set<ApplicationInstanceReference> suspendedApps - = zookeeperStatusService.getAllSuspendedApplications(); + = zkStatusService.getAllSuspendedApplications(); assertThat(suspendedApps.size(), is(0)); - try (MutableStatusRegistry statusRegistry = zookeeperStatusService - .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { + try (ApplicationLock statusRegistry = zkStatusService + .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) { statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN); } - try (MutableStatusRegistry statusRegistry = zookeeperStatusService - .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE2)) { - statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN); + try (ApplicationLock lock = zkStatusService + .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE2)) { + lock.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN); } - suspendedApps = zookeeperStatusService.getAllSuspendedApplications(); + suspendedApps = zkStatusService.getAllSuspendedApplications(); assertThat(suspendedApps.size(), is(2)); assertThat(suspendedApps, hasItem(TestIds.APPLICATION_INSTANCE_REFERENCE)); assertThat(suspendedApps, hasItem(TestIds.APPLICATION_INSTANCE_REFERENCE2)); diff --git a/parent/pom.xml b/parent/pom.xml index 81a7e3360b5..c2b82ff7b46 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -744,7 +744,7 @@ <apache.httpcore.version>4.4.13</apache.httpcore.version> <asm.version>7.0</asm.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <athenz.version>1.8.44</athenz.version> + <athenz.version>1.8.49</athenz.version> <aws.sdk.version>1.11.542</aws.sdk.version> <!-- WARNING: If you change curator version, you also need to update zkfacade/src/main/java/org/apache/curator/**/package-info.java diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index eb4f1f6dc89..6040fc651c2 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -117,6 +117,10 @@ indexing.read.io enum {NORMAL, DIRECTIO} default=DIRECTIO restart ## Control number of threads used for indexing indexing.threads int default=1 restart +## Option to specify what is most important during indexing. +## This is experimental and will most likely be temporary. +indexing.optimize enum {LATENCY, THROUGHPUT} default=LATENCY restart + ## Maximum number of pending operations for each of the internal ## indexing threads. Only used when visibility delay is zero. indexing.tasklimit int default=1000 restart diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 302ffc93f6a..1ee1b703ea9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -140,7 +140,8 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _writeService(sharedExecutor, _writeServiceConfig.indexingThreads(), indexing_thread_stack_size, - _writeServiceConfig.defaultTaskLimit()), + _writeServiceConfig.defaultTaskLimit(), + _writeServiceConfig.optimize()), _initializeThreads(std::move(initializeThreads)), _initConfigSnapshot(), _initConfigSerialNum(0u), diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index 058197f0271..6e7b4967f6d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -10,7 +10,8 @@ using search::SequencedTaskExecutor; namespace proton { ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadStackExecutorBase & sharedExecutor, - uint32_t threads, uint32_t stackSize, uint32_t taskLimit) + uint32_t threads, uint32_t stackSize, uint32_t taskLimit, + OptimizeFor optimize) : _sharedExecutor(sharedExecutor), _masterExecutor(1, stackSize), @@ -21,7 +22,7 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadStackExecutor _summaryService(_summaryExecutor), _indexFieldInverter(SequencedTaskExecutor::create(threads, taskLimit)), _indexFieldWriter(SequencedTaskExecutor::create(threads, taskLimit)), - _attributeFieldWriter(SequencedTaskExecutor::create(threads, taskLimit)) + _attributeFieldWriter(SequencedTaskExecutor::create(threads, taskLimit, optimize)) { } diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h index 7bbe1cb162a..2e4dd2035f3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h @@ -29,6 +29,7 @@ private: std::unique_ptr<search::ISequencedTaskExecutor> _attributeFieldWriter; public: + using OptimizeFor = vespalib::Executor::OptimizeFor; /** * Constructor. * @@ -38,7 +39,8 @@ public: ExecutorThreadingService(vespalib::ThreadStackExecutorBase &sharedExecutor, uint32_t threads = 1, uint32_t stackSize = 128 * 1024, - uint32_t taskLimit = 1000); + uint32_t taskLimit = 1000, + OptimizeFor optimize = OptimizeFor::LATENCY); ~ExecutorThreadingService() override; /** diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp index 55aa1a20ef6..5bc6ef543f3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp @@ -7,13 +7,16 @@ namespace proton { using ProtonConfig = ThreadingServiceConfig::ProtonConfig; +using OptimizeFor = vespalib::Executor::OptimizeFor; ThreadingServiceConfig::ThreadingServiceConfig(uint32_t indexingThreads_, uint32_t defaultTaskLimit_, - uint32_t semiUnboundTaskLimit_) + uint32_t semiUnboundTaskLimit_, + OptimizeFor optimize) : _indexingThreads(indexingThreads_), _defaultTaskLimit(defaultTaskLimit_), - _semiUnboundTaskLimit(semiUnboundTaskLimit_) + _semiUnboundTaskLimit(semiUnboundTaskLimit_), + _optimize(optimize) { } @@ -30,6 +33,16 @@ calculateIndexingThreads(uint32_t cfgIndexingThreads, double concurrency, const return std::max(indexingThreads, 1u); } +OptimizeFor +selectOptimization(ProtonConfig::Indexing::Optimize optimize) { + using CfgOptimize = ProtonConfig::Indexing::Optimize; + switch (optimize) { + case CfgOptimize::LATENCY: return OptimizeFor::LATENCY; + case CfgOptimize::THROUGHPUT: return OptimizeFor::THROUGHPUT; + } + return OptimizeFor::LATENCY; +} + } ThreadingServiceConfig @@ -37,7 +50,8 @@ ThreadingServiceConfig::make(const ProtonConfig &cfg, double concurrency, const { uint32_t indexingThreads = calculateIndexingThreads(cfg.indexing.threads, concurrency, cpuInfo); return ThreadingServiceConfig(indexingThreads, cfg.indexing.tasklimit, - (cfg.indexing.semiunboundtasklimit / indexingThreads)); + (cfg.indexing.semiunboundtasklimit / indexingThreads), + selectOptimization(cfg.indexing.optimize)); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h index be39f516598..149215f97dc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/vespalib/util/executor.h> #include <cstdint> namespace vespa::config::search::core::internal { class InternalProtonType; } @@ -13,14 +14,16 @@ namespace proton { class ThreadingServiceConfig { public: using ProtonConfig = const vespa::config::search::core::internal::InternalProtonType; + using OptimizeFor = vespalib::Executor::OptimizeFor; private: - uint32_t _indexingThreads; - uint32_t _defaultTaskLimit; - uint32_t _semiUnboundTaskLimit; + uint32_t _indexingThreads; + uint32_t _defaultTaskLimit; + uint32_t _semiUnboundTaskLimit; + OptimizeFor _optimize; private: - ThreadingServiceConfig(uint32_t indexingThreads_, uint32_t defaultTaskLimit_, uint32_t semiUnboundTaskLimit_); + ThreadingServiceConfig(uint32_t indexingThreads_, uint32_t defaultTaskLimit_, uint32_t semiUnboundTaskLimit_, OptimizeFor optimize); public: static ThreadingServiceConfig make(const ProtonConfig &cfg, double concurrency, const HwInfo::Cpu &cpuInfo); @@ -28,6 +31,7 @@ public: uint32_t indexingThreads() const { return _indexingThreads; } uint32_t defaultTaskLimit() const { return _defaultTaskLimit; } uint32_t semiUnboundTaskLimit() const { return _semiUnboundTaskLimit; } + OptimizeFor optimize() const { return _optimize;} }; } diff --git a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp index 24be21f65ec..47728c9785c 100644 --- a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp @@ -295,7 +295,7 @@ public: request_ctx.set_query_tensor("query_tensor", tensor_spec); } Blueprint::UP create_blueprint() { - query::NearestNeighborTerm term("query_tensor", attr_name, 0, Weight(0), 7); + query::NearestNeighborTerm term("query_tensor", attr_name, 0, Weight(0), 7, true, 33); return source.createBlueprint(request_ctx, FieldSpec(attr_name, 0, 0), term); } }; diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index a0538952ad9..3ef0ab20cdd 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -226,7 +226,6 @@ struct Fixture while (_attr->getNumDocs() <= docId) { uint32_t newDocId = 0u; _attr->addDoc(newDocId); - _attr->commit(); } } @@ -326,7 +325,6 @@ Fixture::testSetTensorValue() { ensureSpace(4); EXPECT_EQUAL(5u, _attr->getNumDocs()); - EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit()); TEST_DO(assertGetNoTensor(4)); EXPECT_EXCEPTION(set_tensor(4, TensorSpec("double")), WrongTensorTypeException, diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp index b2b15ded274..9491617c135 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp +++ b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp @@ -10,13 +10,19 @@ using ExecutorId = search::ISequencedTaskExecutor::ExecutorId; int main(int argc, char *argv[]) { unsigned long numTasks = 1000000; unsigned numThreads = 4; + unsigned taskLimit = 1000; + vespalib::Executor::OptimizeFor optimize = vespalib::Executor::OptimizeFor::LATENCY; std::atomic<long> counter(0); if (argc > 1) numTasks = atol(argv[1]); if (argc > 2) numThreads = atoi(argv[2]); + if (argc > 3) + taskLimit = atoi(argv[3]); + if (argc > 4) + optimize = vespalib::Executor::OptimizeFor::THROUGHPUT; - auto executor = SequencedTaskExecutor::create(numThreads); + auto executor = SequencedTaskExecutor::create(numThreads, taskLimit, optimize); for (unsigned long tid(0); tid < numTasks; tid++) { executor->executeTask(ExecutorId(tid%numThreads), vespalib::makeLambdaTask([&counter] { counter++; })); } diff --git a/searchlib/src/tests/query/query_visitor_test.cpp b/searchlib/src/tests/query/query_visitor_test.cpp index 39e381c0942..edbc29be784 100644 --- a/searchlib/src/tests/query/query_visitor_test.cpp +++ b/searchlib/src/tests/query/query_visitor_test.cpp @@ -99,7 +99,7 @@ void Test::requireThatAllNodesCanBeVisited() { checkVisit<SuffixTerm>(new SimpleSuffixTerm("t", "field", 0, Weight(0))); checkVisit<PredicateQuery>(new SimplePredicateQuery(PredicateQueryTerm::UP(), "field", 0, Weight(0))); checkVisit<RegExpTerm>(new SimpleRegExpTerm("t", "field", 0, Weight(0))); - checkVisit<NearestNeighborTerm>(new SimpleNearestNeighborTerm("query_tensor", "doc_tensor", 0, Weight(0), 123)); + checkVisit<NearestNeighborTerm>(new SimpleNearestNeighborTerm("query_tensor", "doc_tensor", 0, Weight(0), 123, true, 321)); } } // namespace diff --git a/searchlib/src/tests/query/querybuilder_test.cpp b/searchlib/src/tests/query/querybuilder_test.cpp index 7f496b3493c..8560cb0e091 100644 --- a/searchlib/src/tests/query/querybuilder_test.cpp +++ b/searchlib/src/tests/query/querybuilder_test.cpp @@ -111,7 +111,7 @@ Node::UP createQueryTree() { builder.addStringTerm(str[5], view[5], id[5], weight[6]); builder.addStringTerm(str[6], view[6], id[6], weight[7]); } - builder.add_nearest_neighbor_term("query_tensor", "doc_tensor", id[3], weight[5], 7); + builder.add_nearest_neighbor_term("query_tensor", "doc_tensor", id[3], weight[5], 7, true, 33); } Node::UP node = builder.build(); ASSERT_TRUE(node.get()); @@ -395,8 +395,9 @@ struct MyRegExpTerm : RegExpTerm { }; struct MyNearestNeighborTerm : NearestNeighborTerm { MyNearestNeighborTerm(vespalib::stringref query_tensor_name, vespalib::stringref field_name, - int32_t i, Weight w, uint32_t target_num_hits) - : NearestNeighborTerm(query_tensor_name, field_name, i, w, target_num_hits) + int32_t i, Weight w, uint32_t target_num_hits, + bool allow_approximate, uint32_t explore_additional_hits) + : NearestNeighborTerm(query_tensor_name, field_name, i, w, target_num_hits, allow_approximate, explore_additional_hits) {} }; diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 2516950b0cc..1604a39eaf3 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/tensor/doc_vector_access.h> #include <vespa/searchlib/tensor/hnsw_index.h> #include <vespa/searchlib/tensor/random_level_generator.h> +#include <vespa/searchlib/tensor/inv_log_level_generator.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/generationhandler.h> #include <vector> @@ -65,6 +66,9 @@ public: .set(4, {1, 2}).set(5, {8, 3}).set(6, {7, 2}) .set(7, {3, 5}).set(8, {0, 3}).set(9, {4, 5}); } + + ~HnswIndexTest() {} + void init(bool heuristic_select_neighbors) { auto generator = std::make_unique<LevelGenerator>(); level_generator = generator.get(); @@ -441,5 +445,36 @@ TEST_F(HnswIndexTest, shrink_called_heuristic) EXPECT_TRUE(index->check_link_symmetry()); } +TEST(LevelGeneratorTest, gives_various_levels) +{ + InvLogLevelGenerator generator(4); + EXPECT_EQ(2u, generator.max_level()); + EXPECT_EQ(1u, generator.max_level()); + EXPECT_EQ(0u, generator.max_level()); + EXPECT_EQ(1u, generator.max_level()); + EXPECT_EQ(0u, generator.max_level()); + EXPECT_EQ(1u, generator.max_level()); + EXPECT_EQ(0u, generator.max_level()); + EXPECT_EQ(0u, generator.max_level()); + EXPECT_EQ(0u, generator.max_level()); + + uint32_t left = 1000000; + std::vector<uint32_t> hist; + for (uint32_t i = 0; i < left; ++i) { + uint32_t l = generator.max_level(); + if (hist.size() <= l) { + hist.resize(l+1); + } + hist[l]++; + } + for (uint32_t l = 0; l < hist.size(); ++l) { + double expected = left * 0.75; + EXPECT_TRUE(hist[l] < expected*1.01 + 100); + EXPECT_TRUE(hist[l] > expected*0.99 - 100); + left *= 0.25; + } + EXPECT_TRUE(hist.size() < 14); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 8595b0eff7f..9af05059bef 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -646,7 +646,9 @@ public: query_tensor.release(); setResult(std::make_unique<queryeval::NearestNeighborBlueprint>(_field, *dense_attr_tensor, std::move(dense_query_tensor_up), - n.get_target_num_hits())); + n.get_target_num_hits(), + n.get_allow_approximate(), + n.get_explore_additional_hits())); } }; diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp index 30723a6eb2a..7b0c30ec9d8 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp @@ -2,8 +2,10 @@ #include "sequencedtaskexecutor.h" #include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/util/singleexecutor.h> using vespalib::BlockingThreadStackExecutor; +using vespalib::SingleExecutor; namespace search { @@ -15,12 +17,16 @@ constexpr uint32_t stackSize = 128 * 1024; std::unique_ptr<ISequencedTaskExecutor> -SequencedTaskExecutor::create(uint32_t threads, uint32_t taskLimit) +SequencedTaskExecutor::create(uint32_t threads, uint32_t taskLimit, OptimizeFor optimize) { auto executors = std::make_unique<std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>>>(); executors->reserve(threads); for (uint32_t id = 0; id < threads; ++id) { - executors->push_back(std::make_unique<BlockingThreadStackExecutor>(1, stackSize, taskLimit)); + if (optimize == OptimizeFor::THROUGHPUT) { + executors->push_back(std::make_unique<SingleExecutor>(taskLimit)); + } else { + executors->push_back(std::make_unique<BlockingThreadStackExecutor>(1, stackSize, taskLimit)); + } } return std::unique_ptr<ISequencedTaskExecutor>(new SequencedTaskExecutor(std::move(executors))); } diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h index a29e3d5226c..8568901006f 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h @@ -23,6 +23,7 @@ class SequencedTaskExecutor final : public ISequencedTaskExecutor SequencedTaskExecutor(std::unique_ptr<std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>>> executor); public: using ISequencedTaskExecutor::getExecutorId; + using OptimizeFor = vespalib::Executor::OptimizeFor; ~SequencedTaskExecutor(); @@ -30,7 +31,12 @@ public: void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override; void sync() override; Stats getStats() override; - static std::unique_ptr<ISequencedTaskExecutor> create(uint32_t threads, uint32_t taskLimit = 1000); + + /* + * Note that if you choose Optimize::THROUGHPUT, you must ensure only a single producer, or synchronize on the outside. + */ + static std::unique_ptr<ISequencedTaskExecutor> + create(uint32_t threads, uint32_t taskLimit = 1000, OptimizeFor optimize = OptimizeFor::LATENCY); }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp index 70a3097ae05..c42cf8fc370 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp @@ -21,9 +21,11 @@ SimpleQueryStackDumpIterator::SimpleQueryStackDumpIterator(vespalib::stringref b _currUniqueId(0), _currFlags(0), _currArity(0), - _currArg1(0), - _currArg2(0), - _currArg3(0), + _extraIntArg1(0), + _extraIntArg2(0), + _extraIntArg3(0), + _extraDoubleArg4(0), + _extraDoubleArg5(0), _predicate_query_term(), _curr_index_name(), _curr_term(), @@ -138,7 +140,6 @@ SimpleQueryStackDumpIterator::next() case ParseItem::ITEM_ANY: try { _currArity = readCompressedPositiveInt(p); - _currArg1 = 0; _curr_index_name = vespalib::stringref(); _curr_term = vespalib::stringref(); } catch (...) { @@ -150,7 +151,7 @@ SimpleQueryStackDumpIterator::next() case ParseItem::ITEM_ONEAR: try { _currArity = readCompressedPositiveInt(p); - _currArg1 = readCompressedPositiveInt(p); + _extraIntArg1 = readCompressedPositiveInt(p); _curr_index_name = vespalib::stringref(); _curr_term = vespalib::stringref(); } catch (...) { @@ -161,7 +162,7 @@ SimpleQueryStackDumpIterator::next() case ParseItem::ITEM_WEAK_AND: try { _currArity = readCompressedPositiveInt(p); - _currArg1 = readCompressedPositiveInt(p); + _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits _curr_index_name = read_stringref(p); _curr_term = vespalib::stringref(); } catch (...) { @@ -171,7 +172,6 @@ SimpleQueryStackDumpIterator::next() case ParseItem::ITEM_SAME_ELEMENT: try { _currArity = readCompressedPositiveInt(p); - _currArg1 = 0; _curr_index_name = read_stringref(p); _curr_term = vespalib::stringref(); } catch (...) { @@ -182,7 +182,6 @@ SimpleQueryStackDumpIterator::next() case ParseItem::ITEM_PURE_WEIGHTED_STRING: try { _curr_term = read_stringref(p); - _currArg1 = 0; _currArity = 0; } catch (...) { return false; @@ -196,7 +195,6 @@ SimpleQueryStackDumpIterator::next() p += sizeof(int64_t); if (p > _bufEnd) return false; - _currArg1 = 0; _currArity = 0; break; case ParseItem::ITEM_WORD_ALTERNATIVES: @@ -218,7 +216,6 @@ SimpleQueryStackDumpIterator::next() try { _curr_index_name = read_stringref(p); _curr_term = read_stringref(p); - _currArg1 = 0; _currArity = 0; } catch (...) { return false; @@ -258,11 +255,9 @@ SimpleQueryStackDumpIterator::next() _currArity = readCompressedPositiveInt(p); _curr_index_name = read_stringref(p); if (_currType == ParseItem::ITEM_WAND) { - _currArg1 = readCompressedPositiveInt(p); // targetNumHits - _currArg2 = read_double(p); // scoreThreshold - _currArg3 = read_double(p); // thresholdBoostFactor - } else { - _currArg1 = 0; + _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits + _extraDoubleArg4 = read_double(p); // scoreThreshold + _extraDoubleArg5 = read_double(p); // thresholdBoostFactor } _curr_term = vespalib::stringref(); } catch (...) { @@ -274,7 +269,9 @@ SimpleQueryStackDumpIterator::next() try { _curr_index_name = read_stringref(p); _curr_term = read_stringref(p); // query_tensor_name - _currArg1 = readCompressedPositiveInt(p); // target_num_hits; + _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits + _extraIntArg2 = readCompressedPositiveInt(p); // allow_approximate + _extraIntArg3 = readCompressedPositiveInt(p); // explore_additional_hits _currArity = 0; } catch (...) { return false; diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h index 6eb3fb7777d..73c97bb5fb3 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h @@ -42,12 +42,13 @@ private: /** The arity of the current item */ uint32_t _currArity; - /** The first argument of the current item (length of NEAR/ONEAR area for example) */ - uint32_t _currArg1; - /** The second argument of the current item (score threshold of WAND for example) */ - double _currArg2; - /** The third argument of the current item (threshold boost factor of WAND for example) */ - double _currArg3; + + /* extra arguments */ + uint32_t _extraIntArg1; + uint32_t _extraIntArg2; + uint32_t _extraIntArg3; + double _extraDoubleArg4; + double _extraDoubleArg5; /** The predicate query specification */ query::PredicateQueryTerm::UP _predicate_query_term; /** The index name (field name) in the current item */ @@ -118,11 +119,12 @@ public: uint32_t getArity() const { return _currArity; } - uint32_t getArg1() const { return _currArg1; } - - double getArg2() const { return _currArg2; } - - double getArg3() const { return _currArg3; } + uint32_t getNearDistance() const { return _extraIntArg1; } + uint32_t getTargetNumHits() const { return _extraIntArg1; } + double getScoreThreshold() const { return _extraDoubleArg4; } + double getThresholdBoostFactor() const { return _extraDoubleArg5; } + bool getAllowApproximate() const { return (_extraIntArg2 != 0); } + uint32_t getExploreAdditionalHits() const { return _extraIntArg3; } query::PredicateQueryTerm::UP getPredicateQueryTerm() { return std::move(_predicate_query_term); } diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp index 24c458c7e32..3db6c8e68c8 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp @@ -41,7 +41,7 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor QueryConnector * qc = dynamic_cast<QueryConnector *> (qn.get()); NearQueryNode * nqn = dynamic_cast<NearQueryNode *> (qc); if (nqn) { - nqn->distance(queryRep.getArg1()); + nqn->distance(queryRep.getNearDistance()); } if ((type == ParseItem::ITEM_WEAK_AND) || (type == ParseItem::ITEM_WEIGHTED_SET) || diff --git a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h index 797defc39f5..8e6f2944ec9 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h +++ b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h @@ -205,8 +205,11 @@ createRegExpTerm(vespalib::stringref term, vespalib::stringref view, int32_t id, template <class NodeTypes> typename NodeTypes::NearestNeighborTerm * create_nearest_neighbor_term(vespalib::stringref query_tensor_name, vespalib::stringref field_name, - int32_t id, Weight weight, uint32_t target_num_hits) { - return new typename NodeTypes::NearestNeighborTerm(query_tensor_name, field_name, id, weight, target_num_hits); + int32_t id, Weight weight, uint32_t target_num_hits, + bool allow_approximate, uint32_t explore_additional_hits) +{ + return new typename NodeTypes::NearestNeighborTerm(query_tensor_name, field_name, id, weight, + target_num_hits, allow_approximate, explore_additional_hits); } template <class NodeTypes> @@ -317,9 +320,10 @@ public: return addTerm(createRegExpTerm<NodeTypes>(term, view, id, weight)); } typename NodeTypes::NearestNeighborTerm &add_nearest_neighbor_term(stringref query_tensor_name, stringref field_name, - int32_t id, Weight weight, uint32_t target_num_hits) { + int32_t id, Weight weight, uint32_t target_num_hits, + bool allow_approximate, uint32_t explore_additional_hits) { adjustWeight(weight); - return addTerm(create_nearest_neighbor_term<NodeTypes>(query_tensor_name, field_name, id, weight, target_num_hits)); + return addTerm(create_nearest_neighbor_term<NodeTypes>(query_tensor_name, field_name, id, weight, target_num_hits, allow_approximate, explore_additional_hits)); } }; diff --git a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h index 0bf923960b9..9289df7cbe9 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h @@ -165,7 +165,8 @@ private: void visit(NearestNeighborTerm &node) override { replicate(node, _builder.add_nearest_neighbor_term(node.get_query_tensor_name(), node.getView(), - node.getId(), node.getWeight(), node.get_target_num_hits())); + node.getId(), node.getWeight(), node.get_target_num_hits(), + node.get_allow_approximate(), node.get_explore_additional_hits())); } }; diff --git a/searchlib/src/vespa/searchlib/query/tree/simplequery.h b/searchlib/src/vespa/searchlib/query/tree/simplequery.h index 8663bede4d6..4953f1a5b7c 100644 --- a/searchlib/src/vespa/searchlib/query/tree/simplequery.h +++ b/searchlib/src/vespa/searchlib/query/tree/simplequery.h @@ -105,8 +105,10 @@ struct SimpleRegExpTerm : RegExpTerm { }; struct SimpleNearestNeighborTerm : NearestNeighborTerm { SimpleNearestNeighborTerm(vespalib::stringref query_tensor_name, vespalib::stringref field_name, - int32_t id, Weight weight, uint32_t target_num_hits) - : NearestNeighborTerm(query_tensor_name, field_name, id, weight, target_num_hits) + int32_t id, Weight weight, uint32_t target_num_hits, + bool allow_approximate, uint32_t explore_additional_hits) + : NearestNeighborTerm(query_tensor_name, field_name, id, weight, + target_num_hits, allow_approximate, explore_additional_hits) {} }; diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp index 63acf532144..aafeaa46a22 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp @@ -263,6 +263,8 @@ class QueryNodeConverter : public QueryVisitor { createTermNode(node, ParseItem::ITEM_NEAREST_NEIGHBOR); appendString(node.get_query_tensor_name()); appendCompressedPositiveNumber(node.get_target_num_hits()); + appendCompressedPositiveNumber(node.get_allow_approximate() ? 1 : 0); + appendCompressedPositiveNumber(node.get_explore_additional_hits()); } public: diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h index 791da010720..65d6abeeaad 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h @@ -48,9 +48,6 @@ public: private: static Term * createQueryTerm(search::SimpleQueryStackDumpIterator &queryStack, QueryBuilder<NodeTypes> & builder, vespalib::stringref & pureTermView) { uint32_t arity = queryStack.getArity(); - uint32_t arg1 = queryStack.getArg1(); - double arg2 = queryStack.getArg2(); - double arg3 = queryStack.getArg3(); ParseItem::ItemType type = queryStack.getType(); Node::UP node; Term *t = 0; @@ -68,16 +65,19 @@ private: pureTermView = view; } else if (type == ParseItem::ITEM_WEAK_AND) { vespalib::stringref view = queryStack.getIndexName(); - builder.addWeakAnd(arity, arg1, view); + uint32_t targetNumHits = queryStack.getTargetNumHits(); + builder.addWeakAnd(arity, targetNumHits, view); pureTermView = view; } else if (type == ParseItem::ITEM_EQUIV) { int32_t id = queryStack.getUniqueId(); Weight weight = queryStack.GetWeight(); builder.addEquiv(arity, id, weight); } else if (type == ParseItem::ITEM_NEAR) { - builder.addNear(arity, arg1); + uint32_t nearDistance = queryStack.getNearDistance(); + builder.addNear(arity, nearDistance); } else if (type == ParseItem::ITEM_ONEAR) { - builder.addONear(arity, arg1); + uint32_t nearDistance = queryStack.getNearDistance(); + builder.addONear(arity, nearDistance); } else if (type == ParseItem::ITEM_PHRASE) { vespalib::stringref view = queryStack.getIndexName(); int32_t id = queryStack.getUniqueId(); @@ -104,17 +104,24 @@ private: vespalib::stringref view = queryStack.getIndexName(); int32_t id = queryStack.getUniqueId(); Weight weight = queryStack.GetWeight(); - t = &builder.addWandTerm(arity, view, id, weight, arg1, arg2, arg3); + uint32_t targetNumHits = queryStack.getTargetNumHits(); + double scoreThreshold = queryStack.getScoreThreshold(); + double thresholdBoostFactor = queryStack.getThresholdBoostFactor(); + t = &builder.addWandTerm(arity, view, id, weight, + targetNumHits, scoreThreshold, thresholdBoostFactor); pureTermView = vespalib::stringref(); } else if (type == ParseItem::ITEM_NOT) { builder.addAndNot(arity); } else if (type == ParseItem::ITEM_NEAREST_NEIGHBOR) { vespalib::stringref query_tensor_name = queryStack.getTerm(); vespalib::stringref field_name = queryStack.getIndexName(); - uint32_t target_num_hits = queryStack.getArg1(); + uint32_t target_num_hits = queryStack.getTargetNumHits(); int32_t id = queryStack.getUniqueId(); Weight weight = queryStack.GetWeight(); - builder.add_nearest_neighbor_term(query_tensor_name, field_name, id, weight, target_num_hits); + bool allow_approximate = queryStack.getAllowApproximate(); + uint32_t explore_additional_hits = queryStack.getExploreAdditionalHits(); + builder.add_nearest_neighbor_term(query_tensor_name, field_name, id, weight, + target_num_hits, allow_approximate, explore_additional_hits); } else { vespalib::stringref term = queryStack.getTerm(); vespalib::stringref view = queryStack.getIndexName(); diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.h b/searchlib/src/vespa/searchlib/query/tree/termnodes.h index a82b1e14d76..9af424716fb 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.h +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.h @@ -128,17 +128,24 @@ class NearestNeighborTerm : public QueryNodeMixin<NearestNeighborTerm, TermNode> private: vespalib::string _query_tensor_name; uint32_t _target_num_hits; + bool _allow_approximate; + uint32_t _explore_additional_hits; public: NearestNeighborTerm(vespalib::stringref query_tensor_name, vespalib::stringref field_name, - int32_t id, Weight weight, uint32_t target_num_hits) + int32_t id, Weight weight, uint32_t target_num_hits, + bool allow_approximate, uint32_t explore_additional_hits) : QueryNodeMixinType(field_name, id, weight), _query_tensor_name(query_tensor_name), - _target_num_hits(target_num_hits) + _target_num_hits(target_num_hits), + _allow_approximate(allow_approximate), + _explore_additional_hits(explore_additional_hits) {} virtual ~NearestNeighborTerm() {} const vespalib::string& get_query_tensor_name() const { return _query_tensor_name; } uint32_t get_target_num_hits() const { return _target_num_hits; } + bool get_allow_approximate() const { return _allow_approximate; } + uint32_t get_explore_additional_hits() const { return _explore_additional_hits; } }; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index d4aa2aaa1d7..d3b2925e075 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -13,11 +13,13 @@ namespace search::queryeval { NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& field, const tensor::DenseTensorAttribute& attr_tensor, std::unique_ptr<vespalib::tensor::DenseTensorView> query_tensor, - uint32_t target_num_hits) + uint32_t target_num_hits, bool approximate, uint32_t explore_additional_hits) : ComplexLeafBlueprint(field), _attr_tensor(attr_tensor), _query_tensor(std::move(query_tensor)), _target_num_hits(target_num_hits), + _approximate(approximate), + _explore_additional_hits(explore_additional_hits), _distance_heap(target_num_hits), _found_hits() { @@ -34,15 +36,14 @@ void NearestNeighborBlueprint::perform_top_k() { auto nns_index = _attr_tensor.nearest_neighbor_index(); - if (nns_index) { + if (_approximate && nns_index) { auto lhs_type = _query_tensor->fast_type(); auto rhs_type = _attr_tensor.getTensorType(); // XXX deal with different cell types later if (lhs_type == rhs_type) { auto lhs = _query_tensor->cellsRef(); uint32_t k = _target_num_hits; - uint32_t explore_k = k + 100; // XXX hardcoded for now - _found_hits = nns_index->find_top_k(k, lhs, explore_k); + _found_hits = nns_index->find_top_k(k, lhs, k + _explore_additional_hits); } } } @@ -73,6 +74,8 @@ NearestNeighborBlueprint::visitMembers(vespalib::ObjectVisitor& visitor) const visitor.visitString("attribute_tensor", _attr_tensor.getTensorType().to_spec()); visitor.visitString("query_tensor", _query_tensor->type().to_spec()); visitor.visitInt("target_num_hits", _target_num_hits); + visitor.visitBool("approximate", _approximate); + visitor.visitInt("explore_additional_hits", _explore_additional_hits); } bool diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h index ab4413c487a..39165b066be 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h @@ -21,6 +21,8 @@ private: const tensor::DenseTensorAttribute& _attr_tensor; std::unique_ptr<vespalib::tensor::DenseTensorView> _query_tensor; uint32_t _target_num_hits; + bool _approximate; + uint32_t _explore_additional_hits; mutable NearestNeighborDistanceHeap _distance_heap; std::vector<search::tensor::NearestNeighborIndex::Neighbor> _found_hits; @@ -29,7 +31,7 @@ public: NearestNeighborBlueprint(const queryeval::FieldSpec& field, const tensor::DenseTensorAttribute& attr_tensor, std::unique_ptr<vespalib::tensor::DenseTensorView> query_tensor, - uint32_t target_num_hits); + uint32_t target_num_hits, bool approximate, uint32_t explore_additional_hits); NearestNeighborBlueprint(const NearestNeighborBlueprint&) = delete; NearestNeighborBlueprint& operator=(const NearestNeighborBlueprint&) = delete; ~NearestNeighborBlueprint(); diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 0bdcd53af77..d0f33c6fd0b 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -11,6 +11,7 @@ vespa_add_library(searchlib_tensor OBJECT hnsw_index.cpp imported_tensor_attribute_vector.cpp imported_tensor_attribute_vector_read_guard.cpp + inv_log_level_generator.cpp nearest_neighbor_index.cpp tensor_attribute.cpp tensor_store.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp index 68efe6417c0..8b0c5e931a4 100644 --- a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp @@ -4,6 +4,7 @@ #include "distance_functions.h" #include "hnsw_index.h" #include "random_level_generator.h" +#include "inv_log_level_generator.h" #include <vespa/searchcommon/attribute/config.h> namespace search::tensor { @@ -27,24 +28,24 @@ make_distance_function(ValueType::CellType cell_type) } RandomLevelGenerator::UP -make_random_level_generator() +make_random_level_generator(uint32_t m) { - // TODO: Make generator that results in hierarchical graph. - return std::make_unique<LevelZeroGenerator>(); + return std::make_unique<InvLogLevelGenerator>(m); } -} +} // namespace <unnamed> std::unique_ptr<NearestNeighborIndex> DefaultNearestNeighborIndexFactory::make(const DocVectorAccess& vectors, vespalib::eval::ValueType::CellType cell_type, const search::attribute::HnswIndexParams& params) const { - HnswIndex::Config cfg(params.max_links_per_node() * 2, - params.max_links_per_node(), + uint32_t m = params.max_links_per_node(); + HnswIndex::Config cfg(m * 2, + m, params.neighbors_to_explore_at_insert(), true); - return std::make_unique<HnswIndex>(vectors, make_distance_function(cell_type), make_random_level_generator(), cfg); + return std::make_unique<HnswIndex>(vectors, make_distance_function(cell_type), make_random_level_generator(m), cfg); } } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 229c6c2c34f..1ee54256a03 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -207,7 +207,9 @@ vespalib::tensor::TypedCells DenseTensorAttribute::get_vector(uint32_t docid) const { MutableDenseTensorView tensor_view(_denseTensorStore.type()); - getTensor(docid, tensor_view); + assert(docid < _refVector.size()); + EntryRef ref = _refVector[docid]; + _denseTensorStore.getTensor(ref, tensor_view); return tensor_view.cellsRef(); } diff --git a/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.cpp b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.cpp new file mode 100644 index 00000000000..540edc5e664 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.cpp @@ -0,0 +1,3 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "inv_log_level_generator.h" diff --git a/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h new file mode 100644 index 00000000000..2f7f9f4445e --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "random_level_generator.h" +#include <random> + +namespace search::tensor { + +/** + * Geometric distribution for level selection in HnswIndex. + * Pr(level=k) is (1/M)^k * (1 - 1/M) + * Note that the level is theoretically unbounded, but in + * practice less than 30. + * Generated using floor(ln(U)/ln(1-p)), see + * https://en.wikipedia.org/wiki/Geometric_distribution#Related_distributions + **/ + +class InvLogLevelGenerator : public RandomLevelGenerator { + std::mt19937_64 _rng; + std::uniform_real_distribution<double> _uniform; + double _levelMultiplier; +public: + InvLogLevelGenerator(uint32_t m) + : _rng(0x1234deadbeef5678uLL), + _uniform(0.0, 1.0), + _levelMultiplier(1.0 / log(1.0 * m)) + {} + + uint32_t max_level() override { + double unif = _uniform(_rng); + double r = -log(1.0-unif) * _levelMultiplier; + return (uint32_t) r; + } +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index 43eaf4b57b8..4872a183358 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -257,11 +257,11 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const rc = SkipItem(&iterator); break; case search::ParseItem::ITEM_NEAR: - if (!v->VisitNEAR(&item, iterator.getArity(),iterator.getArg1())) + if (!v->VisitNEAR(&item, iterator.getArity(),iterator.getNearDistance())) rc = SkipItem(&iterator); break; case search::ParseItem::ITEM_ONEAR: - if (!v->VisitWITHIN(&item, iterator.getArity(),iterator.getArg1())) + if (!v->VisitWITHIN(&item, iterator.getArity(),iterator.getNearDistance())) rc = SkipItem(&iterator); break; // Unhandled items are just ignored by juniper diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index 1cfb70560b8..1065ed29dc4 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -2,14 +2,19 @@ package com.yahoo.vespa.service.duper; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; import com.yahoo.log.LogLevel; import com.yahoo.vespa.service.monitor.DuperModelListener; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.TreeMap; +import java.util.Optional; +import java.util.Set; import java.util.logging.Logger; /** @@ -20,37 +25,106 @@ import java.util.logging.Logger; public class DuperModel { private static Logger logger = Logger.getLogger(DuperModel.class.getName()); - private final Map<ApplicationId, ApplicationInfo> applications = new TreeMap<>(); + private final Map<ApplicationId, ApplicationInfo> applicationsById = new HashMap<>(); + private final Map<HostName, ApplicationId> idsByHostname = new HashMap<>(); + private final Map<ApplicationId, Set<HostName>> hostnamesById = new HashMap<>(); + private final List<DuperModelListener> listeners = new ArrayList<>(); private boolean isComplete = false; public void registerListener(DuperModelListener listener) { - applications.values().forEach(listener::applicationActivated); + applicationsById.values().forEach(listener::applicationActivated); listeners.add(listener); } - public void setCompleteness(boolean isComplete) { this.isComplete = isComplete; } + void setComplete() { + if (!isComplete) { + logger.log(LogLevel.INFO, "Bootstrap done - duper model is complete"); + isComplete = true; + + listeners.forEach(DuperModelListener::bootstrapComplete); + } + } + public boolean isComplete() { return isComplete; } + public int numberOfApplications() { + return applicationsById.size(); + } + + public int numberOfHosts() { + return idsByHostname.size(); + } + public boolean contains(ApplicationId applicationId) { - return applications.containsKey(applicationId); + return applicationsById.containsKey(applicationId); + } + + public Optional<ApplicationInfo> getApplicationInfo(ApplicationId applicationId) { + return Optional.ofNullable(applicationsById.get(applicationId)); + } + + public Optional<ApplicationInfo> getApplicationInfo(HostName hostName) { + return Optional.ofNullable(idsByHostname.get(hostName)).map(applicationsById::get); + } + + public List<ApplicationInfo> getApplicationInfos() { + return List.copyOf(applicationsById.values()); } public void add(ApplicationInfo applicationInfo) { - applications.put(applicationInfo.getApplicationId(), applicationInfo); - logger.log(LogLevel.DEBUG, "Added " + applicationInfo.getApplicationId()); + ApplicationId id = applicationInfo.getApplicationId(); + ApplicationInfo oldApplicationInfo = applicationsById.put(id, applicationInfo); + + final String logPrefix; + if (oldApplicationInfo == null) { + logPrefix = isComplete ? "New application " : "Bootstrapped application "; + } else { + logPrefix = isComplete ? "Reactivated application " : "Rebootstrapped application "; + } + logger.log(LogLevel.INFO, logPrefix + id); + + Set<HostName> hostnames = hostnamesById.computeIfAbsent(id, k -> new HashSet<>()); + Set<HostName> removedHosts = new HashSet<>(hostnames); + + applicationInfo.getModel().getHosts().stream() + .map(HostInfo::getHostname) + .map(HostName::from) + .forEach(hostname -> { + if (!removedHosts.remove(hostname)) { + hostnames.add(hostname); + ApplicationId previousId = idsByHostname.put(hostname, id); + + if (previousId != null && !previousId.equals(id)) { + // If an activation contains a host that is currently assigned to a + // different application we will patch up our data structures to remain + // internally consistent. But listeners may be fooled. + logger.log(LogLevel.WARNING, hostname + " has been reassigned from " + + previousId + " to " + id); + + Set<HostName> previousHostnames = hostnamesById.get(previousId); + if (previousHostnames != null) { + previousHostnames.remove(hostname); + } + } + } + }); + + removedHosts.forEach(idsByHostname::remove); + listeners.forEach(listener -> listener.applicationActivated(applicationInfo)); } public void remove(ApplicationId applicationId) { - if (applications.remove(applicationId) != null) { - logger.log(LogLevel.DEBUG, "Removed " + applicationId); - listeners.forEach(listener -> listener.applicationRemoved(applicationId)); + Set<HostName> hostnames = hostnamesById.remove(applicationId); + if (hostnames != null) { + hostnames.forEach(idsByHostname::remove); } - } - public List<ApplicationInfo> getApplicationInfos() { - logger.log(LogLevel.DEBUG, "Applications in duper model: " + applications.values().size()); - return List.copyOf(applications.values()); + ApplicationInfo application = applicationsById.remove(applicationId); + if (application != null) { + logger.log(LogLevel.INFO, "Removed application " + applicationId); + listeners.forEach(listener -> listener.applicationRemoved(applicationId)); + } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index 15c461c7f59..9c93bc1d390 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -173,6 +173,18 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi } } + public Optional<ApplicationInfo> getApplicationInfo(ApplicationId applicationId) { + synchronized (monitor) { + return duperModel.getApplicationInfo(applicationId); + } + } + + public Optional<ApplicationInfo> getApplicationInfo(HostName hostname) { + synchronized (monitor) { + return duperModel.getApplicationInfo(hostname); + } + } + public List<ApplicationInfo> getApplicationInfos() { synchronized (monitor) { return duperModel.getApplicationInfos(); @@ -181,7 +193,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi private void maybeSetDuperModelAsComplete() { if (superModelIsComplete && infraApplicationsIsComplete) { - duperModel.setCompleteness(true); + duperModel.setComplete(); } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java index 09140423010..7b79adf89d0 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -74,7 +75,7 @@ public abstract class InfraApplication implements InfraApplicationApi { @Override public ClusterSpec getClusterSpecWithVersion(Version version) { - return ClusterSpec.request(clusterSpecType, clusterSpecId, version, true); + return ClusterSpec.request(clusterSpecType, clusterSpecId, version, true, Optional.empty()); } public ClusterSpec.Type getClusterSpecType() { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java index 5cc2d538c24..0116b992e23 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java @@ -5,9 +5,13 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.HostName; @@ -24,7 +28,9 @@ import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -37,23 +43,73 @@ public class ApplicationInstanceGenerator { private final ApplicationInfo applicationInfo; private final Zone zone; - private ApplicationId configServerApplicationId; + + // This is cheating a bit, but we don't expect DuperModel's config server application ID to be different. + // We do this to avoid passing through the ID through multiple levels. + private static final ApplicationId configServerApplicationId = new ConfigServerApplication().getApplicationId(); public ApplicationInstanceGenerator(ApplicationInfo applicationInfo, Zone zone) { this.applicationInfo = applicationInfo; this.zone = zone; - - // This is cheating a bit, but we don't expect DuperModel's config server application ID to be different. - // We do this to avoid passing through the ID through multiple levels. - this.configServerApplicationId = new ConfigServerApplication().getApplicationId(); } public ApplicationInstance makeApplicationInstance(ServiceStatusProvider serviceStatusProvider) { + return makeApplicationInstanceLimitedToHosts(serviceStatusProvider, hostname -> true); + } + + public ApplicationInstanceReference toApplicationInstanceReference() { + TenantId tenantId = new TenantId(applicationInfo.getApplicationId().tenant().toString()); + ApplicationInstanceId applicationInstanceId = toApplicationInstanceId(applicationInfo.getApplicationId(), zone); + return new ApplicationInstanceReference(tenantId, applicationInstanceId); + } + + public boolean containsHostname(HostName hostname) { + return applicationInfo.getModel().getHosts().stream() + .map(HostInfo::getHostname) + .anyMatch(hostnameString -> Objects.equals(hostnameString, hostname.s())); + } + + public ApplicationInstance makeApplicationInstanceLimitedTo( + HostName hostname, ServiceStatusProvider serviceStatusProvider) { + return makeApplicationInstanceLimitedToHosts( + serviceStatusProvider, candidateHostname -> candidateHostname.equals(hostname)); + } + + /** Reverse of toApplicationInstanceId, put in this file because it its inverse is. */ + public static ApplicationId toApplicationId(ApplicationInstanceReference reference) { + + String appNameStr = reference.asString(); + String[] appNameParts = appNameStr.split(":"); + + // Env, region and instance seems to be optional due to the hardcoded config server app + // Assume here that first two are tenant and application name. + if (appNameParts.length == 2) { + return ApplicationId.from(TenantName.from(appNameParts[0]), + ApplicationName.from(appNameParts[1]), + InstanceName.defaultName()); + } + + // Other normal application should have 5 parts. + if (appNameParts.length != 5) { + throw new IllegalArgumentException("Application reference not valid (not 5 parts): " + reference); + } + + return ApplicationId.from(TenantName.from(appNameParts[0]), + ApplicationName.from(appNameParts[1]), + InstanceName.from(appNameParts[4])); + } + + private ApplicationInstance makeApplicationInstanceLimitedToHosts(ServiceStatusProvider serviceStatusProvider, + Predicate<HostName> includeHostPredicate) { Map<ServiceClusterKey, Set<ServiceInstance>> groupedServiceInstances = new HashMap<>(); for (HostInfo host : applicationInfo.getModel().getHosts()) { HostName hostName = new HostName(host.getHostname()); + if (!includeHostPredicate.test(hostName)) { + continue; + } + for (ServiceInfo serviceInfo : host.getServices()) { ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo); @@ -77,10 +133,8 @@ public class ApplicationInstanceGenerator { entry.getValue())) .collect(Collectors.toSet()); - ApplicationInstance applicationInstance = new ApplicationInstance( - new TenantId(applicationInfo.getApplicationId().tenant().toString()), - toApplicationInstanceId(applicationInfo, zone), - serviceClusters); + ApplicationInstanceReference reference = toApplicationInstanceReference(); + ApplicationInstance applicationInstance = new ApplicationInstance(reference, serviceClusters); // Fill back-references for (ServiceCluster serviceCluster : applicationInstance.serviceClusters()) { @@ -106,18 +160,18 @@ public class ApplicationInstanceGenerator { return new ServiceInstance(configId, hostName, status); } - private ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) { - if (applicationInfo.getApplicationId().equals(configServerApplicationId)) { + private static ApplicationInstanceId toApplicationInstanceId(ApplicationId applicationId, Zone zone) { + if (applicationId.equals(configServerApplicationId)) { // Removing this historical discrepancy would break orchestration during rollout. // An alternative may be to use a feature flag and flip it between releases, // once that's available. - return new ApplicationInstanceId(applicationInfo.getApplicationId().application().value()); + return new ApplicationInstanceId(applicationId.application().value()); } else { return new ApplicationInstanceId(String.format("%s:%s:%s:%s", - applicationInfo.getApplicationId().application().value(), + applicationId.application().value(), zone.environment().value(), zone.region().value(), - applicationInfo.getApplicationId().instance().value())); + applicationId.instance().value())); } } @@ -129,7 +183,7 @@ public class ApplicationInstanceGenerator { toConfigId(serviceInfo)); } - public static ClusterId getClusterId(ServiceInfo serviceInfo) { + private static ClusterId getClusterId(ServiceInfo serviceInfo) { return new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse("")); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java index a6bf41d6c9b..d9378da957e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java @@ -5,11 +5,13 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -19,13 +21,18 @@ import java.util.stream.Collectors; public class ModelGenerator { public static final String CLUSTER_ID_PROPERTY_NAME = "clustername"; + private final Zone zone; + + public ModelGenerator(Zone zone) { + this.zone = zone; + } + /** * Create service model based primarily on super model. * * If the configServerhosts is non-empty, a config server application is added. */ public ServiceModel toServiceModel(List<ApplicationInfo> allApplicationInfos, - Zone zone, ServiceStatusProvider serviceStatusProvider) { Map<ApplicationInstanceReference, ApplicationInstance> applicationInstances = allApplicationInfos.stream() @@ -36,4 +43,27 @@ public class ModelGenerator { return new ServiceModel(applicationInstances); } + public Set<ApplicationInstanceReference> toApplicationInstanceReferenceSet(List<ApplicationInfo> infos) { + return infos.stream() + .map(info -> new ApplicationInstanceGenerator(info, zone).toApplicationInstanceReference()) + .collect(Collectors.toSet()); + } + + public ApplicationInstance toApplicationInstance(ApplicationInfo applicationInfo, + ServiceStatusProvider serviceStatusProvider) { + var generator = new ApplicationInstanceGenerator(applicationInfo, zone); + return generator.makeApplicationInstance(serviceStatusProvider); + } + + /** + * Make an application instance that contains all services and clusters present on the host, + * but lacking other services and hosts. This is an optimization over + * {@link #toApplicationInstance(ApplicationInfo, ServiceStatusProvider)}. + */ + public ApplicationInstance toApplicationNarrowedToHost(ApplicationInfo applicationInfo, + HostName hostname, + ServiceStatusProvider serviceStatusProvider) { + var generator = new ApplicationInstanceGenerator(applicationInfo, zone); + return generator.makeApplicationInstanceLimitedTo(hostname, serviceStatusProvider); + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java index 1b37555a554..29b2832efd0 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.service.model; import com.yahoo.jdisc.Timer; import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import java.util.function.Supplier; @@ -12,12 +13,11 @@ import java.util.function.Supplier; * * @author hakonhall */ -public class ServiceModelCache implements Supplier<ServiceModel> { +public class ServiceModelCache implements ServiceMonitor { public static final long EXPIRY_MILLIS = 10000; private final Supplier<ServiceModel> expensiveSupplier; private final Timer timer; - private final boolean useCache; private volatile ServiceModel snapshot; private boolean updatePossiblyInProgress = false; @@ -25,18 +25,13 @@ public class ServiceModelCache implements Supplier<ServiceModel> { private final Object updateMonitor = new Object(); private long snapshotMillis; - public ServiceModelCache(Supplier<ServiceModel> expensiveSupplier, Timer timer, boolean useCache) { + public ServiceModelCache(Supplier<ServiceModel> expensiveSupplier, Timer timer) { this.expensiveSupplier = expensiveSupplier; this.timer = timer; - this.useCache = useCache; } @Override - public ServiceModel get() { - if (!useCache) { - return expensiveSupplier.get(); - } - + public ServiceModel getServiceModelSnapshot() { if (snapshot == null) { synchronized (updateMonitor) { if (snapshot == null) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java index c6947810fa0..45ee38ab560 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java @@ -2,33 +2,40 @@ package com.yahoo.vespa.service.model; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; -import com.yahoo.vespa.service.manager.MonitorManager; -import com.yahoo.vespa.service.duper.DuperModelManager; import java.util.List; -import java.util.function.Supplier; +import java.util.Map; +import java.util.Optional; +import java.util.Set; /** * An uncached supplier of ServiceModel based on the DuperModel and MonitorManager. * * @author hakonhall */ -public class ServiceModelProvider implements Supplier<ServiceModel> { +public class ServiceModelProvider implements ServiceMonitor { private final ServiceMonitorMetrics metrics; private final DuperModelManager duperModelManager; private final ModelGenerator modelGenerator; private final Zone zone; - private final MonitorManager monitorManager; + private final ServiceStatusProvider serviceStatusProvider; - public ServiceModelProvider(MonitorManager monitorManager, + public ServiceModelProvider(ServiceStatusProvider serviceStatusProvider, ServiceMonitorMetrics metrics, DuperModelManager duperModelManager, ModelGenerator modelGenerator, Zone zone) { - this.monitorManager = monitorManager; + this.serviceStatusProvider = serviceStatusProvider; this.metrics = metrics; this.duperModelManager = duperModelManager; this.modelGenerator = modelGenerator; @@ -36,13 +43,57 @@ public class ServiceModelProvider implements Supplier<ServiceModel> { } @Override - public ServiceModel get() { + public ServiceModel getServiceModelSnapshot() { try (LatencyMeasurement measurement = metrics.startServiceModelSnapshotLatencyMeasurement()) { - // WARNING: The monitor manager may be out-of-sync with duper model (no locking) - List<ApplicationInfo> applicationInfos = duperModelManager.getApplicationInfos(); + return modelGenerator.toServiceModel(duperModelManager.getApplicationInfos(), serviceStatusProvider); + } + } + + @Override + public Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() { + return modelGenerator.toApplicationInstanceReferenceSet(duperModelManager.getApplicationInfos()); + } - return modelGenerator.toServiceModel(applicationInfos, zone, (ServiceStatusProvider) monitorManager); + @Override + public Optional<ApplicationInstance> getApplication(HostName hostname) { + Optional<ApplicationInfo> applicationInfo = getApplicationInfo(hostname); + if (applicationInfo.isEmpty()) { + return Optional.empty(); } + + return Optional.of(modelGenerator.toApplicationInstance(applicationInfo.get(), serviceStatusProvider)); + } + + @Override + public Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) { + return getApplicationInfo(reference) + .map(applicationInfo -> modelGenerator.toApplicationInstance(applicationInfo, serviceStatusProvider)); } + @Override + public Optional<ApplicationInstance> getApplicationNarrowedTo(HostName hostname) { + Optional<ApplicationInfo> applicationInfo = getApplicationInfo(hostname); + if (applicationInfo.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(modelGenerator.toApplicationNarrowedToHost( + applicationInfo.get(), hostname, serviceStatusProvider)); + } + + @Override + public Map<HostName, List<ServiceInstance>> getServicesByHostname() { + return getServiceModelSnapshot().getServiceInstancesByHostName(); + } + + private Optional<ApplicationInfo> getApplicationInfo(ApplicationInstanceReference reference) { + ApplicationId applicationId = ApplicationInstanceGenerator.toApplicationId(reference); + return duperModelManager.getApplicationInfo(applicationId); + } + + private Optional<ApplicationInfo> getApplicationInfo(HostName hostname) { + // The duper model uses HostName from config.provision, which is more natural than applicationmodel. + var configProvisionHostname = com.yahoo.config.provision.HostName.from(hostname.s()); + return duperModelManager.getApplicationInfo(configProvisionHostname); + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java index 67b4e890c29..d3297d711ff 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java @@ -5,6 +5,10 @@ import com.google.inject.Inject; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.service.duper.DuperModelManager; @@ -12,9 +16,14 @@ import com.yahoo.vespa.service.manager.UnionMonitorManager; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + public class ServiceMonitorImpl implements ServiceMonitor { - private final ServiceModelCache serviceModelProvider; + private final ServiceMonitor delegate; @Inject public ServiceMonitorImpl(DuperModelManager duperModelManager, @@ -25,19 +34,47 @@ public class ServiceMonitorImpl implements ServiceMonitor { FlagSource flagSource) { duperModelManager.registerListener(monitorManager); - ServiceModelProvider uncachedServiceModelProvider = new ServiceModelProvider( + ServiceMonitor serviceMonitor = new ServiceModelProvider( monitorManager, new ServiceMonitorMetrics(metric, timer), duperModelManager, - new ModelGenerator(), + new ModelGenerator(zone), zone); - boolean cache = Flags.SERVICE_MODEL_CACHE.bindTo(flagSource).value(); - serviceModelProvider = new ServiceModelCache(uncachedServiceModelProvider, timer, cache); + + if (Flags.SERVICE_MODEL_CACHE.bindTo(flagSource).value()) { + delegate = new ServiceModelCache(serviceMonitor::getServiceModelSnapshot, timer); + } else { + delegate = serviceMonitor; + } } @Override public ServiceModel getServiceModelSnapshot() { - return serviceModelProvider.get(); + return delegate.getServiceModelSnapshot(); + } + + @Override + public Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() { + return delegate.getAllApplicationInstanceReferences(); + } + + @Override + public Optional<ApplicationInstance> getApplication(HostName hostname) { + return delegate.getApplication(hostname); } + @Override + public Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) { + return delegate.getApplication(reference); + } + + @Override + public Optional<ApplicationInstance> getApplicationNarrowedTo(HostName hostname) { + return delegate.getApplicationNarrowedTo(hostname); + } + + @Override + public Map<HostName, List<ServiceInstance>> getServicesByHostname() { + return delegate.getServicesByHostname(); + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java index 49539c61e5d..deafaaace0e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java @@ -1,6 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.monitor; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + /** * The service monitor interface. A service monitor provides up to date information about the liveness status * (up, down or not known) of each service instance in a Vespa zone @@ -12,7 +22,29 @@ public interface ServiceMonitor { /** * Returns a ServiceModel which contains the current liveness status (up, down or unknown) of all instances * of all services of all clusters of all applications in a zone. + * + * <p>Please use the more specific methods below to avoid the cost of this method.</p> */ ServiceModel getServiceModelSnapshot(); + default Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() { + return getServiceModelSnapshot().getAllApplicationInstances().keySet(); + } + + default Optional<ApplicationInstance> getApplication(HostName hostname) { + return Optional.ofNullable(getServiceModelSnapshot().getApplicationsByHostName().get(hostname)); + } + + default Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) { + return getServiceModelSnapshot().getApplicationInstance(reference); + } + + default Optional<ApplicationInstance> getApplicationNarrowedTo(HostName hostname) { + return getApplication(hostname); + } + + default Map<HostName, List<ServiceInstance>> getServicesByHostname() { + return getServiceModelSnapshot().getServiceInstancesByHostName(); + } + } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java index 67508f14e5a..3448ce46ff9 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java @@ -71,7 +71,6 @@ public class DuperModelManagerTest { verify(duperModel, times(0)).add(any()); manager.infraApplicationActivated(id, proxyHostHosts); verify(duperModel, times(1)).add(any()); - when(duperModel.contains(id)).thenReturn(true); verify(duperModel, times(0)).remove(any()); manager.infraApplicationRemoved(id); @@ -98,7 +97,6 @@ public class DuperModelManagerTest { List<HostName> hostnames1 = Stream.of("node11", "node12").map(HostName::from).collect(Collectors.toList()); manager.infraApplicationActivated(firstId, hostnames1); verify(duperModel, times(1)).add(any()); - when(duperModel.contains(firstId)).thenReturn(true); // Adding the second config server like application will be ignored List<HostName> hostnames2 = Stream.of("node21", "node22").map(HostName::from).collect(Collectors.toList()); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java index dc90035be71..69b6d3d59f3 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java @@ -2,16 +2,20 @@ package com.yahoo.vespa.service.duper; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.monitor.DuperModelListener; import org.junit.Before; import org.junit.Test; import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -23,37 +27,60 @@ import static org.mockito.Mockito.when; */ public class DuperModelTest { private final DuperModel duperModel = new DuperModel(); - private final ApplicationInfo application1 = mock(ApplicationInfo.class); + private final ApplicationId id1 = ApplicationId.fromSerializedForm("tenant:app1:default"); + private final ApplicationInfo application1 = mock(ApplicationInfo.class); + private final HostName hostname1_1 = HostName.from("hostname1-1"); + private final HostName hostname1_2 = HostName.from("hostname1-2"); + private final ApplicationId id2 = ApplicationId.fromSerializedForm("tenant:app2:default"); private final ApplicationInfo application2 = mock(ApplicationInfo.class); + private final HostName hostname2_1 = HostName.from("hostname2-1"); + private final DuperModelListener listener1 = mock(DuperModelListener.class); @Before public void setUp() { - when(application1.getApplicationId()).thenReturn(id1); - when(application2.getApplicationId()).thenReturn(id2); + setUpApplication(id1, application1, hostname1_1, hostname1_2); + setUpApplication(id2, application2, hostname2_1); + } + + private void setUpApplication(ApplicationId id, ApplicationInfo info, HostName... hostnames) { + when(info.getApplicationId()).thenReturn(id); + + Model model = mock(Model.class); + when(info.getModel()).thenReturn(model); + + List<HostInfo> hostInfos = Arrays.stream(hostnames) + .map(hostname -> new HostInfo(hostname.value(), List.of())) + .collect(Collectors.toList()); + when(model.getHosts()).thenReturn(hostInfos); } @Test - public void test() { + public void testListeners() { + assertEquals(0, duperModel.numberOfApplications()); + duperModel.add(application1); - assertTrue(duperModel.contains(id1)); + assertEquals(Optional.of(application1), duperModel.getApplicationInfo(id1)); assertEquals(Arrays.asList(application1), duperModel.getApplicationInfos()); + assertEquals(1, duperModel.numberOfApplications()); duperModel.registerListener(listener1); verify(listener1, times(1)).applicationActivated(application1); verifyNoMoreInteractions(listener1); duperModel.remove(id2); + assertEquals(1, duperModel.numberOfApplications()); verifyNoMoreInteractions(listener1); duperModel.add(application2); + assertEquals(2, duperModel.numberOfApplications()); verify(listener1, times(1)).applicationActivated(application2); verifyNoMoreInteractions(listener1); duperModel.remove(id1); - assertFalse(duperModel.contains(id1)); + assertEquals(Optional.empty(), duperModel.getApplicationInfo(id1)); verify(listener1, times(1)).applicationRemoved(id1); verifyNoMoreInteractions(listener1); assertEquals(Arrays.asList(application2), duperModel.getApplicationInfos()); @@ -61,4 +88,31 @@ public class DuperModelTest { duperModel.remove(id1); verifyNoMoreInteractions(listener1); } + + @Test + public void hostIndices() { + assertEquals(0, duperModel.numberOfHosts()); + + duperModel.add(application1); + assertEquals(2, duperModel.numberOfHosts()); + assertEquals(Optional.of(application1), duperModel.getApplicationInfo(hostname1_1)); + assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname2_1)); + + duperModel.add(application2); + assertEquals(3, duperModel.numberOfHosts()); + assertEquals(Optional.of(application1), duperModel.getApplicationInfo(hostname1_1)); + assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname2_1)); + + duperModel.remove(application1.getApplicationId()); + assertEquals(1, duperModel.numberOfHosts()); + assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname1_1)); + assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname2_1)); + + // Remove hostname2_1 and add hostname1_1 added to id2 + setUpApplication(id2, application2, hostname1_1); + duperModel.add(application2); + assertEquals(1, duperModel.numberOfHosts()); + assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname1_1)); + assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname2_1)); + } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java index ce13cf60082..7be4ff38a7f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java @@ -38,9 +38,9 @@ public class ModelGeneratorTest { @Test public void toApplicationModel() throws Exception { - ModelGenerator modelGenerator = new ModelGenerator(); - Zone zone = new Zone(Environment.from(ENVIRONMENT), RegionName.from(REGION)); + ModelGenerator modelGenerator = new ModelGenerator(zone); + SlobrokMonitorManagerImpl slobrokMonitorManager = mock(SlobrokMonitorManagerImpl.class); when(slobrokMonitorManager.getStatus(any(), any(), any(), any())) @@ -49,7 +49,6 @@ public class ModelGeneratorTest { ServiceModel serviceModel = modelGenerator.toServiceModel( getExampleApplicationInfos(), - zone, slobrokMonitorManager); Map<ApplicationInstanceReference, diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java index c2314be1e0f..21c6a49cc18 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java @@ -16,27 +16,27 @@ import static org.mockito.Mockito.when; public class ServiceModelCacheTest { @SuppressWarnings("unchecked") - private final Supplier<ServiceModel> rawSupplier = mock(Supplier.class); + private final Supplier<ServiceModel> expensiveServiceMonitor = mock(Supplier.class); private final Timer timer = mock(Timer.class); - private final ServiceModelCache cache = new ServiceModelCache(rawSupplier, timer, true); + private final ServiceModelCache cache = new ServiceModelCache(expensiveServiceMonitor, timer); @Test public void sanityCheck() { ServiceModel serviceModel = mock(ServiceModel.class); - when(rawSupplier.get()).thenReturn(serviceModel); + when(expensiveServiceMonitor.get()).thenReturn(serviceModel); long timeMillis = 0; when(timer.currentTimeMillis()).thenReturn(timeMillis); // Will always populate cache the first time - ServiceModel actualServiceModel = cache.get(); + ServiceModel actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel); - verify(rawSupplier, times(1)).get(); + verify(expensiveServiceMonitor, times(1)).get(); // Cache hit timeMillis += ServiceModelCache.EXPIRY_MILLIS / 2; when(timer.currentTimeMillis()).thenReturn(timeMillis); - actualServiceModel = cache.get(); + actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel); // Cache expired @@ -44,17 +44,17 @@ public class ServiceModelCacheTest { when(timer.currentTimeMillis()).thenReturn(timeMillis); ServiceModel serviceModel2 = mock(ServiceModel.class); - when(rawSupplier.get()).thenReturn(serviceModel2); + when(expensiveServiceMonitor.get()).thenReturn(serviceModel2); - actualServiceModel = cache.get(); + actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel2); // '2' because it's cumulative with '1' from the first times(1). - verify(rawSupplier, times(2)).get(); + verify(expensiveServiceMonitor, times(2)).get(); // Cache hit #2 timeMillis += 1; when(timer.currentTimeMillis()).thenReturn(timeMillis); - actualServiceModel = cache.get(); + actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel2); } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java index 13f6da1534d..f34d61970ef 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java @@ -37,8 +37,8 @@ public class ServiceModelProviderTest { .collect(Collectors.toList()); when(duperModelManager.getApplicationInfos()).thenReturn(applications); - ServiceModel serviceModel = provider.get(); + ServiceModel serviceModel = provider.getServiceModelSnapshot(); verify(duperModelManager, times(1)).getApplicationInfos(); - verify(modelGenerator).toServiceModel(applications, zone, slobrokMonitorManager); + verify(modelGenerator).toServiceModel(applications, slobrokMonitorManager); } }
\ No newline at end of file diff --git a/staging_vespalib/CMakeLists.txt b/staging_vespalib/CMakeLists.txt index 3c4e2a9f444..9fb48cbd4fc 100644 --- a/staging_vespalib/CMakeLists.txt +++ b/staging_vespalib/CMakeLists.txt @@ -30,6 +30,7 @@ vespa_define_module( src/tests/shutdownguard src/tests/state_server src/tests/stllike + src/tests/singleexecutor src/tests/timer src/tests/util/process_memory_stats src/tests/xmlserializable diff --git a/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt b/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt new file mode 100644 index 00000000000..c5d42d2c8c5 --- /dev/null +++ b/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_singleexecutor_test_app TEST + SOURCES + singleexecutor_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_singleexecutor_test_app COMMAND staging_vespalib_singleexecutor_test_app) diff --git a/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp b/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp new file mode 100644 index 00000000000..5dacaa5d204 --- /dev/null +++ b/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp @@ -0,0 +1,80 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/vespalib/util/singleexecutor.h> +#include <vespa/vespalib/util/lambdatask.h> +#include <atomic> + +using namespace vespalib; + +TEST("test that all tasks are executed") { + + std::atomic<uint64_t> counter(0); + SingleExecutor executor(10); + + for (uint64_t i(0); i < 10; i++) { + executor.execute(makeLambdaTask([&counter] {counter++;})); + } + executor.sync(); + EXPECT_EQUAL(10u, counter); + + counter = 0; + for (uint64_t i(0); i < 10000; i++) { + executor.execute(makeLambdaTask([&counter] {counter++;})); + } + executor.sync(); + EXPECT_EQUAL(10000u, counter); +} + +void verifyResizeTaskLimit(bool up) { + Monitor lock; + std::atomic<uint64_t> started(0); + std::atomic<uint64_t> allowed(0); + SingleExecutor executor(10); + + uint32_t targetTaskLimit = up ? 20 : 5; + uint32_t roundedTaskLimit = roundUp2inN(targetTaskLimit); + EXPECT_NOT_EQUAL(16u, roundedTaskLimit); + + for (uint64_t i(0); i < 10; i++) { + executor.execute(makeLambdaTask([&lock, &started, &allowed] { + started++; + MonitorGuard guard(lock); + while (allowed < started) { + guard.wait(1ms); + } + })); + } + while (started < 1); + EXPECT_EQUAL(1u, started); + executor.setTaskLimit(targetTaskLimit); + EXPECT_EQUAL(16u, executor.getTaskLimit()); + allowed = 5; + while (started < 6); + EXPECT_EQUAL(6u, started); + EXPECT_EQUAL(16u, executor.getTaskLimit()); + allowed = 10; + while (started < 10); + EXPECT_EQUAL(10u, started); + EXPECT_EQUAL(16u, executor.getTaskLimit()); + executor.execute(makeLambdaTask([&lock, &started, &allowed] { + started++; + MonitorGuard guard(lock); + while (allowed < started) { + guard.wait(1ms); + } + })); + while (started < 11); + EXPECT_EQUAL(11u, started); + EXPECT_EQUAL(roundedTaskLimit, executor.getTaskLimit()); + allowed = 11; +} +TEST("test that resizing up and down works") { + TEST_DO(verifyResizeTaskLimit(true)); + TEST_DO(verifyResizeTaskLimit(false)); + + +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt index 71364a813f6..ba03b77c941 100644 --- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -17,6 +17,7 @@ vespa_add_library(staging_vespalib_vespalib_util OBJECT rusage.cpp shutdownguard.cpp scheduledexecutor.cpp + singleexecutor.cpp xmlserializable.cpp xmlstream.cpp DEPENDS diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp new file mode 100644 index 00000000000..791b0876c19 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp @@ -0,0 +1,131 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "singleexecutor.h" +#include <vespa/vespalib/util/time.h> + +namespace vespalib { + +SingleExecutor::SingleExecutor(uint32_t taskLimit) + : _taskLimit(vespalib::roundUp2inN(taskLimit)), + _wantedTaskLimit(_taskLimit.load()), + _rp(0), + _tasks(std::make_unique<Task::UP[]>(_taskLimit)), + _consumerMonitor(), + _producerMonitor(), + _thread(*this), + _lastAccepted(0), + _maxPending(0), + _wakeupConsumerAt(0), + _producerNeedWakeup(false), + _wp(0) +{ + _thread.start(); +} +SingleExecutor::~SingleExecutor() { + sync(); + _thread.stop().join(); +} + +size_t +SingleExecutor::getNumThreads() const { + return 1; +} + +uint64_t +SingleExecutor::addTask(Task::UP task) { + MonitorGuard guard(_producerMonitor); + wait_for_room(guard); + uint64_t wp = _wp.load(std::memory_order_relaxed); + _tasks[index(wp)] = std::move(task); + _wp.store(wp + 1, std::memory_order_release); + return wp; +} + +Executor::Task::UP +SingleExecutor::execute(Task::UP task) { + uint64_t wp = addTask(std::move(task)); + if (wp == _wakeupConsumerAt.load(std::memory_order_relaxed)) { + MonitorGuard guard(_consumerMonitor); + guard.signal(); + } + return task; +} + +void +SingleExecutor::setTaskLimit(uint32_t taskLimit) { + _wantedTaskLimit = vespalib::roundUp2inN(taskLimit); +} + +SingleExecutor & +SingleExecutor::sync() { + uint64_t wp = _wp.load(std::memory_order_relaxed); + while (wp > _rp.load(std::memory_order_relaxed)) { + std::this_thread::sleep_for(1ms); + } + return *this; +} + +void +SingleExecutor::run() { + while (!_thread.stopped()) { + drain_tasks(); + _wakeupConsumerAt.store(_wp.load(std::memory_order_relaxed) + (_taskLimit.load(std::memory_order_relaxed) >> 2), std::memory_order_relaxed); + MonitorGuard guard(_consumerMonitor); + guard.wait(10ms); + _wakeupConsumerAt.store(0, std::memory_order_relaxed); + } +} + +void +SingleExecutor::drain_tasks() { + while (numTasks() > 0) { + run_tasks_till(_wp.load(std::memory_order_acquire)); + } +} + +void +SingleExecutor::run_tasks_till(uint64_t available) { + uint64_t consumed = _rp.load(std::memory_order_relaxed); + uint64_t left = available - consumed; + if (_maxPending.load(std::memory_order_relaxed) < left) { + _maxPending.store(left, std::memory_order_relaxed); + } + uint64_t wakeupLimit = _producerNeedWakeup.load(std::memory_order_relaxed) + ? (available - (left >> 2)) + : 0; + while (consumed < available) { + Task::UP task = std::move(_tasks[index(consumed)]); + task->run(); + _rp.store(++consumed, std::memory_order_release); + if (wakeupLimit == consumed) { + MonitorGuard guard(_producerMonitor); + guard.broadcast(); + } + } +} + +void +SingleExecutor::wait_for_room(MonitorGuard & producerGuard) { + if (_taskLimit.load(std::memory_order_relaxed) != _wantedTaskLimit.load(std::memory_order_relaxed)) { + sync(); + _tasks = std::make_unique<Task::UP[]>(_wantedTaskLimit); + _taskLimit = _wantedTaskLimit.load(); + } + while (numTasks() >= _taskLimit.load(std::memory_order_relaxed)) { + _producerNeedWakeup.store(true, std::memory_order_relaxed); + producerGuard.wait(10ms); + _producerNeedWakeup.store(false, std::memory_order_relaxed); + } +} + +ThreadExecutor::Stats +SingleExecutor::getStats() { + uint64_t accepted = _wp.load(std::memory_order_relaxed); + Stats stats(_maxPending, (accepted - _lastAccepted), 0); + _lastAccepted = accepted; + _maxPending = 0; + return stats; +} + + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h new file mode 100644 index 00000000000..0ab89355566 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h @@ -0,0 +1,55 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/threadexecutor.h> +#include <vespa/vespalib/util/thread.h> +#include <thread> +#include <atomic> + +namespace vespalib { + +/** + * Has a single thread consuming tasks from a fixed size ringbuffer. + * Made for throughput where the producer has no interaction with the consumer and + * it is hence very cheap to produce a task. High and low watermark at 25%/75% is used + * to reduce ping-pong. + */ +class SingleExecutor final : public vespalib::SyncableThreadExecutor, vespalib::Runnable { +public: + explicit SingleExecutor(uint32_t taskLimit); + ~SingleExecutor() override; + Task::UP execute(Task::UP task) override; + void setTaskLimit(uint32_t taskLimit) override; + SingleExecutor & sync() override; + size_t getNumThreads() const override; + uint32_t getTaskLimit() const { return _taskLimit.load(std::memory_order_relaxed); } + Stats getStats() override; +private: + uint64_t addTask(Task::UP task); + void run() override; + void drain_tasks(); + void run_tasks_till(uint64_t available); + void wait_for_room(MonitorGuard & guard); + uint64_t index(uint64_t counter) const { + return counter & (_taskLimit.load(std::memory_order_relaxed) - 1); + } + + uint64_t numTasks() const { + return _wp.load(std::memory_order_relaxed) - _rp.load(std::memory_order_acquire); + } + std::atomic<uint32_t> _taskLimit; + std::atomic<uint32_t> _wantedTaskLimit; + std::atomic<uint64_t> _rp; + std::unique_ptr<Task::UP[]> _tasks; + vespalib::Monitor _consumerMonitor; + vespalib::Monitor _producerMonitor; + vespalib::Thread _thread; + uint64_t _lastAccepted; + std::atomic<uint64_t> _maxPending; + std::atomic<uint64_t> _wakeupConsumerAt; + std::atomic<bool> _producerNeedWakeup; + std::atomic<uint64_t> _wp; +}; + +} diff --git a/standalone-container/CMakeLists.txt b/standalone-container/CMakeLists.txt index 83c58e09945..6e8bf49d846 100644 --- a/standalone-container/CMakeLists.txt +++ b/standalone-container/CMakeLists.txt @@ -1,2 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_fat_java_artifact(standalone-container) +install(PROGRAMS src/main/sh/standalone-container.sh DESTINATION libexec/vespa) diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java index 28001e8e8d2..13991199a1f 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java @@ -44,7 +44,8 @@ public class AuthorizationResult { DENY_CERT_MISMATCH_ISSUER(AccessCheckStatus.DENY_CERT_MISMATCH_ISSUER), DENY_CERT_MISSING_SUBJECT(AccessCheckStatus.DENY_CERT_MISSING_SUBJECT), DENY_CERT_MISSING_DOMAIN(AccessCheckStatus.DENY_CERT_MISSING_DOMAIN), - DENY_CERT_MISSING_ROLE_NAME(AccessCheckStatus.DENY_CERT_MISSING_ROLE_NAME); + DENY_CERT_MISSING_ROLE_NAME(AccessCheckStatus.DENY_CERT_MISSING_ROLE_NAME), + DENY_CERT_HASH_MISMATCH(AccessCheckStatus.DENY_CERT_HASH_MISMATCH); private final AccessCheckStatus wrappedElement; diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java index 851b1fa214c..cc714f38290 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java @@ -60,25 +60,7 @@ public class DeployMojo extends AbstractVespaDeploymentMojo { } private void tailLogs(ApplicationId id, ZoneId zone, long run) throws MojoFailureException, MojoExecutionException { - long last = -1; - DeploymentLog log; - while (true) { - log = controller.deploymentLog(id, zone, run, last); - for (DeploymentLog.Entry entry : log.entries()) - print(entry); - last = log.last().orElse(last); - - if ( ! log.isActive()) - break; - - try { - Thread.sleep(1000); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } + DeploymentLog log = controller.followDeploymentUntilDone(id, zone, run, this::print); switch (log.status()) { case success: return; case error: throw new MojoExecutionException("Unexpected error during deployment; see log for details"); diff --git a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java index 2e51a4ceafe..176e5044bc2 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java @@ -4,15 +4,14 @@ package com.yahoo.collections; import java.util.Collection; import java.util.Comparator; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Stream; -import static java.util.stream.Collectors.reducing; import static java.util.stream.Collectors.toUnmodifiableList; /** @@ -20,7 +19,7 @@ import static java.util.stream.Collectors.toUnmodifiableList; * * @author jonmv */ -public abstract class AbstractFilteringList<Type, ListType extends AbstractFilteringList<Type, ListType>> { +public abstract class AbstractFilteringList<Type, ListType extends AbstractFilteringList<Type, ListType>> implements Iterable<Type> { private final List<Type> items; private final boolean negate; @@ -84,4 +83,9 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte public final int size() { return items.size(); } + @Override + public Iterator<Type> iterator() { + return items.iterator(); + } + } diff --git a/vespalib/src/vespa/vespalib/util/executor.h b/vespalib/src/vespa/vespalib/util/executor.h index ef5cb4b84fa..97cde4ffbe2 100644 --- a/vespalib/src/vespa/vespalib/util/executor.h +++ b/vespalib/src/vespa/vespalib/util/executor.h @@ -23,6 +23,8 @@ public: virtual ~Task() {} }; + enum class OptimizeFor {LATENCY, THROUGHPUT}; + /** * Execute the given task using one of the internal threads some * time in the future. The task may also be rejected in which case |