diff options
375 files changed, 5591 insertions, 3704 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java index c328b8b6c21..123a9bc7449 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java @@ -59,7 +59,7 @@ public class IdentityDocumentGenerator { zone.environment().value(), identityType); - Set<String> ips = new HashSet<>(node.ipAddresses()); + Set<String> ips = new HashSet<>(node.ipConfig().primary()); PrivateKey privateKey = keyProvider.getPrivateKey(athenzProviderServiceConfig.secretVersion()); AthenzService providerService = new AthenzService(athenzProviderServiceConfig.domain(), athenzProviderServiceConfig.serviceName()); diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index 05bbd790ad7..c258e1be466 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; + import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; @@ -155,9 +156,9 @@ public class InstanceValidator { .map(InetAddresses::forString) .collect(Collectors.toList()); - List<InetAddress> nodeIpAddresses = node.ipAddresses().stream() - .map(InetAddresses::forString) - .collect(Collectors.toList()); + List<InetAddress> nodeIpAddresses = node.ipConfig().primary().stream() + .map(InetAddresses::forString) + .collect(Collectors.toList()); // Validate that ipaddresses in request are valid for node diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java index 19ad9df2d4d..ab81cb8eda5 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java @@ -125,7 +125,7 @@ public class InstanceValidatorTest { Node node = nodeList.get(0); nodeList = allocateNode(nodeList, node, applicationId); when(nodeRepository.getNodes()).thenReturn(nodeList); - String nodeIp = node.ipAddresses().stream().findAny().orElseThrow(() -> new RuntimeException("No ipaddress for mocked node")); + String nodeIp = node.ipConfig().primary().stream().findAny().orElseThrow(() -> new RuntimeException("No ipaddress for mocked node")); InstanceConfirmation instanceConfirmation = createRefreshInstanceConfirmation(applicationId, domain, service, ImmutableList.of(nodeIp)); assertTrue(instanceValidator.isValidRefresh(instanceConfirmation)); @@ -140,7 +140,7 @@ public class InstanceValidatorTest { Node node = nodeList.get(0); nodeList = allocateNode(nodeList, node, applicationId); when(nodeRepository.getNodes()).thenReturn(nodeList); - String nodeIp = node.ipAddresses().stream().findAny().orElseThrow(() -> new RuntimeException("No ipaddress for mocked node")); + String nodeIp = node.ipConfig().primary().stream().findAny().orElseThrow(() -> new RuntimeException("No ipaddress for mocked node")); // Add invalid ip to list of ip addresses InstanceConfirmation instanceConfirmation = createRefreshInstanceConfirmation(applicationId, domain, service, ImmutableList.of(nodeIp, "::ff")); diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java index 892eb9aac05..31c8049b0dd 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java @@ -57,12 +57,12 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo { private String mainClass = null; @Parameter(defaultValue = "false") - private boolean buildVespaPlatformBundle; + private boolean buildLegacyVespaPlatformBundle; public void execute() throws MojoExecutionException { try { - if (discPreInstallBundle != null && ! buildVespaPlatformBundle) - throw new MojoExecutionException("The 'discPreInstallBundle' parameter can only be used by Vespa platform bundles."); + if (discPreInstallBundle != null && !buildLegacyVespaPlatformBundle) + throw new MojoExecutionException("The 'discPreInstallBundle' parameter can only be used by legacy Vespa platform bundles."); Artifacts.ArtifactSet artifactSet = Artifacts.getArtifacts(project); warnOnUnsupportedArtifacts(artifactSet.getNonJarArtifacts()); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java index 957531b9f7f..6888626633b 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java @@ -20,31 +20,28 @@ public class PreGeneratedFileRegistry implements FileRegistry { private final String fileSourceHost; private final Map<String, String> path2Hash = new LinkedHashMap<>(); - private static String entryDelimiter = "\t"; - private static Pattern entryDelimiterPattern = Pattern.compile(entryDelimiter, Pattern.LITERAL); + private static final String entryDelimiter = "\t"; + private static final Pattern entryDelimiterPattern = Pattern.compile(entryDelimiter, Pattern.LITERAL); private PreGeneratedFileRegistry(Reader readerArg) { - BufferedReader reader = new BufferedReader(readerArg); - try { + try (BufferedReader reader = new BufferedReader(readerArg)) { fileSourceHost = reader.readLine(); if (fileSourceHost == null) - throw new RuntimeException("Error while reading pre generated file registry"); + throw new RuntimeException("Error while reading pre-generated file registry"); String line; while ((line = reader.readLine()) != null) { addFromLine(line); } - } catch(IOException e) { - throw new RuntimeException("Error while reading pre generated file registry", e); - } finally { - try { - reader.close(); - } catch(IOException e) {} + } catch (IOException e) { + throw new RuntimeException("Error while reading pre-generated file registry", e); } } private void addFromLine(String line) { String[] parts = entryDelimiterPattern.split(line); + if (parts.length < 2) + throw new IllegalArgumentException("Cannot split '" + line + "' into two parts"); addEntry(parts[0], parts[1]); } @@ -58,8 +55,7 @@ public class PreGeneratedFileRegistry implements FileRegistry { builder.append(registry.fileSourceHost()).append('\n'); for (FileRegistry.Entry entry : entries) { - builder.append(entry.relativePath).append(entryDelimiter).append(entry.reference.value()). - append('\n'); + builder.append(entry.relativePath).append(entryDelimiter).append(entry.reference.value()).append('\n'); } return builder.toString(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java index 0d0da71bd0f..7f1a9933188 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java @@ -6,6 +6,7 @@ import com.yahoo.document.ReferenceDataType; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -30,10 +31,24 @@ public class DocumentReferenceResolver { } public void resolveReferences(SDDocumentType documentType) { - DocumentReferences references = new DocumentReferences(createFieldToDocumentReferenceMapping(documentType)); + var references = new DocumentReferences(createFieldToDocumentReferenceMapping(documentType)); documentType.setDocumentReferences(references); } + public void resolveInheritedReferences(SDDocumentType documentType) { + resolveInheritedReferencesRecursive(documentType, documentType.getInheritedTypes()); + } + + private void resolveInheritedReferencesRecursive(SDDocumentType documentType, + Collection<SDDocumentType> inheritedTypes) { + for (var inheritedType : inheritedTypes) { + documentType.getDocumentReferences().get().mergeFrom(inheritedType.getDocumentReferences().get()); + } + for (var inheritedType : inheritedTypes) { + resolveInheritedReferencesRecursive(documentType, inheritedType.getInheritedTypes()); + } + } + private Map<String, DocumentReference> createFieldToDocumentReferenceMapping(SDDocumentType documentType) { return fieldStream(documentType) .filter(field -> field.getDataType() instanceof ReferenceDataType) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java index 37f5ab1bbde..4a3995b2d40 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java @@ -18,6 +18,10 @@ public class DocumentReferences implements Iterable<Map.Entry<String, DocumentRe this.references = references; } + public void mergeFrom(DocumentReferences other) { + references.putAll(other.references); + } + @Override public Iterator<Map.Entry<String, DocumentReference>> iterator() { return Collections.unmodifiableSet(references.entrySet()).iterator(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index eb68e6af203..1fab30f9ea4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -239,6 +239,7 @@ public class SearchBuilder { var resolver = new DocumentReferenceResolver(searchList); sdocs.forEach(resolver::resolveReferences); + sdocs.forEach(resolver::resolveInheritedReferences); var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList); sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java index d86ed265b77..77064038053 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java @@ -48,11 +48,13 @@ public class MatchedElementsOnlyResolver extends Processor { if (isComplexFieldWithOnlyStructFieldAttributes(sourceField)) { field.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); } - } else if (isSupportedAttributeField(sourceField)) { - field.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } else if (isSupportedMultiValueField(sourceField)) { + if (sourceField.doesAttributing()) { + field.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } } else if (validate) { fail(summary, field, "'matched-elements-only' is not supported for this field type. " + - "Supported field types are: array attribute, weighted set attribute, " + + "Supported field types are: array of primitive, weighted set of primitive, " + "array of simple struct, map of primitive type to simple struct, " + "and map of primitive type to primitive type"); } @@ -60,10 +62,9 @@ public class MatchedElementsOnlyResolver extends Processor { // else case is handled in SummaryFieldsMustHaveValidSource } - private boolean isSupportedAttributeField(ImmutableSDField sourceField) { + private boolean isSupportedMultiValueField(ImmutableSDField sourceField) { var type = sourceField.getDataType(); - return sourceField.doesAttributing() && - (isArrayOfPrimitiveType(type) || isWeightedsetOfPrimitiveType(type)); + return (isArrayOfPrimitiveType(type) || isWeightedsetOfPrimitiveType(type)); } private boolean isArrayOfPrimitiveType(DataType type) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index f7a04c36da0..0192e9d42c6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -66,12 +66,12 @@ public class HostSystem extends AbstractConfigProducer<Host> { * @return the host with the given hostname, or null if no such host */ public HostResource getHostByHostname(String name) { - System.out.println("Getting name=" + name + " all hosts: " + hostname2host); + String localhost = "localhost"; HostResource hostResource = hostname2host.get(name); if (hostResource == null) { - // Create a new HostResource if this is the host this code is running on (as when running tests) - if (HostName.getLocalhost().equals(name)) { - String localhost = "localhost"; + // Create a new HostResource if this is the host this code is running on (as it is when running tests) + // TODO: please eliminate the ugly hack using "localhost.fortestingpurposesonly" + if (HostName.getLocalhost().equals(name) || "localhost.fortestingpurposesonly".equals(name)) { if (! getChildren().containsKey(localhost)) { new Host(this, localhost); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 7484e0cd9a0..1b5be1c2f97 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -264,7 +264,8 @@ public class Admin extends AbstractConfigProducer implements Serializable { FileDistributor fileDistributor = fileDistribution.getFileDistributor(); HostResource hostResource = hostSystem().getHostByHostname(fileDistributor.fileSourceHost()); if (hostResource == null && ! multitenant) - throw new IllegalArgumentException("Could not find " + host + " in the application's " + hostSystem()); + throw new IllegalArgumentException("Could not find " + fileDistributor.fileSourceHost() + + " in the application's " + hostSystem()); FileDistributionConfigProvider configProvider = new FileDistributionConfigProvider(fileDistribution, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 24e88d7ef7d..f9338f9cb35 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -31,7 +31,8 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain @Override public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); - builder.jvm.heapsize(384); + builder.jvm.heapsize(384) + .verbosegc(true); } protected boolean messageBusEnabled() { return false; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 547c05d2c9b..08f4e2fa12f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -5,8 +5,8 @@ import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.container.BundlesConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.model.application.validation.RestartConfigs; @@ -14,18 +14,17 @@ import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.xml.PlatformBundles; import java.util.Set; import java.util.TreeSet; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - /** * Container implementation for cluster-controllers */ @RestartConfigs({FleetcontrollerConfig.class, ZookeeperServerConfig.class}) public class ClusterControllerContainer extends Container implements - BundlesConfig.Producer, + PlatformBundlesConfig.Producer, ZookeeperServerConfig.Producer { private static final ComponentSpecification CLUSTERCONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-apps"); @@ -55,11 +54,12 @@ public class ClusterControllerContainer extends Container implements } addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, "controller", isHosted)); - addFileBundle("lib/jars/clustercontroller-apps-jar-with-dependencies.jar"); - addFileBundle("lib/jars/clustercontroller-apputil-jar-with-dependencies.jar"); - addFileBundle("lib/jars/clustercontroller-core-jar-with-dependencies.jar"); - addFileBundle("lib/jars/clustercontroller-utils-jar-with-dependencies.jar"); - addFileBundle("lib/jars/zookeeper-server-jar-with-dependencies.jar"); + // TODO: Why are bundles added here instead of in the cluster? + addFileBundle("clustercontroller-apps"); + addFileBundle("clustercontroller-apputil"); + addFileBundle("clustercontroller-core"); + addFileBundle("clustercontroller-utils"); + addFileBundle("zookeeper-server"); } @Override @@ -82,8 +82,8 @@ public class ClusterControllerContainer extends Container implements super.addHandler(h); } - private void addFileBundle(String bundlePath) { - bundles.add("file:" + getDefaults().underVespaHome(bundlePath)); + private void addFileBundle(String bundleName) { + bundles.add(PlatformBundles.absoluteBundlePath(bundleName).toString()); } private ComponentModel createComponentModel(String id, String className, ComponentSpecification bundle) { @@ -102,10 +102,8 @@ public class ClusterControllerContainer extends Container implements } @Override - public void getConfig(BundlesConfig.Builder builder) { - for (String bundle : bundles) { - builder.bundle(bundle); - } + public void getConfig(PlatformBundlesConfig.Builder builder) { + bundles.forEach(builder::bundlePaths); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 86b6ab8a25c..4dc9811a024 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -38,9 +38,9 @@ import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.xml.PlatformBundles; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -58,8 +58,6 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.getDefaultPublicConsumer; import static com.yahoo.vespa.model.admin.monitoring.MetricSet.emptyMetricSet; import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.getVespaMetricsConsumer; -import static com.yahoo.vespa.model.container.xml.BundleMapper.JarSuffix.JAR_WITH_DEPS; -import static com.yahoo.vespa.model.container.xml.BundleMapper.absoluteBundlePath; /** * Container cluster for metrics proxy containers. @@ -76,7 +74,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC public static final Logger log = Logger.getLogger(MetricsProxyContainerCluster.class.getName()); private static final String METRICS_PROXY_NAME = "metrics-proxy"; - static final Path METRICS_PROXY_BUNDLE_FILE = absoluteBundlePath((Paths.get(METRICS_PROXY_NAME + JAR_WITH_DEPS.suffix))); + static final Path METRICS_PROXY_BUNDLE_FILE = PlatformBundles.absoluteBundlePath(METRICS_PROXY_NAME); static final String METRICS_PROXY_BUNDLE_NAME = "com.yahoo.vespa." + METRICS_PROXY_NAME; static final class AppDimensionNames { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 941870e980b..20a7a6cc856 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -642,6 +642,7 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.puts.sum.failures.total.rate")); metrics.add(new Metric("vds.distributor.puts.sum.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.puts.sum.failures.test_and_set_failed.rate")); + metrics.add(new Metric("vds.distributor.puts.sum.failures.concurrent_mutations.rate")); metrics.add(new Metric("vds.distributor.removes.sum.latency.max")); metrics.add(new Metric("vds.distributor.removes.sum.latency.sum")); metrics.add(new Metric("vds.distributor.removes.sum.latency.count")); @@ -650,6 +651,7 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.removes.sum.failures.total.rate")); metrics.add(new Metric("vds.distributor.removes.sum.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.removes.sum.failures.test_and_set_failed.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.failures.concurrent_mutations.rate")); metrics.add(new Metric("vds.distributor.updates.sum.latency.max")); metrics.add(new Metric("vds.distributor.updates.sum.latency.sum")); metrics.add(new Metric("vds.distributor.updates.sum.latency.count")); @@ -658,6 +660,7 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.updates.sum.failures.total.rate")); metrics.add(new Metric("vds.distributor.updates.sum.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.updates.sum.failures.test_and_set_failed.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.failures.concurrent_mutations.rate")); metrics.add(new Metric("vds.distributor.updates.sum.diverging_timestamp_updates.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.ok.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.failures.total.rate")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 04fe77d9e05..b0ac02d0fe8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -8,8 +8,8 @@ import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.container.BundlesConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.di.config.ApplicationBundlesConfig; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.handler.metrics.MetricsProxyApiConfig; import com.yahoo.container.handler.metrics.MetricsV2Handler; @@ -21,7 +21,6 @@ import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ConfigProducerGroup; @@ -29,9 +28,9 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.Servlet; import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; import com.yahoo.vespa.model.container.jersey.RestApi; +import com.yahoo.vespa.model.container.xml.PlatformBundles; import com.yahoo.vespa.model.utils.FileSender; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; @@ -47,7 +46,7 @@ import java.util.stream.Stream; * @author gjoranv */ public final class ApplicationContainerCluster extends ContainerCluster<ApplicationContainer> implements - BundlesConfig.Producer, + ApplicationBundlesConfig.Producer, QrStartConfig.Producer, RankProfilesConfig.Producer, RankingConstantsConfig.Producer, @@ -135,11 +134,11 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private void addTestrunnerComponentsIfTester(DeployState deployState) { if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) { - addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-testrunner-components-jar-with-dependencies.jar"))); - addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-osgi-testrunner-jar-with-dependencies.jar"))); - addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/tenant-cd-api-jar-with-dependencies.jar"))); + addPlatformBundle(PlatformBundles.absoluteBundlePath("vespa-testrunner-components")); + addPlatformBundle(PlatformBundles.absoluteBundlePath("vespa-osgi-testrunner")); + addPlatformBundle(PlatformBundles.absoluteBundlePath("tenant-cd-api")); if(deployState.zone().system().isPublic()) { - addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/cloud-tenant-cd-jar-with-dependencies.jar"))); + addPlatformBundle(PlatformBundles.absoluteBundlePath("cloud-tenant-cd")); } } } @@ -189,10 +188,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public Optional<Integer> getMemoryPercentage() { return Optional.ofNullable(memoryPercentage); } @Override - public void getConfig(BundlesConfig.Builder builder) { + public void getConfig(ApplicationBundlesConfig.Builder builder) { applicationBundles.stream().map(FileReference::value) - .forEach(builder::bundle); - super.getConfig(builder); + .forEach(builder::bundles); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 5127616ad5e..240157fb7aa 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -12,12 +12,12 @@ import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.Zone; -import com.yahoo.container.BundlesConfig; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.core.document.ContainerDocumentConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; @@ -69,8 +69,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import static com.yahoo.container.core.BundleLoaderProperties.DISK_BUNDLE_PREFIX; - /** * Parent class for all container cluster types. * @@ -87,7 +85,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> ContainerDocumentConfig.Producer, HealthMonitorConfig.Producer, ApplicationMetadataConfig.Producer, - BundlesConfig.Producer, + PlatformBundlesConfig.Producer, IndexInfoConfig.Producer, IlscriptsConfig.Producer, SchemamappingConfig.Producer, @@ -464,6 +462,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> /** * Adds a bundle present at a known location at the target container nodes. + * Note that the set of platform bundles cannot change during the jdisc container's lifetime. * * @param bundlePath usually an absolute path, e.g. '$VESPA_HOME/lib/jars/foo.jar' */ @@ -472,13 +471,10 @@ public abstract class ContainerCluster<CONTAINER extends Container> } @Override - public void getConfig(BundlesConfig.Builder builder) { - platformBundles.stream() .map(ContainerCluster::toFileReferenceString) - .forEach(builder::bundle); - } - - private static String toFileReferenceString(Path path) { - return DISK_BUNDLE_PREFIX + path.toString(); + public void getConfig(PlatformBundlesConfig.Builder builder) { + platformBundles.stream() + .map(Path::toString) + .forEach(builder::bundlePaths); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 58d12663c86..51f526d5efd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -1,9 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.QrSearchersConfig; -import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; @@ -25,7 +23,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import static com.yahoo.vespa.model.container.xml.BundleMapper.searchAndDocprocBundle; +import static com.yahoo.vespa.model.container.xml.PlatformBundles.searchAndDocprocBundle; /** * @author gjoranv diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java index 284aa3b46c0..232e8fcbd1a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.xml.BundleMapper; +import com.yahoo.vespa.model.container.xml.PlatformBundles; import com.yahoo.vespa.model.search.IndexedSearchCluster; /** @@ -32,7 +32,7 @@ public class DispatcherComponent extends Component<AbstractConfigProducer<?>, Co String dispatcherComponentId = "dispatcher." + indexedSearchCluster.getClusterName(); // used by ClusterSearcher return new ComponentModel(dispatcherComponentId, com.yahoo.search.dispatch.Dispatcher.class.getName(), - BundleMapper.searchAndDocprocBundle); + PlatformBundles.searchAndDocprocBundle); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java index 2689c2ce71b..248b30eafa7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.container.search; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.xml.BundleMapper; +import com.yahoo.vespa.model.container.xml.PlatformBundles; public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent, ComponentModel> { @@ -13,6 +13,6 @@ public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent private static ComponentModel toComponentModel(String clusterName) { String componentId = "rpcresourcepool." + clusterName; - return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), BundleMapper.searchAndDocprocBundle); + return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), PlatformBundles.searchAndDocprocBundle); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index 19d1b6546a6..4e0bff1c1fc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -26,13 +26,14 @@ public class BundleInstantiationSpecificationBuilder { BundleInstantiationSpecification instSpec = new BundleInstantiationSpecification(id, classId, bundle); validate(instSpec); - return bundle == null ? setBundleForKnownClass(instSpec) : instSpec; + return bundle == null ? setBundleForSearchAndDocprocComponents(instSpec) : instSpec; } - private static BundleInstantiationSpecification setBundleForKnownClass(BundleInstantiationSpecification spec) { - return BundleMapper.getBundle(spec.getClassName()). - map(spec::inBundle). - orElse(spec); + private static BundleInstantiationSpecification setBundleForSearchAndDocprocComponents(BundleInstantiationSpecification spec) { + if (PlatformBundles.isSearchAndDocprocClass(spec.getClassName())) + return spec.inBundle(PlatformBundles.searchAndDocprocBundle); + else + return spec; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java deleted file mode 100644 index aa7cb5eb539..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.xml; - -import com.yahoo.vespa.defaults.Defaults; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -/** - * @author gjoranv - * @author Ulf Lilleengen - */ -public class BundleMapper { - - public enum JarSuffix { - JAR_WITH_DEPS("-jar-with-dependencies.jar"), - DEPLOY("-deploy.jar"); - - public final String suffix; - - JarSuffix(String suffix) { - this.suffix = suffix; - } - } - - public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); - - public static final String searchAndDocprocBundle = "container-search-and-docproc"; - - private static final Map<String, String> bundleFromClass; - private static final Map<String, Path> bundleFileFromClass; - - public static Optional<String> getBundle(String className) { - return Optional.ofNullable(bundleFromClass.get(className)); - } - - public static Optional<Path> getBundlePath(String className) { - return Optional.ofNullable(absoluteBundlePath(bundleFileFromClass.get(className))); - } - - public static Path absoluteBundlePath(Path fileName) { - if (fileName == null) return null; - return LIBRARY_PATH.resolve(fileName); - } - - /** - * TODO: This is a temporary hack to ensure that users can use our internal components without - * specifying the bundle in which the components reside. Ideally, this information - * should be generated during vespamodel build time. - * - * The container_maven_plugin has much of the logic in place, but needs to be extended. - */ - static { - bundleFromClass = new HashMap<>(); - bundleFileFromClass = new HashMap<>(); - - bundleFromClass.put("com.yahoo.docproc.AbstractConcreteDocumentFactory", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.docproc.DocumentProcessor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.docproc.SimpleDocumentProcessor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.docproc.util.JoinerDocumentProcessor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.docproc.util.SplitterDocumentProcessor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.example.TimingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.language.simple.SimpleLinguistics", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.cluster.ClusterSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.fastsearch.FastSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.fastsearch.VespaBackEndSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.CJKSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.CollapsePhraseSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.LiteralBoostSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.NoRankingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.NonPhrasingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.NormalizingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.PhrasingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.RecallSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.StemmingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.BlendingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.FieldCollapsingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.FillSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.JSONDebugSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.JuniperSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.MultipleResultsSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.PosSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.QuotingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.ValidateSortingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.semantics.SemanticSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.statistics.StatisticsSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.templates.SearchRendererAdaptor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.Searcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.cluster.ClusterSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.cluster.PingableSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.FederationSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.ForwardingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.http.ConfiguredHTTPClientSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.http.HTTPClientSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.http.HTTPProviderSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.http.HTTPSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.news.NewsSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.vespa.VespaSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.grouping.GroupingQueryParser", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.grouping.GroupingValidator", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.grouping.vespa.GroupingExecutor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.handler.SearchWithRendererHandler", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.PageTemplate", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.PageTemplateSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.engine.Resolver", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.model.Renderer", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.query.rewrite.QueryRewriteSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.query.rewrite.SearchChainDispatcherSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.GenericExpansionRewriter", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.MisspellRewriter", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.NameRewriter", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.AllLowercasingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.DefaultPositionSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.LowercasingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.NGramSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.VespaLowercasingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.rendering.Renderer", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.rendering.SectionedRenderer", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.searchchain.ForkingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.searchchain.example.ExampleSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.searchers.CacheControlSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.statistics.PeakQpsSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.statistics.TimingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.vespa.streamingvisitors.MetricsSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher", searchAndDocprocBundle); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index b83632a58a0..41e092c7ea5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -154,21 +154,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ApplicationContainerCluster cluster = createContainerCluster(spec, modelContext); addClusterContent(cluster, spec, modelContext); - addBundlesForPlatformComponents(cluster); cluster.setMessageBusEnabled(rpcServerEnabled); cluster.setRpcServerEnabled(rpcServerEnabled); cluster.setHttpServerEnabled(httpServerEnabled); model.setCluster(cluster); } - private void addBundlesForPlatformComponents(ApplicationContainerCluster cluster) { - for (Component<?, ?> component : cluster.getAllComponents()) { - String componentClass = component.model.bundleInstantiationSpec.getClassName(); - BundleMapper.getBundlePath(componentClass). - ifPresent(cluster::addPlatformBundle); - } - } - private ApplicationContainerCluster createContainerCluster(Element spec, ConfigModelContext modelContext) { return new VespaDomBuilder.DomConfigProducerBuilder<ApplicationContainerCluster>() { @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/PlatformBundles.java new file mode 100644 index 00000000000..dc2437c1834 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/PlatformBundles.java @@ -0,0 +1,119 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.vespa.defaults.Defaults; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Set; + +/** + * @author gjoranv + * @author Ulf Lilleengen + */ +public class PlatformBundles { + + private enum JarSuffix { + JAR_WITH_DEPS("-jar-with-dependencies.jar"), + DEPLOY("-deploy.jar"); + + public final String suffix; + + JarSuffix(String suffix) { + this.suffix = suffix; + } + } + + public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); + public static final String searchAndDocprocBundle = "container-search-and-docproc"; + + private static final Set<String> searchAndDocprocComponents; + + public static boolean isSearchAndDocprocClass(String className) { + return searchAndDocprocComponents.contains(className); + } + + public static Path absoluteBundlePath(String fileName) { + if (fileName == null) return null; + return LIBRARY_PATH.resolve(Paths.get(fileName + JarSuffix.JAR_WITH_DEPS.suffix)); + } + + // This is a hack to allow users to declare components from the search-and-docproc bundle without naming the bundle. + static { + searchAndDocprocComponents = Set.of( + "com.yahoo.docproc.AbstractConcreteDocumentFactory", + "com.yahoo.docproc.DocumentProcessor", + "com.yahoo.docproc.SimpleDocumentProcessor", + "com.yahoo.docproc.util.JoinerDocumentProcessor", + "com.yahoo.docproc.util.SplitterDocumentProcessor", + "com.yahoo.example.TimingSearcher", + "com.yahoo.language.simple.SimpleLinguistics", + "com.yahoo.prelude.cluster.ClusterSearcher", + "com.yahoo.prelude.fastsearch.FastSearcher", + "com.yahoo.prelude.fastsearch.VespaBackEndSearcher", + "com.yahoo.prelude.querytransform.CJKSearcher", + "com.yahoo.prelude.querytransform.CollapsePhraseSearcher", + "com.yahoo.prelude.querytransform.LiteralBoostSearcher", + "com.yahoo.prelude.querytransform.NoRankingSearcher", + "com.yahoo.prelude.querytransform.NonPhrasingSearcher", + "com.yahoo.prelude.querytransform.NormalizingSearcher", + "com.yahoo.prelude.querytransform.PhrasingSearcher", + "com.yahoo.prelude.querytransform.RecallSearcher", + "com.yahoo.prelude.querytransform.StemmingSearcher", + "com.yahoo.prelude.searcher.BlendingSearcher", + "com.yahoo.prelude.searcher.FieldCollapsingSearcher", + "com.yahoo.prelude.searcher.FillSearcher", + "com.yahoo.prelude.searcher.JSONDebugSearcher", + "com.yahoo.prelude.searcher.JuniperSearcher", + "com.yahoo.prelude.searcher.MultipleResultsSearcher", + "com.yahoo.prelude.searcher.PosSearcher", + "com.yahoo.prelude.searcher.QuotingSearcher", + "com.yahoo.prelude.searcher.ValidateSortingSearcher", + "com.yahoo.prelude.semantics.SemanticSearcher", + "com.yahoo.prelude.statistics.StatisticsSearcher", + "com.yahoo.prelude.templates.SearchRendererAdaptor", + "com.yahoo.search.Searcher", + "com.yahoo.search.cluster.ClusterSearcher", + "com.yahoo.search.cluster.PingableSearcher", + "com.yahoo.search.federation.FederationSearcher", + "com.yahoo.search.federation.ForwardingSearcher", + "com.yahoo.search.federation.http.ConfiguredHTTPClientSearcher", + "com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher", + "com.yahoo.search.federation.http.HTTPClientSearcher", + "com.yahoo.search.federation.http.HTTPProviderSearcher", + "com.yahoo.search.federation.http.HTTPSearcher", + "com.yahoo.search.federation.news.NewsSearcher", + "com.yahoo.search.federation.vespa.VespaSearcher", + "com.yahoo.search.grouping.GroupingQueryParser", + "com.yahoo.search.grouping.GroupingValidator", + "com.yahoo.search.grouping.vespa.GroupingExecutor", + "com.yahoo.search.handler.SearchWithRendererHandler", + "com.yahoo.search.pagetemplates.PageTemplate", + "com.yahoo.search.pagetemplates.PageTemplateSearcher", + "com.yahoo.search.pagetemplates.engine.Resolver", + "com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver", + "com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver", + "com.yahoo.search.pagetemplates.model.Renderer", + "com.yahoo.search.query.rewrite.QueryRewriteSearcher", + "com.yahoo.search.query.rewrite.SearchChainDispatcherSearcher", + "com.yahoo.search.query.rewrite.rewriters.GenericExpansionRewriter", + "com.yahoo.search.query.rewrite.rewriters.MisspellRewriter", + "com.yahoo.search.query.rewrite.rewriters.NameRewriter", + "com.yahoo.search.querytransform.AllLowercasingSearcher", + "com.yahoo.search.querytransform.DefaultPositionSearcher", + "com.yahoo.search.querytransform.LowercasingSearcher", + "com.yahoo.search.querytransform.NGramSearcher", + "com.yahoo.search.querytransform.VespaLowercasingSearcher", + "com.yahoo.search.rendering.Renderer", + "com.yahoo.search.rendering.SectionedRenderer", + "com.yahoo.search.searchchain.ForkingSearcher", + "com.yahoo.search.searchchain.example.ExampleSearcher", + "com.yahoo.search.searchers.CacheControlSearcher", + "com.yahoo.search.statistics.PeakQpsSearcher", + "com.yahoo.search.statistics.TimingSearcher", + "com.yahoo.vespa.streamingvisitors.MetricsSearcher", + "com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher" + ); + } + +} diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg new file mode 100644 index 00000000000..1f4d5619248 --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg @@ -0,0 +1,108 @@ +attribute[].name "ref_from_a" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "ref_from_b" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "from_a_int_field" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "from_b_int_field" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/child_a.sd b/config-model/src/test/derived/imported_fields_inherited_reference/child_a.sd new file mode 100644 index 00000000000..75c16d1eefe --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/child_a.sd @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +schema child_a { + document child_a { + field ref_from_a type reference<parent> { + indexing: attribute + } + } +} diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/child_b.sd b/config-model/src/test/derived/imported_fields_inherited_reference/child_b.sd new file mode 100644 index 00000000000..b4349fcc65d --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/child_b.sd @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +schema child_b { + document child_b inherits child_a { + field ref_from_b type reference<parent> { + indexing: attribute + } + } +} diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/child_c.sd b/config-model/src/test/derived/imported_fields_inherited_reference/child_c.sd new file mode 100644 index 00000000000..f798b8c3446 --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/child_c.sd @@ -0,0 +1,7 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +schema child_c { + document child_c inherits child_b { + } + import field ref_from_a.int_field as from_a_int_field {} + import field ref_from_b.int_field as from_b_int_field {} +} diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg new file mode 100644 index 00000000000..ca490b053f7 --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg @@ -0,0 +1,110 @@ +enablecompression false +documenttype[].id -94853056 +documenttype[].name "child_a" +documenttype[].version 0 +documenttype[].headerstruct 867409663 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].datatype[].id 867409663 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "child_a.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "ref_from_a" +documenttype[].datatype[].sstruct.field[].id 300427062 +documenttype[].datatype[].sstruct.field[].datatype 427398467 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].fieldsets{[]}.fields[] "ref_from_a" +documenttype[].referencetype[].id 427398467 +documenttype[].referencetype[].target_type_id 1175161836 +documenttype[].id -94852095 +documenttype[].name "child_b" +documenttype[].version 0 +documenttype[].headerstruct 670896158 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].inherits[].id -94853056 +documenttype[].datatype[].id 670896158 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "child_b.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "ref_from_b" +documenttype[].datatype[].sstruct.field[].id 185778735 +documenttype[].datatype[].sstruct.field[].datatype 427398467 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].fieldsets{[]}.fields[] "ref_from_a" +documenttype[].fieldsets{[]}.fields[] "ref_from_b" +documenttype[].id -94851134 +documenttype[].name "child_c" +documenttype[].version 0 +documenttype[].headerstruct 474382653 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].inherits[].id -94852095 +documenttype[].datatype[].id 474382653 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "child_c.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].fieldsets{[]}.fields[] "ref_from_a" +documenttype[].fieldsets{[]}.fields[] "ref_from_b" +documenttype[].importedfield[].name "from_a_int_field" +documenttype[].importedfield[].name "from_b_int_field" +documenttype[].id 1175161836 +documenttype[].name "parent" +documenttype[].version 0 +documenttype[].headerstruct 836075987 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].datatype[].id 836075987 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "parent.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "int_field" +documenttype[].datatype[].sstruct.field[].id 2128822283 +documenttype[].datatype[].sstruct.field[].datatype 0 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].fieldsets{[]}.fields[] "int_field" + diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/imported-fields.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/imported-fields.cfg new file mode 100644 index 00000000000..e574bfa2393 --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/imported-fields.cfg @@ -0,0 +1,6 @@ +attribute[].name "from_a_int_field" +attribute[].referencefield "ref_from_a" +attribute[].targetfield "int_field" +attribute[].name "from_b_int_field" +attribute[].referencefield "ref_from_b" +attribute[].targetfield "int_field" diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/parent.sd b/config-model/src/test/derived/imported_fields_inherited_reference/parent.sd new file mode 100644 index 00000000000..0509f1a8565 --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/parent.sd @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +schema parent { + document parent { + field int_field type int { + indexing: attribute + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java index b06745f9a75..a0dd89229dd 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java @@ -30,4 +30,9 @@ public class ImportedFieldsTestCase extends AbstractExportingTestCase { public void configs_for_imported_position_field_summary_are_derived() throws IOException, ParseException { assertCorrectDeriving("imported_position_field_summary", "child"); } + + @Test + public void derives_configs_for_imported_fields_when_reference_fields_are_inherited() throws IOException, ParseException { + assertCorrectDeriving("imported_fields_inherited_reference", "child_c"); + } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java index 0ef696df6cf..ab98706fd44 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java @@ -149,10 +149,10 @@ public class MatchedElementsOnlyResolverTestCase { exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage("For search 'test', document summary 'default', summary field 'my_field': " + "'matched-elements-only' is not supported for this field type. " + - "Supported field types are: array attribute, weighted set attribute, " + + "Supported field types are: array of primitive, weighted set of primitive, " + "array of simple struct, map of primitive type to simple struct, " + "and map of primitive type to primitive type"); - buildSearch(joinLines("field my_field type array<string> {", + buildSearch(joinLines("field my_field type string {", " indexing: summary", " summary: matched-elements-only", "}")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index bed77bd5c77..34f06519ac9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -15,8 +15,8 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.Zone; -import com.yahoo.container.BundlesConfig; import com.yahoo.container.core.ApplicationMetadataConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames; @@ -44,7 +44,6 @@ import static org.hamcrest.CoreMatchers.endsWith; import static org.hamcrest.CoreMatchers.hasItem; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -56,11 +55,11 @@ public class MetricsProxyContainerClusterTest { @Test public void metrics_proxy_bundle_is_included_in_bundles_config() { VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); - var builder = new BundlesConfig.Builder(); + var builder = new PlatformBundlesConfig.Builder(); model.getConfig(builder, CLUSTER_CONFIG_ID); - BundlesConfig config = builder.build(); - assertEquals(1, config.bundle().size()); - assertThat(config.bundle(0).value(), endsWith(METRICS_PROXY_BUNDLE_FILE.toString())); + PlatformBundlesConfig config = builder.build(); + assertEquals(1, config.bundlePaths().size()); + assertThat(config.bundlePaths(0), endsWith(METRICS_PROXY_BUNDLE_FILE.toString())); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 03c05af1145..97359b392a5 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.container; import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.RoutingProviderConfig; -import com.yahoo.config.FileReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -16,7 +15,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; -import com.yahoo.container.BundlesConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.Host; @@ -292,9 +291,9 @@ public class ContainerClusterTest { .zone(zone).build(); MockRoot root = new MockRoot("foo", state); ApplicationContainerCluster cluster = new ApplicationContainerCluster(root, "container0", "container1", state); - BundlesConfig.Builder bundleBuilder = new BundlesConfig.Builder(); + var bundleBuilder = new PlatformBundlesConfig.Builder(); cluster.getConfig(bundleBuilder); - List<String> installedBundles = bundleBuilder.build().bundle().stream().map(FileReference::value).collect(Collectors.toList()); + List<String> installedBundles = bundleBuilder.build().bundlePaths(); assertEquals(expectedBundleNames.size(), installedBundles.size()); assertThat(installedBundles, containsInAnyOrder( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java index d2a840d1fbc..70ae6a27324 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -33,7 +33,7 @@ public class BundleInstantiationSpecificationBuilderTest { @Test public void bundle_is_replaced_for_internal_class() throws IOException, SAXException { String internalClass = GroupingValidator.class.getName(); - verifyExpectedBundle(internalClass, null, BundleMapper.searchAndDocprocBundle); + verifyExpectedBundle(internalClass, null, PlatformBundles.searchAndDocprocBundle); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index eda90b03147..e9048cf7863 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -3,17 +3,16 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.docproc.DocprocConfig; import com.yahoo.config.docproc.SchemamappingConfig; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; -import com.yahoo.container.BundlesConfig; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.HostPorts; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ApplicationContainer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.docproc.DocprocChain; import com.yahoo.vespa.model.container.docproc.DocumentProcessor; @@ -30,8 +29,8 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.core.IsNull.notNullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** @@ -46,7 +45,6 @@ public class DocprocBuilderTest extends DomBuilderTest { private ContainerMbusConfig containerMbusConfig; private ComponentsConfig componentsConfig; private ChainsConfig chainsConfig; - private BundlesConfig bundlesConfig; private SchemamappingConfig schemamappingConfig; private DocprocConfig docprocConfig; private QrStartConfig qrStartConfig; @@ -64,7 +62,6 @@ public class DocprocBuilderTest extends DomBuilderTest { cluster.getConfigId() + "/component/com.yahoo.docproc.jdisc.DocumentProcessingHandler"); documentmanagerConfig = root.getConfig(DocumentmanagerConfig.class, cluster.getConfigId()); - bundlesConfig = root.getConfig(BundlesConfig.class, cluster.getConfigId()); schemamappingConfig = root.getConfig(SchemamappingConfig.class, cluster.getContainers().get(0).getConfigId()); qrStartConfig = root.getConfig(QrStartConfig.class, cluster.getConfigId()); docprocConfig = root.getConfig(DocprocConfig.class, cluster.getConfigId()); @@ -207,11 +204,6 @@ public class DocprocBuilderTest extends DomBuilderTest { } @Test - public void testBundlesConfig() { - assertTrue(bundlesConfig.bundle().isEmpty()); - } - - @Test public void testSchemaMappingConfig() { assertTrue(schemamappingConfig.fieldmapping().isEmpty()); } diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 0d216480c4e..cde539f25f4 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -53,7 +53,7 @@ ztsUrl string default="" maintainerIntervalMinutes int default=30 # TODO: Default set to a high value (1 year) => maintainer will not run, change when maintainer verified out in prod tenantsMaintainerIntervalMinutes int default=525600 -keepUnusedFileReferencesHours int default=12 +keepUnusedFileReferencesHours int default=4 # Bootstrapping # How long bootstrapping can take before giving up (in seconds) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 41ec7626049..58b7d0ec5bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -206,6 +206,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.metric = metric; } + public Clock clock() { + return clock; + } + + public Metric metric() { + return metric; + } + // ---------------- Deploying ---------------------------------------------------------------- public PrepareResult prepare(Tenant tenant, long sessionId, PrepareParams prepareParams, Instant now) { @@ -677,7 +685,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye tenantRepository.getAllTenants().forEach(tenant -> sessionsPerTenant.put(tenant, tenant.getSessionRepository().getLocalSessions())); Set<ApplicationId> applicationIds = new HashSet<>(); - sessionsPerTenant.values().forEach(sessionList -> sessionList.forEach(s -> applicationIds.add(s.getApplicationId()))); + sessionsPerTenant.values() + .forEach(sessionList -> sessionList.stream() + .map(Session::getApplicationId) + .filter(Objects::nonNull) + .forEach(applicationIds::add)); Map<ApplicationId, Long> activeSessions = new HashMap<>(); applicationIds.forEach(applicationId -> { @@ -862,12 +874,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // We make no validation that the hostname is actually allocated to the given application since // most applications under hosted-vespa are not known to the model and it's OK for a user to get // logs for any host if they are authorized for the hosted-vespa tenant. - if (hostname.isPresent()) { - if (HOSTED_VESPA_TENANT.equals(applicationId.tenant())) - return "http://" + hostname.get() + ":8080/logs"; - else - throw new IllegalArgumentException("Using hostname parameter when getting logs is not supported for application " - + applicationId); + if (hostname.isPresent() && HOSTED_VESPA_TENANT.equals(applicationId.tenant())) { + return "http://" + hostname.get() + ":8080/logs"; } Application application = getApplication(applicationId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index def629f738c..466138c817f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -134,13 +134,13 @@ public class Deployment implements com.yahoo.config.provision.Deployment { try (ActionTimer timer = applicationRepository.timerFor(session.getApplicationId(), "deployment.activateMillis")) { TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + validateSessionStatus(session); ApplicationId applicationId = session.getApplicationId(); if ( ! timeoutBudget.hasTimeLeft()) throw new RuntimeException("Timeout exceeded when trying to activate '" + applicationId + "'"); RemoteSession previousActiveSession; try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { - validateSessionStatus(session); NestedTransaction transaction = new NestedTransaction(); previousActiveSession = applicationRepository.getActiveSession(applicationId); transaction.add(deactivateCurrentActivateNew(previousActiveSession, session, ignoreSessionStaleFailure)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 7e83d7013e0..c634d82010e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -36,16 +36,14 @@ public class ZooKeeperClient { private final ConfigCurator configCurator; private final DeployLogger logger; - private final boolean logFine; /* This is the generation that will be used for reading and writing application data. (1 more than last deployed application) */ private final Path rootPath; private static final ApplicationFile.PathFilter xmlFilter = path -> path.getName().endsWith(".xml"); - public ZooKeeperClient(ConfigCurator configCurator, DeployLogger logger, boolean logFine, Path rootPath) { + public ZooKeeperClient(ConfigCurator configCurator, DeployLogger logger, Path rootPath) { this.configCurator = configCurator; this.logger = logger; - this.logFine = logFine; this.rootPath = rootPath; } @@ -62,7 +60,6 @@ public class ZooKeeperClient { try { while (retries > 0) { try { - logFine("Setting up ZooKeeper nodes for this application"); createZooKeeperNodes(); break; } catch (RuntimeException e) { @@ -105,16 +102,11 @@ public class ZooKeeperClient { * @param app the application package to feed to zookeeper */ void write(ApplicationPackage app) { - logFine("Feeding application config into ZooKeeper"); try { - logFine("Feeding user def files into ZooKeeper"); writeUserDefs(app); - logFine("Feeding application package into ZooKeeper"); writeSomeOf(app); writeSearchDefinitions(app); writeUserIncludeDirs(app, app.getUserIncludeDirs()); - logFine("Feeding sd from docproc bundle into ZooKeeper"); - logFine("Write application metadata into ZooKeeper"); write(app.getMetaData()); } catch (Exception e) { throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" + @@ -269,7 +261,6 @@ public class ZooKeeperClient { } private void write(Version vespaVersion, FileRegistry fileRegistry) { - logFine("Feeding file registry data into ZooKeeper"); String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); configCurator.putData(getZooKeeperAppPath(null).append(ZKApplicationPackage.fileRegistryNode).getAbsolute(), @@ -288,7 +279,6 @@ public class ZooKeeperClient { } void cleanupZooKeeper() { - logFine("Exception occurred. Cleaning up ZooKeeper"); try { for (String subPath : Arrays.asList( ConfigCurator.DEFCONFIGS_ZK_SUBPATH, @@ -317,12 +307,6 @@ public class ZooKeeperClient { } } - private void logFine(String msg) { - if (logFine) { - logger.log(Level.FINE, msg); - } - } - public void write(AllocatedHosts hosts) throws IOException { configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), AllocatedHostsSerializer.toJson(hosts)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java index 1cfe30270c3..ae18c3e6e95 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionFactory.java @@ -16,7 +16,7 @@ import java.io.File; @SuppressWarnings("WeakerAccess") public class FileDistributionFactory { - private final ConfigserverConfig configserverConfig; + protected final ConfigserverConfig configserverConfig; private final Supervisor supervisor = new Supervisor(new Transport()); @Inject diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java index c06e4da2b7b..de3a2b47233 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java @@ -1,6 +1,7 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.FileReference; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; @@ -50,6 +51,10 @@ public class FileDistributionUtil { return configServers.size() > 0 ? new JRTConnectionPool(new ConfigSourceSet(configServers)) : emptyConnectionPool(); } + public static boolean fileReferenceExistsOnDisk(File downloadDirectory, FileReference applicationPackageReference) { + return getFileReferencesOnDisk(downloadDirectory).contains(applicationPackageReference.value()); + } + static ConnectionPool emptyConnectionPool() { return new EmptyConnectionPool(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java deleted file mode 100644 index db70a51b2b4..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.filedistribution; - -import com.yahoo.config.model.api.FileDistribution; -import com.yahoo.config.model.application.provider.MockFileRegistry; - -import java.io.File; - -/** - * @author Ulf Lilleengen - */ -public class MockFileDistributionProvider extends FileDistributionProvider { - public int timesCalled = 0; - - public MockFileDistributionProvider(File fileReferencesDir) { - super(new MockFileRegistry(), new MockFileDistribution(fileReferencesDir)); - } - - public FileDistribution getFileDistribution() { - timesCalled++; - return super.getFileDistribution(); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index d1861184c24..749f57b3104 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.inject.Inject; import com.yahoo.component.Version; +import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -24,8 +25,11 @@ import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; import com.yahoo.vespa.config.server.http.NotFoundException; +import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.vespa.defaults.Defaults; +import java.io.File; import java.io.IOException; import java.time.Duration; import java.util.List; @@ -34,6 +38,8 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; + /** * Operations on applications (delete, wait for config convergence, restart, application content etc.) * @@ -154,11 +160,25 @@ public class ApplicationHandler extends HttpHandler { } } + return getApplicationResponse(applicationId); + } + + GetApplicationResponse getApplicationResponse(ApplicationId applicationId) { Tenant tenant = applicationRepository.getTenant(applicationId); Optional<ApplicationSet> applicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, applicationId); + String applicationPackage = ""; + RemoteSession session = applicationRepository.getActiveSession(applicationId); + if (session != null) { + FileReference applicationPackageReference = session.getApplicationPackageReference(); + File downloadDirectory = new File(Defaults.getDefaults().underVespaHome(applicationRepository.configserverConfig().fileReferencesDir())); + if (applicationPackageReference != null && ! fileReferenceExistsOnDisk(downloadDirectory, applicationPackageReference)) + applicationPackage = applicationPackageReference.value(); + } + return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId), - applicationSet.get().getAllVersions(applicationId)); + applicationSet.get().getAllVersions(applicationId), + applicationPackage); } @Override @@ -316,9 +336,10 @@ public class ApplicationHandler extends HttpHandler { } private static class GetApplicationResponse extends JSONResponse { - GetApplicationResponse(int status, long generation, List<Version> modelVersions) { + GetApplicationResponse(int status, long generation, List<Version> modelVersions, String applicationPackageReference) { super(status); object.setLong("generation", generation); + object.setString("applicationPackageFileReference", applicationPackageReference); Cursor modelVersionArray = object.setArray("modelVersions"); modelVersions.forEach(version -> modelVersionArray.addString(version.toFullString())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 7c983ab48a0..e539acba916 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -16,11 +16,10 @@ import com.yahoo.vespa.flags.Flags; import java.io.File; import java.time.Duration; -import java.util.Set; import java.util.logging.Logger; -import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.createConnectionPool; +import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; /** * Verifies that all active sessions has an application package on local disk. @@ -40,19 +39,19 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { ApplicationPackageMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, - ConfigserverConfig configserverConfig, FlagSource flagSource) { super(applicationRepository, curator, flagSource, interval, interval); this.applicationRepository = applicationRepository; - this.configserverConfig = configserverConfig; + this.configserverConfig = applicationRepository.configserverConfig(); distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); downloadDirectory = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())); } @Override - protected void maintain() { - if (! distributeApplicationPackage.value()) return; + protected boolean maintain() { + boolean success = true; + if (! distributeApplicationPackage.value()) return success; try (var fileDownloader = new FileDownloader(createConnectionPool(configserverConfig), downloadDirectory)) { for (var applicationId : applicationRepository.listApplications()) { @@ -65,10 +64,11 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { log.fine(() -> "Verifying application package file reference " + applicationPackage + " for session " + sessionId); if (applicationPackage != null) { - if (missingOnDisk(applicationPackage)) { + if (! fileReferenceExistsOnDisk(downloadDirectory, applicationPackage)) { log.fine(() -> "Downloading missing application package for application " + applicationId + " - session " + sessionId); if (fileDownloader.getFile(applicationPackage).isEmpty()) { + success = false; log.warning("Failed to download application package for application " + applicationId + " - session " + sessionId); continue; } @@ -77,6 +77,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { } } } + return success; } private void createLocalSessionIfMissing(ApplicationId applicationId, long sessionId) { @@ -86,9 +87,4 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { sessionRepository.createLocalSessionUsingDistributedApplicationPackage(sessionId); } - private boolean missingOnDisk(FileReference applicationPackageReference) { - Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(downloadDirectory); - return ! fileReferencesOnDisk.contains(applicationPackageReference.value()); - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java index 5369bbef366..5854b1d85da 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.config.server.maintenance; import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.concurrent.maintenance.JobControlState; +import com.yahoo.concurrent.maintenance.JobMetrics; import com.yahoo.concurrent.maintenance.Maintainer; +import com.yahoo.jdisc.Metric; import com.yahoo.path.Path; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.config.server.ApplicationRepository; @@ -13,6 +15,7 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; import java.time.Duration; +import java.util.Map; import java.util.Set; /** @@ -26,16 +29,24 @@ public abstract class ConfigServerMaintainer extends Maintainer { ConfigServerMaintainer(ApplicationRepository applicationRepository, Curator curator, FlagSource flagSource, Duration initialDelay, Duration interval) { - super(null, interval, initialDelay, new JobControl(new JobControlFlags(curator, flagSource))); + super(null, interval, initialDelay, new JobControl(new JobControlFlags(curator, flagSource)), + jobMetrics(applicationRepository.metric())); this.applicationRepository = applicationRepository; } + private static JobMetrics jobMetrics(Metric metric) { + return new JobMetrics((job, consecutiveFailures) -> { + metric.set("maintenance.consecutiveFailures", consecutiveFailures, metric.createContext(Map.of("job", job))); + }); + } + private static class JobControlFlags implements JobControlState { private static final Path root = Path.fromString("/configserver/v1/"); - private static final Path lockRoot = root.append("locks"); + private static final Path lockRoot = root.append("locks"); private final Curator curator; + private final ListFlag<String> inactiveJobsFlag; public JobControlFlags(Curator curator, FlagSource flagSource) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index a6585be391c..ecdca39dc72 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -4,9 +4,8 @@ package com.yahoo.vespa.config.server.maintenance; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.SystemName; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; @@ -30,19 +29,18 @@ public class ConfigServerMaintenance extends AbstractComponent { public ConfigServerMaintenance(ConfigserverConfig configserverConfig, ApplicationRepository applicationRepository, Curator curator, - FileDistributionFactory fileDistributionFactory, - FlagSource flagSource) { + FlagSource flagSource, + Metric metric) { DefaultTimes defaults = new DefaultTimes(configserverConfig); - // TODO: Disabled until we have application metadata + // TODO: Disabled until we have application metadata per tenant //tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval); - fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig, flagSource); + fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, flagSource); sessionsMaintainer = new SessionsMaintainer(applicationRepository, curator, Duration.ofMinutes(1), flagSource); - applicationPackageMaintainer = new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofMinutes(1), configserverConfig, flagSource); + applicationPackageMaintainer = new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofMinutes(1), flagSource); } @Override public void deconstruct() { - //tenantsMaintainer.close(); fileDistributionMaintainer.close(); sessionsMaintainer.close(); applicationPackageMaintainer.close(); @@ -55,16 +53,9 @@ public class ConfigServerMaintenance extends AbstractComponent { private static class DefaultTimes { private final Duration defaultInterval; - private final Duration tenantsMaintainerInterval; DefaultTimes(ConfigserverConfig configserverConfig) { this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes()); - boolean isCd = configserverConfig.system().equals(SystemName.cd.value()); - // TODO: Want job control or feature flag to control when to run this, for now use a very - // long interval to avoid running the maintainer except in CD - this.tenantsMaintainerInterval = isCd - ? defaultInterval - : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java index ed323438e3f..3980ae9d980 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -21,22 +21,21 @@ public class FileDistributionMaintainer extends ConfigServerMaintainer { private final ApplicationRepository applicationRepository; private final File fileReferencesDir; - private final ConfigserverConfig configserverConfig; + private final Duration maxUnusedFileReferenceAge; FileDistributionMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, - ConfigserverConfig configserverConfig, FlagSource flagSource) { super(applicationRepository, curator, flagSource, interval, interval); this.applicationRepository = applicationRepository; - this.configserverConfig = configserverConfig; - this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())); + this.maxUnusedFileReferenceAge = Duration.ofHours(applicationRepository.configserverConfig().keepUnusedFileReferencesHours()); + this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(applicationRepository.configserverConfig().fileReferencesDir())); } @Override - protected void maintain() { - applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, - Duration.ofHours(configserverConfig.keepUnusedFileReferencesHours())); + protected boolean maintain() { + applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge); + return true; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java index 4adf287448d..72e8e08e38f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -26,13 +26,13 @@ public class SessionsMaintainer extends ConfigServerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { applicationRepository.deleteExpiredLocalSessions(); // Expired remote sessions are sessions that belong to an application that have external deployments that // are no longer active if (hostedVespa) { - Duration expiryTime = Duration.ofDays(1); + Duration expiryTime = Duration.ofHours(12); int deleted = applicationRepository.deleteExpiredRemoteSessions(expiryTime); log.log(LogLevel.FINE, "Deleted " + deleted + " expired remote sessions, expiry time " + expiryTime); } @@ -41,5 +41,7 @@ public class SessionsMaintainer extends ConfigServerMaintainer { int deleted = applicationRepository.deleteExpiredLocks(lockExpiryTime); if (deleted > 0) log.log(LogLevel.INFO, "Deleted " + deleted + " locks older than " + lockExpiryTime); + + return true; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java index 9a81d9f7547..d29eea842f5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java @@ -28,8 +28,9 @@ public class TenantsMaintainer extends ConfigServerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { applicationRepository.deleteUnusedTenants(ttlForUnusedTenant, clock.instant()); + return true; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index c520094e294..23300239d17 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -15,6 +15,7 @@ import org.apache.zookeeper.KeeperException; import java.time.Clock; import java.util.Optional; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -91,15 +92,22 @@ public class RemoteSession extends Session { try { completionWaiter.notifyCompletion(); } catch (RuntimeException e) { - // Throw only if we get something else than NoNodeException -- NoNodeException might happen when - // the session is no longer in use (e.g. the app using this session has been deleted) and this method - // has not been called yet for the previous session operation - // on a minority of the config servers (see awaitInternal() method in this class) - if (e.getCause().getClass() != KeeperException.NoNodeException.class) { + // Throw only if we get something else than NoNodeException or NodeExistsException. + // NoNodeException might happen when the session is no longer in use (e.g. the app using this session + // has been deleted) and this method has not been called yet for the previous session operation on a + // minority of the config servers. + // NodeExistsException might happen if an event for this node is delivered more than once, in that case + // this is a no-op + Set<Class<? extends KeeperException>> acceptedExceptions = Set.of(KeeperException.NoNodeException.class, + KeeperException.NodeExistsException.class); + Class<? extends Throwable> exceptionClass = e.getCause().getClass(); + if (acceptedExceptions.contains(exceptionClass)) + log.log(Level.INFO, "Not able to notify completion for session: " + getSessionId() + ", node " + + (exceptionClass.equals(KeeperException.NoNodeException.class) + ? "has been deleted" + : "already exists")); + else throw e; - } else { - log.log(Level.INFO, "Not able to notify completion for session: " + getSessionId() + ", node has been deleted"); - } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index 3643b237d7e..a6818d1e43f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -70,9 +70,18 @@ public abstract class Session implements Comparable<Session> { * @return log preamble */ public String logPre() { - return getApplicationId().equals(ApplicationId.defaultId()) - ? TenantRepository.logPre(getTenantName()) - : TenantRepository.logPre(getApplicationId()); + Optional<ApplicationId> applicationId; + // We might not be able to read application id from zookeeper + // e.g. when the app has been deleted. Use tenant name in that case. + try { + applicationId = Optional.of(getApplicationId()); + } catch (Exception e) { + applicationId = Optional.empty(); + } + return applicationId + .filter(appId -> ! appId.equals(ApplicationId.defaultId())) + .map(TenantRepository::logPre) + .orElse(TenantRepository.logPre(getTenantName())); } public Instant getCreateTime() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index b6b0ac45bb5..35bbc1a8233 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -120,10 +120,10 @@ public class SessionPreparer { * @param tenantPath Zookeeper path for the tenant for this session * @return the config change actions that must be done to handle the activation of the models prepared. */ - public ConfigChangeActions prepare(HostValidator<ApplicationId> hostValidator, DeployLogger logger, PrepareParams params, - Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, - Instant now, File serverDbSessionDir, ApplicationPackage applicationPackage, - SessionZooKeeperClient sessionZooKeeperClient) { + public PrepareResult prepare(HostValidator<ApplicationId> hostValidator, DeployLogger logger, PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, + Instant now, File serverDbSessionDir, ApplicationPackage applicationPackage, + SessionZooKeeperClient sessionZooKeeperClient) { Preparation preparation = new Preparation(hostValidator, logger, params, currentActiveApplicationSet, tenantPath, serverDbSessionDir, applicationPackage, sessionZooKeeperClient); @@ -313,8 +313,8 @@ public class SessionPreparer { checkTimeout("distribute files"); } - ConfigChangeActions result() { - return prepareResult.getConfigChangeActions(); + PrepareResult result() { + return prepareResult; } private List<ContainerEndpoint> readEndpointsIfNull(List<ContainerEndpoint> endpoints) { @@ -352,7 +352,7 @@ public class SessionPreparer { } /** The result of preparation over all model versions */ - private static class PrepareResult { + static class PrepareResult { private final AllocatedHosts allocatedHosts; private final ImmutableList<PreparedModelsBuilder.PreparedModelResult> results; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 0f6ba37cf51..98620877b1f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -164,7 +164,8 @@ public class SessionRepository { ConfigChangeActions actions = sessionPreparer.prepare(applicationRepo.getHostValidator(), logger, params, currentActiveApplicationSet, tenantPath, now, getSessionAppDir(sessionId), - session.getApplicationPackage(), sessionZooKeeperClient); + session.getApplicationPackage(), sessionZooKeeperClient) + .getConfigChangeActions(); session.setPrepared(); waiter.awaitCompletion(params.getTimeoutBudget().timeLeft()); return actions; @@ -571,11 +572,18 @@ public class SessionRepository { throw new IllegalArgumentException(sourceDir.getAbsolutePath() + " is not a directory"); // Copy app atomically: Copy to a temp dir and move to destination - java.nio.file.Path tempDestinationDir = Files.createTempDirectory(destinationDir.getParentFile().toPath(), "app-package"); - log.log(Level.FINE, "Copying dir " + sourceDir.getAbsolutePath() + " to " + tempDestinationDir.toFile().getAbsolutePath()); - IOUtils.copyDirectory(sourceDir, tempDestinationDir.toFile()); - log.log(Level.FINE, "Moving " + tempDestinationDir + " to " + destinationDir.getAbsolutePath()); - Files.move(tempDestinationDir, destinationDir.toPath(), StandardCopyOption.ATOMIC_MOVE); + java.nio.file.Path tempDestinationDir = null; + try { + tempDestinationDir = Files.createTempDirectory(destinationDir.getParentFile().toPath(), "app-package"); + log.log(Level.FINE, "Copying dir " + sourceDir.getAbsolutePath() + " to " + tempDestinationDir.toFile().getAbsolutePath()); + IOUtils.copyDirectory(sourceDir, tempDestinationDir.toFile()); + log.log(Level.FINE, "Moving " + tempDestinationDir + " to " + destinationDir.getAbsolutePath()); + Files.move(tempDestinationDir, destinationDir.toPath(), StandardCopyOption.ATOMIC_MOVE); + } finally { + // In case some of the operations above fail + if (tempDestinationDir != null) + IOUtils.recursiveDeleteDir(tempDestinationDir.toFile()); + } } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 807629a2148..1b9527f4376 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -150,8 +150,8 @@ public class SessionZooKeeperClient { } public ApplicationId readApplicationId() { - if ( ! configCurator.exists(applicationIdPath())) return ApplicationId.defaultId(); - return ApplicationId.fromSerializedForm(configCurator.getData(applicationIdPath())); + String idString = configCurator.getData(applicationIdPath()); + return idString == null ? null : ApplicationId.fromSerializedForm(idString); } void writeApplicationPackageReference(FileReference applicationPackageReference) { @@ -214,7 +214,7 @@ public class SessionZooKeeperClient { } public ZooKeeperDeployer createDeployer(DeployLogger logger) { - ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, logger, true, sessionPath); + ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, logger, sessionPath); return new ZooKeeperDeployer(zkClient); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index 685856d5cf8..11cec9efd95 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -93,8 +93,7 @@ public class ZKApplicationPackage implements ApplicationPackage { try { return PreGeneratedFileRegistry.importRegistry(zkApplication.getDataReader(fileRegistryNode)); } catch (Exception e) { - throw new RuntimeException("Could not determine which files to distribute. " + - "Please try redeploying the application", e); + throw new RuntimeException("Could not determine which files to distribute", e); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index a4b2bfb8902..f74378b32a9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -121,14 +121,15 @@ public class ApplicationRepositoryTest { public void setup(FlagSource flagSource) throws IOException { Curator curator = new MockCurator(); configCurator = ConfigCurator.create(curator); + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .curator(curator) - .configServerConfig(new ConfigserverConfig.Builder() - .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .flagSource(flagSource) .clock(clock) .build(); @@ -142,7 +143,11 @@ public class ApplicationRepositoryTest { applicationRepository = new ApplicationRepository(tenantRepository, provisioner, orchestrator, - clock); + configserverConfig, + new MockLogRetriever(), + clock, + new MockTesterClient(), + new NullMetric()); timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); } @@ -234,14 +239,6 @@ public class ApplicationRepositoryTest { assertEquals(200, response.getStatus()); } - @Test(expected = IllegalArgumentException.class) - public void refuseToGetLogsFromHostnameNotInApplication() { - applicationRepository = createApplicationRepository(); - deployApp(testAppLogServerWithContainer); - HttpResponse response = applicationRepository.getLogs(applicationId(), Optional.of("host123.fake.yahoo.com"), ""); - assertEquals(200, response.getStatus()); - } - @Test public void deleteUnusedFileReferences() throws IOException { File fileReferencesDir = temporaryFolder.newFolder(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index f03550c0a80..68dd5396cf1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -51,7 +51,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private final TenantListener tenantListener; private final PermanentApplicationPackage permanentApplicationPackage; private final HostRegistries hostRegistries; - private final FileDistributionFactory fileDistributionProvider; + private final FileDistributionFactory fileDistributionFactory; private final ModelFactoryRegistry modelFactoryRegistry; private final Optional<Provisioner> hostProvisioner; private final Zone zone; @@ -65,7 +65,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private TestComponentRegistry(Curator curator, ConfigCurator configCurator, Metrics metrics, ModelFactoryRegistry modelFactoryRegistry, PermanentApplicationPackage permanentApplicationPackage, - FileDistributionFactory fileDistributionProvider, + FileDistributionFactory fileDistributionFactory, HostRegistries hostRegistries, ConfigserverConfig configserverConfig, SessionPreparer sessionPreparer, @@ -86,7 +86,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { this.defRepo = defRepo; this.permanentApplicationPackage = permanentApplicationPackage; this.hostRegistries = hostRegistries; - this.fileDistributionProvider = fileDistributionProvider; + this.fileDistributionFactory = fileDistributionFactory; this.modelFactoryRegistry = modelFactoryRegistry; this.hostProvisioner = hostProvisioner; this.sessionPreparer = sessionPreparer; @@ -247,6 +247,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { return secretStore; } - public FileDistributionFactory getFileDistributionProvider() { return fileDistributionProvider; } + public FileDistributionFactory getFileDistributionFactory() { return fileDistributionFactory; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/a.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/a.def index bf99c65ec14..e2f22a38fc3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/a.def +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/a.def @@ -29,7 +29,7 @@ config[].role string config[].id reference ## Wether the NC should start the corresponding role using the -## slavewrapper utility application or not. +## wrapper utility application or not. config[].usewrapper bool default=false routingtable[].hop[].name string diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index 8394611737e..a4fce5e37ba 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -53,7 +53,7 @@ public class ZooKeeperClientTest { @Before public void setupZK() throws IOException { zk = ConfigCurator.create(new MockCurator()); - ZooKeeperClient zkc = new ZooKeeperClient(zk, new BaseDeployLogger(), true, Path.fromString(appPath)); + ZooKeeperClient zkc = new ZooKeeperClient(zk, new BaseDeployLogger(), Path.fromString(appPath)); ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(new File("src/test/apps/zkfeed"), new DeployData("foo", "/bar/baz", @@ -85,7 +85,7 @@ public class ZooKeeperClientTest { ConfigCurator zk = ConfigCurator.create(new MockCurator()); BaseDeployLogger logger = new BaseDeployLogger(); long generation = 1L; - ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, true, Path.fromString("/1")); + ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, Path.fromString("/1")); zooKeeperClient.setupZooKeeper(); String appPath = "/"; assertThat(zk.getChildren(appPath).size(), is(1)); @@ -120,7 +120,7 @@ public class ZooKeeperClientTest { ConfigCurator zk = ConfigCurator.create(new MockCurator()); BaseDeployLogger logger = new BaseDeployLogger(); Path app = Path.fromString("/1"); - ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, true, app); + ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, app); zooKeeperClient.setupZooKeeper(); String currentAppPath = app.getAbsolute(); @@ -191,7 +191,7 @@ public class ZooKeeperClientTest { ConfigCurator zk = ConfigCurator.create(new MockCurator()); BaseDeployLogger logger = new BaseDeployLogger(); Path app = Path.fromString("/1"); - ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, true, app); + ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, app); zooKeeperClient.setupZooKeeper(); HostSpec host1 = new HostSpec("host1.yahoo.com", Collections.emptyList(), Optional.empty()); HostSpec host2 = new HostSpec("host2.yahoo.com", Collections.emptyList(), Optional.empty()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java index 4825ccc1328..641fbe5bf41 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java @@ -48,7 +48,7 @@ public class ZooKeeperDeployerTest { public void deploy(ApplicationPackage applicationPackage, ConfigCurator configCurator, Path appPath) throws IOException { MockDeployLogger logger = new MockDeployLogger(); - ZooKeeperClient client = new ZooKeeperClient(configCurator, logger, true, appPath); + ZooKeeperClient client = new ZooKeeperClient(configCurator, logger, appPath); ZooKeeperDeployer deployer = new ZooKeeperDeployer(client); deployer.deploy(applicationPackage, Collections.singletonMap(new Version(1, 0, 0), new MockFileRegistry()), AllocatedHosts.withHosts(Collections.emptySet())); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java index 78729156b93..af55dc6a90e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionFactory.java @@ -10,16 +10,13 @@ import java.io.File; */ public class MockFileDistributionFactory extends FileDistributionFactory { - public final MockFileDistributionProvider mockFileDistributionProvider; - public MockFileDistributionFactory(ConfigserverConfig configserverConfig) { super(configserverConfig); - mockFileDistributionProvider = new MockFileDistributionProvider(new File(configserverConfig.fileReferencesDir())); } @Override public com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider createProvider(File applicationFile) { - return mockFileDistributionProvider; + return new MockFileDistributionProvider(applicationFile, new File(configserverConfig.fileReferencesDir())); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java new file mode 100644 index 00000000000..45e00e2ece8 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.model.api.FileDistribution; + +import java.io.File; + +/** + * @author hmusum + */ +public class MockFileDistributionProvider extends FileDistributionProvider { + + public MockFileDistributionProvider(File applicationDir, File fileReferencesDir) { + super(new MockFileRegistry(applicationDir, fileReferencesDir.toPath()), + new MockFileDistribution(fileReferencesDir)); + } + + public FileDistribution getFileDistribution() { + return super.getFileDistribution(); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java new file mode 100644 index 00000000000..343e0c50520 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.net.HostName; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +/** + * A file registry for config server tests + * + * @author hmusum + */ +public class MockFileRegistry implements FileRegistry { + + private final List<Entry> entries = new ArrayList<>(); + private final AddFileInterface addFileInterface; + + public MockFileRegistry(File applicationDir, Path rootPath) { + FileDirectory fileDirectory = new FileDirectory(rootPath.toFile()); + this.addFileInterface = new ApplicationFileManager(applicationDir, fileDirectory); + } + + public FileReference addFile(String relativePath) { + if (relativePath.isEmpty()) + relativePath = "./"; + addFileInterface.addFile(relativePath); + + FileReference fileReference = new FileReference(relativePath); + entries.add(new Entry(relativePath, fileReference)); + return fileReference; + } + + @Override + public String fileSourceHost() { return HostName.getLocalhost(); } + + public List<Entry> export() { return entries; } + + @Override + public FileReference addUri(String uri) { + throw new IllegalArgumentException("FileReference addUri(String uri) is not implemented for " + getClass().getCanonicalName()); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index eca32cd364a..1a558f89284 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -353,7 +353,9 @@ public class ApplicationHandlerTest { HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(url, GET)); assertEquals(200, response.getStatus()); String renderedString = SessionHandlerTest.getRenderedString(response); - assertEquals("{\"generation\":" + expectedGeneration + ",\"modelVersions\":[\"" + expectedVersion.toFullString() + "\"]}", renderedString); + assertEquals("{\"generation\":" + expectedGeneration + + ",\"applicationPackageFileReference\":\"\"" + + ",\"modelVersions\":[\"" + expectedVersion.toFullString() + "\"]}", renderedString); } private void assertApplicationExists(ApplicationId applicationId, Zone zone) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index c1377ae439b..a2ef6aeb578 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -62,7 +62,7 @@ public class LocalSessionTest { .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) .build()) .build(); - tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName); configCurator = ConfigCurator.create(curator); } @@ -119,19 +119,21 @@ public class LocalSessionTest { Optional<AllocatedHosts> allocatedHosts) throws Exception { SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenant, sessionId, allocatedHosts); zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); - ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, + ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); if (allocatedHosts.isPresent()) { zkClient.write(allocatedHosts.get()); } zkClient.write(Collections.singletonMap(new Version(0, 0, 0), new MockFileRegistry())); TenantApplications applications = tenantRepository.getTenant(tenantName).getApplicationRepo(); - applications.createApplication(zkc.readApplicationId()); - return new LocalSession(tenant, sessionId, FilesApplicationPackage.fromFile(testApp), zkc, applications); + applications.createApplication(applicationId()); + LocalSession session = new LocalSession(tenant, sessionId, FilesApplicationPackage.fromFile(testApp), zkc, applications); + session.setApplicationId(applicationId()); + return session; } private void doPrepare(LocalSession session) { - doPrepare(session, new PrepareParams.Builder().build()); + doPrepare(session, new PrepareParams.Builder().applicationId(applicationId()).build()); } private void doPrepare(LocalSession session, PrepareParams params) { @@ -140,8 +142,11 @@ public class LocalSessionTest { } private DeployHandlerLogger getLogger() { - return new DeployHandlerLogger(new Slime().get(), false, - new ApplicationId.Builder().tenant(tenantName).applicationName("testapp").build()); + return new DeployHandlerLogger(new Slime().get(), false, applicationId()); + } + + private ApplicationId applicationId() { + return new ApplicationId.Builder().tenant(tenantName).applicationName("testapp").build(); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 46b8754ebe0..7d1554c3e19 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ContainerEndpoint; @@ -33,7 +34,6 @@ import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudgetTest; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; -import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.model.TestModelFactory; @@ -59,7 +59,6 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -68,6 +67,7 @@ import java.util.Set; import java.util.logging.Level; import static com.yahoo.vespa.config.server.session.SessionZooKeeperClient.APPLICATION_PACKAGE_REFERENCE_PATH; +import static com.yahoo.vespa.config.server.session.SessionPreparer.PrepareResult; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -85,27 +85,31 @@ public class SessionPreparerTest { private static final File invalidTestApp = new File("src/test/apps/illegalApp"); private static final Version version123 = new Version(1, 2, 3); private static final Version version321 = new Version(3, 2, 1); - private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); - private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), + private final KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + private final X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build(); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private MockCurator curator; private ConfigCurator configCurator; private SessionPreparer preparer; private TestComponentRegistry componentRegistry; - private MockFileDistributionFactory fileDistributionFactory; - private MockSecretStore secretStore = new MockSecretStore(); + private final MockSecretStore secretStore = new MockSecretStore(); @Rule public TemporaryFolder folder = new TemporaryFolder(); @Before - public void setUp() { + public void setUp() throws IOException { curator = new MockCurator(); configCurator = ConfigCurator.create(curator); - componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - fileDistributionFactory = (MockFileDistributionFactory)componentRegistry.getFileDistributionProvider(); + componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .fileReferencesDir(folder.newFolder().getAbsolutePath()) + .configServerDBDir(folder.newFolder().getAbsolutePath()) + .configDefinitionsDir(folder.newFolder().getAbsolutePath()) + .build()) + .build(); preparer = createPreparer(); } @@ -115,7 +119,7 @@ public class SessionPreparerTest { private SessionPreparer createPreparer(HostProvisionerProvider hostProvisionerProvider) { ModelFactoryRegistry modelFactoryRegistry = - new ModelFactoryRegistry(Arrays.asList(new TestModelFactory(version123), new TestModelFactory(version321))); + new ModelFactoryRegistry(List.of(new TestModelFactory(version123), new TestModelFactory(version321))); return createPreparer(modelFactoryRegistry, hostProvisionerProvider); } @@ -123,7 +127,7 @@ public class SessionPreparerTest { HostProvisionerProvider hostProvisionerProvider) { return new SessionPreparer( modelFactoryRegistry, - componentRegistry.getFileDistributionProvider(), + componentRegistry.getFileDistributionFactory(), hostProvisionerProvider, new PermanentApplicationPackage(componentRegistry.getConfigserverConfig()), componentRegistry.getConfigserverConfig(), @@ -152,14 +156,13 @@ public class SessionPreparerTest { @Test public void require_that_filedistribution_is_ignored_on_dryrun() throws IOException { - prepare(testApp, new PrepareParams.Builder().dryRun(true).timeoutBudget(TimeoutBudgetTest.day()).build()); - assertThat(fileDistributionFactory.mockFileDistributionProvider.timesCalled, is(0)); + PrepareResult result = prepare(testApp, new PrepareParams.Builder().dryRun(true).build()); + assertTrue(result.getFileRegistries().get(version321).export().isEmpty()); } @Test public void require_that_application_is_prepared() throws Exception { prepare(testApp); - assertThat(fileDistributionFactory.mockFileDistributionProvider.timesCalled, is(1)); // Only builds the newest version assertTrue(configCurator.exists(sessionsPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); } @@ -327,11 +330,11 @@ public class SessionPreparerTest { prepare(app, new PrepareParams.Builder().build()); } - private void prepare(File app, PrepareParams params) throws IOException { + private PrepareResult prepare(File app, PrepareParams params) throws IOException { FilesApplicationPackage applicationPackage = getApplicationPackage(app); - preparer.prepare(new HostRegistry<>(), getLogger(), params, - Optional.empty(), tenantPath, Instant.now(), applicationPackage.getAppDir(), - applicationPackage, new SessionZooKeeperClient(curator, sessionsPath)); + return preparer.prepare(new HostRegistry<>(), getLogger(), params, + Optional.empty(), tenantPath, Instant.now(), applicationPackage.getAppDir(), + applicationPackage, new SessionZooKeeperClient(curator, sessionsPath)); } private FilesApplicationPackage getApplicationPackage(File testFile) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java index 5ae5910d827..7f38083797e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java @@ -3,17 +3,18 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.curator.mock.MockCurator; import java.io.File; import java.time.Instant; -import java.util.ArrayList; +import java.util.List; import java.util.Optional; +import java.util.Set; /** * @author Ulf Lilleengen @@ -28,12 +29,12 @@ public class SessionTest { } @Override - public ConfigChangeActions prepare(HostValidator<ApplicationId> hostValidator, DeployLogger logger, PrepareParams params, - Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, - Instant now, File serverDbSessionDir, ApplicationPackage applicationPackage, - SessionZooKeeperClient sessionZooKeeperClient) { + public PrepareResult prepare(HostValidator<ApplicationId> hostValidator, DeployLogger logger, PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, + Instant now, File serverDbSessionDir, ApplicationPackage applicationPackage, + SessionZooKeeperClient sessionZooKeeperClient) { isPrepared = true; - return new ConfigChangeActions(new ArrayList<>()); + return new PrepareResult(AllocatedHosts.withHosts(Set.of()), List.of()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java index 5633ec2c5f8..1d7df7acfd0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java @@ -85,11 +85,6 @@ public class SessionZooKeeperClientTest { } @Test - public void require_that_default_name_is_returned_if_node_does_not_exist() { - assertThat(createSessionZKClient("3").readApplicationId().application().value(), is("default")); - } - - @Test public void require_that_create_time_can_be_written_and_read() { SessionZooKeeperClient zkc = createSessionZKClient("3"); curator.delete(Path.fromString("3")); diff --git a/container-core/src/main/java/com/yahoo/container/core/BundleLoaderProperties.java b/container-core/src/main/java/com/yahoo/container/core/BundleLoaderProperties.java deleted file mode 100644 index ee12c7d4c9f..00000000000 --- a/container-core/src/main/java/com/yahoo/container/core/BundleLoaderProperties.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.core; - -/** - * @author gjoranv - */ -public interface BundleLoaderProperties { - - // TODO: This should be removed. The prefix is used to separate the bundles in BundlesConfig - // into those that are transferred with filedistribution and those that are preinstalled - // on disk. Instead, the model should have put them in two different configs. I.e. create a new - // config 'preinstalled-bundles.def'. - String DISK_BUNDLE_PREFIX = "file:"; - -} diff --git a/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java new file mode 100644 index 00000000000..f87dd3f42d2 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java @@ -0,0 +1,132 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.core.config; + +import com.yahoo.config.FileReference; +import com.yahoo.osgi.Osgi; +import org.osgi.framework.Bundle; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Manages the set of installed and active/inactive bundles. + * + * @author gjoranv + * @author Tony Vaagenes + */ +public class ApplicationBundleLoader { + private static final Logger log = Logger.getLogger(ApplicationBundleLoader.class.getName()); + + /* Map of file refs of active bundles (not scheduled for uninstall) to the installed bundle. + * + * Used to: + * 1. Avoid installing already installed bundles. Just an optimization, installing the same bundle location is a NOP + * 2. Start bundles (all are started every time) + * 3. Calculate the set of bundles to uninstall + */ + private final Map<FileReference, Bundle> reference2Bundle = new LinkedHashMap<>(); + + private final Osgi osgi; + private final FileAcquirerBundleInstaller bundleInstaller; + + public ApplicationBundleLoader(Osgi osgi, FileAcquirerBundleInstaller bundleInstaller) { + this.osgi = osgi; + this.bundleInstaller = bundleInstaller; + } + + /** + * Installs the given set of bundles and returns the set of bundles that is no longer used + * by the application, and should therefore be scheduled for uninstall. + */ + public synchronized Set<Bundle> useBundles(List<FileReference> newFileReferences) { + + Set<FileReference> obsoleteReferences = getObsoleteFileReferences(newFileReferences); + Set<Bundle> bundlesToUninstall = getObsoleteBundles(obsoleteReferences); + log.info("Bundles to schedule for uninstall: " + bundlesToUninstall); + + osgi.allowDuplicateBundles(bundlesToUninstall); + removeInactiveFileReferences(obsoleteReferences); + + installBundles(newFileReferences); + BundleStarter.startBundles(reference2Bundle.values()); + log.info(installedBundlesMessage()); + + return bundlesToUninstall; + } + + private Set<FileReference> getObsoleteFileReferences(List<FileReference> newReferences) { + Set<FileReference> obsoleteReferences = new HashSet<>(reference2Bundle.keySet()); + obsoleteReferences.removeAll(newReferences); + return obsoleteReferences; + } + + /** + * Returns the bundles that will not be retained by the new application generation. + */ + private Set<Bundle> getObsoleteBundles(Set<FileReference> obsoleteReferences) { + return obsoleteReferences.stream().map(reference2Bundle::get).collect(Collectors.toSet()); + } + + private void removeInactiveFileReferences(Set<FileReference> fileReferencesToRemove) { + fileReferencesToRemove.forEach(reference2Bundle::remove); + } + + private void installBundles(List<FileReference> references) { + Set<FileReference> bundlesToInstall = new HashSet<>(references); + + // This is just an optimization, as installing a bundle with the same location id returns the already installed bundle. + bundlesToInstall.removeAll(reference2Bundle.keySet()); + + if (!bundlesToInstall.isEmpty()) { + if (bundleInstaller.hasFileDistribution()) { + installWithFileDistribution(bundlesToInstall, bundleInstaller); + } else { + log.warning("Can't retrieve bundles since file distribution is disabled."); + } + } + } + + private void installWithFileDistribution(Set<FileReference> bundlesToInstall, + FileAcquirerBundleInstaller bundleInstaller) { + for (FileReference reference : bundlesToInstall) { + try { + log.info("Installing bundle with reference '" + reference.value() + "'"); + List<Bundle> bundles = bundleInstaller.installBundles(reference, osgi); + + // If more than one bundle was installed, and the OSGi framework is the real Felix one, + // it means that the X-JDisc-Preinstall-Bundle header was used. + // However, test osgi frameworks may return multiple bundles when installing a single bundle. + if (bundles.size() > 1 && osgi.hasFelixFramework()) { + // TODO: remove if-statement below when the last model with preinstall has rolled out of hosted + if (! bundles.get(0).getSymbolicName().equals("config-model-fat-amended")) + throw new RuntimeException("Bundle '" + bundles.get(0).getSymbolicName() + "' tried to pre-install bundles from disk."); + } + reference2Bundle.put(reference, bundles.get(0)); + } + catch(Exception e) { + throw new RuntimeException("Could not install bundle with reference '" + reference + "'", e); + } + } + } + + private String installedBundlesMessage() { + StringBuilder sb = new StringBuilder("Installed bundles: {" ); + for (Bundle b : osgi.getBundles()) + sb.append("[" + b.getBundleId() + "]" + b.getSymbolicName() + ":" + b.getVersion() + ", "); + sb.setLength(sb.length() - 2); + sb.append("}"); + return sb.toString(); + } + + // Only for testing + List<FileReference> getActiveFileReferences() { + return new ArrayList<>(reference2Bundle.keySet()); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleInstaller.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleInstaller.java deleted file mode 100644 index fc919571b6c..00000000000 --- a/container-core/src/main/java/com/yahoo/container/core/config/BundleInstaller.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.core.config; - -import com.yahoo.config.FileReference; -import com.yahoo.osgi.Osgi; -import org.osgi.framework.Bundle; - -import java.util.List; - -/** - * @author gjoranv - */ -public interface BundleInstaller { - - /** - * Installs the bundle with the given file reference, plus all bundles in its X-JDisc-Preinstall-Bundle directive. - * Returns all bundles installed to the given OSGi framework as a result of this call. - */ - List<Bundle> installBundles(FileReference reference, Osgi osgi) throws InterruptedException; - -} diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleManager.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleManager.java deleted file mode 100644 index 5e9b42a5fda..00000000000 --- a/container-core/src/main/java/com/yahoo/container/core/config/BundleManager.java +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.core.config; - -import com.yahoo.collections.PredicateSplit; -import com.yahoo.config.FileReference; -import com.yahoo.container.Container; -import com.yahoo.filedistribution.fileacquirer.FileAcquirer; -import com.yahoo.osgi.Osgi; -import org.osgi.framework.Bundle; -import org.osgi.framework.wiring.BundleRevision; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static com.yahoo.collections.PredicateSplit.partition; -import static com.yahoo.container.core.BundleLoaderProperties.DISK_BUNDLE_PREFIX; - -/** - * Manages the set of installed and active/inactive bundles. - * - * @author gjoranv - * @author Tony Vaagenes - */ -public class BundleManager { - - /* Map of file refs of active bundles (not scheduled for uninstall) to a list of all bundles that were installed - * (pre-install directive) by the bundle pointed to by the file ref (including itself). - * - * Used to: - * 1. Avoid installing already installed bundles. Just an optimization, installing the same bundle location is a NOP - * 2. Start bundles (all are started every time) - * 3. Calculate the set of bundles to uninstall - */ - private final Map<FileReference, List<Bundle>> reference2Bundles = new LinkedHashMap<>(); - - private final Logger log = Logger.getLogger(BundleManager.class.getName()); - private final Osgi osgi; - - // A custom bundle installer for non-disk bundles, to be used for testing - private BundleInstaller customBundleInstaller = null; - - public BundleManager(Osgi osgi) { - this.osgi = osgi; - } - - /** - * Installs the given set of bundles and returns the set of bundles that is no longer used - * by the application, and should therefore be scheduled for uninstall. - */ - public synchronized Set<Bundle> use(List<FileReference> newFileReferences) { - // Must be done before allowing duplicates because allowed duplicates affect osgi.getCurrentBundles - Set<Bundle> bundlesToUninstall = getObsoleteBundles(newFileReferences); - - Set<FileReference> obsoleteReferences = getObsoleteFileReferences(newFileReferences); - allowDuplicateBundles(obsoleteReferences); - removeInactiveFileReferences(obsoleteReferences); - - installBundles(newFileReferences); - startBundles(); - - bundlesToUninstall.removeAll(allActiveBundles()); - log.info("Bundles to schedule for uninstall: " + bundlesToUninstall); - - log.info(installedBundlesMessage()); - return bundlesToUninstall; - } - - /** - * Returns the bundles that are not assumed to be retained by the new application generation. - * Note that at this point we don't yet know the full set of new bundles, because of the potential - * pre-install directives in the new bundles. However, only "disk bundles" (file:) can be listed - * in the pre-install directive, so we know about all the obsolete application bundles. - */ - private Set<Bundle> getObsoleteBundles(List<FileReference> newReferences) { - Set<Bundle> bundlesToRemove = new HashSet<>(osgi.getCurrentBundles()); - - for (FileReference fileReferenceToKeep : newReferences) { - if (reference2Bundles.containsKey(fileReferenceToKeep)) { - bundlesToRemove.removeAll(reference2Bundles.get(fileReferenceToKeep)); - } - } - bundlesToRemove.removeAll(osgi.getInitialBundles()); - return bundlesToRemove; - } - - - private Set<FileReference> getObsoleteFileReferences(List<FileReference> newReferences) { - Set<FileReference> obsoleteReferences = new HashSet<>(reference2Bundles.keySet()); - obsoleteReferences.removeAll(newReferences); - return obsoleteReferences; - } - - /** - * Allow duplicates (bsn+version) for each bundle that corresponds to obsolete file references, - * and avoid allowing duplicates for bundles that were installed via the - * X-JDisc-Preinstall-Bundle directive. These bundles are always "disk bundles" (library - * bundles installed on the node, and not transferred via file distribution). - * Such bundles will never have duplicates because they always have the same location id. - */ - private void allowDuplicateBundles(Set<FileReference> obsoleteReferences) { - // The bundle at index 0 for each file reference always corresponds to the bundle at the file reference location - Set<Bundle> allowedDuplicates = obsoleteReferences.stream() - .filter(reference -> ! isDiskBundle(reference)) - .map(reference -> reference2Bundles.get(reference).get(0)) - .collect(Collectors.toSet()); - - log.info(() -> allowedDuplicates.isEmpty() ? "" : "Adding bundles to allowed duplicates: " + allowedDuplicates); - osgi.allowDuplicateBundles(allowedDuplicates); - } - - /** - * Cleans up the map of active file references - */ - private void removeInactiveFileReferences(Set<FileReference> fileReferencesToRemove) { - // Clean up the map of active bundles - fileReferencesToRemove.forEach(reference2Bundles::remove); - } - - private void installBundles(List<FileReference> references) { - Set<FileReference> bundlesToInstall = new HashSet<>(references); - - // This is just an optimization, as installing a bundle with the same location id returns the already installed bundle. - bundlesToInstall.removeAll(reference2Bundles.keySet()); - - PredicateSplit<FileReference> bundlesToInstall_isDisk = partition(bundlesToInstall, BundleManager::isDiskBundle); - installBundlesFromDisk(bundlesToInstall_isDisk.trueValues); - installBundlesFromFileDistribution(bundlesToInstall_isDisk.falseValues); - - // TODO: Remove. Bundles are also started in use() - startBundles(); - } - - private static boolean isDiskBundle(FileReference fileReference) { - return fileReference.value().startsWith(DISK_BUNDLE_PREFIX); - } - - private void installBundlesFromDisk(List<FileReference> bundlesToInstall) { - for (FileReference reference : bundlesToInstall) { - try { - installBundleFromDisk(reference); - } - catch(Exception e) { - throw new RuntimeException("Could not install bundle '" + reference + "'", e); - } - } - } - - private void installBundlesFromFileDistribution(List<FileReference> bundlesToInstall) { - if (!bundlesToInstall.isEmpty()) { - FileAcquirer fileAcquirer = Container.get().getFileAcquirer(); - boolean hasFileDistribution = (fileAcquirer != null); - if (hasFileDistribution) { - installWithFileDistribution(bundlesToInstall, new FileAcquirerBundleInstaller(fileAcquirer)); - } else if (customBundleInstaller != null) { - installWithFileDistribution(bundlesToInstall, customBundleInstaller); - } else { - log.warning("Can't retrieve bundles since file distribution is disabled."); - } - } - } - - private void installBundleFromDisk(FileReference reference) { - log.info("Installing bundle from disk with reference '" + reference.value() + "'"); - - var bundleInstaller = new DiskBundleInstaller(); - List<Bundle> bundles = bundleInstaller.installBundles(reference, osgi); - reference2Bundles.put(reference, bundles); - } - - private void installWithFileDistribution(List<FileReference> bundlesToInstall, BundleInstaller bundleInstaller) { - for (FileReference reference : bundlesToInstall) { - try { - log.info("Installing bundle with reference '" + reference.value() + "'"); - List<Bundle> bundles = bundleInstaller.installBundles(reference, osgi); - reference2Bundles.put(reference, bundles); - } - catch(Exception e) { - throw new RuntimeException("Could not install bundle '" + reference + "'", e); - } - } - } - - /** - * Resolves and starts (calls the Bundles BundleActivator) all bundles. Bundle resolution must take place - * after all bundles are installed to ensure that the framework can resolve dependencies between bundles. - */ - private void startBundles() { - for (List<Bundle> bundles : reference2Bundles.values()) { - for (Bundle bundle : bundles) { - try { - if ( ! isFragment(bundle)) - bundle.start(); // NOP for already ACTIVE bundles - } catch(Exception e) { - throw new RuntimeException("Could not start bundle '" + bundle.getSymbolicName() + "'", e); - } - } - } - } - - private boolean isFragment(Bundle bundle) { - BundleRevision bundleRevision = bundle.adapt(BundleRevision.class); - if (bundleRevision == null) - throw new NullPointerException("Null bundle revision means that bundle has probably been uninstalled: " + - bundle.getSymbolicName() + ":" + bundle.getVersion()); - return (bundleRevision.getTypes() & BundleRevision.TYPE_FRAGMENT) != 0; - } - - private Set<Bundle> allActiveBundles() { - return reference2Bundles.keySet().stream() - .flatMap(reference -> reference2Bundles.get(reference).stream()) - .collect(Collectors.toSet()); - } - - private String installedBundlesMessage() { - StringBuilder sb = new StringBuilder("Installed bundles: {" ); - for (Bundle b : osgi.getBundles()) - sb.append("[" + b.getBundleId() + "]" + b.getSymbolicName() + ":" + b.getVersion() + ", "); - sb.setLength(sb.length() - 2); - sb.append("}"); - return sb.toString(); - } - - // Only for testing - void useCustomBundleInstaller(BundleInstaller bundleInstaller) { - customBundleInstaller = bundleInstaller; - } - - // Only for testing - List<FileReference> getActiveFileReferences() { - return new ArrayList<>(reference2Bundles.keySet()); - } - -} diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleStarter.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleStarter.java new file mode 100644 index 00000000000..4a87c27b990 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/config/BundleStarter.java @@ -0,0 +1,40 @@ +package com.yahoo.container.core.config; + +import org.osgi.framework.Bundle; +import org.osgi.framework.wiring.BundleRevision; + +import java.util.Collection; + +/** + * Utility to start a collection of bundles. + * + * @author gjoranv + */ +public class BundleStarter { + + private BundleStarter() { } + + /** + * Resolves and starts (calls the Bundles BundleActivator) all bundles. Bundle resolution must take place + * after all bundles are installed to ensure that the framework can resolve dependencies between bundles. + */ + static void startBundles(Collection<Bundle> bundles) { + for (var bundle : bundles) { + try { + if ( ! isFragment(bundle)) + bundle.start(); // NOP for already ACTIVE bundles + } catch(Exception e) { + throw new RuntimeException("Could not start bundle '" + bundle.getSymbolicName() + "'", e); + } + } + } + + private static boolean isFragment(Bundle bundle) { + BundleRevision bundleRevision = bundle.adapt(BundleRevision.class); + if (bundleRevision == null) + throw new NullPointerException("Null bundle revision means that bundle has probably been uninstalled: " + + bundle.getSymbolicName() + ":" + bundle.getVersion()); + return (bundleRevision.getTypes() & BundleRevision.TYPE_FRAGMENT) != 0; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/core/config/DiskBundleInstaller.java b/container-core/src/main/java/com/yahoo/container/core/config/DiskBundleInstaller.java index 3edabe9f861..a4cce2e38db 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/DiskBundleInstaller.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/DiskBundleInstaller.java @@ -1,28 +1,21 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.core.config; -import com.yahoo.config.FileReference; import com.yahoo.osgi.Osgi; import org.osgi.framework.Bundle; import java.io.File; import java.util.List; -import static com.yahoo.container.core.BundleLoaderProperties.DISK_BUNDLE_PREFIX; - /** * @author gjoranv */ -public class DiskBundleInstaller implements BundleInstaller { - - @Override - public List<Bundle> installBundles(FileReference reference, Osgi osgi) { - assert(reference.value().startsWith(DISK_BUNDLE_PREFIX)); - String referenceFileName = reference.value().substring(DISK_BUNDLE_PREFIX.length()); +public class DiskBundleInstaller { - File file = new File(referenceFileName); + public List<Bundle> installBundles(String bundlePath, Osgi osgi) { + File file = new File(bundlePath); if ( ! file.exists()) { - throw new IllegalArgumentException("Reference '" + reference.value() + "' not found on disk."); + throw new IllegalArgumentException("Bundle file '" + bundlePath + "' not found on disk."); } return osgi.install(file.getAbsolutePath()); diff --git a/container-core/src/main/java/com/yahoo/container/core/config/FileAcquirerBundleInstaller.java b/container-core/src/main/java/com/yahoo/container/core/config/FileAcquirerBundleInstaller.java index 72951e67b4e..51d77462652 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/FileAcquirerBundleInstaller.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/FileAcquirerBundleInstaller.java @@ -13,9 +13,11 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Logger; /** + * Retrieves bundles with file distribution, and installs them to the OSGi framework. + * * @author gjoranv */ -public class FileAcquirerBundleInstaller implements BundleInstaller { +public class FileAcquirerBundleInstaller { private static Logger log = Logger.getLogger(FileAcquirerBundleInstaller.class.getName()); private final FileAcquirer fileAcquirer; @@ -24,7 +26,6 @@ public class FileAcquirerBundleInstaller implements BundleInstaller { this.fileAcquirer = fileAcquirer; } - @Override public List<Bundle> installBundles(FileReference reference, Osgi osgi) throws InterruptedException { File file = fileAcquirer.waitFor(reference, 7, TimeUnit.DAYS); @@ -46,6 +47,10 @@ public class FileAcquirerBundleInstaller implements BundleInstaller { return osgi.install(file.getAbsolutePath()); } + public boolean hasFileDistribution() { + return fileAcquirer != null; + } + private static boolean notReadable(File file) { return ! Files.isReadable(file.toPath()); } diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index b983cbcfe37..13d50b9b30f 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -16,6 +16,7 @@ import com.yahoo.container.di.config.SubscriberFactory; import com.yahoo.container.di.osgi.BundleClasses; import com.yahoo.container.di.osgi.OsgiUtil; import com.yahoo.container.logging.AccessLog; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.service.ClientProvider; @@ -68,7 +69,6 @@ public class HandlersConfigurerDi { } private final com.yahoo.container.Container vespaContainer; - private final OsgiWrapper osgiWrapper; private final Container container; private volatile ComponentGraph currentGraph = new ComponentGraph(0); @@ -81,7 +81,7 @@ public class HandlersConfigurerDi { OsgiFramework osgiFramework) { this(subscriberFactory, vespaContainer, configId, deconstructor, discInjector, - new ContainerAndDiOsgi(osgiFramework)); + new ContainerAndDiOsgi(osgiFramework, vespaContainer.getFileAcquirer())); } // Only public for testing @@ -93,7 +93,6 @@ public class HandlersConfigurerDi { OsgiWrapper osgiWrapper) { this.vespaContainer = vespaContainer; - this.osgiWrapper = osgiWrapper; container = new Container(subscriberFactory, configId, deconstructor, osgiWrapper); getNewComponentGraph(discInjector, false); } @@ -101,12 +100,16 @@ public class HandlersConfigurerDi { private static class ContainerAndDiOsgi extends OsgiImpl implements OsgiWrapper { private final OsgiFramework osgiFramework; - private final BundleManager bundleManager; + private final ApplicationBundleLoader applicationBundleLoader; + private final PlatformBundleLoader platformBundleLoader; - public ContainerAndDiOsgi(OsgiFramework osgiFramework) { + public ContainerAndDiOsgi(OsgiFramework osgiFramework, FileAcquirer fileAcquirer) { super(osgiFramework); this.osgiFramework = osgiFramework; - bundleManager = new BundleManager(new OsgiImpl(osgiFramework)); + + OsgiImpl osgi = new OsgiImpl(osgiFramework); + applicationBundleLoader = new ApplicationBundleLoader(osgi, new FileAcquirerBundleInstaller(fileAcquirer)); + platformBundleLoader = new PlatformBundleLoader(osgi); } @@ -131,9 +134,15 @@ public class HandlersConfigurerDi { } @Override - public Set<Bundle> useBundles(Collection<FileReference> bundles) { + public void installPlatformBundles(Collection<String> bundlePaths) { + log.fine("Installing platform bundles."); + platformBundleLoader.useBundles(new ArrayList<>(bundlePaths)); + } + + @Override + public Set<Bundle> useApplicationBundles(Collection<FileReference> bundles) { log.info("Installing bundles from the latest application"); - return bundleManager.use(new ArrayList<>(bundles)); + return applicationBundleLoader.useBundles(new ArrayList<>(bundles)); } } diff --git a/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java new file mode 100644 index 00000000000..0ab89e223f6 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java @@ -0,0 +1,72 @@ +package com.yahoo.container.core.config; + +import com.yahoo.osgi.Osgi; +import org.osgi.framework.Bundle; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; + +/** + * Used to install the bundles that are added as platform bundles by the config-model. + * + * All platform bundles reside on disk, and they are never uninstalled. + * Platform bundles are allowed to pre-install other bundles on disk via the + * X-JDisc-Preinstall-Bundle manifest header. + * + * Attempts to install additional bundles, after the first call, should be a NOP. + * + * @author gjoranv + */ +public class PlatformBundleLoader { + private static final Logger log = Logger.getLogger(PlatformBundleLoader.class.getName()); + + private final Osgi osgi; + private final DiskBundleInstaller installer; + + private Set<Bundle> installedBundles; + private boolean hasLoadedBundles = false; + + public PlatformBundleLoader(Osgi osgi) { + this(osgi, new DiskBundleInstaller()); + } + + PlatformBundleLoader(Osgi osgi, DiskBundleInstaller installer) { + this.osgi = osgi; + this.installer = installer; + } + + public void useBundles(List<String> bundlePaths) { + if (hasLoadedBundles) { + log.fine(() -> "Platform bundles have already been installed." + + "\nInstalled bundles: " + installedBundles + + "\nGiven files: " + bundlePaths); + return; + } + installedBundles = install(bundlePaths); + BundleStarter.startBundles(installedBundles); + hasLoadedBundles = true; + } + + private Set<Bundle> install(List<String> bundlesToInstall) { + var allInstalled = new LinkedHashSet<Bundle>(); + for (String bundlePath : bundlesToInstall) { + try { + allInstalled.addAll(installBundleFromDisk(bundlePath)); + } + catch(Exception e) { + throw new RuntimeException("Could not install bundle '" + bundlePath + "'", e); + } + } + return allInstalled; + } + + private List<Bundle> installBundleFromDisk(String bundlePath) { + log.info("Installing bundle from disk: " + bundlePath); + List<Bundle> bundles = installer.installBundles(bundlePath, osgi); + log.fine("Installed " + bundles.size() + " bundles for file " + bundlePath); + return bundles; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java index 9f49b016b68..503bf2f2db1 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java @@ -41,7 +41,8 @@ public class HandlersConfigurerTestWrapper { private final static String testFiles[] = { "components.cfg", "handlers.cfg", - "bundles.cfg", + "platform-bundles.cfg", + "application-bundles.cfg", "string.cfg", "int.cfg", "renderers.cfg", diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java index ac0fbd71671..98c927b8efd 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java @@ -16,11 +16,6 @@ import static java.util.Collections.emptyList; public class MockOsgiWrapper implements OsgiWrapper { @Override - public List<Bundle> getInitialBundles() { - return emptyList(); - } - - @Override public Bundle[] getBundles() { return new Bundle[0]; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index f3149ed4998..b2a156862eb 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -35,11 +35,13 @@ public class LogHandler extends ThreadedHttpRequestHandler { Instant to = Optional.ofNullable(request.getProperty("to")) .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MAX); + Optional<String> hostname = Optional.ofNullable(request.getProperty("hostname")); + return new HttpResponse(200) { @Override public void render(OutputStream outputStream) { try { - logReader.writeLogs(outputStream, from, to); + logReader.writeLogs(outputStream, from, to, hostname); } catch (Throwable t) { log.log(Level.WARNING, "Failed reading logs from " + from + " to " + to, t); diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 3cf849a6835..8e4b9aea9b8 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -57,7 +58,7 @@ class LogReader { this.logFilePattern = logFilePattern; } - void writeLogs(OutputStream out, Instant from, Instant to) { + void writeLogs(OutputStream out, Instant from, Instant to, Optional<String> hostname) { double fromSeconds = from.getEpochSecond() + from.getNano() / 1e9; double toSeconds = to.getEpochSecond() + to.getNano() / 1e9; BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out)); @@ -67,7 +68,7 @@ class LogReader { try { // Logs in each sub-list contain entries covering the same time interval, so do a merge sort while reading for (Path log : logs) - logLineIterators.add(new LogLineIterator(log, fromSeconds, toSeconds)); + logLineIterators.add(new LogLineIterator(log, fromSeconds, toSeconds, hostname)); Iterator<LineWithTimestamp> lines = Iterators.mergeSorted(logLineIterators, Comparator.comparingDouble(LineWithTimestamp::timestamp)); @@ -96,14 +97,16 @@ class LogReader { private final BufferedReader reader; private final double from; private final double to; + private final Optional<String> hostname; private LineWithTimestamp next; - private LogLineIterator(Path log, double from, double to) throws IOException { + private LogLineIterator(Path log, double from, double to, Optional<String> hostname) throws IOException { boolean zipped = log.toString().endsWith(".gz"); InputStream in = Files.newInputStream(log); this.reader = new BufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(in) : in, UTF_8)); this.from = from; this.to = to; + this.hostname = hostname; this.next = readNext(); } @@ -131,6 +134,9 @@ class LogReader { if (parts.length != 7) continue; + if (hostname.map(host -> !host.equals(parts[1])).orElse(false)) + continue; + double timestamp = Double.parseDouble(parts[0]); if (timestamp > to) return null; diff --git a/container-core/src/main/java/com/yahoo/osgi/MockOsgi.java b/container-core/src/main/java/com/yahoo/osgi/MockOsgi.java index d809c493565..6a700a65a03 100644 --- a/container-core/src/main/java/com/yahoo/osgi/MockOsgi.java +++ b/container-core/src/main/java/com/yahoo/osgi/MockOsgi.java @@ -12,15 +12,11 @@ import java.util.List; /** * @author Tony Vaagenes + * @author gjoranv */ public class MockOsgi extends NonWorkingOsgiFramework implements Osgi { @Override - public List<Bundle> getInitialBundles() { - return Collections.emptyList(); - } - - @Override public Bundle[] getBundles() { return new Bundle[0]; } diff --git a/container-core/src/main/java/com/yahoo/osgi/Osgi.java b/container-core/src/main/java/com/yahoo/osgi/Osgi.java index 8f0acf41f30..513e7883594 100644 --- a/container-core/src/main/java/com/yahoo/osgi/Osgi.java +++ b/container-core/src/main/java/com/yahoo/osgi/Osgi.java @@ -9,11 +9,10 @@ import java.util.List; /** * @author Tony Vaagenes + * @author gjoranv */ public interface Osgi { - List<Bundle> getInitialBundles(); - Bundle[] getBundles(); /** Returns all bundles that have not been scheduled for uninstall. */ @@ -25,4 +24,7 @@ public interface Osgi { void allowDuplicateBundles(Collection<Bundle> bundles); + default boolean hasFelixFramework() { + return false; + } } diff --git a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java index ed93d15c975..b34442d50a9 100644 --- a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java +++ b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java @@ -16,6 +16,7 @@ import java.util.logging.Logger; /** * @author Tony Vaagenes * @author bratseth + * @author gjoranv */ public class OsgiImpl implements Osgi { private static final Logger log = Logger.getLogger(OsgiImpl.class.getName()); @@ -42,11 +43,6 @@ public class OsgiImpl implements Osgi { } @Override - public List<Bundle> getInitialBundles() { - return initialBundles; - } - - @Override public Bundle[] getBundles() { List<Bundle> bundles = jdiscOsgi.bundles(); return bundles.toArray(new Bundle[bundles.size()]); @@ -155,6 +151,11 @@ public class OsgiImpl implements Osgi { jdiscOsgi.allowDuplicateBundles(bundles); } + @Override + public boolean hasFelixFramework() { + return jdiscOsgi.isFelixFramework(); + } + private static Bundle firstNonFrameworkBundle(List<Bundle> bundles) { for (Bundle b : bundles) { if (! (b instanceof Framework)) diff --git a/container-core/src/test/java/com/yahoo/container/core/config/BundleManagerTest.java b/container-core/src/test/java/com/yahoo/container/core/config/ApplicationBundleLoaderTest.java index 414e6b05128..b56e5d99123 100644 --- a/container-core/src/test/java/com/yahoo/container/core/config/BundleManagerTest.java +++ b/container-core/src/test/java/com/yahoo/container/core/config/ApplicationBundleLoaderTest.java @@ -2,6 +2,9 @@ package com.yahoo.container.core.config; import com.yahoo.config.FileReference; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; +import com.yahoo.osgi.Osgi; import org.junit.Before; import org.junit.Test; import org.osgi.framework.Bundle; @@ -16,27 +19,27 @@ import static org.junit.Assert.assertTrue; /** * @author gjoranv */ -public class BundleManagerTest { +public class ApplicationBundleLoaderTest { private static final FileReference BUNDLE_1_REF = new FileReference("bundle-1"); private static final Bundle BUNDLE_1 = new TestBundle(BUNDLE_1_REF.value()); private static final FileReference BUNDLE_2_REF = new FileReference("bundle-2"); private static final Bundle BUNDLE_2 = new TestBundle(BUNDLE_2_REF.value()); - private BundleManager bundleLoader; + private ApplicationBundleLoader bundleLoader; private TestOsgi osgi; @Before public void setup() { osgi = new TestOsgi(testBundles()); - var bundleInstaller = new TestBundleInstaller(); - bundleLoader = new BundleManager(osgi); - bundleLoader.useCustomBundleInstaller(bundleInstaller); + var bundleInstaller = new TestBundleInstaller(MockFileAcquirer.returnFile(null)); + + bundleLoader = new ApplicationBundleLoader(osgi, bundleInstaller); } @Test public void bundles_are_installed_and_started() { - bundleLoader.use(List.of(BUNDLE_1_REF)); + bundleLoader.useBundles(List.of(BUNDLE_1_REF)); assertEquals(1, osgi.getInstalledBundles().size()); // The bundle is installed and started @@ -51,8 +54,8 @@ public class BundleManagerTest { @Test public void new_bundle_can_be_installed_in_reconfig() { - bundleLoader.use(List.of(BUNDLE_1_REF)); - Set<Bundle> obsoleteBundles = bundleLoader.use(List.of(BUNDLE_1_REF, BUNDLE_2_REF)); + bundleLoader.useBundles(List.of(BUNDLE_1_REF)); + Set<Bundle> obsoleteBundles = bundleLoader.useBundles(List.of(BUNDLE_1_REF, BUNDLE_2_REF)); // No bundles are obsolete assertTrue(obsoleteBundles.isEmpty()); @@ -76,8 +79,8 @@ public class BundleManagerTest { @Test public void unused_bundle_is_marked_obsolete_after_reconfig() { - bundleLoader.use(List.of(BUNDLE_1_REF)); - Set<Bundle> obsoleteBundles = bundleLoader.use(List.of(BUNDLE_2_REF)); + bundleLoader.useBundles(List.of(BUNDLE_1_REF)); + Set<Bundle> obsoleteBundles = bundleLoader.useBundles(List.of(BUNDLE_2_REF)); // The returned set of obsolete bundles contains bundle-1 assertEquals(1, obsoleteBundles.size()); @@ -103,4 +106,17 @@ public class BundleManagerTest { BUNDLE_2_REF.value(), BUNDLE_2); } + static class TestBundleInstaller extends FileAcquirerBundleInstaller { + + TestBundleInstaller(FileAcquirer fileAcquirer) { + super(fileAcquirer); + } + + @Override + public List<Bundle> installBundles(FileReference reference, Osgi osgi) { + return osgi.install(reference.value()); + } + + } + } diff --git a/container-core/src/test/java/com/yahoo/container/core/config/PlatformBundleLoaderTest.java b/container-core/src/test/java/com/yahoo/container/core/config/PlatformBundleLoaderTest.java new file mode 100644 index 00000000000..931b0c547fd --- /dev/null +++ b/container-core/src/test/java/com/yahoo/container/core/config/PlatformBundleLoaderTest.java @@ -0,0 +1,64 @@ +package com.yahoo.container.core.config; + +import com.yahoo.osgi.Osgi; +import org.junit.Before; +import org.junit.Test; +import org.osgi.framework.Bundle; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author gjoranv + */ +public class PlatformBundleLoaderTest { + + private static final String BUNDLE_1_REF = "bundle-1"; + private static final Bundle BUNDLE_1 = new TestBundle(BUNDLE_1_REF); + private static final String BUNDLE_2_REF = "bundle-2"; + private static final Bundle BUNDLE_2 = new TestBundle(BUNDLE_2_REF); + + private PlatformBundleLoader bundleLoader; + private TestOsgi osgi; + + @Before + public void setup() { + osgi = new TestOsgi(testBundles()); + bundleLoader = new PlatformBundleLoader(osgi, new TestBundleInstaller()); + } + + @Test + public void bundles_are_installed_and_started() { + bundleLoader.useBundles(List.of(BUNDLE_1_REF)); + assertEquals(1, osgi.getInstalledBundles().size()); + + // The bundle is installed and started + TestBundle installedBundle = (TestBundle)osgi.getInstalledBundles().get(0); + assertEquals(BUNDLE_1.getSymbolicName(), installedBundle.getSymbolicName()); + assertTrue(installedBundle.started); + } + + @Test + public void bundles_cannot_be_added_by_later_calls() { + bundleLoader.useBundles(List.of(BUNDLE_1_REF)); + bundleLoader.useBundles(List.of(BUNDLE_2_REF)); // Should be a NOP + + assertEquals(1, osgi.getInstalledBundles().size()); + assertEquals(BUNDLE_1.getSymbolicName(), osgi.getInstalledBundles().get(0).getSymbolicName()); + } + + private static Map<String, Bundle> testBundles() { + return Map.of(BUNDLE_1_REF, BUNDLE_1, + BUNDLE_2_REF, BUNDLE_2); + } + + static class TestBundleInstaller extends DiskBundleInstaller { + @Override + public List<Bundle> installBundles(String bundlePath, Osgi osgi) { + return osgi.install(bundlePath); + } + } +} diff --git a/container-core/src/test/java/com/yahoo/container/core/config/TestBundleInstaller.java b/container-core/src/test/java/com/yahoo/container/core/config/TestBundleInstaller.java deleted file mode 100644 index 43a5268eabf..00000000000 --- a/container-core/src/test/java/com/yahoo/container/core/config/TestBundleInstaller.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.core.config; - -import com.yahoo.config.FileReference; -import com.yahoo.osgi.Osgi; -import org.osgi.framework.Bundle; - -import java.util.List; - -/** - * @author gjoranv - */ -class TestBundleInstaller implements BundleInstaller { - - @Override - public List<Bundle> installBundles(FileReference reference, Osgi osgi) { - return osgi.install(reference.value()); - } - -} diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index ab0d0d54675..97aa8864eae 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -9,6 +9,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.time.Instant; +import java.util.Optional; import java.util.concurrent.Executor; import static org.junit.Assert.assertEquals; @@ -47,7 +48,7 @@ public class LogHandlerTest { } @Override - protected void writeLogs(OutputStream out, Instant from, Instant to) { + protected void writeLogs(OutputStream out, Instant from, Instant to, Optional<String> hostname) { try { if (to.isAfter(Instant.ofEpochMilli(1000))) { out.write("newer log".getBytes()); diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index 3f7a78e13be..ad9398a5eec 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Before; import org.junit.Test; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -14,8 +13,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; +import java.util.Optional; import java.util.regex.Pattern; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import static java.nio.charset.StandardCharsets.UTF_8; @@ -26,12 +25,12 @@ public class LogReaderTest { private final FileSystem fileSystem = TestFileSystem.create(); private final Path logDirectory = fileSystem.getPath("/opt/vespa/logs"); - private static final String logv11 = "3600.2\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tfourth\n"; - private static final String logv = "90000.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tlast\n"; - private static final String log100 = "0.2\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tsecond\n"; - private static final String log101 = "0.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"; - private static final String log110 = "3600.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tthird\n"; - private static final String log200 = "86400.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)\n"; + private static final String logv11 = "3600.2\tnode1.com\t5480\tcontainer\tstdout\tinfo\tfourth\n"; + private static final String logv = "90000.1\tnode1.com\t5480\tcontainer\tstdout\tinfo\tlast\n"; + private static final String log100 = "0.2\tnode2.com\t5480\tcontainer\tstdout\tinfo\tsecond\n"; + private static final String log101 = "0.1\tnode2.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"; + private static final String log110 = "3600.1\tnode1.com\t5480\tcontainer\tstderr\twarning\tthird\n"; + private static final String log200 = "86400.1\tnode2.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)\n"; @Before public void setup() throws IOException { @@ -52,32 +51,41 @@ public class LogReaderTest { } @Test - public void testThatLogsOutsideRangeAreExcluded() throws Exception { + public void testThatLogsOutsideRangeAreExcluded() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050)); + logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050), Optional.empty()); assertEquals(log100 + logv11 + log110, baos.toString(UTF_8)); } @Test - public void testThatLogsNotMatchingRegexAreExcluded() throws Exception { + public void testThatLogsNotMatchingRegexAreExcluded() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*-1.*")); - logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2))); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty()); assertEquals(log101 + logv11, baos.toString(UTF_8)); } @Test - public void testZippedStreaming() throws IOException { + public void testZippedStreaming() { ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2))); + logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.empty()); assertEquals(log101 + log100 + logv11 + log110 + log200 + logv, zippedBaos.toString(UTF_8)); } + @Test + public void logsForSingeNodeIsRetrieved() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2)), Optional.of("node2.com")); + + assertEquals(log101 + log100 + log200, baos.toString(UTF_8)); + } + private byte[] compress(String input) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream zip = new GZIPOutputStream(baos); diff --git a/container-di/CMakeLists.txt b/container-di/CMakeLists.txt index c2b033baa92..02b2b0d34d9 100644 --- a/container-di/CMakeLists.txt +++ b/container-di/CMakeLists.txt @@ -1,5 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_config_definition(src/main/resources/configdefinitions/bundles.def container.bundles.def) +install_config_definition(src/main/resources/configdefinitions/application-bundles.def com.yahoo.container.di.config.application-bundles.def) +install_config_definition(src/main/resources/configdefinitions/platform-bundles.def com.yahoo.container.di.config.platform-bundles.def) install_config_definition(src/main/resources/configdefinitions/components.def container.components.def) install_config_definition(src/main/resources/configdefinitions/jersey-bundles.def container.di.config.jersey-bundles.def) install_config_definition(src/main/resources/configdefinitions/jersey-injection.def container.di.config.jersey-injection.def) diff --git a/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java b/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java index 6ed9c2a2994..d8350ec2f6b 100644 --- a/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java +++ b/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java @@ -74,9 +74,7 @@ public final class ConfigRetriever { return getConfigs(componentConfigKeys, leastGeneration, false); } - /** - * Try to get config just once - */ + // TODO: duplicate code, let getConfigs call this. Optional<ConfigSnapshot> getConfigsOnce(Set<ConfigKey<? extends ConfigInstance>> componentConfigKeys, long leastGeneration, boolean restartOnRedeploy) { diff --git a/container-di/src/main/java/com/yahoo/container/di/Container.java b/container-di/src/main/java/com/yahoo/container/di/Container.java index ef7813ce368..af580767a17 100644 --- a/container-di/src/main/java/com/yahoo/container/di/Container.java +++ b/container-di/src/main/java/com/yahoo/container/di/Container.java @@ -5,15 +5,17 @@ import com.google.inject.Injector; import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.ConfigInterruptedException; -import com.yahoo.container.BundlesConfig; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.ConfigRetriever.BootstrapConfigs; +import com.yahoo.container.di.ConfigRetriever.ComponentsConfigs; import com.yahoo.container.di.ConfigRetriever.ConfigSnapshot; import com.yahoo.container.di.componentgraph.core.ComponentGraph; import com.yahoo.container.di.componentgraph.core.ComponentNode; import com.yahoo.container.di.componentgraph.core.JerseyNode; import com.yahoo.container.di.componentgraph.core.Node; +import com.yahoo.container.di.config.ApplicationBundlesConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.di.config.RestApiContext; import com.yahoo.container.di.config.SubscriberFactory; import com.yahoo.vespa.config.ConfigKey; @@ -23,6 +25,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.IdentityHashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Level; @@ -40,12 +43,14 @@ public class Container { private static final Logger log = Logger.getLogger(Container.class.getName()); private final SubscriberFactory subscriberFactory; - private ConfigKey<BundlesConfig> bundlesConfigKey; + private ConfigKey<ApplicationBundlesConfig> applicationBundlesConfigKey; + private ConfigKey<PlatformBundlesConfig> platformBundlesConfigKey; private ConfigKey<ComponentsConfig> componentsConfigKey; private final ComponentDeconstructor componentDeconstructor; private final Osgi osgi; private final ConfigRetriever configurer; + private List<String> platformBundles; // Used to verify that platform bundles don't change. private long previousConfigGeneration = -1L; private long leastGeneration = -1L; @@ -54,9 +59,10 @@ public class Container { this.componentDeconstructor = componentDeconstructor; this.osgi = osgi; - bundlesConfigKey = new ConfigKey<>(BundlesConfig.class, configId); + applicationBundlesConfigKey = new ConfigKey<>(ApplicationBundlesConfig.class, configId); + platformBundlesConfigKey = new ConfigKey<>(PlatformBundlesConfig.class, configId); componentsConfigKey = new ConfigKey<>(ComponentsConfig.class, configId); - var bootstrapKeys = Set.of(bundlesConfigKey, componentsConfigKey); + var bootstrapKeys = Set.of(applicationBundlesConfigKey, platformBundlesConfigKey, componentsConfigKey); this.configurer = new ConfigRetriever(bootstrapKeys, subscriberFactory::getSubscriber); } @@ -74,7 +80,6 @@ public class Container { deconstructObsoleteComponents(oldGraph, newGraph, obsoleteBundles); return newGraph; } catch (Throwable t) { - // TODO: Wrap ComponentConstructorException in an Error when generation==0 (+ unit test that Error is thrown) invalidateGeneration(oldGraph.generation(), t); throw t; } @@ -98,28 +103,26 @@ public class Container { "Got bootstrap configs out of sequence for old config generation %d.\n" + "Previous config generation is %d", getBootstrapGeneration(), previousConfigGeneration)); } - log.log(FINE, - String.format( - "Got new bootstrap generation\n" + "bootstrap generation = %d\n" + "components generation: %d\n" - + "previous generation: %d\n", - getBootstrapGeneration(), getComponentsGeneration(), previousConfigGeneration)); + log.log(FINE, "Got new bootstrap generation\n" + configGenerationsString()); - Collection<Bundle> bundlesToRemove = installBundles(snapshot.configs()); + if (graph.generation() == 0) { + platformBundles = getConfig(platformBundlesConfigKey, snapshot.configs()).bundlePaths(); + osgi.installPlatformBundles(platformBundles); + } else { + throwIfPlatformBundlesChanged(snapshot); + } + Collection<Bundle> bundlesToRemove = installApplicationBundles(snapshot.configs()); obsoleteBundles.addAll(bundlesToRemove); graph = createComponentsGraph(snapshot.configs(), getBootstrapGeneration(), fallbackInjector); // Continues loop - } else if (snapshot instanceof ConfigRetriever.ComponentsConfigs) { + } else if (snapshot instanceof ComponentsConfigs) { break; } } - log.log(FINE, - String.format( - "Got components configs,\n" + "bootstrap generation = %d\n" + "components generation: %d\n" - + "previous generation: %d", - getBootstrapGeneration(), getComponentsGeneration(), previousConfigGeneration)); + log.log(FINE, "Got components configs,\n" + configGenerationsString()); return createAndConfigureComponentsGraph(snapshot.configs(), fallbackInjector); } @@ -131,6 +134,17 @@ public class Container { return configurer.getComponentsGeneration(); } + private String configGenerationsString() { + return String.format("bootstrap generation = %d\n" + "components generation: %d\n" + "previous generation: %d", + getBootstrapGeneration(), getComponentsGeneration(), previousConfigGeneration); + } + + private void throwIfPlatformBundlesChanged(ConfigSnapshot snapshot) { + var checkPlatformBundles = getConfig(platformBundlesConfigKey, snapshot.configs()).bundlePaths(); + if (! checkPlatformBundles.equals(platformBundles)) + throw new RuntimeException("Platform bundles are not allowed to change!\nOld: " + platformBundles + "\nNew: " + checkPlatformBundles); + } + private ComponentGraph createAndConfigureComponentsGraph(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> componentsConfigs, Injector fallbackInjector) { ComponentGraph componentGraph = createComponentsGraph(componentsConfigs, getComponentsGeneration(), fallbackInjector); @@ -151,9 +165,9 @@ public class Container { componentDeconstructor.deconstruct(oldComponents.keySet(), obsoleteBundles); } - private Set<Bundle> installBundles(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configsIncludingBootstrapConfigs) { - BundlesConfig bundlesConfig = getConfig(bundlesConfigKey, configsIncludingBootstrapConfigs); - return osgi.useBundles(bundlesConfig.bundle()); + private Set<Bundle> installApplicationBundles(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configsIncludingBootstrapConfigs) { + ApplicationBundlesConfig applicationBundlesConfig = getConfig(applicationBundlesConfigKey, configsIncludingBootstrapConfigs); + return osgi.useApplicationBundles(applicationBundlesConfig.bundles()); } private ComponentGraph createComponentsGraph(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configsIncludingBootstrapConfigs, diff --git a/container-di/src/main/java/com/yahoo/container/di/Osgi.java b/container-di/src/main/java/com/yahoo/container/di/Osgi.java index ab7da7665b6..940986e2f38 100644 --- a/container-di/src/main/java/com/yahoo/container/di/Osgi.java +++ b/container-di/src/main/java/com/yahoo/container/di/Osgi.java @@ -25,11 +25,15 @@ public interface Osgi { return new BundleClasses(new MockBundle(), Collections.emptySet()); } + default void installPlatformBundles(Collection<String> bundlePaths) { + System.out.println("installPlatformBundles " + bundlePaths); + } + /** * Returns the set of bundles that is not used by the current application generation, * and therefore should be scheduled for uninstalling. */ - default Set<Bundle> useBundles(Collection<FileReference> bundles) { + default Set<Bundle> useApplicationBundles(Collection<FileReference> bundles) { System.out.println("useBundles " + bundles.stream().map(Object::toString).collect(Collectors.joining(", "))); return emptySet(); } diff --git a/container-di/src/main/resources/configdefinitions/application-bundles.def b/container-di/src/main/resources/configdefinitions/application-bundles.def new file mode 100644 index 00000000000..7e03b1e3ac8 --- /dev/null +++ b/container-di/src/main/resources/configdefinitions/application-bundles.def @@ -0,0 +1,5 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package=com.yahoo.container.di.config + +# References to user bundles to install. +bundles[] file diff --git a/container-di/src/main/resources/configdefinitions/bundles.def b/container-di/src/main/resources/configdefinitions/bundles.def index 9e10d863106..79e24742398 100644 --- a/container-di/src/main/resources/configdefinitions/bundles.def +++ b/container-di/src/main/resources/configdefinitions/bundles.def @@ -1,5 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=container -# References to all 3rd-party bundles to be installed. +# References to both application and platform bundles to install. bundle[] file diff --git a/container-di/src/main/resources/configdefinitions/platform-bundles.def b/container-di/src/main/resources/configdefinitions/platform-bundles.def new file mode 100644 index 00000000000..a30a846b565 --- /dev/null +++ b/container-di/src/main/resources/configdefinitions/platform-bundles.def @@ -0,0 +1,5 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package=com.yahoo.container.di.config + +# Paths to platform bundles to install. +bundlePaths[] string diff --git a/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java b/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java index eac64c20274..19f277ff8fb 100644 --- a/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java +++ b/container-di/src/test/java/com/yahoo/container/di/ContainerTest.java @@ -267,7 +267,8 @@ public class ContainerTest extends ContainerTestBase { "inject[0].forClass \"" + injectedClass.getName() + "\"\n"; dirConfigSource.writeConfig("components", componentsConfig); - dirConfigSource.writeConfig("bundles", ""); + dirConfigSource.writeConfig("platform-bundles", ""); + dirConfigSource.writeConfig("application-bundles", ""); dirConfigSource.writeConfig("jersey-bundles", "bundles[0].spec \"mock-entry-to-enforce-a-MockBundle\""); dirConfigSource.writeConfig("jersey-injection", injectionConfig); diff --git a/container-di/src/test/java/com/yahoo/container/di/ContainerTestBase.java b/container-di/src/test/java/com/yahoo/container/di/ContainerTestBase.java index 18236a6bde9..f1f3c4a2ae4 100644 --- a/container-di/src/test/java/com/yahoo/container/di/ContainerTestBase.java +++ b/container-di/src/test/java/com/yahoo/container/di/ContainerTestBase.java @@ -61,7 +61,7 @@ public class ContainerTestBase { } @Override - public Set<Bundle> useBundles(Collection<FileReference> bundles) { + public Set<Bundle> useApplicationBundles(Collection<FileReference> bundles) { return emptySet(); } @@ -81,7 +81,8 @@ public class ContainerTestBase { } protected void writeBootstrapConfigs(ComponentEntry... componentEntries) { - dirConfigSource.writeConfig("bundles", ""); + dirConfigSource.writeConfig("platform-bundles", ""); + dirConfigSource.writeConfig("application-bundles", ""); StringBuilder components = new StringBuilder(); for (int i = 0; i < componentEntries.length; i++) { components.append(componentEntries[i].asConfig(i)); diff --git a/container-disc/pom.xml b/container-disc/pom.xml index 48872d0665b..1caf66e29dc 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -181,7 +181,8 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> <configuration> - <discApplicationClass>com.yahoo.container.jdisc.ConfiguredApplication</discApplicationClass> + <discApplicationClass>com.yahoo.container.jdisc.ConfiguredApplication</discApplicationClass> + <buildLegacyVespaPlatformBundle>true</buildLegacyVespaPlatformBundle> <discPreInstallBundle> <!-- Vespa bundles --> config-bundle-jar-with-dependencies.jar, diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index fcbe23aeb61..a338bb5d8e7 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -527,6 +527,31 @@ ], "fields": [] }, + "com.yahoo.prelude.query.GeoLocationItem": { + "superClass": "com.yahoo.prelude.query.TermItem", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.prelude.Location)", + "public void <init>(com.yahoo.prelude.Location, java.lang.String)", + "public com.yahoo.prelude.Location getLocation()", + "public java.lang.String getRawWord()", + "public com.yahoo.prelude.query.Item$ItemType getItemType()", + "public java.lang.String getName()", + "public java.lang.String stringValue()", + "public void setValue(java.lang.String)", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public java.lang.String getIndexedString()", + "protected void encodeThis(java.nio.ByteBuffer)", + "public int getNumWords()", + "public boolean isStemmed()", + "public boolean isWords()" + ], + "fields": [] + }, "com.yahoo.prelude.query.HasIndexItem": { "superClass": "java.lang.Object", "interfaces": [], @@ -697,7 +722,7 @@ "public static final enum com.yahoo.prelude.query.Item$ItemType REGEXP", "public static final enum com.yahoo.prelude.query.Item$ItemType WORD_ALTERNATIVES", "public static final enum com.yahoo.prelude.query.Item$ItemType NEAREST_NEIGHBOR", - "public static final enum com.yahoo.prelude.query.Item$ItemType LOCATION_TERM", + "public static final enum com.yahoo.prelude.query.Item$ItemType GEO_LOCATION_TERM", "public final int code" ] }, @@ -6122,8 +6147,6 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.search.query.profile.compiled.DimensionalValue$Value)", - "public void <init>(java.util.List)", "public java.lang.Object get(java.util.Map)", "public boolean isEmpty()", "public java.lang.String toString()" @@ -6145,6 +6168,8 @@ "public com.yahoo.search.query.profile.types.QueryProfileType queryProfileType()", "public com.yahoo.search.query.profile.compiled.ValueWithSource withValue(java.lang.Object)", "public java.util.Optional variant()", + "public int hashCode()", + "public boolean equals(java.lang.Object)", "public java.lang.String toString()" ], "fields": [] diff --git a/container-search/src/main/java/com/yahoo/prelude/Location.java b/container-search/src/main/java/com/yahoo/prelude/Location.java index 908bf835e3c..3d3eed3b3df 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Location.java +++ b/container-search/src/main/java/com/yahoo/prelude/Location.java @@ -9,7 +9,7 @@ import java.util.StringTokenizer; /** * Location data for a geographical query. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen * @author arnej27959 */ public class Location { @@ -127,7 +127,7 @@ public class Location { throw new IllegalArgumentException("n/s location must be in range [-90,+90]"); } if (radius_in_degrees < 0) { - pr = 512 * 1024 * 1024; + pr = -1; } x = px; y = py; @@ -142,7 +142,7 @@ public class Location { throw new IllegalArgumentException("can only set geo circle once"); } if (radius_in_units < 0) { - throw new IllegalArgumentException("radius must be positive"); + radius_in_units = -1; } x = px; y = py; @@ -248,6 +248,13 @@ public class Location { } public String toString() { + return render(false); + } + public String backendString() { + return render(true); + } + + private String render(boolean forBackend) { StringBuilder ser = new StringBuilder(); if (attribute != null) { ser.append(attribute).append(':'); @@ -271,7 +278,7 @@ public class Location { if (dimensions == 2) { ser.append(",").append(y); } - ser.append(",").append(r). + ser.append(",").append(forBackend ? backendRadius() : r). append(",").append(tableId). append(",").append(s). append(",").append(replace); @@ -358,11 +365,16 @@ public class Location { /** * Obtain circle radius (in degrees). + * Note that "no radius" or "infinite radius" is represented as -1. * May only be called when isGeoCircle() returns true. **/ public double degRadius() { checkGeoCircle(); - return 0.000001 * r; + return (r < 0) ? -1.0 : (0.000001 * r); + } + + private int backendRadius() { + return (r < 0) ? (512 * 1024 * 1024) : r; } /** @@ -370,7 +382,7 @@ public class Location { * For internal use. */ public int encode(ByteBuffer buffer) { - byte[] loc = Utf8.toBytes(toString()); + byte[] loc = Utf8.toBytes(backendString()); buffer.put(loc); return loc.length; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java new file mode 100644 index 00000000000..8202c8fb279 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java @@ -0,0 +1,119 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.prelude.query; + +import com.google.common.annotations.Beta; +import com.yahoo.prelude.Location; +import java.nio.ByteBuffer; + +/** + * This represents a geo-location in the query tree. + * Used for closeness(fieldname) and distance(fieldname) rank features. + * @author arnej + */ +@Beta +public class GeoLocationItem extends TermItem { + + private Location location; + + /** + * Construct from a Location, which must be geo circle with an attribute set. + **/ + public GeoLocationItem(Location location) { + this(location, location.getAttribute()); + if (! location.hasAttribute()) { + throw new IllegalArgumentException("missing attribute on location: "+location); + } + } + + /** + * Construct from a Location and a field name. + * The Location must be a geo circle. + * If the Location has an attribute set, it must match the field name. + **/ + public GeoLocationItem(Location location, String fieldName) { + super(fieldName, false); + if (location.hasAttribute() && ! location.getAttribute().equals(fieldName)) { + throw new IllegalArgumentException("inconsistent attribute on location: "+location.getAttribute()+" versus fieldName: "+fieldName); + } + if (! location.isGeoCircle()) { + throw new IllegalArgumentException("GeoLocationItem only supports Geo Circles, got: "+location); + } + if (location.hasBoundingBox()) { + throw new IllegalArgumentException("GeoLocationItem does not support bounding box yet, got: "+location); + } + this.location = new Location(location.toString()); + this.location.setAttribute(null); // keep this in (superclass) indexName only + setNormalizable(false); + } + + public Location getLocation() { + return location; + } + + @Override + public String getRawWord() { + return stringValue(); + } + + @Override + public ItemType getItemType() { + return ItemType.GEO_LOCATION_TERM; + } + + @Override + public String getName() { + return "GEO_LOCATION"; + } + + @Override + public String stringValue() { + return location.toString(); + } + + @Override + public void setValue(String value) { + throw new UnsupportedOperationException("Cannot setValue("+value+") on "+getName()); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(super.hashCode(), location); + } + + @Override + public boolean equals(Object object) { + if ( ! super.equals(object)) return false; + GeoLocationItem other = (GeoLocationItem) object; // Ensured by superclass + if ( ! location.equals(other.location)) return false; + return true; + } + + @Override + public String getIndexedString() { + return location.toString(); + } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); // takes care of index bytes + // TODO: use a better format for encoding the location on the wire. + putString(location.backendString(), buffer); + } + + @Override + public int getNumWords() { + return 1; + } + + @Override + public boolean isStemmed() { + return true; + } + + @Override + public boolean isWords() { + return false; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index bd368864e9a..c4978b2a378 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -61,7 +61,7 @@ public abstract class Item implements Cloneable { REGEXP(24), WORD_ALTERNATIVES(25), NEAREST_NEIGHBOR(26), - LOCATION_TERM(27); + GEO_LOCATION_TERM(27); public final int code; diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index b4b23a0e94a..1867da0317b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -84,6 +84,7 @@ public class StatisticsSearcher extends Searcher { private enum DegradedReason { match_phase, adaptive_timeout, timeout, non_ideal_state } private Metric metric; + private Map<String, Metric.Context> chainContexts = new CopyOnWriteHashMap<>(); private Map<String, Metric.Context> statePageOnlyContexts = new CopyOnWriteHashMap<>(); private Map<String, Map<DegradedReason, Metric.Context>> degradedReasonContexts = new CopyOnWriteHashMap<>(); private Map<String, Map<String, Metric.Context>> relevanceContexts = new CopyOnWriteHashMap<>(); @@ -152,11 +153,15 @@ public class StatisticsSearcher extends Searcher { peakQpsReporter.countQuery(); } - private Metric.Context getChainMetricContext(String chainName, String endpoint) { - Map<String, String> dimensions = new HashMap<>(); - dimensions.put("chain", chainName); - dimensions.put("endpoint", endpoint); - return this.metric.createContext(dimensions); + private Metric.Context getChainMetricContext(String chainName) { + Metric.Context context = chainContexts.get(chainName); + if (context == null) { + Map<String, String> dimensions = new HashMap<>(); + dimensions.put("chain", chainName); + context = this.metric.createContext(dimensions); + chainContexts.put(chainName, context); + } + return context; } private Metric.Context getDegradedMetricContext(String chainName, Coverage coverage) { @@ -223,7 +228,7 @@ public class StatisticsSearcher extends Searcher { return execution.search(query); } - Metric.Context metricContext = getChainMetricContext(execution.chain().getId().stringValue(), query.getHttpRequest().getHeader("Host")); + Metric.Context metricContext = getChainMetricContext(execution.chain().getId().stringValue()); incrQueryCount(metricContext); logQuery(query); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java index 9b661368972..0e8759f740e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java @@ -137,7 +137,7 @@ public class RpcFillInvoker extends FillInvoker { root.setString("ranking", rankProfile); } if (location != null) { - root.setString("location", location.toString()); + root.setString("location", location.backendString()); } Cursor gids = root.setArray("gids"); for (FastHit hit : hits) { diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 9910eb9532d..0d9acea7643 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -3,9 +3,12 @@ package com.yahoo.search.query; import com.google.common.base.Preconditions; import com.yahoo.collections.LazyMap; +import com.yahoo.geo.DistanceParser; +import com.yahoo.geo.ParsedDegree; import com.yahoo.language.Language; import com.yahoo.language.process.Normalizer; import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.Location; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.BoolItem; import com.yahoo.prelude.query.CompositeItem; @@ -15,6 +18,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; @@ -47,6 +51,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -61,6 +66,65 @@ import static com.yahoo.slime.Type.LONG; import static com.yahoo.slime.Type.OBJECT; import static com.yahoo.slime.Type.STRING; +import static com.yahoo.search.yql.YqlParser.ACCENT_DROP; +import static com.yahoo.search.yql.YqlParser.ALTERNATIVES; +import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING; +import static com.yahoo.search.yql.YqlParser.ANNOTATIONS; +import static com.yahoo.search.yql.YqlParser.APPROXIMATE; +import static com.yahoo.search.yql.YqlParser.ASCENDING_HITS_ORDER; +import static com.yahoo.search.yql.YqlParser.BOUNDS; +import static com.yahoo.search.yql.YqlParser.BOUNDS_LEFT_OPEN; +import static com.yahoo.search.yql.YqlParser.BOUNDS_OPEN; +import static com.yahoo.search.yql.YqlParser.BOUNDS_RIGHT_OPEN; +import static com.yahoo.search.yql.YqlParser.CONNECTION_ID; +import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT; +import static com.yahoo.search.yql.YqlParser.CONNECTIVITY; +import static com.yahoo.search.yql.YqlParser.DEFAULT_TARGET_NUM_HITS; +import static com.yahoo.search.yql.YqlParser.DESCENDING_HITS_ORDER; +import static com.yahoo.search.yql.YqlParser.DISTANCE; +import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; +import static com.yahoo.search.yql.YqlParser.END_ANCHOR; +import static com.yahoo.search.yql.YqlParser.EQUIV; +import static com.yahoo.search.yql.YqlParser.FILTER; +import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; +import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; +import static com.yahoo.search.yql.YqlParser.HNSW_EXPLORE_ADDITIONAL_HITS; +import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; +import static com.yahoo.search.yql.YqlParser.LABEL; +import static com.yahoo.search.yql.YqlParser.NEAR; +import static com.yahoo.search.yql.YqlParser.NEAREST_NEIGHBOR; +import static com.yahoo.search.yql.YqlParser.NFKC; +import static com.yahoo.search.yql.YqlParser.NORMALIZE_CASE; +import static com.yahoo.search.yql.YqlParser.ONEAR; +import static com.yahoo.search.yql.YqlParser.ORIGIN; +import static com.yahoo.search.yql.YqlParser.ORIGIN_LENGTH; +import static com.yahoo.search.yql.YqlParser.ORIGIN_OFFSET; +import static com.yahoo.search.yql.YqlParser.ORIGIN_ORIGINAL; +import static com.yahoo.search.yql.YqlParser.PHRASE; +import static com.yahoo.search.yql.YqlParser.PREDICATE; +import static com.yahoo.search.yql.YqlParser.PREFIX; +import static com.yahoo.search.yql.YqlParser.RANGE; +import static com.yahoo.search.yql.YqlParser.RANK; +import static com.yahoo.search.yql.YqlParser.RANKED; +import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT; +import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD; +import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE; +import static com.yahoo.search.yql.YqlParser.START_ANCHOR; +import static com.yahoo.search.yql.YqlParser.STEM; +import static com.yahoo.search.yql.YqlParser.SUBSTRING; +import static com.yahoo.search.yql.YqlParser.SUFFIX; +import static com.yahoo.search.yql.YqlParser.TARGET_HITS; +import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS; +import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR; +import static com.yahoo.search.yql.YqlParser.UNIQUE_ID; +import static com.yahoo.search.yql.YqlParser.URI; +import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA; +import static com.yahoo.search.yql.YqlParser.USER_INPUT_LANGUAGE; +import static com.yahoo.search.yql.YqlParser.WAND; +import static com.yahoo.search.yql.YqlParser.WEAK_AND; +import static com.yahoo.search.yql.YqlParser.WEIGHT; +import static com.yahoo.search.yql.YqlParser.WEIGHTED_SET; + /** * The Select query language. * @@ -70,6 +134,14 @@ import static com.yahoo.slime.Type.STRING; */ public class SelectParser implements Parser { + private static final String AND = "and"; + private static final String AND_NOT = "and_not"; + private static final String CALL = "call"; + private static final String CONTAINS = "contains"; + private static final String EQ = "equals"; + private static final String MATCHES = "matches"; + private static final String OR = "or"; + Parsable query; private final IndexFacts indexFacts; private final Map<Integer, TaggableItem> identifiedItems = LazyMap.newHashMap(); @@ -77,65 +149,7 @@ public class SelectParser implements Parser { private final Normalizer normalizer; private IndexFacts.Session indexFactsSession; - // YQL parameters and functions - private static final String DESCENDING_HITS_ORDER = "descending"; - private static final String ASCENDING_HITS_ORDER = "ascending"; - private static final Integer DEFAULT_TARGET_NUM_HITS = 10; - private static final String ORIGIN_LENGTH = "length"; - private static final String ORIGIN_OFFSET = "offset"; - private static final String ORIGIN = "origin"; - private static final String ORIGIN_ORIGINAL = "original"; - private static final String CONNECTION_ID = "id"; - private static final String CONNECTION_WEIGHT = "weight"; - private static final String CONNECTIVITY = "connectivity"; - private static final String ANNOTATIONS = "annotations"; - private static final String NFKC = "nfkc"; - private static final String USER_INPUT_LANGUAGE = "language"; - private static final String ACCENT_DROP = "accentDrop"; - private static final String ALTERNATIVES = "alternatives"; - private static final String AND_SEGMENTING = "andSegmenting"; - private static final String APPROXIMATE = "approximate"; - private static final String DISTANCE = "distance"; - private static final String DOT_PRODUCT = "dotProduct"; - private static final String EQUIV = "equiv"; - private static final String FILTER = "filter"; - private static final String HIT_LIMIT = "hitLimit"; - private static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; - private static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; - private static final String LABEL = "label"; - private static final String NEAR = "near"; - private static final String NEAREST_NEIGHBOR = "nearestNeighbor"; - private static final String NORMALIZE_CASE = "normalizeCase"; - private static final String ONEAR = "onear"; - private static final String PHRASE = "phrase"; - private static final String PREDICATE = "predicate"; - private static final String PREFIX = "prefix"; - private static final String RANKED = "ranked"; - private static final String RANK = "rank"; - private static final String SAME_ELEMENT = "sameElement"; - private static final String SCORE_THRESHOLD = "scoreThreshold"; - private static final String SIGNIFICANCE = "significance"; - private static final String STEM = "stem"; - private static final String SUBSTRING = "substring"; - private static final String SUFFIX = "suffix"; - private static final String TARGET_HITS = "targetHits"; - private static final String TARGET_NUM_HITS = "targetNumHits"; - private static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; - private static final String UNIQUE_ID = "id"; - private static final String USE_POSITION_DATA = "usePositionData"; - private static final String WAND = "wand"; - private static final String WEAK_AND = "weakAnd"; - private static final String WEIGHTED_SET = "weightedSet"; - private static final String WEIGHT = "weight"; - private static final String AND = "and"; - private static final String AND_NOT = "and_not"; - private static final String OR = "or"; - private static final String EQ = "equals"; - private static final String RANGE = "range"; - private static final String CONTAINS = "contains"; - private static final String MATCHES = "matches"; - private static final String CALL = "call"; - private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, NEAREST_NEIGHBOR, PREDICATE, RANK, WEAK_AND); + private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, GEO_LOCATION, NEAREST_NEIGHBOR, PREDICATE, RANK, WEAK_AND); public SelectParser(ParserEnvironment environment) { indexFacts = environment.getIndexFacts(); @@ -153,7 +167,7 @@ public class SelectParser implements Parser { } private QueryTree buildTree() { - Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString().getBytes()).get(); + Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString()).get(); if (inspector.field("error_message").valid()) { throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); @@ -213,7 +227,7 @@ public class SelectParser implements Parser { /** Translates a list of grouping requests on JSON form to a list in the grouping language form */ private List<String> toGroupingRequests(String groupingJson) { - Inspector inspector = SlimeUtils.jsonToSlime(groupingJson.getBytes()).get(); + Inspector inspector = SlimeUtils.jsonToSlime(groupingJson).get(); if (inspector.field("error_message").valid()) { throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); @@ -264,6 +278,8 @@ public class SelectParser implements Parser { return buildWeightedSet(key, value); case DOT_PRODUCT: return buildDotProduct(key, value); + case GEO_LOCATION: + return buildGeoLocation(key, value); case NEAREST_NEIGHBOR: return buildNearestNeighbor(key, value); case PREDICATE: @@ -410,6 +426,47 @@ public class SelectParser implements Parser { return orItem; } + private Item buildGeoLocation(String key, Inspector value) { + HashMap<Integer, Inspector> children = childMap(value); + Preconditions.checkArgument(children.size() == 4, "Expected 4 arguments, got %s.", children.size()); + String field = children.get(0).asString(); + var arg1 = children.get(1); + var arg2 = children.get(2); + var arg3 = children.get(3); + var loc = new Location(); + if (arg3.type() != Type.STRING) { + throw new IllegalArgumentException("Invalid geoLocation radius type "+arg3.type()+" for "+arg3); + } + double radius = DistanceParser.parse(arg3.asString()); + if (arg1.type() == Type.STRING && arg2.type() == Type.STRING) { + var c1input = children.get(1).asString(); + var c2input = children.get(2).asString(); + var coord_1 = ParsedDegree.fromString(c1input, true, false); + var coord_2 = ParsedDegree.fromString(c2input, false, true); + if (coord_1.isLatitude && coord_2.isLongitude) { + loc.setGeoCircle(coord_1.degrees, coord_2.degrees, radius); + } else if (coord_2.isLatitude && coord_1.isLongitude) { + loc.setGeoCircle(coord_2.degrees, coord_1.degrees, radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinates '"+c1input+"' and '"+c2input+"'"); + } + } else if (arg1.type() == Type.DOUBLE && arg2.type() == Type.DOUBLE) { + loc.setGeoCircle(arg1.asDouble(), arg2.asDouble(), radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinate types "+arg1.type()+" and "+arg2.type()); + } + var item = new GeoLocationItem(loc, field); + Inspector annotations = getAnnotations(value); + if (annotations != null){ + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (LABEL.equals(annotation_name)) { + item.setLabel(annotation_value.asString()); + } + }); + } + return item; + } + private Item buildNearestNeighbor(String key, Inspector value) { HashMap<Integer, Inspector> children = childMap(value); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java index 88014eef46d..27f600f9ad6 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java @@ -57,27 +57,44 @@ public class Binding implements Comparable<Binding> { return new Binding(generality, context); } - private Binding(int generality, Map<String, String> binding) { + /** Creates a binding from a map containing the exact bindings this will have */ + private Binding(int generality, Map<String, String> bindings) { this.generality = generality; // Map -> arrays to limit memory consumption and speed up evaluation - dimensions = new String[binding.size()]; - dimensionValues = new String[binding.size()]; + dimensions = new String[bindings.size()]; + dimensionValues = new String[bindings.size()]; int i = 0; - int bindingHash = 0; - for (Map.Entry<String,String> entry : binding.entrySet()) { + for (Map.Entry<String,String> entry : bindings.entrySet()) { dimensions[i] = entry.getKey(); dimensionValues[i] = entry.getValue(); - bindingHash += i * entry.getKey().hashCode() + 11 * i * entry.getValue().hashCode(); i++; } - this.hashCode = bindingHash; + this.hashCode = Arrays.hashCode(dimensions) + 11 * Arrays.hashCode(dimensionValues); } + Binding(DimensionalValue.BindingSpec spec, Map<String, String> bindings) { + this.generality = 0; // Not used here + + // Map -> arrays to limit memory consumption and speed up evaluation + dimensions = spec.dimensions(); + dimensionValues = new String[spec.dimensions().length]; + for (int i = 0; i < dimensions.length; i++) { + dimensionValues[i] = bindings.get(dimensions[i]); + } + this.hashCode = Arrays.hashCode(dimensions) + 11 * Arrays.hashCode(dimensionValues); + } + + /** Returns true only if this binding is null (contains no values for its dimensions (if any) */ public boolean isNull() { return dimensions.length == 0; } + /** Do not change the returtned array */ + String[] dimensions() { return dimensions; } + + String[] dimensionValues() { return dimensionValues; } + @Override public String toString() { StringBuilder b = new StringBuilder("Binding["); @@ -116,7 +133,7 @@ public class Binding implements Comparable<Binding> { /** * Implements a partial ordering where more specific bindings come before less specific ones, * taking both the number of bindings and their positions into account (earlier dimensions - * take precedence over later ones. + * take precedence over later ones). * <p> * The order is not well defined for bindings in different dimensional spaces. */ diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java index 0137b848dac..c1ee49913fe 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java @@ -6,6 +6,7 @@ import com.yahoo.search.query.profile.DimensionBinding; import com.yahoo.search.query.profile.SubstituteString; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -20,20 +21,22 @@ import java.util.Set; */ public class DimensionalValue<VALUE> { - private final List<Value<VALUE>> values; + private final Map<Binding, VALUE> indexedVariants; + private final List<BindingSpec> bindingSpecs; - /** Create a set of variants which is a single value regardless of dimensions */ - public DimensionalValue(Value<VALUE> value) { - this.values = Collections.singletonList(value); - } + private DimensionalValue(List<Value<VALUE>> variants) { + Collections.sort(variants); - public DimensionalValue(List<Value<VALUE>> valueVariants) { - if (valueVariants.size() == 1) { // special cased for efficiency - this.values = Collections.singletonList(valueVariants.get(0)); - } - else { - this.values = new ArrayList<>(valueVariants); - Collections.sort(this.values); + // If there are inconsistent definitions of the same property, we should pick the first in the sort order + this.indexedVariants = new HashMap<>(); + for (Value<VALUE> variant : variants) + indexedVariants.putIfAbsent(variant.binding(), variant.value()); + + this.bindingSpecs = new ArrayList<>(); + for (Value<VALUE> variant : variants) { + BindingSpec spec = new BindingSpec(variant.binding()); + if ( ! bindingSpecs.contains(spec)) + bindingSpecs.add(spec); } } @@ -41,18 +44,21 @@ public class DimensionalValue<VALUE> { public VALUE get(Map<String, String> context) { if (context == null) context = Collections.emptyMap(); - for (Value<VALUE> value : values) { - if (value.matches(context)) - return value.value(); + + for (BindingSpec spec : bindingSpecs) { + if ( ! spec.matches(context)) continue; + VALUE value = indexedVariants.get(new Binding(spec, context)); + if (value != null) + return value; } return null; } - public boolean isEmpty() { return values.isEmpty(); } + public boolean isEmpty() { return indexedVariants.isEmpty(); } @Override public String toString() { - return values.toString(); + return indexedVariants.toString(); } public static class Builder<VALUE> { @@ -93,7 +99,7 @@ public class DimensionalValue<VALUE> { /** A value for a particular binding */ private static class Value<VALUE> implements Comparable<Value> { - private VALUE value = null; + private VALUE value; /** The minimal binding this holds for */ private Binding binding; @@ -126,9 +132,6 @@ public class DimensionalValue<VALUE> { return " value '" + value + "' for " + binding; } - /** - * A single value with the minimal set of dimension combinations it holds for. - */ private static class Builder<VALUE> { private final VALUE value; @@ -212,4 +215,38 @@ public class DimensionalValue<VALUE> { } + /** A list of dimensions for which there exist one or more bindings in this */ + static class BindingSpec { + + /** The dimensions of this. Unenforced invariant: Content never changes. */ + private final String[] dimensions; + + public BindingSpec(Binding binding) { + this.dimensions = binding.dimensions(); + } + + /** Do not change the returned array */ + String[] dimensions() { return dimensions; } + + /** Returns whether this context contains all the keys of this */ + public boolean matches(Map<String, String> context) { + for (int i = 0; i < dimensions.length; i++) + if ( ! context.containsKey(dimensions[i])) return false; + return true; + } + + @Override + public int hashCode() { + return Arrays.hashCode(dimensions); + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof BindingSpec)) return false; + return Arrays.equals(((BindingSpec)other).dimensions, this.dimensions); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java index bc49e116c6e..d2c4eaaec9b 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java @@ -2,9 +2,9 @@ package com.yahoo.search.query.profile.compiled; import com.yahoo.search.query.profile.DimensionValues; -import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.types.QueryProfileType; +import java.util.Objects; import java.util.Optional; /** @@ -68,6 +68,24 @@ public class ValueWithSource { public Optional<DimensionValues> variant() { return Optional.ofNullable(variant); } @Override + public int hashCode() { + // Value is always a value object. Don't include source in identity. + return Objects.hash(value, isUnoverridable, type); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof ValueWithSource)) return false; + + ValueWithSource other = (ValueWithSource)o; + if ( ! Objects.equals(this.value, other.value)) return false; + if ( ! Objects.equals(this.isUnoverridable, other.isUnoverridable)) return false; + if ( ! Objects.equals(this.type, other.type)) return false; + return true; + } + + @Override public String toString() { return value + " (from query profile '" + source + "'" + diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index dd52b9e19b8..22328fb026e 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -16,6 +16,7 @@ import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; import static com.yahoo.search.yql.YqlParser.END_ANCHOR; import static com.yahoo.search.yql.YqlParser.EQUIV; import static com.yahoo.search.yql.YqlParser.FILTER; +import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; import static com.yahoo.search.yql.YqlParser.LABEL; @@ -72,6 +73,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; @@ -689,6 +691,26 @@ public class VespaSerializer { } + private static class GeoLocationSerializer extends Serializer<GeoLocationItem> { + @Override + void onExit(StringBuilder destination, GeoLocationItem item) { } + @Override + boolean serialize(StringBuilder destination, GeoLocationItem item) { + String annotations = leafAnnotations(item); + if (annotations.length() > 0) { + destination.append("([{").append(annotations).append("}]"); + } + destination.append(GEO_LOCATION).append('('); + destination.append(item.getIndexName()).append(", "); + var loc = item.getLocation(); + destination.append(loc.degNS()).append(", "); + destination.append(loc.degEW()).append(", "); + destination.append('"').append(loc.degRadius()).append(" deg").append('"'); + destination.append(')'); + return false; + } + } + private static class NearestNeighborSerializer extends Serializer<NearestNeighborItem> { @Override @@ -1163,6 +1185,7 @@ public class VespaSerializer { dispatchBuilder.put(EquivItem.class, new EquivSerializer()); dispatchBuilder.put(ExactStringItem.class, new WordSerializer()); dispatchBuilder.put(IntItem.class, new NumberSerializer()); + dispatchBuilder.put(GeoLocationItem.class, new GeoLocationSerializer()); dispatchBuilder.put(BoolItem.class, new BoolSerializer()); dispatchBuilder.put(MarkerWordItem.class, new WordSerializer()); // gotcha dispatchBuilder.put(NearItem.class, new NearSerializer()); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 7d17fe4f09d..6a464a1503b 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -19,11 +19,14 @@ import com.google.common.annotations.Beta; import com.google.common.base.Preconditions; import com.yahoo.collections.LazyMap; import com.yahoo.collections.LazySet; +import com.yahoo.geo.DistanceParser; +import com.yahoo.geo.ParsedDegree; import com.yahoo.language.Language; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.Location; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.BoolItem; @@ -34,6 +37,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; @@ -94,8 +98,8 @@ import com.yahoo.search.query.parser.ParserFactory; */ public class YqlParser implements Parser { - private static final String DESCENDING_HITS_ORDER = "descending"; - private static final String ASCENDING_HITS_ORDER = "ascending"; + public static final String DESCENDING_HITS_ORDER = "descending"; + public static final String ASCENDING_HITS_ORDER = "ascending"; private enum SegmentWhen { NEVER, POSSIBLY, ALWAYS; @@ -107,12 +111,12 @@ public class YqlParser implements Parser { private static final Integer DEFAULT_HITS = 10; private static final Integer DEFAULT_OFFSET = 0; - private static final Integer DEFAULT_TARGET_NUM_HITS = 10; + public static final Integer DEFAULT_TARGET_NUM_HITS = 10; private static final String ACCENT_DROP_DESCRIPTION = "setting for whether to remove accents if field implies it"; - private static final String ANNOTATIONS = "annotations"; + public static final String ANNOTATIONS = "annotations"; private static final String FILTER_DESCRIPTION = "term filter setting"; private static final String IMPLICIT_TRANSFORMS_DESCRIPTION = "setting for whether built-in query transformers should touch the term"; - private static final String NFKC = "nfkc"; + public static final String NFKC = "nfkc"; private static final String NORMALIZE_CASE_DESCRIPTION = "setting for whether to do case normalization if field implies it"; private static final String ORIGIN_DESCRIPTION = "string origin for a term"; private static final String RANKED_DESCRIPTION = "setting for whether to use term for ranking"; @@ -121,7 +125,7 @@ public class YqlParser implements Parser { private static final String USER_INPUT_ALLOW_EMPTY = "allowEmpty"; private static final String USER_INPUT_DEFAULT_INDEX = "defaultIndex"; private static final String USER_INPUT_GRAMMAR = "grammar"; - private static final String USER_INPUT_LANGUAGE = "language"; + public static final String USER_INPUT_LANGUAGE = "language"; private static final String USER_INPUT_RAW = "raw"; private static final String USER_INPUT_SEGMENT = "segment"; private static final String USER_INPUT = "userInput"; @@ -134,55 +138,56 @@ public class YqlParser implements Parser { public static final String SORTING_LOCALE = "locale"; public static final String SORTING_STRENGTH = "strength"; - static final String ACCENT_DROP = "accentDrop"; - static final String ALTERNATIVES = "alternatives"; - static final String AND_SEGMENTING = "andSegmenting"; - static final String APPROXIMATE = "approximate"; - static final String BOUNDS = "bounds"; - static final String BOUNDS_LEFT_OPEN = "leftOpen"; - static final String BOUNDS_OPEN = "open"; - static final String BOUNDS_RIGHT_OPEN = "rightOpen"; - static final String CONNECTION_ID = "id"; - static final String CONNECTION_WEIGHT = "weight"; - static final String CONNECTIVITY = "connectivity"; - static final String DISTANCE = "distance"; - static final String DOT_PRODUCT = "dotProduct"; - static final String EQUIV = "equiv"; - static final String FILTER = "filter"; - static final String HIT_LIMIT = "hitLimit"; - static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; - static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; - static final String LABEL = "label"; - static final String NEAR = "near"; - static final String NEAREST_NEIGHBOR = "nearestNeighbor"; - static final String NORMALIZE_CASE = "normalizeCase"; - static final String ONEAR = "onear"; - static final String ORIGIN_LENGTH = "length"; - static final String ORIGIN_OFFSET = "offset"; - static final String ORIGIN = "origin"; - static final String ORIGIN_ORIGINAL = "original"; - static final String PHRASE = "phrase"; - static final String PREDICATE = "predicate"; - static final String PREFIX = "prefix"; - static final String RANGE = "range"; - static final String RANKED = "ranked"; - static final String RANK = "rank"; - static final String SAME_ELEMENT = "sameElement"; - static final String SCORE_THRESHOLD = "scoreThreshold"; - static final String SIGNIFICANCE = "significance"; - static final String STEM = "stem"; - static final String SUBSTRING = "substring"; - static final String SUFFIX = "suffix"; - static final String TARGET_HITS = "targetHits"; - static final String TARGET_NUM_HITS = "targetNumHits"; - static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; - static final String UNIQUE_ID = "id"; - static final String USE_POSITION_DATA = "usePositionData"; - static final String WAND = "wand"; - static final String WEAK_AND = "weakAnd"; - static final String WEIGHTED_SET = "weightedSet"; - static final String WEIGHT = "weight"; - static final String URI = "uri"; + public static final String ACCENT_DROP = "accentDrop"; + public static final String ALTERNATIVES = "alternatives"; + public static final String AND_SEGMENTING = "andSegmenting"; + public static final String APPROXIMATE = "approximate"; + public static final String BOUNDS = "bounds"; + public static final String BOUNDS_LEFT_OPEN = "leftOpen"; + public static final String BOUNDS_OPEN = "open"; + public static final String BOUNDS_RIGHT_OPEN = "rightOpen"; + public static final String CONNECTION_ID = "id"; + public static final String CONNECTION_WEIGHT = "weight"; + public static final String CONNECTIVITY = "connectivity"; + public static final String DISTANCE = "distance"; + public static final String DOT_PRODUCT = "dotProduct"; + public static final String EQUIV = "equiv"; + public static final String FILTER = "filter"; + public static final String GEO_LOCATION = "geoLocation"; + public static final String HIT_LIMIT = "hitLimit"; + public static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; + public static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; + public static final String LABEL = "label"; + public static final String NEAR = "near"; + public static final String NEAREST_NEIGHBOR = "nearestNeighbor"; + public static final String NORMALIZE_CASE = "normalizeCase"; + public static final String ONEAR = "onear"; + public static final String ORIGIN_LENGTH = "length"; + public static final String ORIGIN_OFFSET = "offset"; + public static final String ORIGIN = "origin"; + public static final String ORIGIN_ORIGINAL = "original"; + public static final String PHRASE = "phrase"; + public static final String PREDICATE = "predicate"; + public static final String PREFIX = "prefix"; + public static final String RANGE = "range"; + public static final String RANKED = "ranked"; + public static final String RANK = "rank"; + public static final String SAME_ELEMENT = "sameElement"; + public static final String SCORE_THRESHOLD = "scoreThreshold"; + public static final String SIGNIFICANCE = "significance"; + public static final String STEM = "stem"; + public static final String SUBSTRING = "substring"; + public static final String SUFFIX = "suffix"; + public static final String TARGET_HITS = "targetHits"; + public static final String TARGET_NUM_HITS = "targetNumHits"; + public static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; + public static final String UNIQUE_ID = "id"; + public static final String USE_POSITION_DATA = "usePositionData"; + public static final String WAND = "wand"; + public static final String WEAK_AND = "weakAnd"; + public static final String WEIGHTED_SET = "weightedSet"; + public static final String WEIGHT = "weight"; + public static final String URI = "uri"; private final IndexFacts indexFacts; private final List<ConnectedItem> connectedItems = new ArrayList<>(); @@ -372,6 +377,8 @@ public class YqlParser implements Parser { return buildWeightedSet(ast); case DOT_PRODUCT: return buildDotProduct(ast); + case GEO_LOCATION: + return buildGeoLocation(ast); case NEAREST_NEIGHBOR: return buildNearestNeighbor(ast); case PREDICATE: @@ -413,6 +420,29 @@ public class YqlParser implements Parser { return fillWeightedSet(ast, args.get(1), new DotProductItem(getIndex(args.get(0)))); } + private Item buildGeoLocation(OperatorNode<ExpressionOperator> ast) { + List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); + Preconditions.checkArgument(args.size() == 4, "Expected 4 arguments, got %s.", args.size()); + String field = fetchFieldRead(args.get(0)); + var coord_1 = ParsedDegree.fromString(fetchFieldRead(args.get(1)), true, false); + var coord_2 = ParsedDegree.fromString(fetchFieldRead(args.get(2)), false, true); + double radius = DistanceParser.parse(fetchFieldRead(args.get(3))); + var loc = new Location(); + if (coord_1.isLatitude && coord_2.isLongitude) { + loc.setGeoCircle(coord_1.degrees, coord_2.degrees, radius); + } else if (coord_2.isLatitude && coord_1.isLongitude) { + loc.setGeoCircle(coord_2.degrees, coord_1.degrees, radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinates '"+coord_1+"' and '"+coord_2+"'"); + } + var item = new GeoLocationItem(loc, field); + String label = getAnnotation(ast, LABEL, String.class, null, "item label"); + if (label != null) { + item.setLabel(label); + } + return item; + } + private Item buildNearestNeighbor(OperatorNode<ExpressionOperator> ast) { List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); Preconditions.checkArgument(args.size() == 2, "Expected 2 arguments, got %s.", args.size()); @@ -438,7 +468,7 @@ public class YqlParser implements Parser { item.setAllowApproximate(allowApproximate); String label = getAnnotation(ast, LABEL, String.class, null, "item label"); if (label != null) { - item.setLabel(label); + item.setLabel(label); } return item; } @@ -902,6 +932,8 @@ public class YqlParser implements Parser { private static String fetchFieldRead(OperatorNode<ExpressionOperator> ast) { switch (ast.getOperator()) { + case LITERAL: + return ast.getArgument(0).toString(); case READ_FIELD: return ast.getArgument(1); case PROPREF: diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java index aa3fa53119e..aa48e8494f2 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java @@ -123,7 +123,8 @@ public class PosSearcherTestCase { q.properties().set("pos.ll", "N0;E0"); q.properties().set("pos.radius", "-1"); doSearch(searcher, q, 0, 10); - assertEquals("(2,0,0,536870912,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + assertEquals("(2,0,0,-1,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + assertEquals("(2,0,0,536870912,0,1,0,4294967295)", q.getRanking().getLocation().backendString()); } /** diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java index e85940278e3..445073ced3a 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java @@ -14,7 +14,6 @@ import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; -import org.junit.Ignore; import org.junit.Test; import java.util.HashMap; diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index d770b08d31a..f8e930fa19d 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -119,6 +119,14 @@ public class VespaSerializerTestCase { } @Test + public void testGeoLocation() { + parseAndConfirm("geoLocation(workplace, 63.418417, 10.433033, \"0.5 deg\")"); + parseAndConfirm("geoLocation(headquarters, 37.41638, -122.024683, \"180.0 deg\")"); + parseAndConfirm("geoLocation(home, -17.0, 42.0, \"0.0 deg\")"); + parseAndConfirm("geoLocation(workplace, -12.0, -34.0, \"-1.0 deg\")"); + } + + @Test public void testNear() { parseAndConfirm("title contains near(\"a\", \"b\")"); parseAndConfirm("title contains ([{\"distance\": 50}]near(\"a\", \"b\"))"); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index a151244525a..62a9e27cd96 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -548,6 +548,24 @@ public class YqlParserTestCase { } @Test + public void testGeoLocation() { + assertParse("select foo from bar where geoLocation(workplace, 63.418417, 10.433033, \"0.5 deg\");", + "GEO_LOCATION workplace:(2,10433033,63418417,500000,0,1,0,1921876103)"); + assertParse("select foo from bar where geoLocation(headquarters, \"37.416383\", \"-122.024683\", \"100 miles\");", + "GEO_LOCATION headquarters:(2,-122024683,37416383,1450561,0,1,0,3411238761)"); + assertParse("select foo from bar where geoLocation(home, \"E10.433033\", \"N63.418417\", \"5km\");", + "GEO_LOCATION home:(2,10433033,63418417,45066,0,1,0,1921876103)"); + + assertParseFail("select foo from bar where geoLocation(qux, 1, 2);", + new IllegalArgumentException("Expected 4 arguments, got 3.")); + assertParseFail("select foo from bar where geoLocation(qux, 2.0, \"N5.0\", \"0.5 deg\");", + new IllegalArgumentException( + "Invalid geoLocation coordinates 'Latitude: 2.0 degrees' and 'Latitude: 5.0 degrees'")); + assertParse("select foo from bar where geoLocation(workplace, -12, -34, \"-77 d\");", + "GEO_LOCATION workplace:(2,-34000000,-12000000,-1,0,1,0,4201111954)"); + } + + @Test public void testNearestNeighbor() { assertParse("select foo from bar where nearestNeighbor(semantic_embedding, my_vector);", "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=0,approximate=true,targetHits=0}"); diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 4691ef42e55..f297fd69f24 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -522,6 +522,18 @@ public class SelectTestCase { } @Test + public void testGeoLocation() { + assertParse("{ \"geoLocation\": [ \"workplace\", 63.418417, 10.433033, \"0.5 deg\" ] }", + "GEO_LOCATION workplace:(2,10433033,63418417,500000,0,1,0,1921876103)"); + assertParse("{ \"geoLocation\": [ \"headquarters\", \"37.416383\", \"-122.024683\", \"100 miles\" ] }", + "GEO_LOCATION headquarters:(2,-122024683,37416383,1450561,0,1,0,3411238761)"); + assertParse("{ \"geoLocation\": [ \"home\", \"E10.433033\", \"N63.418417\", \"5km\" ] }", + "GEO_LOCATION home:(2,10433033,63418417,45066,0,1,0,1921876103)"); + assertParse("{ \"geoLocation\": [ \"workplace\", -12.0, -34.0, \"-77 deg\" ] }", + "GEO_LOCATION workplace:(2,-34000000,-12000000,-1,0,1,0,4201111954)"); + } + + @Test public void testNearestNeighbor() { assertParse("{ \"nearestNeighbor\": [ \"f1field\", \"q2prop\" ] }", "NEAREST_NEIGHBOR {field=f1field,queryTensorName=q2prop,hnsw.exploreAdditionalHits=0,approximate=true,targetHits=0}"); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 16464f7cea6..c04adcec594 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.io.InputStream; @@ -32,7 +33,7 @@ public interface ConfigServer { PreparedApplication deploy(DeploymentData deployment); - void restart(DeploymentId deployment, Optional<Hostname> hostname); + void restart(DeploymentId deployment, RestartFilter restartFilter); void deactivate(DeploymentId deployment) throws NotFoundException; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java index 44db38c3ec2..a86dc6895ae 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationStore.java @@ -22,10 +22,7 @@ public interface ApplicationStore { byte[] get(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion); /** Find application package by given build number */ - default Optional<byte[]> find(TenantName tenant, ApplicationName application, long buildNumber) { - // TODO(mpolden): Remove default once all implemenations catch up - return Optional.empty(); - } + Optional<byte[]> find(TenantName tenant, ApplicationName application, long buildNumber); /** Stores the given tenant application package of the given version. */ void put(TenantName tenant, ApplicationName application, ApplicationVersion applicationVersion, byte[] applicationPackage); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/Jira.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/Jira.java index 30bf23f18a9..6848c67fed2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/Jira.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/Jira.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.jira; +import java.io.InputStream; import java.util.List; /** @@ -14,5 +15,5 @@ public interface Jira { void commentIssue(JiraIssue issue, JiraComment comment); - void addAttachment(JiraIssue issue, String filename, String fileContent); + void addAttachment(JiraIssue issue, String filename, InputStream fileContent); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/RestartFilter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/RestartFilter.java new file mode 100644 index 00000000000..685ed392c00 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/RestartFilter.java @@ -0,0 +1,51 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; + +import java.util.Optional; + +/** + * Attributes to filter when restarting nodes in a deployment. + * If all attributes are empty, all nodes are restarted. + * Used in {@link ConfigServer#restart(DeploymentId, RestartFilter)} + * + * @author olaa + */ +public class RestartFilter { + + private Optional<HostName> hostName = Optional.empty(); + private Optional<ClusterSpec.Type> clusterType = Optional.empty(); + private Optional<ClusterSpec.Id> clusterId = Optional.empty(); + + public Optional<HostName> getHostName() { + return hostName; + } + + public Optional<ClusterSpec.Type> getClusterType() { + return clusterType; + } + + public Optional<ClusterSpec.Id> getClusterId() { + return clusterId; + } + + public RestartFilter withHostName(Optional<HostName> hostName) { + this.hostName = hostName; + return this; + } + + public RestartFilter withClusterType(Optional<ClusterSpec.Type> clusterType) { + this.clusterType = clusterType; + return this; + } + + public RestartFilter withClusterId(Optional<ClusterSpec.Id> clusterId) { + this.clusterId = clusterId; + return this; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index d644cf21638..c03b38867f9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -46,6 +46,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepo import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationPackageValidator; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -680,10 +681,10 @@ public class ApplicationController { /** * Tells config server to schedule a restart of all nodes in this deployment * - * @param hostname If non-empty, restart will only be scheduled for this host + * @param restartFilter Variables to filter which nodes to restart. */ - public void restart(DeploymentId deploymentId, Optional<Hostname> hostname) { - configServer.restart(deploymentId, hostname); + public void restart(DeploymentId deploymentId, RestartFilter restartFilter) { + configServer.restart(deploymentId, restartFilter); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 3b1926fa6e9..26270c092d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -5,13 +5,11 @@ import com.google.common.base.Suppliers; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -64,14 +62,12 @@ public class RoutingController { private final Controller controller; private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; - private final BooleanFlag allowDirectRouting; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); this.routingPolicies = new RoutingPolicies(controller); this.rotationRepository = new RotationRepository(rotationsConfig, controller.applications(), controller.curator()); - this.allowDirectRouting = Flags.ALLOW_DIRECT_ROUTING.bindTo(controller.flagSource()); } public RoutingPolicies policies() { @@ -93,7 +89,7 @@ public class RoutingController { for (var routingMethod : controller.zoneRegistry().routingMethods(policy.id().zone())) { if (routingMethod.isDirect() && !isSystemApplication && !canRouteDirectlyTo(deployment, application.get())) continue; endpoints.add(policy.endpointIn(controller.system(), routingMethod, controller.zoneRegistry())); - endpoints.add(policy.weightedEndpointIn(controller.system(), routingMethod)); + endpoints.add(policy.regionEndpointIn(controller.system(), routingMethod)); } } return EndpointList.copyOf(endpoints); @@ -115,22 +111,23 @@ public class RoutingController { var deployments = rotation.regions().stream() .map(region -> new DeploymentId(instance.id(), ZoneId.from(Environment.prod, region))) .collect(Collectors.toList()); - computeGlobalEndpoints(RoutingId.of(instance.id(), rotation.endpointId()), + computeGlobalEndpoints(RoutingId.of(instance.id(), rotation.endpointId()), rotation.clusterId(), application, deployments).requiresRotation() .forEach(endpoints::add); } // Add global endpoints provided by routing policies - var deploymentsByRoutingId = new LinkedHashMap<RoutingId, List<DeploymentId>>(); + var deploymentsByEndpointKey = new LinkedHashMap<EndpointKey, List<DeploymentId>>(); for (var policy : routingPolicies.get(instance.id()).values()) { if (!policy.status().isActive()) continue; for (var endpointId : policy.endpoints()) { - var routingId = RoutingId.of(instance.id(), endpointId); - deploymentsByRoutingId.putIfAbsent(routingId, new ArrayList<>()); - deploymentsByRoutingId.get(routingId).add(new DeploymentId(instance.id(), policy.id().zone())); + var endpointKey = new EndpointKey(RoutingId.of(instance.id(), endpointId), policy.id().cluster()); + deploymentsByEndpointKey.computeIfAbsent(endpointKey, (k) -> new ArrayList<>()) + .add(new DeploymentId(instance.id(), policy.id().zone())); } } - deploymentsByRoutingId.forEach((routingId, deployments) -> { - computeGlobalEndpoints(routingId, application, deployments).not().requiresRotation().forEach(endpoints::add); + deploymentsByEndpointKey.forEach((endpointKey, deployments) -> { + computeGlobalEndpoints(endpointKey.routingId, endpointKey.cluster, application, + deployments).not().requiresRotation().forEach(endpoints::add); }); return EndpointList.copyOf(endpoints); } @@ -265,7 +262,7 @@ public class RoutingController { private boolean canRouteDirectlyTo(DeploymentId deploymentId, Application application) { if (controller.system().isPublic()) return true; // Public always supports direct routing if (controller.system().isCd()) return true; // CD deploys directly so we cannot enforce all requirements below - if(deploymentId.zoneId().environment().isManuallyDeployed()) return true; // Manually deployed zones does not include any use cases where direct routing is not supported + if (deploymentId.zoneId().environment().isManuallyDeployed()) return true; // Manually deployed zones always support direct routing // Check Athenz service presence. The test framework uses this identity when sending requests to the // deployment's container(s). @@ -285,39 +282,34 @@ public class RoutingController { .or(() -> application.latestVersion().flatMap(ApplicationVersion::compileVersion)); if (compileVersion.isEmpty()) return false; if (compileVersion.get().isBefore(DIRECT_ROUTING_MIN_VERSION)) return false; - - // Check feature flag - // TODO(mpolden): Remove once we make this default - return this.allowDirectRouting.with(FetchVector.Dimension.APPLICATION_ID, - deploymentId.applicationId().serializedForm()) - .value(); + return true; } /** Compute global endpoints for given routing ID, application and deployments */ - private EndpointList computeGlobalEndpoints(RoutingId routingId, Application application, List<DeploymentId> deployments) { + private EndpointList computeGlobalEndpoints(RoutingId routingId, ClusterSpec.Id cluster, Application application, List<DeploymentId> deployments) { var endpoints = new ArrayList<Endpoint>(); var directMethods = 0; - var targets = deployments.stream().map(DeploymentId::zoneId).collect(Collectors.toList()); + var zones = deployments.stream().map(DeploymentId::zoneId).collect(Collectors.toList()); for (var method : routingMethodsOfAll(deployments, application)) { if (method.isDirect() && ++directMethods > 1) { throw new IllegalArgumentException("Invalid routing methods for " + routingId + ": Exceeded maximum " + "direct methods"); } endpoints.add(Endpoint.of(routingId.application()) - .named(routingId.endpointId(), targets) + .target(routingId.endpointId(), cluster, zones) .on(Port.fromRoutingMethod(method)) .routingMethod(method) .in(controller.system())); // TODO(mpolden): Remove this once all applications have migrated away from legacy endpoints if (method == RoutingMethod.shared) { endpoints.add(Endpoint.of(routingId.application()) - .named(routingId.endpointId(), targets) + .target(routingId.endpointId(), cluster, zones) .on(Port.plain(4080)) .legacy() .routingMethod(method) .in(controller.system())); endpoints.add(Endpoint.of(routingId.application()) - .named(routingId.endpointId(), targets) + .target(routingId.endpointId(), cluster, zones) .on(Port.tls(4443)) .legacy() .routingMethod(method) @@ -327,4 +319,30 @@ public class RoutingController { return EndpointList.copyOf(endpoints); } + private static class EndpointKey { + + private final RoutingId routingId; + private final ClusterSpec.Id cluster; + + public EndpointKey(RoutingId routingId, ClusterSpec.Id cluster) { + this.routingId = routingId; + this.cluster = cluster; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EndpointKey that = (EndpointKey) o; + return routingId.equals(that.routingId) && + cluster.equals(that.cluster); + } + + @Override + public int hashCode() { + return Objects.hash(routingId, cluster); + } + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java index 083984bd13c..d0e8a0f2816 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java @@ -5,11 +5,13 @@ import com.google.common.collect.ImmutableMap; import com.google.common.hash.Hashing; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayInputStream; @@ -19,7 +21,10 @@ import java.io.InputStreamReader; import java.io.Reader; import java.io.UncheckedIOException; import java.security.cert.X509Certificate; +import java.time.Duration; import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -184,4 +189,27 @@ public class ApplicationPackage { } + /** Creates a valid application package that will remove all application's deployments */ + public static ApplicationPackage deploymentRemoval() { + DeploymentSpec deploymentSpec = DeploymentSpec.empty; + ValidationOverrides validationOverrides = allValidationOverrides(); + try (ZipBuilder zipBuilder = new ZipBuilder(deploymentSpec.xmlForm().length() + validationOverrides.xmlForm().length() + 500)) { + zipBuilder.add("validation-overrides.xml", validationOverrides.xmlForm().getBytes(UTF_8)); + zipBuilder.add("deployment.xml", deploymentSpec.xmlForm().getBytes(UTF_8)); + + zipBuilder.close(); + return new ApplicationPackage(zipBuilder.toByteArray()); + } + } + + private static ValidationOverrides allValidationOverrides() { + String until = DateTimeFormatter.ISO_LOCAL_DATE.format(Instant.now().plus(Duration.ofDays(25)).atZone(ZoneOffset.UTC)); + StringBuilder validationOverridesContents = new StringBuilder(1000); + validationOverridesContents.append("<validation-overrides version=\"1.0\">\n"); + for (ValidationId validationId: ValidationId.values()) + validationOverridesContents.append("\t<allow until=\"").append(until).append("\">").append(validationId.value()).append("</allow>\n"); + validationOverridesContents.append("</validation-overrides>\n"); + + return ValidationOverrides.fromXml(validationOverridesContents.toString()); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java index b245718171f..3e72936575d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java @@ -51,7 +51,8 @@ public class ApplicationPackageValidator { /** Verify that we have the security/clients.pem file for public systems */ private void validateSecurityClientsPem(ApplicationPackage applicationPackage) { - if (controller.system().isPublic() && applicationPackage.trustedCertificates().isEmpty()) + if (!controller.system().isPublic() || applicationPackage.deploymentSpec().steps().isEmpty()) return; + if (applicationPackage.trustedCertificates().isEmpty()) throw new IllegalArgumentException("Missing required file 'security/clients.pem'"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 62804074337..341bf7de9d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -17,8 +17,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; /** - * Represents an application's endpoint. The endpoint scope can either be global or a specific zone. This is visible to - * the tenant and is used by the tenant when accessing deployments. + * Represents an application's endpoint in hosted Vespa. This encapsulates all logic for building URLs and DNS names for + * application endpoints. * * @author mpolden */ @@ -29,7 +29,8 @@ public class Endpoint { private static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud"; private static final String PUBLIC_CD_DNS_SUFFIX = ".public-cd.vespa.oath.cloud"; - private final String name; + private final EndpointId id; + private final ClusterSpec.Id cluster; private final URI url; private final List<ZoneId> zones; private final Scope scope; @@ -37,16 +38,20 @@ public class Endpoint { private final RoutingMethod routingMethod; private final boolean tls; - private Endpoint(String name, URI url, List<ZoneId> zones, Scope scope, Port port, boolean legacy, RoutingMethod routingMethod) { - Objects.requireNonNull(name, "name must be non-null"); + private Endpoint(EndpointId id, ClusterSpec.Id cluster, URI url, List<ZoneId> zones, Scope scope, Port port, boolean legacy, RoutingMethod routingMethod) { + Objects.requireNonNull(cluster, "cluster must be non-null"); Objects.requireNonNull(zones, "zones must be non-null"); Objects.requireNonNull(scope, "scope must be non-null"); Objects.requireNonNull(port, "port must be non-null"); Objects.requireNonNull(routingMethod, "routingMethod must be non-null"); - if ((scope == Scope.zone || scope == Scope.weighted) && zones.size() != 1) { - throw new IllegalArgumentException("A single zone must be given for " + scope + "-scoped endpoints"); + if (scope == Scope.global) { + if (id == null) throw new IllegalArgumentException("Endpoint ID must be set for global endpoints"); + } else { + if (id != null) throw new IllegalArgumentException("Endpoint ID cannot be set for " + scope + " endpoints"); + if (zones.size() != 1) throw new IllegalArgumentException("A single zone must be given for " + scope + " endpoints"); } - this.name = name; + this.id = id; + this.cluster = cluster; this.url = url; this.zones = List.copyOf(zones); this.scope = scope; @@ -55,10 +60,11 @@ public class Endpoint { this.tls = port.tls; } - private Endpoint(String name, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system, + private Endpoint(EndpointId id, ClusterSpec.Id cluster, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system, Port port, boolean legacy, RoutingMethod routingMethod) { - this(name, - createUrl(name, + this(id, + cluster, + createUrl(endpointOrClusterAsString(id, cluster), Objects.requireNonNull(application, "application must be non-null"), zones, scope, @@ -70,14 +76,19 @@ public class Endpoint { } /** - * Returns the name of this endpoint (the first component of the DNS name). Depending on the endpoint type, this - * can be one of the following: - * - A wildcard (any scope) - * - A cluster name (only zone scope) - * - An endpoint ID (only global scope) + * Returns the name of this endpoint (the first component of the DNS name). This can be one of the following: + * + * - The wildcard character '*' (for wildcard endpoints, with any scope) + * - The cluster ID (zone scope) + * - The endpoint ID (global scope) */ public String name() { - return name; + return endpointOrClusterAsString(id, cluster); + } + + /** Returns the cluster ID to which this routes traffic */ + public ClusterSpec.Id cluster() { + return cluster; } /** Returns the URL used to access this */ @@ -106,7 +117,7 @@ public class Endpoint { return legacy; } - /** Returns the routing used for this */ + /** Returns the routing method used for this */ public RoutingMethod routingMethod() { return routingMethod; } @@ -125,7 +136,7 @@ public class Endpoint { public String upstreamIdOf(DeploymentId deployment) { if (scope != Scope.global) throw new IllegalArgumentException("Scope " + scope + " does not have upstream name"); if (!routingMethod.isShared()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not have upstream name"); - return upstreamIdOf(name, deployment.applicationId(), deployment.zoneId()); + return upstreamIdOf(name(), deployment.applicationId(), deployment.zoneId()); } @Override @@ -151,6 +162,10 @@ public class Endpoint { return dnsSuffix(system, false); } + private static String endpointOrClusterAsString(EndpointId id, ClusterSpec.Id cluster) { + return id == null ? cluster.value() : id.id(); + } + private static URI createUrl(String name, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system, Port port, boolean legacy, RoutingMethod routingMethod) { String scheme = port.tls ? "https" : "http"; @@ -190,7 +205,7 @@ public class Endpoint { if (scope == Scope.global) return "global"; var zone = zones.get(0); var region = zone.region().value(); - if (scope == Scope.weighted) region += "-w"; + if (scope == Scope.region) region += "-w"; if (!legacy && zone.environment().isProduction()) return region; // Skip prod environment for non-legacy endpoints return region + "." + zone.environment().value(); } @@ -261,15 +276,15 @@ public class Endpoint { /** An endpoint's scope */ public enum Scope { - /** Endpoint points to all zones */ + /** Endpoint points to one or more zones. Traffic is routed to the zone closest to the client */ global, + /** Endpoint points to one more zones in the same geographical region. Traffic is routed equally across zones */ + region, + /** Endpoint points to a single zone */ zone, - /** Endpoint points to a single region */ - weighted, - } /** Represents an endpoint's HTTP port */ @@ -323,7 +338,7 @@ public class Endpoint { if (!systemApplication.hasEndpoint()) throw new IllegalArgumentException(systemApplication + " has no endpoint"); RoutingMethod routingMethod = RoutingMethod.exclusive; Port port = url.getPort() == -1 ? Port.tls() : Port.tls(url.getPort()); // System application endpoints are always TLS - return new Endpoint("", url, List.of(zone), Scope.zone, port, false, routingMethod); + return new Endpoint(null, ClusterSpec.Id.from("admin"), url, List.of(zone), Scope.zone, port, false, routingMethod); } public static class EndpointBuilder { @@ -337,60 +352,55 @@ public class Endpoint { private Port port; private RoutingMethod routingMethod = RoutingMethod.shared; private boolean legacy = false; - private boolean wildcard = false; private EndpointBuilder(ApplicationId application) { this.application = application; } - /** Sets the cluster target for this */ + /** Sets the zone target for this */ public EndpointBuilder target(ClusterSpec.Id cluster, ZoneId zone) { - if (endpointId != null || wildcard) { - throw new IllegalArgumentException("Cannot set multiple target types"); - } + checkScope(); this.cluster = cluster; this.scope = Scope.zone; this.zones = List.of(zone); return this; } - /** Sets the endpoint target ID for this (as defined in deployments.xml) */ - public EndpointBuilder named(EndpointId endpointId) { - return named(endpointId, List.of()); + /** Sets the global target with given ID and pointing to the default cluster */ + public EndpointBuilder target(EndpointId endpointId) { + return target(endpointId, ClusterSpec.Id.from("default"), List.of()); } - /** Sets the endpoint ID for this (as defined in deployments.xml) */ - public EndpointBuilder named(EndpointId endpointId, List<ZoneId> targets) { - if (cluster != null || wildcard) { - throw new IllegalArgumentException("Cannot set multiple target types"); - } + /** Sets the global target with given ID, zones and cluster (as defined in deployments.xml) */ + public EndpointBuilder target(EndpointId endpointId, ClusterSpec.Id cluster, List<ZoneId> zones) { + checkScope(); this.endpointId = endpointId; - this.zones = targets; + this.cluster = cluster; + this.zones = zones; this.scope = Scope.global; return this; } /** Sets the global wildcard target for this */ public EndpointBuilder wildcard() { - return wildcard(Scope.global, List.of()); + return target(EndpointId.of("*"), ClusterSpec.Id.from("*"), List.of()); } /** Sets the zone wildcard target for this */ public EndpointBuilder wildcard(ZoneId zone) { - return wildcard(Scope.zone, List.of(zone)); + return target(ClusterSpec.Id.from("*"), zone); } - private EndpointBuilder wildcard(Scope scope, List<ZoneId> zones) { - if (endpointId != null || cluster != null) { - throw new IllegalArgumentException("Cannot set multiple target types"); - } - this.wildcard = true; - this.scope = scope; - this.zones = zones; + /** Sets the region target for this, deduced from given zone */ + public EndpointBuilder targetRegion(ClusterSpec.Id cluster, ZoneId zone) { + checkScope(); + this.cluster = cluster; + this.scope = Scope.region; + this.zones = List.of(effectiveZone(zone)); return this; } - /** Sets the port of this */ + /** Sets the port of this */ public EndpointBuilder on(Port port) { this.port = port; return this; @@ -408,35 +418,21 @@ public class Endpoint { return this; } - /** Make this a weighted endpoint */ - public EndpointBuilder weighted() { - if (scope != Scope.zone && scope != Scope.weighted) { - throw new IllegalArgumentException("Endpoint must target zone before making it weighted"); - } - this.scope = Scope.weighted; - this.zones = List.of(effectiveZone(zones.get(0))); - return this; - } - /** Sets the system that owns this */ public Endpoint in(SystemName system) { - String name; - if (wildcard) { - name = "*"; - } else if (endpointId != null) { - name = endpointId.id(); - } else if (cluster != null) { - name = cluster.value(); - } else { - throw new IllegalArgumentException("Must set either cluster, rotation or wildcard target"); - } if (system.isPublic() && routingMethod != RoutingMethod.exclusive) { throw new IllegalArgumentException("Public system only supports routing method " + RoutingMethod.exclusive); } if (routingMethod.isDirect() && !port.isDefault()) { throw new IllegalArgumentException("Routing method " + routingMethod + " can only use default port"); } - return new Endpoint(name, application, zones, scope, system, port, legacy, routingMethod); + return new Endpoint(endpointId, cluster, application, zones, scope, system, port, legacy, routingMethod); + } + + private void checkScope() { + if (scope != null) { + throw new IllegalArgumentException("Cannot change endpoint scope. Already set to " + scope); + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index 4da34dad737..e847667bf45 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.collections.AbstractFilteringList; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.zone.ZoneId; import java.util.Collection; @@ -33,6 +34,11 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> return matching(endpoint -> endpoint.name().equals(id.id())); } + /** Returns the subset of endpoints pointing to given cluster */ + public EndpointList cluster(ClusterSpec.Id cluster) { + return matching(endpoint -> endpoint.cluster().equals(cluster)); + } + /** Returns the subset of endpoints which target all of the given zones */ public EndpointList targets(List<ZoneId> zones) { return matching(endpoint -> endpoint.zones().containsAll(zones)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index 425364f6741..1cf42cf0073 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -12,10 +12,6 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; - -import java.util.LinkedHashSet; -import java.util.logging.Level; - import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.flags.BooleanFlag; @@ -24,8 +20,8 @@ import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; @@ -41,6 +37,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -48,6 +45,7 @@ import java.util.OptionalInt; import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -303,8 +301,8 @@ public class EndpointCertificateManager { List<Endpoint.EndpointBuilder> endpoints = new ArrayList<>(); - if(zone.environment().isProduction()) { - endpoints.add(Endpoint.of(applicationId).named(EndpointId.defaultId())); + if (zone.environment().isProduction()) { + endpoints.add(Endpoint.of(applicationId).target(EndpointId.defaultId())); endpoints.add(Endpoint.of(applicationId).wildcard()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 6bc0ec6baf8..89f57f6ea9b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -38,6 +37,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -244,10 +244,10 @@ public class InternalStepRunner implements StepRunner { .flatMap(action -> action.services.stream()) .map(service -> service.hostName) .sorted().distinct() - .map(Hostname::new) + .map(HostName::from) .forEach(hostname -> { - controller.applications().restart(new DeploymentId(id, type.zone(controller.system())), Optional.of(hostname)); - logger.log("Schedule service restart on host " + hostname.id() + "."); + controller.applications().restart(new DeploymentId(id, type.zone(controller.system())), new RestartFilter().withHostName(Optional.of(hostname))); + logger.log("Schedule service restart on host " + hostname.value() + "."); }); logger.log("Deployment successful."); if (prepareResponse.message != null) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 9a5f9b1074a..ac19b5dd5e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -47,7 +47,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; -import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -58,7 +57,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toUnmodifiableList; -import static java.util.stream.Collectors.toUnmodifiableMap; /** * A singleton owned by the controller, which contains the state and methods for controlling deployment jobs. @@ -379,8 +377,8 @@ public class JobController { * Accepts and stores a new application package and test jar pair under a generated application version key. */ public ApplicationVersion submit(TenantAndApplicationId id, Optional<SourceRevision> revision, Optional<String> authorEmail, - Optional<String> sourceUrl, Optional<String> commit, - long projectId, ApplicationPackage applicationPackage, byte[] testPackageBytes) { + Optional<String> sourceUrl, long projectId, ApplicationPackage applicationPackage, + byte[] testPackageBytes) { AtomicReference<ApplicationVersion> version = new AtomicReference<>(); controller.applications().lockApplicationOrThrow(id, application -> { long run = 1 + application.get().latestVersion() @@ -390,7 +388,7 @@ public class JobController { applicationPackage.compileVersion(), applicationPackage.buildTime(), sourceUrl, - commit)); + revision.map(SourceRevision::commit))); controller.applications().applicationStore().put(id.tenant(), id.application(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 786819d9442..1f20e48edf5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -17,6 +17,7 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.util.HashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; /** @@ -38,14 +39,15 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { } @Override - protected void maintain() { - confirmApplicationOwnerships(); - ensureConfirmationResponses(); - updateConfirmedApplicationOwners(); + protected boolean maintain() { + return confirmApplicationOwnerships() & + ensureConfirmationResponses() & + updateConfirmedApplicationOwners(); } /** File an ownership issue with the owners of all applications we know about. */ - private void confirmApplicationOwnerships() { + private boolean confirmApplicationOwnerships() { + AtomicBoolean success = new AtomicBoolean(true); applications() .withProjectId() .withProductionDeployment() @@ -63,10 +65,11 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { }).ifPresent(newIssueId -> store(newIssueId, application.id())); } catch (RuntimeException e) { // Catch errors due to wrong data in the controller, or issues client timeout. + success.set(false); log.log(Level.INFO, "Exception caught when attempting to file an issue for '" + application.id() + "': " + Exceptions.toMessageString(e)); } }); - + return success.get(); } private ApplicationSummary summaryOf(TenantAndApplicationId application) { @@ -85,7 +88,8 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { } /** Escalate ownership issues which have not been closed before a defined amount of time has passed. */ - private void ensureConfirmationResponses() { + private boolean ensureConfirmationResponses() { + AtomicBoolean success = new AtomicBoolean(true); for (Application application : applications()) application.ownershipIssueId().ifPresent(issueId -> { try { @@ -93,12 +97,14 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { ownershipIssues.ensureResponse(issueId, tenant.contact()); } catch (RuntimeException e) { + success.set(false); log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } }); + return success.get(); } - private void updateConfirmedApplicationOwners() { + private boolean updateConfirmedApplicationOwners() { applications() .withProjectId() .withProductionDeployment() @@ -112,6 +118,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { controller().applications().store(lockedApplication.withOwner(owner))); }); }); + return true; } private ApplicationList applications() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java index 4b96bd404ee..c0d79861fae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java @@ -13,9 +13,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Issue; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandler; import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -24,6 +24,7 @@ import java.util.stream.Collectors; * Automatically fetches and handles scheduled events from AWS: * 1. Deprovisions the affected hosts if applicable * 2. Submits an issue detailing the event if some hosts are not processed by 1. + * * @author mgimle */ public class CloudEventReporter extends ControllerMaintainer { @@ -44,40 +45,47 @@ public class CloudEventReporter extends ControllerMaintainer { } @Override - protected void maintain() { - log.log(Level.INFO, "Fetching events for cloud hosts."); + protected boolean maintain() { for (var awsRegion : zonesByCloudNativeRegion.keySet()) { List<CloudEvent> events = eventFetcher.getEvents(awsRegion); for (var event : events) { log.info(String.format("Retrieved event %s, affecting the following instances: %s", event.instanceEventId, event.affectedInstances)); - List<String> deprovisionedHosts = deprovisionHosts(awsRegion, event); - submitIssue(event, deprovisionedHosts); + List<Node> needsManualIntervention = handleInstances(awsRegion, event); + if (!needsManualIntervention.isEmpty()) + submitIssue(event); } } + return true; } - private List<String> deprovisionHosts(String awsRegion, CloudEvent event) { - return zonesByCloudNativeRegion.get(awsRegion) - .stream() - .flatMap(zone -> - nodeRepository.list(zone.getId()) - .stream() - .filter(shouldDeprovisionHost(event)) - .map(node -> { - if (!node.wantToDeprovision() || !node.wantToRetire()) - log.info(String.format("Setting host %s to wantToRetire and wantToDeprovision", node.hostname().value())); - nodeRepository.retireAndDeprovision(zone.getId(), node.hostname().value()); - return node.hostname().value(); - }) - ) - .collect(Collectors.toList()); + /** + * Handles affected instances in the following way: + * 1. Ignore if unknown instance, presumably belongs to different system + * 2. Retire and deprovision if tenant host + * 3. Submit issue if infrastructure host, as it requires manual intervention + */ + private List<Node> handleInstances(String awsRegion, CloudEvent event) { + List<Node> needsManualIntervention = new ArrayList<>(); + for (var zone : zonesByCloudNativeRegion.get(awsRegion)) { + for (var node : nodeRepository.list(zone.getId())) { + if (!isAffected(node, event)){ + continue; + } + if (node.type() == NodeType.host) { + log.info(String.format("Setting host %s to wantToRetire and wantToDeprovision", node.hostname().value())); + nodeRepository.retireAndDeprovision(zone.getId(), node.hostname().value()); + } + else { + needsManualIntervention.add(node); + } + } + } + return needsManualIntervention; } - private void submitIssue(CloudEvent event, List<String> deprovisionedHosts) { - if (event.affectedInstances.size() == deprovisionedHosts.size()) - return; + private void submitIssue(CloudEvent event) { Issue issue = eventFetcher.createIssue(event); if (!issueHandler.issueExists(issue)) { issueHandler.file(issue); @@ -85,11 +93,9 @@ public class CloudEventReporter extends ControllerMaintainer { } } - private Predicate<Node> shouldDeprovisionHost(CloudEvent event) { - return node -> - node.type() == NodeType.host && - event.affectedInstances.stream() - .anyMatch(instance -> node.hostname().value().contains(instance)); + private boolean isAffected(Node node, CloudEvent event) { + return event.affectedInstances.stream() + .anyMatch(instance -> node.hostname().value().contains(instance)); } private Map<String, List<ZoneApi>> getZonesByCloudNativeRegion() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index 4aba8d881bf..e19f3b4f9a2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -35,8 +35,9 @@ public class ContactInformationMaintainer extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { TenantController tenants = controller().tenants(); + boolean success = true; for (Tenant tenant : tenants.asList()) { log.log(INFO, "Updating contact information for " + tenant); try { @@ -55,11 +56,13 @@ public class ContactInformationMaintainer extends ControllerMaintainer { throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } } catch (Exception e) { + success = false; log.log(Level.WARNING, "Failed to update contact information for " + tenant + ": " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } } + return success; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java index 2b7c78f96d0..9bf6352813a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java @@ -1,12 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.concurrent.maintenance.JobMetrics; import com.yahoo.concurrent.maintenance.Maintainer; import com.yahoo.config.provision.SystemName; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; import java.time.Duration; import java.util.EnumSet; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.logging.Logger; @@ -30,7 +33,8 @@ public abstract class ControllerMaintainer extends Maintainer { } public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems) { - super(name, interval, controller.clock().instant(), controller.jobControl(), controller.curator().cluster()); + super(name, interval, controller.clock().instant(), controller.jobControl(), + jobMetrics(controller.metric()), controller.curator().cluster()); this.controller = controller; this.activeSystems = Set.copyOf(Objects.requireNonNull(activeSystems)); } @@ -43,4 +47,10 @@ public abstract class ControllerMaintainer extends Maintainer { super.run(); } + private static JobMetrics jobMetrics(Metric metric) { + return new JobMetrics((job, consecutiveFailures) -> { + metric.set("maintenance.consecutiveFailures", consecutiveFailures, metric.createContext(Map.of("job", job))); + }); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index ca695a2d234..10b21ece233 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -69,7 +69,7 @@ public class ControllerMaintenance extends AbstractComponent { nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10)); costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), controller.serviceRegistry().costReportConsumer()); resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(1), metric, controller.serviceRegistry().meteringService()); - cloudEventReporter = new CloudEventReporter(controller, Duration.ofDays(1)); + cloudEventReporter = new CloudEventReporter(controller, Duration.ofMinutes(30)); rotationStatusUpdater = new RotationStatusUpdater(controller, maintenanceInterval); resourceTagMaintainer = new ResourceTagMaintainer(controller, Duration.ofMinutes(30), controller.serviceRegistry().resourceTagger()); systemRoutingPolicyMaintainer = new SystemRoutingPolicyMaintainer(controller, Duration.ofMinutes(10)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index d028a88fb92..28b64b5bfe0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -31,9 +31,10 @@ public class CostReportMaintainer extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations()); consumer.consume(csv); + return true; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index bb2161bca1d..7bd2c737fcb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -24,7 +23,8 @@ public class DeploymentExpirer extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { + boolean success = true; for (Application application : controller().applications().readable()) for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) { @@ -34,11 +34,13 @@ public class DeploymentExpirer extends ControllerMaintainer { log.log(Level.INFO, "Expiring deployment of " + instance.id() + " in " + deployment.zone()); controller().applications().deactivate(instance.id(), deployment.zone()); } catch (Exception e) { + success = false; log.log(Level.WARNING, "Could not expire " + deployment + " of " + instance + ": " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } } + return success; } /** Returns whether given deployment has expired according to its TTL */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 89f1e0fe840..a94e7407898 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; -import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Application; @@ -20,6 +19,7 @@ import java.time.Duration; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.broken; @@ -44,10 +44,10 @@ public class DeploymentIssueReporter extends ControllerMaintainer { } @Override - protected void maintain() { - maintainDeploymentIssues(applications()); - maintainPlatformIssue(applications()); - escalateInactiveDeploymentIssues(applications()); + protected boolean maintain() { + return maintainDeploymentIssues(applications()) & + maintainPlatformIssue(applications()) & + escalateInactiveDeploymentIssues(applications()); } /** Returns the applications to maintain issue status for. */ @@ -62,7 +62,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { * and store the issue id for the filed issues. Also, clear the issueIds of applications * where deployment has not failed for this amount of time. */ - private void maintainDeploymentIssues(List<Application> applications) { + private boolean maintainDeploymentIssues(List<Application> applications) { List<TenantAndApplicationId> failingApplications = controller().jobController().deploymentStatuses(ApplicationList.from(applications)) .failingApplicationChangeSince(controller().clock().instant().minus(maxFailureAge)) .mapToList(status -> status.application().id()); @@ -72,6 +72,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { fileDeploymentIssueFor(application); else store(application.id(), null); + return true; } /** @@ -79,24 +80,26 @@ public class DeploymentIssueReporter extends ControllerMaintainer { * applications that have been failing the upgrade to the system version for * longer than the set grace period, or update this list if the issue already exists. */ - private void maintainPlatformIssue(List<Application> applications) { + private boolean maintainPlatformIssue(List<Application> applications) { + boolean success = true; if (controller().system() == SystemName.cd) - return; + return success; Version systemVersion = controller().systemVersion(); if ((controller().versionStatus().version(systemVersion).confidence() != broken)) - return; + return success; DeploymentStatusList statuses = controller().jobController().deploymentStatuses(ApplicationList.from(applications)); if (statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant().minus(upgradeGracePeriod)).isEmpty()) - return; + return success; List<ApplicationId> failingApplications = statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant()) .mapToList(status -> status.application().id().defaultInstance()); // TODO jonmv: Send only tenant and application, here and elsewhere in this. deploymentIssues.fileUnlessOpen(failingApplications, systemVersion); + return success; } private Tenant ownerOf(TenantAndApplicationId applicationId) { @@ -121,7 +124,8 @@ public class DeploymentIssueReporter extends ControllerMaintainer { } /** Escalate issues for which there has been no activity for a certain amount of time. */ - private void escalateInactiveDeploymentIssues(Collection<Application> applications) { + private boolean escalateInactiveDeploymentIssues(Collection<Application> applications) { + AtomicBoolean success = new AtomicBoolean(true); applications.forEach(application -> application.deploymentIssueId().ifPresent(issueId -> { try { Tenant tenant = ownerOf(application.id()); @@ -130,9 +134,11 @@ public class DeploymentIssueReporter extends ControllerMaintainer { tenant.type() == Tenant.Type.athenz ? tenant.contact() : Optional.empty()); } catch (RuntimeException e) { + success.set(false); log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } })); + return success.get(); } private void store(TenantAndApplicationId id, IssueId issueId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index c03be2ca1d1..c8416578932 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -39,7 +38,7 @@ public class DeploymentMetricsMaintainer extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { AtomicInteger failures = new AtomicInteger(0); AtomicInteger attempts = new AtomicInteger(0); AtomicReference<Exception> lastException = new AtomicReference<>(null); @@ -91,6 +90,7 @@ public class DeploymentMetricsMaintainer extends ControllerMaintainer { } catch (InterruptedException e) { throw new RuntimeException(e); } + return lastException.get() == null; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 7006458538d..7952355d5fb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -35,8 +35,9 @@ public abstract class InfrastructureUpgrader<VERSION> extends ControllerMaintain } @Override - protected void maintain() { + protected boolean maintain() { targetVersion().ifPresent(target -> upgradeAll(target, SystemApplication.all())); + return true; } /** Deploy a list of system applications until they converge on the given version */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index cfe9257bdf8..e0f2f0718ef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -48,9 +48,10 @@ public class JobRunner extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { jobs.active().forEach(this::advance); jobs.collectGarbage(); + return true; } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index cc4a8c628eb..0c5ef123eef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -68,12 +68,13 @@ public class MetricsReporter extends ControllerMaintainer { } @Override - public void maintain() { + public boolean maintain() { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); reportInfrastructureUpgradeMetrics(); reportAuditLog(); + return true; } private void reportAuditLog() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java index 9febc73a5a7..e223809a211 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java @@ -38,12 +38,13 @@ public class NameServiceDispatcher extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { + boolean success = true; try (var lock = db.lockNameServiceQueue()) { var queue = db.readNameServiceQueue(); var instant = clock.instant(); var remaining = queue.dispatchTo(nameService, requestCount); - if (queue == remaining) return; // Queue unchanged + if (queue == remaining) return success; // Queue unchanged var dispatched = queue.first(requestCount); if (!dispatched.requests().isEmpty()) { @@ -53,6 +54,7 @@ public class NameServiceDispatcher extends ControllerMaintainer { } db.writeNameServiceQueue(remaining); } + return success; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index a62b1745145..20febfaea1d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.yolean.Exceptions; @@ -19,14 +18,16 @@ public class OsVersionStatusUpdater extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { try { OsVersionStatus newStatus = OsVersionStatus.compute(controller()); controller().updateOsVersionStatus(newStatus); + return true; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } + return false; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java index 5dd62251759..a032f266de5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java @@ -19,12 +19,13 @@ public class OutstandingChangeDeployer extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { for (Application application : ApplicationList.from(controller().applications().readable()) .withProductionDeployment() .withDeploymentSpec() .asList()) controller().applications().deploymentTrigger().triggerNewRevision(application.id()); + return true; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java index 32b65f05cac..a626f21359a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReadyJobsTrigger.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.Controller; import java.time.Duration; @@ -18,8 +17,9 @@ public class ReadyJobsTrigger extends ControllerMaintainer { } @Override - public void maintain() { + public boolean maintain() { controller().applications().deploymentTrigger().triggerReadyJobs(); + return true; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 76a186a2f6b..f460561df08 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -50,13 +50,15 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { try { collectResourceSnapshots(); + return true; } catch (Exception e) { log.log(Level.WARNING, "Failed to collect resource snapshots. Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); } + return false; } private void collectResourceSnapshots() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index 31434de472d..863302223ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -1,7 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; @@ -27,7 +26,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer { } @Override - public void maintain() { + public boolean maintain() { controller().zoneRegistry().zones() .ofCloud(CloudName.from("aws")) .reachable() @@ -37,8 +36,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer { if (taggedResources > 0) log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId()); }); - - + return true; } private Map<HostName, ApplicationId> getTenantOfParentHosts(ZoneId zoneId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java index 245747a882f..935bcbec597 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java @@ -41,7 +41,7 @@ public class RotationStatusUpdater extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { var failures = new AtomicInteger(0); var attempts = new AtomicInteger(0); var lastException = new AtomicReference<Exception>(null); @@ -78,6 +78,7 @@ public class RotationStatusUpdater extends ControllerMaintainer { } catch (InterruptedException e) { throw new RuntimeException(e); } + return lastException.get() == null; } private RotationStatus getStatus(Instance instance) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java index 0fe6f7e0bfb..3b0a1fca4af 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java @@ -21,13 +21,14 @@ public class SystemRoutingPolicyMaintainer extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { for (var zone : controller().zoneRegistry().zones().all().ids()) { for (var application : SystemApplication.values()) { if (!application.hasEndpoint()) continue; controller().routing().policies().refresh(application.id(), DeploymentSpec.empty, zone); } } + return true; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index bf44c796f34..d9c78c8a442 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -62,15 +63,12 @@ public class SystemUpgrader extends InfrastructureUpgrader<Version> { if (application.hasApplicationPackage()) { // For applications with package we do not have a zone-wide version target. This means that we must check // the wanted version of each node. + boolean zoneHasSharedRouting = controller().zoneRegistry().routingMethods(zone.getId()).stream() + .anyMatch(RoutingMethod::isShared); return minVersion(zone, application, Node::wantedVersion) - // Upgrade if target is after any wanted version - .map(target::isAfter) - // Skip upgrade if there are no nodes allocated. This is overloaded to mean that the zone is not - // expected to have a deployment of this application. - // TODO(mpolden): Once all zones are either directly routed or not: Change this to - // always deploy proxy app and wait for convergence in zones that are not directly - // routed. - .orElse(false); + .map(target::isAfter) // Upgrade if target is after any wanted version + .orElse(zoneHasSharedRouting); // Always upgrade if zone uses shared routing, but has no nodes allocated yet + } return controller().serviceRegistry().configServer().nodeRepository() .targetVersionsOf(zone.getId()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 5f0f2e4ba4e..9ab2b0e77e8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -51,7 +51,7 @@ public class Upgrader extends ControllerMaintainer { * Schedule application upgrades. Note that this implementation must be idempotent. */ @Override - public void maintain() { + public boolean maintain() { // Determine target versions for each upgrade policy Version canaryTarget = controller().systemVersion(); Collection<Version> defaultTargets = targetVersions(Confidence.normal); @@ -89,6 +89,7 @@ public class Upgrader extends ControllerMaintainer { upgrade(instances.with(UpgradePolicy.canary), canaryTarget, instances.size()); defaultTargets.forEach(target -> upgrade(instances.with(UpgradePolicy.defaultPolicy), target, numberOfApplicationsToUpgrade())); conservativeTargets.forEach(target -> upgrade(instances.with(UpgradePolicy.conservative), target, numberOfApplicationsToUpgrade())); + return true; } /** Returns the target versions for given confidence, one per major version in the system */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index d8b74a4ae99..a3e9672b715 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -29,7 +29,7 @@ public class VersionStatusUpdater extends ControllerMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { try { VersionStatus newStatus = VersionStatus.compute(controller()); controller().updateVersionStatus(newStatus); @@ -37,10 +37,12 @@ public class VersionStatusUpdater extends ControllerMaintainer { controller().serviceRegistry().systemMonitor().reportSystemVersion(version.versionNumber(), convert(version.confidence())); }); + return true; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } + return false; } static SystemMonitor.Confidence convert(VespaVersion.Confidence confidence) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 2429c5ee8c5..2697651f61b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -95,7 +95,6 @@ public class RoutingPolicySerializer { } public GlobalRouting globalRoutingFromSlime(Inspector object) { - if (!object.valid()) return GlobalRouting.DEFAULT_STATUS; var status = GlobalRouting.Status.valueOf(object.field(statusField).asString()); var agent = GlobalRouting.Agent.valueOf(object.field(agentField).asString()); var changedAt = Serializers.optionalInstant(object.field(changedAtField)).orElse(Instant.EPOCH); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index dc3c14c76b7..5b35a1aa5c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -10,7 +10,9 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; @@ -42,7 +44,6 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Application; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; @@ -54,6 +55,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; @@ -284,6 +286,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/key")) return removeDeveloperKey(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deployment")) return removeAllProdDeployments(path.get("tenant"), path.get("application")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", "all"); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", path.get("choice")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return removeDeployKey(path.get("tenant"), path.get("application"), request); @@ -953,8 +956,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .ifPresent(version -> toSlime(version, object.setObject("revision"))); } - private void toSlime(Endpoint endpoint, String cluster, Cursor object) { - object.setString("cluster", cluster); + private void toSlime(Endpoint endpoint, Cursor object) { + object.setString("cluster", endpoint.cluster().value()); object.setBool("tls", endpoint.tls()); object.setString("url", endpoint.url().toString()); object.setString("scope", endpointScopeString(endpoint.scope())); @@ -971,16 +974,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Add zone endpoints var endpointArray = response.setArray("endpoints"); - for (var endpoint : controller.routing().endpointsOf(deploymentId).scope(Endpoint.Scope.zone)) { - toSlime(endpoint, endpoint.name(), endpointArray.addObject()); + for (var endpoint : controller.routing().endpointsOf(deploymentId) + .scope(Endpoint.Scope.zone) + .not().legacy()) { + toSlime(endpoint, endpointArray.addObject()); } // Add global endpoints var globalEndpoints = controller.routing().endpointsOf(application, deploymentId.applicationId().instance()) .not().legacy() .targets(deploymentId.zoneId()); for (var endpoint : globalEndpoints) { - // TODO(mpolden): Pass cluster name. Cluster that a global endpoint points to is not available at this level. - toSlime(endpoint, "", endpointArray.addObject()); + toSlime(endpoint, endpointArray.addObject()); } response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); @@ -1385,11 +1389,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse restart(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), ZoneId.from(environment, region)); + RestartFilter restartFilter = new RestartFilter() + .withHostName(Optional.ofNullable(request.getProperty("hostname")).map(HostName::from)) + .withClusterType(Optional.ofNullable(request.getProperty("clusterType")).map(ClusterSpec.Type::from)) + .withClusterId(Optional.ofNullable(request.getProperty("clusterId")).map(ClusterSpec.Id::from)); - // TODO: Propagate all filters - Optional<Hostname> hostname = Optional.ofNullable(request.getProperty("hostname")).map(Hostname::new); - controller.applications().restart(deploymentId, hostname); - + controller.applications().restart(deploymentId, restartFilter); return new MessageResponse("Requested restart of " + deploymentId); } @@ -1900,12 +1905,18 @@ public class ApplicationApiHandler extends LoggingRequestHandler { sourceRevision, authorEmail, sourceUrl, - commit, projectId, applicationPackage, dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); } + private HttpResponse removeAllProdDeployments(String tenant, String application) { + JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, + Optional.empty(), Optional.empty(), Optional.empty(), 1, + ApplicationPackage.deploymentRemoval(), new byte[0]); + return new MessageResponse("All deployments removed"); + } + private static Map<String, byte[]> parseDataParts(HttpRequest request) { String contentHash = request.getHeader("x-Content-Hash"); if (contentHash == null) @@ -1946,6 +1957,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private static String endpointScopeString(Endpoint.Scope scope) { switch (scope) { + case region: return "region"; case global: return "global"; case zone: return "zone"; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index f72439b694a..0e25ee1fe85 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -525,13 +525,12 @@ class JobControllerApiHandlerHelper { */ static HttpResponse submitResponse(JobController jobController, String tenant, String application, Optional<SourceRevision> sourceRevision, Optional<String> authorEmail, - Optional<String> sourceUrl, Optional<String> commit, long projectId, + Optional<String> sourceUrl, long projectId, ApplicationPackage applicationPackage, byte[] testPackage) { ApplicationVersion version = jobController.submit(TenantAndApplicationId.from(tenant, application), sourceRevision, authorEmail, sourceUrl, - commit, projectId, applicationPackage, testPackage); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 523f3533a7f..0356e11ae36 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -7,9 +7,6 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; @@ -48,12 +45,10 @@ public class RoutingPolicies { private final Controller controller; private final CuratorDb db; - private final BooleanFlag weightedDnsPerRegion; public RoutingPolicies(Controller controller) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); this.db = controller.curator(); - this.weightedDnsPerRegion = Flags.WEIGHTED_DNS_PER_REGION.bindTo(controller.flagSource()); try (var lock = db.lockRoutingPolicies()) { // Update serialized format for (var policy : db.readRoutingPolicies().entrySet()) { db.writeRoutingPolicies(policy.getKey(), policy.getValue()); @@ -92,7 +87,7 @@ public class RoutingPolicies { removeGlobalDnsUnreferencedBy(allocation, lock); storePoliciesOf(allocation, lock); removePoliciesUnreferencedBy(allocation, lock); - updateGlobalDnsOf(application, get(allocation.deployment.applicationId()).values(), inactiveZones, lock); + updateGlobalDnsOf(get(allocation.deployment.applicationId()).values(), inactiveZones, lock); } } @@ -102,8 +97,8 @@ public class RoutingPolicies { db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, GlobalRouting.status(status, GlobalRouting.Agent.operator, controller.clock().instant()))); Map<ApplicationId, Map<RoutingPolicyId, RoutingPolicy>> allPolicies = db.readRoutingPolicies(); - for (var kv : allPolicies.entrySet()) { - updateGlobalDnsOf(kv.getKey(), kv.getValue().values(), Set.of(), lock); + for (var applicationPolicies : allPolicies.values()) { + updateGlobalDnsOf(applicationPolicies.values(), Set.of(), lock); } } } @@ -120,56 +115,7 @@ public class RoutingPolicies { newPolicies.put(policy.id(), newPolicy); } db.writeRoutingPolicies(deployment.applicationId(), newPolicies); - updateGlobalDnsOf(deployment.applicationId(), newPolicies.values(), Set.of(), lock); - } - } - - /** Update global DNS record for given policies */ - private void legacyUpdateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { - // Create DNS record for each routing ID - var routingTable = routingTableFrom(routingPolicies); - for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { - var targets = new LinkedHashSet<AliasTarget>(); - var staleTargets = new LinkedHashSet<AliasTarget>(); - for (var policy : routeEntry.getValue()) { - if (policy.dnsZone().isEmpty()) continue; - if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(RoutingMethod.exclusive)) continue; - var target = new LatencyAliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone()); - var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); - // Remove target zone if global routing status is set out at: - // - zone level (ZoneRoutingPolicy) - // - deployment level (RoutingPolicy) - // - application package level (deployment.xml) - if (isConfiguredOut(policy, zonePolicy, inactiveZones)) { - staleTargets.add(target); - } else { - targets.add(target); - } - } - // If all targets are configured out, all targets are set in. We do this because otherwise removing 100% of - // the ALIAS records would cause the global endpoint to stop resolving entirely (NXDOMAIN). - if (targets.isEmpty() && !staleTargets.isEmpty()) { - targets.addAll(staleTargets); - staleTargets.clear(); - } - if (!targets.isEmpty()) { - var endpoints = controller.routing().endpointsOf(routeEntry.getKey().application()) - .named(routeEntry.getKey().endpointId()) - .not().requiresRotation(); - endpoints.forEach(endpoint -> controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal)); - } - staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, - RecordData.fqdn(t.name().value()), - Priority.normal)); - } - } - - // TODO(mpolden): Remove and inline call to updateGlobalDnsOf when feature flag disappears - private void updateGlobalDnsOf(ApplicationId application, Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { - if (useWeightedDnsPerRegion(application)) { - updateGlobalDnsOf(routingPolicies, inactiveZones, lock); - } else { - legacyUpdateGlobalDnsOf(routingPolicies, inactiveZones, lock); + updateGlobalDnsOf(newPolicies.values(), Set.of(), lock); } } @@ -220,7 +166,7 @@ public class RoutingPolicies { for (var policy : policies) { if (policy.dnsZone().isEmpty()) continue; if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(routingMethod)) continue; - Endpoint weighted = policy.weightedEndpointIn(controller.system(), routingMethod); + Endpoint regionEndpoint = policy.regionEndpointIn(controller.system(), routingMethod); var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); long weight = 1; if (isConfiguredOut(policy, zonePolicy, inactiveZones)) { @@ -229,9 +175,9 @@ public class RoutingPolicies { } var weightedTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone(), weight); - endpoints.computeIfAbsent(weighted, (k) -> new RegionEndpoint(new LatencyAliasTarget(HostName.from(weighted.dnsName()), - policy.dnsZone().get(), - policy.id().zone()))) + endpoints.computeIfAbsent(regionEndpoint, (k) -> new RegionEndpoint(new LatencyAliasTarget(HostName.from(regionEndpoint.dnsName()), + policy.dnsZone().get(), + policy.id().zone()))) .zoneTargets() .add(weightedTarget); } @@ -245,7 +191,7 @@ public class RoutingPolicies { var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), allocation.deployment.zoneId()); var existingPolicy = policies.get(policyId); var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.dnsZone(), - allocation.endpointIdsOf(loadBalancer, useWeightedDnsPerRegion(loadBalancer.application())), + allocation.endpointIdsOf(loadBalancer), new Status(isActive(loadBalancer), GlobalRouting.DEFAULT_STATUS)); // Preserve global routing status for existing policy if (existingPolicy != null) { @@ -301,10 +247,10 @@ public class RoutingPolicies { } /** Compute routing IDs from given load balancers */ - private Set<RoutingId> routingIdsFrom(LoadBalancerAllocation allocation) { + private static Set<RoutingId> routingIdsFrom(LoadBalancerAllocation allocation) { Set<RoutingId> routingIds = new LinkedHashSet<>(); for (var loadBalancer : allocation.loadBalancers) { - for (var endpointId : allocation.endpointIdsOf(loadBalancer, useWeightedDnsPerRegion(loadBalancer.application()))) { + for (var endpointId : allocation.endpointIdsOf(loadBalancer)) { routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); } } @@ -344,10 +290,6 @@ public class RoutingPolicies { return false; } - private boolean useWeightedDnsPerRegion(ApplicationId application) { - return weightedDnsPerRegion.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value(); - } - /** Represents records for a region-wide endpoint */ private static class RegionEndpoint { @@ -409,7 +351,7 @@ public class RoutingPolicies { } /** Compute all endpoint IDs for given load balancer */ - private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer, boolean useWeightedDnsPerRegion) { + private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { if (!deployment.zoneId().environment().isProduction()) { // Only production deployments have configurable endpoints return Set.of(); } @@ -417,7 +359,7 @@ public class RoutingPolicies { if (instanceSpec.isEmpty()) { return Set.of(); } - if (useWeightedDnsPerRegion && instanceSpec.get().globalServiceId().filter(id -> id.equals(loadBalancer.cluster().value())).isPresent()) { + if (instanceSpec.get().globalServiceId().filter(id -> id.equals(loadBalancer.cluster().value())).isPresent()) { // Legacy assignment always has the default endpoint Id return Set.of(EndpointId.defaultId()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 7f4a707949b..f87c6e2d11c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -69,17 +69,17 @@ public class RoutingPolicy { return new RoutingPolicy(id, canonicalName, dnsZone, endpoints, status); } - /** Returns the endpoint of this */ + /** Returns the zone endpoint of this */ public Endpoint endpointIn(SystemName system, RoutingMethod routingMethod, ZoneRegistry zoneRegistry) { Optional<Endpoint> infraEndpoint = SystemApplication.matching(id.owner()) .flatMap(app -> app.endpointIn(id.zone(), zoneRegistry)); if (infraEndpoint.isPresent()) return infraEndpoint.get(); - return endpoint(routingMethod).in(system); + return endpoint(routingMethod).target(id.cluster(), id.zone()).in(system); } - /** Returns the weighted endpoint of this */ - public Endpoint weightedEndpointIn(SystemName system, RoutingMethod routingMethod) { - return endpoint(routingMethod).weighted().in(system); + /** Returns the region endpoint of this */ + public Endpoint regionEndpointIn(SystemName system, RoutingMethod routingMethod) { + return endpoint(routingMethod).targetRegion(id.cluster(), id.zone()).in(system); } @Override @@ -104,7 +104,6 @@ public class RoutingPolicy { private Endpoint.EndpointBuilder endpoint(RoutingMethod routingMethod) { return Endpoint.of(id.owner()) - .target(id.cluster(), id.zone()) .on(Port.fromRoutingMethod(routingMethod)) .routingMethod(routingMethod); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 9e6eb9ca2e1..26f718ae5ff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -18,8 +18,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -29,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.LatencyAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; @@ -822,10 +821,16 @@ public class ControllerTest { context.submit(applicationPackage).deploy(); var expectedRecords = List.of( - // The 'east' global endpoint, pointing to zone 2 with exclusive routing + // The weighted record for zone 2's region + new Record(Record.Type.ALIAS, + RecordName.from("application.tenant.us-east-3-w.vespa.oath.cloud"), + new WeightedAliasTarget(HostName.from("lb-0--tenant:application:default--prod.us-east-3"), + "dns-zone-1", ZoneId.from("prod.us-east-3"), 1).pack()), + + // The 'east' global endpoint, pointing to the weighted record for zone 2's region new Record(Record.Type.ALIAS, RecordName.from("east.application.tenant.global.vespa.oath.cloud"), - new LatencyAliasTarget(HostName.from("lb-0--tenant:application:default--prod.us-east-3"), + new LatencyAliasTarget(HostName.from("application.tenant.us-east-3-w.vespa.oath.cloud"), "dns-zone-1", ZoneId.from("prod.us-east-3")).pack()), // The 'default' global endpoint, pointing to both zones with shared routing, via rotation @@ -861,26 +866,20 @@ public class ControllerTest { .stream() .map(Endpoint::routingMethod) .collect(Collectors.toSet()); - ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.ALLOW_DIRECT_ROUTING.id(), false); - // Without everything + // Without satisfying any requirement context.submit(applicationPackageBuilder.build()).deploy(); assertEquals(Set.of(RoutingMethod.shared), routingMethods.get()); - // Without Athenz service + // Without satisfying Athenz service requirement context.submit(applicationPackageBuilder.compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION).build()) .deploy(); assertEquals(Set.of(RoutingMethod.shared), routingMethods.get()); - // Without feature flag - applicationPackageBuilder = applicationPackageBuilder.compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION) - .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")); - context.submit(applicationPackageBuilder.build()).deploy(); - assertEquals(Set.of(RoutingMethod.shared), routingMethods.get()); - - // With everything required - ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.ALLOW_DIRECT_ROUTING.id(), true); - context.submit(applicationPackageBuilder.build()).deploy(); + // Satisfying all requirements + context.submit(applicationPackageBuilder.compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION) + .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) + .build()).deploy(); assertEquals(Set.of(RoutingMethod.shared, RoutingMethod.sharedLayer4), routingMethods.get()); // Global endpoint is configured and includes directly routed endpoint name diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java new file mode 100644 index 00000000000..f7f0c9ce58e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java @@ -0,0 +1,27 @@ +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.ValidationId; +import org.junit.Test; + +import java.time.Instant; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author valerijf + */ +public class ApplicationPackageTest { + @Test + public void test_createEmptyForDeploymentRemoval() { + ApplicationPackage app = ApplicationPackage.deploymentRemoval(); + assertEquals(DeploymentSpec.empty, app.deploymentSpec()); + assertEquals(List.of(), app.trustedCertificates()); + + for (ValidationId validationId : ValidationId.values()) { + assertTrue(app.validationOverrides().allows(validationId, Instant.now())); + } + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index 2e57a5eaaa1..eb97fa0725c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -29,43 +29,43 @@ public class EndpointTest { Map<String, Endpoint> tests = Map.of( // Legacy endpoint "http://a1.t1.global.vespa.yahooapis.com:4080/", - Endpoint.of(app1).named(endpointId).on(Port.plain(4080)).legacy().in(SystemName.main), + Endpoint.of(app1).target(endpointId).on(Port.plain(4080)).legacy().in(SystemName.main), // Legacy endpoint with TLS "https://a1--t1.global.vespa.yahooapis.com:4443/", - Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).legacy().in(SystemName.main), + Endpoint.of(app1).target(endpointId).on(Port.tls(4443)).legacy().in(SystemName.main), // Main endpoint "https://a1--t1.global.vespa.oath.cloud:4443/", - Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).in(SystemName.main), + Endpoint.of(app1).target(endpointId).on(Port.tls(4443)).in(SystemName.main), // Main endpoint in CD "https://cd--a1--t1.global.vespa.oath.cloud:4443/", - Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).in(SystemName.cd), + Endpoint.of(app1).target(endpointId).on(Port.tls(4443)).in(SystemName.cd), // Main endpoint in CD "https://cd--i2--a2--t2.global.vespa.oath.cloud:4443/", - Endpoint.of(app2).named(endpointId).on(Port.tls(4443)).in(SystemName.cd), + Endpoint.of(app2).target(endpointId).on(Port.tls(4443)).in(SystemName.cd), // Main endpoint with direct routing and default TLS port "https://a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), + Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint with custom rotation name "https://r1.a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), + Endpoint.of(app1).target(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance in default rotation "https://i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), + Endpoint.of(app2).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance with custom rotation name "https://r2.i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), + Endpoint.of(app2).target(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint in public system "https://a1.t1.global.public.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) + Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } @@ -77,43 +77,43 @@ public class EndpointTest { Map<String, Endpoint> tests = Map.of( // Legacy endpoint "http://a1.t1.global.vespa.yahooapis.com:4080/", - Endpoint.of(app1).named(endpointId).on(Port.plain(4080)).legacy().in(SystemName.main), + Endpoint.of(app1).target(endpointId).on(Port.plain(4080)).legacy().in(SystemName.main), // Legacy endpoint with TLS "https://a1--t1.global.vespa.yahooapis.com:4443/", - Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).legacy().in(SystemName.main), + Endpoint.of(app1).target(endpointId).on(Port.tls(4443)).legacy().in(SystemName.main), // Main endpoint "https://a1--t1.global.vespa.oath.cloud:4443/", - Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).in(SystemName.main), + Endpoint.of(app1).target(endpointId).on(Port.tls(4443)).in(SystemName.main), // Main endpoint in CD "https://cd--i2--a2--t2.global.vespa.oath.cloud:4443/", - Endpoint.of(app2).named(endpointId).on(Port.tls(4443)).in(SystemName.cd), + Endpoint.of(app2).target(endpointId).on(Port.tls(4443)).in(SystemName.cd), // Main endpoint in CD "https://cd--a1--t1.global.vespa.oath.cloud:4443/", - Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).in(SystemName.cd), + Endpoint.of(app1).target(endpointId).on(Port.tls(4443)).in(SystemName.cd), // Main endpoint with direct routing and default TLS port "https://a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), + Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint with custom rotation name "https://r1.a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), + Endpoint.of(app1).target(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance in default rotation "https://i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), + Endpoint.of(app2).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance with custom rotation name "https://r2.i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), + Endpoint.of(app2).target(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint in public system "https://a1.t1.global.public.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) + Endpoint.of(app1).target(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } @@ -178,7 +178,7 @@ public class EndpointTest { // Default rotation "https://a1.t1.global.public.vespa.oath.cloud/", Endpoint.of(app1) - .named(EndpointId.defaultId()) + .target(EndpointId.defaultId()) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), @@ -233,30 +233,26 @@ public class EndpointTest { Map<String, Endpoint> tests = Map.of( "https://a1.t1.us-north-1-w.public.vespa.oath.cloud/", Endpoint.of(app1) - .target(cluster, ZoneId.from("prod", "us-north-1a")) - .weighted() + .targetRegion(cluster, ZoneId.from("prod", "us-north-1a")) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), "https://a1.t1.us-north-2-w.public.vespa.oath.cloud/", Endpoint.of(app1) - .target(cluster, ZoneId.from("prod", "us-north-2")) - .weighted() + .targetRegion(cluster, ZoneId.from("prod", "us-north-2")) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), "https://a1.t1.us-north-2-w.test.public.vespa.oath.cloud/", Endpoint.of(app1) - .target(cluster, ZoneId.from("test", "us-north-2")) - .weighted() + .targetRegion(cluster, ZoneId.from("test", "us-north-2")) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); Endpoint endpoint = Endpoint.of(app1) - .target(cluster, ZoneId.from("prod", "us-north-1a")) - .weighted() + .targetRegion(cluster, ZoneId.from("prod", "us-north-1a")) .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.main); @@ -271,20 +267,20 @@ public class EndpointTest { var tests1 = Map.of( // With default cluster "a1.t1.us-north-1.prod", - Endpoint.of(app1).named(EndpointId.defaultId()).on(Port.tls(4443)).in(SystemName.main), + Endpoint.of(app1).target(EndpointId.defaultId()).on(Port.tls(4443)).in(SystemName.main), // With non-default cluster "c1.a1.t1.us-north-1.prod", - Endpoint.of(app1).named(EndpointId.of("c1")).on(Port.tls(4443)).in(SystemName.main) + Endpoint.of(app1).target(EndpointId.of("c1")).on(Port.tls(4443)).in(SystemName.main) ); var tests2 = Map.of( // With non-default instance "i2.a2.t2.us-north-1.prod", - Endpoint.of(app2).named(EndpointId.defaultId()).on(Port.tls(4443)).in(SystemName.main), + Endpoint.of(app2).target(EndpointId.defaultId()).on(Port.tls(4443)).in(SystemName.main), // With non-default instance and cluster "c2.i2.a2.t2.us-north-1.prod", - Endpoint.of(app2).named(EndpointId.of("c2")).on(Port.tls(4443)).in(SystemName.main) + Endpoint.of(app2).target(EndpointId.of("c2")).on(Port.tls(4443)).in(SystemName.main) ); tests1.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(new DeploymentId(app1, zone)))); tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(new DeploymentId(app2, zone)))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index acb8cf1a2a9..d90eb715499 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -13,8 +13,6 @@ import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; @@ -113,7 +111,6 @@ public class DeploymentContext { this.runner = tester.runner(); this.tester = tester; createTenantAndApplication(); - ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.ALLOW_DIRECT_ROUTING.id(), true); } private void createTenantAndApplication() { @@ -247,7 +244,7 @@ public class DeploymentContext { .projectId() .orElse(1000); // These are really set through submission, so just pick one if it hasn't been set. lastSubmission = jobs.submit(applicationId, sourceRevision, Optional.of("a@b"), Optional.empty(), - Optional.empty(), projectId, applicationPackage, new byte[0]); + projectId, applicationPackage, new byte[0]); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java index 6a12c4457db..2e8b3e46a87 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.SlimeUtils; @@ -12,7 +11,6 @@ import com.yahoo.vespa.hosted.controller.application.EndpointId; import org.junit.Test; import java.io.IOException; -import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; @@ -33,7 +31,7 @@ public class TestConfigSerializerTest { JobType.systemTest, true, Map.of(zone, List.of(Endpoint.of(ApplicationId.defaultId()) - .named(EndpointId.of("ai")) + .target(EndpointId.of("ai")) .on(Endpoint.Port.tls()) .in(SystemName.main))), Map.of(zone, List.of("facts"))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index fff63e1954e..251a5ce9acb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -19,8 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; -import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; @@ -33,6 +31,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundEx import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.serviceview.bindings.ApplicationView; @@ -402,8 +401,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public void restart(DeploymentId deployment, Optional<Hostname> hostname) { - nodeRepository().requestRestart(deployment, hostname.map(Identifier::id).map(HostName::from)); + public void restart(DeploymentId deployment, RestartFilter restartFilter) { + nodeRepository().requestRestart(deployment, restartFilter.getHostName()); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java index 1151fdd07f0..6a2feba1d47 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java @@ -2,7 +2,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import org.junit.Before; import org.junit.Test; @@ -27,19 +29,47 @@ public class ControllerMaintainerTest { @Test public void only_runs_in_permitted_systems() { AtomicInteger executions = new AtomicInteger(); - maintainerIn(SystemName.cd, executions).run(); - maintainerIn(SystemName.main, executions).run(); + new TestControllerMaintainer(tester.controller(), SystemName.cd, executions).run(); + new TestControllerMaintainer(tester.controller(), SystemName.main, executions).run(); assertEquals(1, executions.get()); } - private ControllerMaintainer maintainerIn(SystemName system, AtomicInteger executions) { - return new ControllerMaintainer(tester.controller(), Duration.ofDays(1), - "MockMaintainer", EnumSet.of(system)) { - @Override - protected void maintain() { - executions.incrementAndGet(); - } - }; + @Test + public void records_metric() { + TestControllerMaintainer maintainer = new TestControllerMaintainer(tester.controller(), SystemName.main, new AtomicInteger()); + maintainer.run(); + assertEquals(0L, consecutiveFailuresMetric()); + maintainer.success = false; + maintainer.run(); + maintainer.run(); + assertEquals(2L, consecutiveFailuresMetric()); + maintainer.success = true; + maintainer.run();; + assertEquals(0, consecutiveFailuresMetric()); + } + + private long consecutiveFailuresMetric() { + MetricsMock metrics = (MetricsMock) tester.controller().metric(); + return metrics.getMetric((context) -> "TestControllerMaintainer".equals(context.get("job")), + "maintenance.consecutiveFailures").get().longValue(); + } + + private static class TestControllerMaintainer extends ControllerMaintainer { + + private final AtomicInteger executions; + private boolean success = true; + + public TestControllerMaintainer(Controller controller, SystemName system, AtomicInteger executions) { + super(controller, Duration.ofDays(1), null, EnumSet.of(system)); + this.executions = executions; + } + + @Override + protected boolean maintain() { + executions.incrementAndGet(); + return success; + } + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index fe33d728b7c..88eab642a60 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -91,7 +91,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); jobs.start(id, systemTest, versions); try { @@ -122,7 +122,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); Supplier<Run> run = () -> jobs.last(id, systemTest).get(); jobs.start(id, systemTest, versions); @@ -229,7 +229,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); RunId runId = new RunId(id, systemTest, 1); jobs.start(id, systemTest, versions); @@ -266,7 +266,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId instanceId = appId.defaultInstance(); JobId jobId = new JobId(instanceId, systemTest); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); assertFalse(jobs.lastSuccess(jobId).isPresent()); for (int i = 0; i < jobs.historyLength(); i++) { @@ -361,7 +361,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); jobs.start(id, systemTest, versions); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); @@ -378,7 +378,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); for (RunStatus status : RunStatus.values()) { if (status == success) continue; // Status not used for steps. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index d5c26408b23..98412b8147b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -275,11 +275,12 @@ public class SystemUpgraderTest { } @Test - public void does_not_deploy_proxy_app_in_zones_without_proxy() { + public void does_not_deploy_proxy_app_in_zone_without_shared_routing() { var applications = List.of(SystemApplication.configServerHost, SystemApplication.configServer, SystemApplication.tenantHost); tester.configServer().bootstrap(List.of(zone1.getId()), applications); tester.configServer().disallowConvergenceCheck(SystemApplication.proxy.id()); + tester.zoneRegistry().exclusiveRoutingIn(zone1); var systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1)); // System begins upgrade @@ -382,8 +383,7 @@ public class SystemUpgraderTest { private SystemUpgrader systemUpgrader(UpgradePolicy upgradePolicy) { tester.zoneRegistry().setUpgradePolicy(upgradePolicy); - return new SystemUpgrader(tester.controller(), Duration.ofDays(1) - ); + return new SystemUpgrader(tester.controller(), Duration.ofDays(1)); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index 7ca964e06dd..d67e9c3e432 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; @@ -16,7 +15,6 @@ import org.junit.Test; import java.time.Instant; import java.util.Iterator; -import java.util.Map; import java.util.Optional; import java.util.Set; @@ -63,16 +61,4 @@ public class RoutingPolicySerializerTest { } } - // TODO(mpolden): Remove after January 2020 - @Test - public void legacy_serialization() { - var json = "{\"routingPolicies\":[{\"cluster\":\"default\",\"zone\":\"prod.us-north-1\",\"canonicalName\":\"lb-host\",\"dnsZone\":\"dnsZoneId\",\"rotations\":[\"default\"],\"active\":true}]}"; - var owner = ApplicationId.defaultId(); - var serialized = serializer.fromSlime(owner, SlimeUtils.jsonToSlime(json)); - var id = new RoutingPolicyId(owner, ClusterSpec.Id.from("default"), ZoneId.from("prod", "us-north-1")); - var expected = Map.of(id, new RoutingPolicy(id, HostName.from("lb-host"), Optional.of("dnsZoneId"), - Set.of(EndpointId.defaultId()), new Status(true, GlobalRouting.DEFAULT_STATUS))); - assertEquals(expected, serialized); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 388ca65dc40..10682218353 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -70,11 +70,9 @@ import org.junit.Before; import org.junit.Test; import java.io.File; -import java.math.BigDecimal; import java.net.URI; import java.time.Duration; import java.time.Instant; -import java.time.YearMonth; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Base64; @@ -84,7 +82,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.TreeSet; import java.util.function.Supplier; import static com.yahoo.application.container.handler.Request.Method.DELETE; @@ -975,6 +972,36 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test + public void testRemovingAllDeployments() { + createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1") + .region("us-west-1") + .region("us-east-3") + .region("eu-west-1") + .endpoint("eu", "default", "eu-west-1") + .endpoint("default", "default", "us-west-1", "us-east-3") + .build(); + + deploymentTester.controllerTester().createTenant("tenant1", ATHENZ_TENANT_DOMAIN.getName(), 432L); + + // Create tenant and deploy + var app = deploymentTester.newDeploymentContext("tenant1", "application1", "instance1"); + app.submit(applicationPackage).deploy(); + tester.controller().jobController().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage); + + assertEquals(Set.of(ZoneId.from("prod.us-west-1"), ZoneId.from("prod.us-east-3"), ZoneId.from("prod.eu-west-1"), ZoneId.from("dev.us-east-1")), + app.instance().deployments().keySet()); + + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deployment", DELETE) + .userIdentity(USER_ID) + .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), + "{\"message\":\"All deployments removed\"}"); + + assertEquals(Set.of(ZoneId.from("dev.us-east-1")), app.instance().deployments().keySet()); + } + + @Test public void testErrorResponses() throws Exception { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 928525a20d1..c74092c4ae8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -13,7 +13,7 @@ "routingMethod": "shared" }, { - "cluster": "", + "cluster": "foo", "tls": true, "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", "scope": "global", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 4ffe809297d..7d2def6c479 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -16,7 +16,7 @@ "routingMethod": "shared" }, { - "cluster": "", + "cluster": "foo", "tls": true, "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", "scope": "global", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesLegacyTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesLegacyTest.java deleted file mode 100644 index d5a50d98c8d..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesLegacyTest.java +++ /dev/null @@ -1,711 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.routing; - -import com.google.common.collect.Sets; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.config.provision.AthenzService; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.RoutingController; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; -import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -/** - * @author mortent - * @author mpolden - */ -// TODO(mpolden): Remove when weighted-dns-per-region flag is removed -public class RoutingPoliciesLegacyTest { - - private final ZoneId zone1 = ZoneId.from("prod", "us-west-1"); - private final ZoneId zone2 = ZoneId.from("prod", "us-central-1"); - private final ZoneId zone3 = ZoneId.from("prod", "us-east-3"); - - private final ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) - .region(zone2.region()) - .build(); - - @Test - public void global_routing_policies() { - var tester = new RoutingPoliciesTester(); - var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); - var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); - int clustersPerZone = 2; - int numberOfDeployments = 2; - var applicationPackage = applicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .endpoint("r0", "c0") - .endpoint("r1", "c0", "us-west-1") - .endpoint("r2", "c1") - .build(); - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); - - // Creates alias records - context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); - tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2); - assertEquals("Routing policy count is equal to cluster count", - numberOfDeployments * clustersPerZone, - tester.policiesOf(context1.instance().id()).size()); - - // Applications gains a new deployment - ApplicationPackage applicationPackage2 = applicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .region(zone3.region()) - .endpoint("r0", "c0") - .endpoint("r1", "c0", "us-west-1") - .endpoint("r2", "c1") - .build(); - numberOfDeployments++; - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3); - context1.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - - // Endpoints are updated to contain cluster in new deployment - tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3); - tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); - tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2, zone3); - - // Another application is deployed with a single cluster and global endpoint - var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud"; - tester.provisionLoadBalancers(1, context2.instanceId(), zone1, zone2); - var applicationPackage3 = applicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .endpoint("r0", "c0") - .build(); - context2.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - - // All endpoints for app1 are removed - ApplicationPackage applicationPackage4 = applicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .region(zone3.region()) - .allow(ValidationId.globalEndpointChange) - .build(); - context1.submit(applicationPackage4).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0); - tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0); - tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0); - var policies = tester.policiesOf(context1.instanceId()); - assertEquals(clustersPerZone * numberOfDeployments, policies.size()); - assertTrue("Rotation membership is removed from all policies", - policies.stream().allMatch(policy -> policy.endpoints().isEmpty())); - assertEquals("Rotations for " + context2.application() + " are not removed", 2, tester.aliasDataOf(endpoint4).size()); - } - - @Test - public void zone_routing_policies() { - zone_routing_policies(false); - zone_routing_policies(true); - } - - private void zone_routing_policies(boolean sharedRoutingLayer) { - var tester = new RoutingPoliciesTester(); - var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); - var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); - - // Deploy application - int clustersPerZone = 2; - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2); - context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - - // Deployment creates records and policies for all clusters in all zones - Set<String> expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, tester.recordNames()); - assertEquals(4, tester.policiesOf(context1.instanceId()).size()); - - // Next deploy does nothing - context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - assertEquals(expectedRecords, tester.recordNames()); - assertEquals(4, tester.policiesOf(context1.instanceId()).size()); - - // Add 1 cluster in each zone and deploy - tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), sharedRoutingLayer, zone1, zone2); - context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c2.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud", - "c2.app1.tenant1.us-central-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, tester.recordNames()); - assertEquals(6, tester.policiesOf(context1.instanceId()).size()); - - // Deploy another application - tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), sharedRoutingLayer, zone1, zone2); - context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c2.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud", - "c2.app1.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-central-1.vespa.oath.cloud", - "c1.app2.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-west-1.vespa.oath.cloud", - "c1.app2.tenant1.us-west-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords.stream().sorted().collect(Collectors.toList()), tester.recordNames().stream().sorted().collect(Collectors.toList())); - assertEquals(4, tester.policiesOf(context2.instanceId()).size()); - - // Deploy removes cluster from app1 - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2); - context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-central-1.vespa.oath.cloud", - "c1.app2.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-west-1.vespa.oath.cloud", - "c1.app2.tenant1.us-west-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, tester.recordNames()); - - // Remove app2 completely - tester.controllerTester().controller().applications().requireInstance(context2.instanceId()).deployments().keySet() - .forEach(zone -> { - tester.controllerTester().configServer().removeLoadBalancers(context2.instanceId(), zone); - tester.controllerTester().controller().applications().deactivate(context2.instanceId(), zone); - }); - context2.flushDnsUpdates(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, tester.recordNames()); - assertTrue("Removes stale routing policies " + context2.application(), tester.routingPolicies().get(context2.instanceId()).isEmpty()); - assertEquals("Keeps routing policies for " + context1.application(), 4, tester.routingPolicies().get(context1.instanceId()).size()); - } - - @Test - public void global_routing_policies_in_rotationless_system() { - var tester = new RoutingPoliciesTester(new DeploymentTester(new ControllerTester(new RotationsConfig.Builder().build()))); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); - - var applicationPackage = applicationPackageBuilder() - .region(zone1.region().value()) - .endpoint("r0", "c0") - .build(); - context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - - var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud"; - assertEquals(endpoint + " points to c0 in all regions", - List.of("latency/lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint)); - assertTrue("No rotations assigned", context.application().instances().values().stream() - .map(Instance::rotations) - .allMatch(List::isEmpty)); - } - - @Test - public void manual_deployment_creates_routing_policy() { - // Empty application package is valid in manually deployed environments - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - var emptyApplicationPackage = new ApplicationPackageBuilder().build(); - var zone = ZoneId.from("dev", "us-east-1"); - var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); - tester.controllerTester().serviceRegistry().zoneRegistry() - .setZones(zoneApi) - .exclusiveRoutingIn(zoneApi); - - // Deploy to dev - tester.controllerTester().controller().applications().deploy(context.instanceId(), zone, Optional.of(emptyApplicationPackage), DeployOptions.none()); - assertEquals("DeploymentSpec is not persisted", DeploymentSpec.empty, context.application().deploymentSpec()); - context.flushDnsUpdates(); - - // Routing policy is created and DNS is updated - assertEquals(1, tester.policiesOf(context.instanceId()).size()); - assertEquals(Set.of("app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); - } - - @Test - public void manual_deployment_creates_routing_policy_with_non_empty_spec() { - // Initial deployment - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - context.submit(applicationPackage).deploy(); - var zone = ZoneId.from("dev", "us-east-1"); - var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); - tester.controllerTester().serviceRegistry().zoneRegistry() - .setZones(zoneApi) - .exclusiveRoutingIn(zoneApi); - var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud"); - assertEquals(prodRecords, tester.recordNames()); - - // Deploy to dev under different instance - var devInstance = context.application().id().instance("user"); - tester.controllerTester().controller().applications().deploy(devInstance, zone, Optional.of(applicationPackage), DeployOptions.none()); - assertEquals("DeploymentSpec is persisted", applicationPackage.deploymentSpec(), context.application().deploymentSpec()); - context.flushDnsUpdates(); - - // Routing policy is created and DNS is updated - assertEquals(1, tester.policiesOf(devInstance).size()); - assertEquals(Sets.union(prodRecords, Set.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames()); - } - - @Test - public void reprovisioning_load_balancer_preserves_cname_record() { - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - - // Initial load balancer is provisioned - tester.provisionLoadBalancers(1, context.instanceId(), zone1); - var applicationPackage = applicationPackageBuilder() - .region(zone1.region()) - .build(); - - // Application is deployed - context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - var expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, tester.recordNames()); - assertEquals(1, tester.policiesOf(context.instanceId()).size()); - - // Application is removed and the load balancer is deprovisioned - tester.controllerTester().controller().applications().deactivate(context.instanceId(), zone1); - tester.controllerTester().configServer().removeLoadBalancers(context.instanceId(), zone1); - - // Load balancer for the same application is provisioned again, but with a different hostname - var newHostname = HostName.from("new-hostname"); - var loadBalancer = new LoadBalancer("LB-0-Z-" + zone1.value(), - context.instanceId(), - ClusterSpec.Id.from("c0"), - newHostname, - LoadBalancer.State.active, - Optional.of("dns-zone-1")); - tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer)); - - // Application redeployment preserves DNS record - context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - assertEquals(expectedRecords, tester.recordNames()); - assertEquals(1, tester.policiesOf(context.instanceId()).size()); - assertEquals("CNAME points to current load blancer", newHostname.value() + ".", - tester.cnameDataOf(expectedRecords.iterator().next()).get(0)); - } - - @Test - public void set_global_endpoint_status() { - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - - // Provision load balancers and deploy application - tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); - var applicationPackage = applicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) - .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) - .build(); - context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - - // Global DNS record is created - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); - - // Global routing status is overridden in one zone - var changedAt = tester.controllerTester().clock().instant(); - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, - GlobalRouting.Agent.tenant); - context.flushDnsUpdates(); - - // Inactive zone is removed from global DNS record - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); - tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); - - // Status details is stored in policy - var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); - assertEquals(GlobalRouting.Status.out, policy1.status().globalRouting().status()); - assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); - assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); - - // Other zone remains in - var policy2 = tester.routingPolicies().get(context.deploymentIdIn(zone2)).values().iterator().next(); - assertEquals(GlobalRouting.Status.in, policy2.status().globalRouting().status()); - assertEquals(GlobalRouting.Agent.system, policy2.status().globalRouting().agent()); - assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt()); - - // Next deployment does not affect status - context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); - tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); - - // Deployment is set back in - tester.controllerTester().clock().advance(Duration.ofHours(1)); - changedAt = tester.controllerTester().clock().instant(); - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); - context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); - - policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); - assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status()); - assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); - assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); - - // Deployment is set out through a new deployment.xml - var applicationPackage2 = applicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region(), false) - .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) - .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) - .build(); - context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); - tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1); - - // ... back in - var applicationPackage3 = applicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) - .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) - .build(); - context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); - } - - @Test - public void set_zone_global_endpoint_status() { - var tester = new RoutingPoliciesTester(); - var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); - var context2 = tester.newDeploymentContext("tenant2", "app2", "default"); - var contexts = List.of(context1, context2); - - // Deploy applications - var applicationPackage = applicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .endpoint("default", "c0", zone1.region().value(), zone2.region().value()) - .build(); - for (var context : contexts) { - tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); - context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); - } - - // Set zone out - tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out); - context1.flushDnsUpdates(); - tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); - tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); - for (var context : contexts) { - var policies = tester.routingPolicies().get(context.instanceId()); - assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in, - policies.values().stream() - .map(RoutingPolicy::status) - .map(Status::globalRouting) - .map(GlobalRouting::status) - .allMatch(status -> status == GlobalRouting.Status.in)); - } - var changedAt = tester.controllerTester().clock().instant(); - var zonePolicy = tester.controllerTester().controller().curator().readZoneRoutingPolicy(zone2); - assertEquals(GlobalRouting.Status.out, zonePolicy.globalRouting().status()); - assertEquals(GlobalRouting.Agent.operator, zonePolicy.globalRouting().agent()); - assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.globalRouting().changedAt()); - - // Setting status per deployment does not affect status as entire zone is out - tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); - context1.flushDnsUpdates(); - tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); - tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); - - // Set single deployment out - tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant); - context1.flushDnsUpdates(); - - // Set zone back in. Deployment set explicitly out, remains out, the rest are in - tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in); - context1.flushDnsUpdates(); - tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); - tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); - } - - @Test - public void non_production_deployment_is_not_registered_in_global_endpoint() { - var tester = new RoutingPoliciesTester(SystemName.Public); - - // Configure the system to use the same region for test, staging and prod - var sharedRegion = RegionName.from("aws-us-east-1c"); - var prodZone = ZoneId.from(Environment.prod, sharedRegion); - var stagingZone = ZoneId.from(Environment.staging, sharedRegion); - var testZone = ZoneId.from(Environment.test, sharedRegion); - var zones = List.of(ZoneApiMock.from(prodZone), - ZoneApiMock.from(stagingZone), - ZoneApiMock.from(testZone)); - tester.controllerTester().zoneRegistry() - .setZones(zones) - .setRoutingMethod(zones, RoutingMethod.exclusive); - tester.controllerTester().configServer().bootstrap(List.of(prodZone, stagingZone, testZone), - SystemApplication.all()); - - var context = tester.tester.newDeploymentContext(); - var endpointId = EndpointId.of("r0"); - var applicationPackage = applicationPackageBuilder() - .trustDefaultCertificate() - .region(sharedRegion) - .endpoint(endpointId.id(), "default") - .build(); - - // Application starts deployment - context = context.submit(applicationPackage); - for (var testJob : List.of(JobType.systemTest, JobType.stagingTest)) { - context = context.runJob(testJob); - // Since runJob implicitly tears down the deployment and immediately deletes DNS records associated with the - // deployment, we consume only one DNS update at a time here - do { - context = context.flushDnsUpdates(1); - tester.assertTargets(context.instanceId(), endpointId, 0); - } while (!tester.recordNames().isEmpty()); - } - - // Deployment completes - context.completeRollout(); - tester.assertTargets(context.instanceId(), endpointId, 0, prodZone); - } - - @Test - public void changing_global_routing_status_never_removes_all_members() { - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - - // Provision load balancers and deploy application - tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); - var applicationPackage = applicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) - .build(); - context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - - // Global DNS record is created, pointing to all configured zones - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - - // Global routing status is overridden for one deployment - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, - GlobalRouting.Agent.tenant); - context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); - - // Setting other deployment out implicitly sets all deployments in - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out, - GlobalRouting.Agent.tenant); - context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - - // One inactive deployment is put back in. Global DNS record now points to the only active deployment - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, - GlobalRouting.Agent.tenant); - context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); - - // Setting zone (containing active deployment) out puts all deployments in - tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out); - context.flushDnsUpdates(); - assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status()); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - - // Setting zone back in removes the currently inactive deployment - tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in); - context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); - - // Inactive deployment is set in - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in, - GlobalRouting.Agent.tenant); - context.flushDnsUpdates(); - for (var policy : tester.routingPolicies().get(context.instanceId()).values()) { - assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status()); - } - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - } - - @Test - public void config_server_routing_policy() { - var tester = new RoutingPoliciesTester(); - var app = SystemApplication.configServer.id(); - RecordName name = RecordName.from("cfg.prod.us-west-1.test.vip"); - - tester.provisionLoadBalancers(1, app, zone1); - tester.routingPolicies().refresh(app, DeploymentSpec.empty, zone1); - new NameServiceDispatcher(tester.tester.controller(), Duration.ofDays(1), Integer.MAX_VALUE).run(); - - List<Record> records = tester.controllerTester().nameService().findRecords(Record.Type.CNAME, name); - assertEquals(1, records.size()); - assertEquals(RecordData.from("lb-0--hosted-vespa:zone-config-servers:default--prod.us-west-1."), - records.get(0).data()); - } - - /** Returns an application package builder that satisfies requirements for a directly routed endpoint */ - private static ApplicationPackageBuilder applicationPackageBuilder() { - return new ApplicationPackageBuilder() - .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) - .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION); - } - - private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, boolean shared, int count) { - List<LoadBalancer> loadBalancers = new ArrayList<>(); - for (int i = 0; i < count; i++) { - HostName lbHostname; - if (shared) { - lbHostname = HostName.from("shared-lb--" + zone.value()); - } else { - lbHostname = HostName.from("lb-" + i + "--" + application.serializedForm() + - "--" + zone.value()); - } - loadBalancers.add( - new LoadBalancer("LB-" + i + "-Z-" + zone.value(), - application, - ClusterSpec.Id.from("c" + i), - lbHostname, - LoadBalancer.State.active, - Optional.of("dns-zone-1"))); - } - return loadBalancers; - } - - private static class RoutingPoliciesTester { - - private final DeploymentTester tester; - - public RoutingPoliciesTester() { - this(SystemName.main); - } - - public RoutingPoliciesTester(SystemName system) { - this(new DeploymentTester(new ControllerTester(new ServiceRegistryMock(system)))); - } - - public RoutingPolicies routingPolicies() { - return tester.controllerTester().controller().routing().policies(); - } - - public DeploymentContext newDeploymentContext(String tenant, String application, String instance) { - return tester.newDeploymentContext(tenant, application, instance); - } - - public ControllerTester controllerTester() { - return tester.controllerTester(); - } - - public RoutingPoliciesTester(DeploymentTester tester) { - this.tester = tester; - // Make all zones directly routed - tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones()); - } - - private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, boolean shared, ZoneId... zones) { - for (ZoneId zone : zones) { - tester.configServer().removeLoadBalancers(application, zone); - tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, shared, clustersPerZone)); - } - } - - private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { - provisionLoadBalancers(clustersPerZone, application, false, zones); - } - - private Collection<RoutingPolicy> policiesOf(ApplicationId instance) { - return tester.controller().curator().readRoutingPolicies(instance).values(); - } - - private Set<String> recordNames() { - return tester.controllerTester().nameService().records().stream() - .map(Record::name) - .map(RecordName::asString) - .collect(Collectors.toSet()); - } - - private List<String> aliasDataOf(String name) { - return tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from(name)).stream() - .map(Record::data) - .map(RecordData::asString) - .collect(Collectors.toList()); - } - - private List<String> cnameDataOf(String name) { - return tester.controllerTester().nameService().findRecords(Record.Type.CNAME, RecordName.from(name)).stream() - .map(Record::data) - .map(RecordData::asString) - .collect(Collectors.toList()); - } - - private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) { - var endpoint = tester.controller().routing().endpointsOf(application) - .named(endpointId) - .targets(List.of(zone)) - .primary() - .map(Endpoint::dnsName) - .orElse("<none>"); - var zoneTargets = Arrays.stream(zone) - .map(z -> "latency/lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + - z.value() + "/dns-zone-1/" + z.value()) - .collect(Collectors.toSet()); - assertEquals("Global endpoint " + endpoint + " points to expected zones", zoneTargets, - Set.copyOf(aliasDataOf(endpoint))); - } - - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 6679fc112a3..fab61eeaec3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -16,8 +16,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.RoutingController; @@ -749,7 +747,6 @@ public class RoutingPoliciesTest { } tester.controllerTester().configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.all()); - ((InMemoryFlagSource) tester.controllerTester().controller().flagSource()).withBooleanFlag(Flags.WEIGHTED_DNS_PER_REGION.id(), true); } private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, boolean shared, ZoneId... zones) { @@ -788,16 +785,16 @@ public class RoutingPoliciesTest { .collect(Collectors.toList()); } - private void assertTargets(ApplicationId application, EndpointId endpointId, ClusterSpec.Id clusterId, int loadBalancerId, Map<ZoneId, Long> zoneWeights) { + private void assertTargets(ApplicationId application, EndpointId endpointId, ClusterSpec.Id cluster, int loadBalancerId, Map<ZoneId, Long> zoneWeights) { Set<String> latencyTargets = new HashSet<>(); Map<String, List<ZoneId>> zonesByRegionEndpoint = new HashMap<>(); for (var zone : zoneWeights.keySet()) { - Endpoint weighted = tester.controller().routing().endpointsOf(new DeploymentId(application, zone)) - .scope(Endpoint.Scope.weighted) - .named(EndpointId.of(clusterId.value())) - .asList() - .get(0); - zonesByRegionEndpoint.computeIfAbsent(weighted.dnsName(), (k) -> new ArrayList<>()) + Endpoint regionEndpoint = tester.controller().routing().endpointsOf(new DeploymentId(application, zone)) + .scope(Endpoint.Scope.region) + .cluster(cluster) + .asList() + .get(0); + zonesByRegionEndpoint.computeIfAbsent(regionEndpoint.dnsName(), (k) -> new ArrayList<>()) .add(zone); } zonesByRegionEndpoint.forEach((regionEndpoint, zonesInRegion) -> { @@ -806,7 +803,7 @@ public class RoutingPoliciesTest { application.serializedForm() + "--" + z.value() + "/dns-zone-1/" + z.value() + "/" + zoneWeights.get(z)) .collect(Collectors.toSet()); - assertEquals("Weighted endpoint " + regionEndpoint + " points to load balancer", + assertEquals("Region endpoint " + regionEndpoint + " points to load balancer", weightedTargets, aliasDataOf(regionEndpoint)); ZoneId zone = zonesInRegion.get(0); diff --git a/dist/vespa.spec b/dist/vespa.spec index 59b7121e9d2..e49bd74e545 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -56,10 +56,12 @@ BuildRequires: cmake3 BuildRequires: llvm7.0-devel BuildRequires: vespa-boost-devel >= 1.59.0-6 BuildRequires: vespa-gtest >= 1.8.1-1 -BuildRequires: vespa-protobuf-devel >= 3.7.0-4 +BuildRequires: vespa-icu-devel >= 65.1.0-1 +BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel >= 1.3.0-1 BuildRequires: vespa-openssl-devel >= 1.1.1g-1 -BuildRequires: vespa-icu-devel >= 65.1.0-1 +BuildRequires: vespa-protobuf-devel >= 3.7.0-4 +BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %endif %if 0%{?el8} BuildRequires: cmake >= 3.11.4-3 @@ -67,14 +69,18 @@ BuildRequires: llvm-devel >= 9.0.1 BuildRequires: boost-devel >= 1.66 BuildRequires: openssl-devel BuildRequires: vespa-gtest >= 1.8.1-1 -BuildRequires: vespa-protobuf-devel >= 3.7.0-4 +BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel >= 1.3.0-1 +BuildRequires: vespa-protobuf-devel >= 3.7.0-4 +BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %endif %if 0%{?fedora} BuildRequires: cmake >= 3.9.1 BuildRequires: maven -BuildRequires: vespa-onnxruntime-devel >= 1.3.0-1 BuildRequires: openssl-devel +BuildRequires: vespa-lz4-devel >= 1.9.2-2 +BuildRequires: vespa-onnxruntime-devel >= 1.3.0-1 +BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %if 0%{?fc31} BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: llvm-devel >= 9.0.0 @@ -99,8 +105,6 @@ BuildRequires: gmock-devel %endif BuildRequires: xxhash-devel >= 0.7.3 BuildRequires: openblas-devel -BuildRequires: lz4-devel -BuildRequires: libzstd-devel BuildRequires: zlib-devel BuildRequires: re2-devel %if ! 0%{?el7} @@ -146,8 +150,6 @@ Requires: openblas %else Requires: openblas-serial %endif -Requires: lz4 -Requires: libzstd Requires: zlib Requires: re2 %if ! 0%{?el7} @@ -158,12 +160,14 @@ Requires: gdb Requires: net-tools %if 0%{?el7} Requires: llvm7.0 +Requires: vespa-icu >= 65.1.0-1 +Requires: vespa-lz4 >= 1.9.2-2 Requires: vespa-onnxruntime >= 1.3.0-1 Requires: vespa-openssl >= 1.1.1g-1 -Requires: vespa-icu >= 65.1.0-1 Requires: vespa-protobuf >= 3.7.0-4 Requires: vespa-telegraf >= 1.1.1-1 Requires: vespa-valgrind >= 3.16.0-1 +Requires: vespa-zstd >= 1.4.5-2 %define _vespa_llvm_version 7 %define _extra_link_directory /usr/lib64/llvm7.0/lib;%{_vespa_deps_prefix}/lib64 %define _extra_include_directory /usr/include/llvm7.0;%{_vespa_deps_prefix}/include;/usr/include/openblas @@ -171,15 +175,19 @@ Requires: vespa-valgrind >= 3.16.0-1 %if 0%{?el8} Requires: llvm-libs >= 9.0.1 %define _vespa_llvm_version 9 -Requires: vespa-protobuf >= 3.7.0-4 -Requires: vespa-onnxruntime >= 1.3.0-1 Requires: openssl-libs +Requires: vespa-lz4 >= 1.9.2-2 +Requires: vespa-onnxruntime >= 1.3.0-1 +Requires: vespa-protobuf >= 3.7.0-4 +Requires: vespa-zstd >= 1.4.5-2 %define _extra_link_directory %{_vespa_deps_prefix}/lib64 %define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas %endif %if 0%{?fedora} -Requires: vespa-onnxruntime >= 1.3.0-1 Requires: openssl-libs +Requires: vespa-lz4 >= 1.9.2-2 +Requires: vespa-onnxruntime >= 1.3.0-1 +Requires: vespa-zstd >= 1.4.5-2 %if 0%{?fc31} Requires: vespa-protobuf >= 3.7.0-4 Requires: llvm-libs >= 9.0.0 @@ -207,8 +215,8 @@ Requires: %{name}-malloc = %{version}-%{release} Requires: %{name}-tools = %{version}-%{release} # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private -# _dl_sym function. -%global __requires_exclude ^libc\\.so\\.6\\(GLIBC_PRIVATE\\)\\(64bit\\)$ +# _dl_sym function. Exclude automated reqires for libraries in /opt/vespa-deps/lib64. +%global __requires_exclude ^lib(c\\.so\\.6\\(GLIBC_PRIVATE\\)|(crypto|icui18n|icuuc|lz4|protobuf|ssl|zstd)\\.so\\.[0-9.]*\\((OPENSSL_1_1_0)?\\))\\(64bit\\)$ %description @@ -233,13 +241,13 @@ Vespa - The open big data serving engine - base Summary: Vespa - The open big data serving engine - base C++ libs Requires: xxhash-libs >= 0.7.3 -Requires: lz4 -Requires: libzstd %if 0%{?el7} Requires: vespa-openssl >= 1.1.1g-1 %else Requires: openssl-libs %endif +Requires: vespa-lz4 >= 1.9.2-2 +Requires: vespa-libzstd >= 1.4.5-2 %description base-libs diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index d5063a42605..cc3cb3adc85 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -71,7 +71,8 @@ public class Flags { public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag( "disabled-host-admin-tasks", List.of(), String.class, - "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask) that should be skipped", + "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask), or some node-agent " + + "functionality (see NodeAgentTask), that should be skipped", "Takes effect on next host admin tick", HOSTNAME, NODE_TYPE); @@ -264,12 +265,6 @@ public class Flags { "Takes effect on redeploy", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag ALLOW_DIRECT_ROUTING = defineFeatureFlag( - "publish-direct-routing-endpoint", true, - "Whether an application should receive a directly routed endpoint in its endpoint list", - "Takes effect immediately", - APPLICATION_ID); - public static final UnboundBooleanFlag NLB_PROXY_PROTOCOL = defineFeatureFlag( "nlb-proxy-protocol", false, "Configure NLB to use proxy protocol", @@ -321,7 +316,7 @@ public class Flags { ); public static final UnboundBooleanFlag WEIGHTED_DNS_PER_REGION = defineFeatureFlag( - "weighted-dns-per-region", false, + "weighted-dns-per-region", true, "Whether to create weighted DNS records per region in global endpoints", "Takes effect on next deployment through controller", APPLICATION_ID @@ -345,6 +340,13 @@ public class Flags { "Takes effect on next tick" ); + public static final UnboundBooleanFlag USE_CONFIG_SERVER_LOCK = defineFeatureFlag( + "use-config-server-lock", + false, + "Whether the node-repository should take the same application lock as the config server when making changes to nodes", + "Takes effect on config server restart" + ); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/fnet/src/tests/examples/examples_test.cpp b/fnet/src/tests/examples/examples_test.cpp index c704c58abc9..8f13aca0898 100644 --- a/fnet/src/tests/examples/examples_test.cpp +++ b/fnet/src/tests/examples/examples_test.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/thread.h> #include <atomic> @@ -10,9 +10,9 @@ static const int PORT0 = 18570; static const int PORT1 = 18571; -using vespalib::SlaveProc; +using vespalib::ChildProcess; -bool runProc(SlaveProc &proc, std::atomic<bool> &done) { +bool runProc(ChildProcess &proc, std::atomic<bool> &done) { char buf[4096]; proc.close(); // close stdin while (proc.running() && !done) { @@ -38,7 +38,7 @@ bool runProc(const std::string &cmd) { vespalib::Thread::sleep(500); } std::atomic<bool> done(false); - SlaveProc proc(cmd.c_str()); + ChildProcess proc(cmd.c_str()); ok = runProc(proc, done); } return ok; @@ -47,60 +47,60 @@ bool runProc(const std::string &cmd) { TEST("usage") { std::atomic<bool> done(false); { - SlaveProc proc("exec ../../examples/proxy/fnet_proxy_app"); + ChildProcess proc("exec ../../examples/proxy/fnet_proxy_app"); EXPECT_FALSE(runProc(proc, done)); } { - SlaveProc proc("exec ../../examples/ping/fnet_pingserver_app"); + ChildProcess proc("exec ../../examples/ping/fnet_pingserver_app"); EXPECT_FALSE(runProc(proc, done)); } { - SlaveProc proc("exec ../../examples/ping/fnet_pingclient_app"); + ChildProcess proc("exec ../../examples/ping/fnet_pingclient_app"); EXPECT_FALSE(runProc(proc, done)); } { - SlaveProc proc("exec ../../examples/frt/rpc/fnet_rpc_client_app"); + ChildProcess proc("exec ../../examples/frt/rpc/fnet_rpc_client_app"); EXPECT_FALSE(runProc(proc, done)); } { - SlaveProc proc("exec ../../examples/frt/rpc/fnet_rpc_server_app"); + ChildProcess proc("exec ../../examples/frt/rpc/fnet_rpc_server_app"); EXPECT_FALSE(runProc(proc, done)); } { - SlaveProc proc("exec ../../examples/frt/rpc/fnet_echo_client_app"); + ChildProcess proc("exec ../../examples/frt/rpc/fnet_echo_client_app"); EXPECT_FALSE(runProc(proc, done)); } { - SlaveProc proc("exec ../../examples/frt/rpc/vespa-rpc-info"); + ChildProcess proc("exec ../../examples/frt/rpc/vespa-rpc-info"); EXPECT_FALSE(runProc(proc, done)); } { - SlaveProc proc("exec ../../examples/frt/rpc/vespa-rpc-invoke-bin"); + ChildProcess proc("exec ../../examples/frt/rpc/vespa-rpc-invoke-bin"); EXPECT_FALSE(runProc(proc, done)); } { - SlaveProc proc("exec ../../examples/frt/rpc/fnet_rpc_callback_server_app"); + ChildProcess proc("exec ../../examples/frt/rpc/fnet_rpc_callback_server_app"); EXPECT_FALSE(runProc(proc, done)); } { - SlaveProc proc("exec ../../examples/frt/rpc/fnet_rpc_callback_client_app"); + ChildProcess proc("exec ../../examples/frt/rpc/fnet_rpc_callback_client_app"); EXPECT_FALSE(runProc(proc, done)); } { - SlaveProc proc("exec ../../examples/frt/rpc/vespa-rpc-proxy"); + ChildProcess proc("exec ../../examples/frt/rpc/vespa-rpc-proxy"); EXPECT_FALSE(runProc(proc, done)); } } TEST("timeout") { std::string out; - EXPECT_TRUE(SlaveProc::run("exec ../../examples/timeout/fnet_timeout_app", out)); + EXPECT_TRUE(ChildProcess::run("exec ../../examples/timeout/fnet_timeout_app", out)); fprintf(stderr, "%s\n", out.c_str()); } TEST_MT_F("ping", 2, std::atomic<bool>()) { if (thread_id == 0) { - SlaveProc proc(vespalib::make_string("exec ../../examples/ping/fnet_pingserver_app tcp/%d", + ChildProcess proc(vespalib::make_string("exec ../../examples/ping/fnet_pingserver_app tcp/%d", PORT0).c_str()); TEST_BARRIER(); EXPECT_TRUE(runProc(proc, f1)); @@ -114,7 +114,7 @@ TEST_MT_F("ping", 2, std::atomic<bool>()) { TEST_MT_F("ping times out", 2, std::atomic<bool>()) { if (thread_id == 0) { - SlaveProc proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", + ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", PORT0).c_str()); TEST_BARRIER(); EXPECT_TRUE(runProc(proc, f1)); @@ -128,12 +128,12 @@ TEST_MT_F("ping times out", 2, std::atomic<bool>()) { TEST_MT_F("ping with proxy", 3, std::atomic<bool>()) { if (thread_id == 0) { - SlaveProc proc(vespalib::make_string("exec ../../examples/ping/fnet_pingserver_app tcp/%d", + ChildProcess proc(vespalib::make_string("exec ../../examples/ping/fnet_pingserver_app tcp/%d", PORT0).c_str()); TEST_BARRIER(); EXPECT_TRUE(runProc(proc, f1)); } else if (thread_id == 1) { - SlaveProc proc(vespalib::make_string("exec ../../examples/proxy/fnet_proxy_app tcp/%d tcp/localhost:%d", + ChildProcess proc(vespalib::make_string("exec ../../examples/proxy/fnet_proxy_app tcp/%d tcp/localhost:%d", PORT1, PORT0).c_str()); TEST_BARRIER(); EXPECT_TRUE(runProc(proc, f1)); @@ -147,7 +147,7 @@ TEST_MT_F("ping with proxy", 3, std::atomic<bool>()) { TEST_MT_F("rpc client server", 2, std::atomic<bool>()) { if (thread_id == 0) { - SlaveProc proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", + ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", PORT0).c_str()); TEST_BARRIER(); EXPECT_TRUE(runProc(proc, f1)); @@ -161,7 +161,7 @@ TEST_MT_F("rpc client server", 2, std::atomic<bool>()) { TEST_MT_F("rpc echo client", 2, std::atomic<bool>()) { if (thread_id == 0) { - SlaveProc proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", + ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", PORT0).c_str()); TEST_BARRIER(); EXPECT_TRUE(runProc(proc, f1)); @@ -175,7 +175,7 @@ TEST_MT_F("rpc echo client", 2, std::atomic<bool>()) { TEST_MT_F("rpc info", 2, std::atomic<bool>()) { if (thread_id == 0) { - SlaveProc proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", + ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", PORT0).c_str()); TEST_BARRIER(); EXPECT_TRUE(runProc(proc, f1)); @@ -191,7 +191,7 @@ TEST_MT_F("rpc info", 2, std::atomic<bool>()) { TEST_MT_F("rpc invoke", 2, std::atomic<bool>()) { if (thread_id == 0) { - SlaveProc proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", + ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_server_app tcp/%d", PORT0).c_str()); TEST_BARRIER(); EXPECT_TRUE(runProc(proc, f1)); @@ -206,7 +206,7 @@ TEST_MT_F("rpc invoke", 2, std::atomic<bool>()) { TEST_MT_F("rpc callback client server", 2, std::atomic<bool>()) { if (thread_id == 0) { - SlaveProc proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_callback_server_app tcp/%d", + ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_callback_server_app tcp/%d", PORT0).c_str()); TEST_BARRIER(); EXPECT_TRUE(runProc(proc, f1)); @@ -220,12 +220,12 @@ TEST_MT_F("rpc callback client server", 2, std::atomic<bool>()) { TEST_MT_F("rpc callback client server with proxy", 3, std::atomic<bool>()) { if (thread_id == 0) { - SlaveProc proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_callback_server_app tcp/%d", + ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/fnet_rpc_callback_server_app tcp/%d", PORT0).c_str()); TEST_BARRIER(); EXPECT_TRUE(runProc(proc, f1)); } else if (thread_id == 1) { - SlaveProc proc(vespalib::make_string("exec ../../examples/frt/rpc/vespa-rpc-proxy tcp/%d tcp/localhost:%d", + ChildProcess proc(vespalib::make_string("exec ../../examples/frt/rpc/vespa-rpc-proxy tcp/%d tcp/localhost:%d", PORT1, PORT0).c_str()); TEST_BARRIER(); EXPECT_TRUE(runProc(proc, f1)); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java index b1aceb81bc6..12168663205 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java @@ -113,4 +113,11 @@ public interface OsgiFramework { */ void stop() throws BundleException; + /** + * Returns true if this is a Felix based framework and not e.g. a test framework. + */ + default boolean isFelixFramework() { + return false; + } + } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java index c14e513fb98..bd189f8b898 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java @@ -169,6 +169,11 @@ public class FelixFramework implements OsgiFramework { collisionHook.allowDuplicateBundles(bundles); } + @Override + public boolean isFelixFramework() { + return true; + } + private void installBundle(String bundleLocation, Set<String> mask, List<Bundle> out) throws BundleException { bundleLocation = BundleLocationResolver.resolve(bundleLocation); if (mask.contains(bundleLocation)) { diff --git a/jdisc_http_service/pom.xml b/jdisc_http_service/pom.xml index bd8c77bc9cc..b140e66f28a 100644 --- a/jdisc_http_service/pom.xml +++ b/jdisc_http_service/pom.xml @@ -140,6 +140,7 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> <configuration> + <buildLegacyVespaPlatformBundle>true</buildLegacyVespaPlatformBundle> <discPreInstallBundle> javax.servlet-api-3.1.0.jar, jetty-continuation-${jetty.version}.jar, diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java index f5f1eb5791d..f8a6f47f946 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java @@ -5,10 +5,9 @@ package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.http.TextResponse; import ai.vespa.metricsproxy.metric.model.ConsumerId; +import ai.vespa.metricsproxy.metric.model.DimensionId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; -import ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil; -import ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil; import com.google.inject.Inject; import com.yahoo.container.handler.metrics.ErrorResponse; import com.yahoo.container.handler.metrics.HttpHandlerBase; @@ -25,7 +24,6 @@ import java.util.stream.Collectors; import static ai.vespa.metricsproxy.http.ValuesFetcher.getConsumerOrDefault; import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericApplicationModel; -import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericJsonModel; import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toMetricsPackets; import static ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil.toPrometheusModel; import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; @@ -84,7 +82,9 @@ public class ApplicationMetricsHandler extends HttpHandlerBase { List<GenericJsonModel> genericNodes = toGenericApplicationModel(metricsByNode).nodes; List<MetricsPacket> metricsForAllNodes = genericNodes.stream() - .flatMap(element -> toMetricsPackets(element).stream() + .flatMap(element -> toMetricsPackets(element) + .stream() + .map(builder -> builder.putDimension(DimensionId.toDimensionId("hostname"), element.hostname)) .map(MetricsPacket.Builder::build)) .collect(Collectors.toList()); return new TextResponse(200, toPrometheusModel(metricsForAllNodes).serialize()); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java index cc6b6b36057..0fa6fea7d11 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java @@ -11,6 +11,8 @@ import ai.vespa.metricsproxy.metric.model.json.GenericService; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.json.JSONArray; import org.json.JSONObject; import org.junit.Before; @@ -62,6 +64,8 @@ public class ApplicationMetricsHandlerTest { private static final String MOCK_METRICS_PATH = "/node0"; + private static final Pattern PROMETHEUS_REGEX_FORMAT = Pattern.compile("[a-z_]+([{]([A-Za-z_]+=\"[A-Za-z.\\-\\/0.9_]+\",)*[}])?( [0-9E]+(\\.[0-9E]+)?){2}"); + private int port; private static RequestHandlerTestDriver testDriver; @@ -112,13 +116,20 @@ public class ApplicationMetricsHandlerTest { @Ignore @Test - public void visually_inspect_values_response() throws Exception { + public void visually_inspect_values_response_metrics() throws Exception { String response = testDriver.sendRequest(METRICS_VALUES_URI).readAll(); ObjectMapper mapper = createObjectMapper(); var jsonModel = mapper.readValue(response, GenericApplicationModel.class); System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel)); } + @Ignore + @Test + public void visually_inspect_values_response_prometheus() throws Exception { + String response = testDriver.sendRequest(PROMETHEUS_VALUES_URI).readAll(); + System.out.println(response); + } + @Test public void response_contains_node() { GenericApplicationModel jsonModel = getResponseAsJsonModel(DEFAULT_PUBLIC_CONSUMER_ID.id); @@ -132,6 +143,23 @@ public class ApplicationMetricsHandlerTest { } @Test + public void prometheus_response_contains_hostname() { + String response = testDriver.sendRequest(PROMETHEUS_VALUES_URI).readAll(); + Arrays.stream(response.split("\n")) + .filter(line -> line.contains("{")) + .forEach(line -> assertTrue(line.contains("hostname"))); + } + + @Test + public void prometheus_response_obeys_format() { + String response = testDriver.sendRequest(PROMETHEUS_VALUES_URI).readAll(); + Arrays.stream(response.split("\n")) + .filter(line -> !line.startsWith("#")) + .filter(line -> !line.contains("{")) + .forEach(line -> assertTrue(PROMETHEUS_REGEX_FORMAT.matcher(line).find())); + } + + @Test public void response_contains_services_with_metrics() { GenericApplicationModel jsonModel = getResponseAsJsonModel(DEFAULT_PUBLIC_CONSUMER_ID.id); diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index 7ac912416de..84e92c2c316 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -94,6 +94,9 @@ MetricManager::~MetricManager() void MetricManager::stop() { + if (!running()) { + return; // Let stop() be idempotent. + } Runnable::stop(); { vespalib::MonitorGuard sync(_waiter); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 30ca2e0d218..a5efec1dcb7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -5,7 +5,6 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeType; -import java.util.logging.Level; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; @@ -16,8 +15,9 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule; import com.yahoo.vespa.hosted.node.admin.maintenance.disk.LinearCleanupRule; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentTask; import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; @@ -37,8 +37,8 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Function; +import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Stream; import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; import static com.yahoo.yolean.Exceptions.uncheck; @@ -107,6 +107,8 @@ public class StorageMaintainer { } public boolean cleanDiskIfFull(NodeAgentContext context) { + if (context.isDisabled(NodeAgentTask.DiskCleanup)) return false; + double totalBytes = context.node().diskSize().bytes(); // Delete enough bytes to get below 70% disk usage, but only if we are already using more than 80% disk long bytesToRemove = diskUsageFor(context) @@ -148,6 +150,7 @@ public class StorageMaintainer { /** Checks if container has any new coredumps, reports and archives them if so */ public void handleCoreDumpsForContainer(NodeAgentContext context, Optional<Container> container) { + if (context.isDisabled(NodeAgentTask.CoreDumps)) return; coredumpHandler.converge(context, () -> getCoredumpNodeAttributes(context, container)); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index 360cea8a60d..fe6b29402b5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -2,9 +2,9 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.acl; import com.google.common.net.InetAddresses; -import java.util.logging.Level; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentTask; import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses; @@ -18,6 +18,7 @@ import java.nio.file.Path; import java.util.List; import java.util.function.Consumer; import java.util.function.Supplier; +import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.yolean.Exceptions.uncheck; @@ -51,6 +52,8 @@ public class AclMaintainer { // ip(6)tables operate while having the xtables lock, run with synchronized to prevent multiple NodeAgents // invoking ip(6)tables concurrently. public synchronized void converge(NodeAgentContext context) { + if (context.isDisabled(NodeAgentTask.AclMaintainer)) return; + // Apply acl to the filter table editFlushOnError(context, IPVersion.IPv4, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv4)); editFlushOnError(context, IPVersion.IPv6, "filter", FilterTableLineEditor.from(context.acl(), IPVersion.IPv6)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 3320851a36c..d6c08a820cd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.identity; -import java.util.logging.Level; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; @@ -24,6 +23,7 @@ import com.yahoo.vespa.athenz.utils.SiaUtils; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentTask; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; @@ -46,6 +46,7 @@ import java.time.Instant; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -108,6 +109,8 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } public boolean converge(NodeAgentContext context) { + if (context.isDisabled(NodeAgentTask.CredentialsMaintainer)) return false; + try { context.log(logger, Level.FINE, "Checking certificate"); Path containerSiaDirectory = context.pathOnHostFromPathInNode(CONTAINER_SIA_DIRECTORY); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java index d589000c07e..872b8a8096b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java @@ -44,6 +44,10 @@ public interface NodeAgentContext extends TaskContext { String vespaUserOnHost(); + default boolean isDisabled(NodeAgentTask task) { + return false; + }; + /** * The vcpu value in NodeSpec is multiplied by the speedup factor per cpu core compared to a historical baseline * for a particular cpu generation of the host (see flavors.def cpuSpeedup). @@ -52,7 +56,7 @@ public interface NodeAgentContext extends TaskContext { */ double unscaledVcpu(); - /** The file system used by the NodeAgentContext. All paths must have the same provider. */ + /** The file system used by the NodeAgentContext. All paths must have the same provider. */ FileSystem fileSystem(); /** diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index 9f0c8d47d64..c7c0675c30e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -7,6 +7,10 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; @@ -18,6 +22,7 @@ import java.nio.file.Path; import java.nio.file.ProviderMismatchException; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; @@ -40,10 +45,11 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final String vespaUser; private final String vespaUserOnHost; private final double cpuSpeedup; + private final Set<NodeAgentTask> disabledNodeAgentTasks; public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzIdentity identity, DockerNetworking dockerNetworking, ZoneApi zone, - FileSystem fileSystem, + FileSystem fileSystem, FlagSource flagSource, Path pathToContainerStorage, Path pathToVespaHome, String vespaUser, String vespaUserOnHost, double cpuSpeedup) { if (cpuSpeedup <= 0) @@ -55,13 +61,15 @@ public class NodeAgentContextImpl implements NodeAgentContext { this.identity = Objects.requireNonNull(identity); this.dockerNetworking = Objects.requireNonNull(dockerNetworking); this.zone = Objects.requireNonNull(zone); - this.fileSystem = fileSystem; + this.fileSystem = Objects.requireNonNull(fileSystem); this.pathToNodeRootOnHost = requireValidPath(pathToContainerStorage).resolve(containerName.asString()); this.pathToVespaHome = requireValidPath(pathToVespaHome); this.logPrefix = containerName.asString() + ": "; this.vespaUser = vespaUser; this.vespaUserOnHost = vespaUserOnHost; this.cpuSpeedup = cpuSpeedup; + this.disabledNodeAgentTasks = NodeAgentTask.fromString( + Flags.DISABLED_HOST_ADMIN_TASKS.bindTo(flagSource).with(FetchVector.Dimension.HOSTNAME, node.hostname()).value()); } @Override @@ -105,6 +113,11 @@ public class NodeAgentContextImpl implements NodeAgentContext { } @Override + public boolean isDisabled(NodeAgentTask task) { + return disabledNodeAgentTasks.contains(task); + } + + @Override public double unscaledVcpu() { return node.vcpu() / cpuSpeedup; } @@ -212,6 +225,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { private String vespaUser; private String vespaUserOnHost; private FileSystem fileSystem = FileSystems.getDefault(); + private FlagSource flagSource; private double cpuSpeedUp = 1; public Builder(NodeSpec node) { @@ -268,6 +282,11 @@ public class NodeAgentContextImpl implements NodeAgentContext { return this; } + public Builder flagSource(FlagSource flagSource) { + this.flagSource = flagSource; + return this; + } + public Builder cpuSpeedUp(double cpuSpeedUp) { this.cpuSpeedUp = cpuSpeedUp; return this; @@ -301,6 +320,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { } }), fileSystem, + Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new), fileSystem.getPath("/home/docker/container-storage"), fileSystem.getPath("/opt/vespa"), Optional.ofNullable(vespaUser).orElse("vespa"), diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java new file mode 100644 index 00000000000..d57c680e190 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java @@ -0,0 +1,30 @@ +package com.yahoo.vespa.hosted.node.admin.nodeagent; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public enum NodeAgentTask { + + // The full task name is prefixed with 'node>', e.g. 'node>DiskCleanup' + DiskCleanup, + CoreDumps, + CredentialsMaintainer, + AclMaintainer; + + private static final Map<String, NodeAgentTask> tasksByName = Arrays.stream(NodeAgentTask.values()) + .collect(Collectors.toUnmodifiableMap(NodeAgentTask::taskName, n -> n)); + + private final String taskName; + NodeAgentTask() { + this.taskName = "node>" + name(); + } + + public String taskName() { return taskName; } + + public static Set<NodeAgentTask> fromString(List<String> tasks) { + return tasks.stream().filter(tasksByName::containsKey).map(tasksByName::get).collect(Collectors.toUnmodifiableSet()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java index 9bcbce849af..b7e0a2a1d97 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java @@ -2,14 +2,19 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.config.provision.DockerImage; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; import java.nio.file.FileSystem; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author freva @@ -84,6 +89,23 @@ public class NodeAgentContextImplTest { assertRewrite("docker.tld/vespa/hosted:1.2.3", "/opt/vespa/log", "/opt/vespa/log"); } + @Test + public void disabledTasksTest() { + NodeAgentContext context1 = createContextWithDisabledTasks(); + assertFalse(context1.isDisabled(NodeAgentTask.DiskCleanup)); + assertFalse(context1.isDisabled(NodeAgentTask.CoreDumps)); + + NodeAgentContext context2 = createContextWithDisabledTasks("root>UpgradeTask", "DiskCleanup", "node>CoreDumps"); + assertFalse(context2.isDisabled(NodeAgentTask.DiskCleanup)); + assertTrue(context2.isDisabled(NodeAgentTask.CoreDumps)); + } + + private static NodeAgentContext createContextWithDisabledTasks(String... tasks) { + InMemoryFlagSource flagSource = new InMemoryFlagSource(); + flagSource.withListFlag(Flags.DISABLED_HOST_ADMIN_TASKS.id(), List.of(tasks), String.class); + return new NodeAgentContextImpl.Builder("node123").flagSource(flagSource).build(); + } + private static void assertRewrite(String dockerImage, String path, String expected) { NodeAgentContext context = new NodeAgentContextImpl.Builder("node123") .nodeSpecBuilder(ns -> ns.wantedDockerImage(DockerImage.fromString(dockerImage))) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index be4f551ca29..e146583ae04 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -94,15 +94,6 @@ public final class Node { throw new IllegalArgumentException("Only hosts can be reserved to a tenant"); } - /** Returns the IP addresses of this node */ - // TODO: Remove and make callers access this through ipConfig() - public Set<String> ipAddresses() { return ipConfig.primary(); } - - /** Returns the IP address pool available on this node. These IP addresses are available for use by containers - * running on this node */ - // TODO: Remove and make callers access this through ipConfig() - public IP.Pool ipAddressPool() { return ipConfig.pool(); } - /** Returns the IP config of this node */ public IP.Config ipConfig() { return ipConfig; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 99b6c48c90b..7bf76b7c2c2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -18,6 +18,7 @@ import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.Node.State; import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; @@ -46,6 +47,7 @@ import com.yahoo.vespa.hosted.provision.restapi.NotFoundException; import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -58,6 +60,8 @@ import java.util.Set; import java.util.TreeSet; import java.util.function.BiFunction; import java.util.function.Predicate; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -92,6 +96,8 @@ import java.util.stream.Stream; // Nodes might have an application assigned in dirty. public class NodeRepository extends AbstractComponent { + private static final Logger log = Logger.getLogger(NodeRepository.class.getName()); + private final CuratorDatabaseClient db; private final Clock clock; private final Zone zone; @@ -106,6 +112,7 @@ public class NodeRepository extends AbstractComponent { private final Applications applications; private final boolean canProvisionHosts; private final int spareCount; + private final boolean useConfigServerLock; /** * Creates a node repository from a zookeeper provider. @@ -146,7 +153,9 @@ public class NodeRepository extends AbstractComponent { boolean useCuratorClientCache, boolean canProvisionHosts, int spareCount) { - this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache); + // Flag is read once here as it shouldn't not change at runtime + this.useConfigServerLock = Flags.USE_CONFIG_SERVER_LOCK.bindTo(flagSource).value(); + this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache, useConfigServerLock); this.zone = zone; this.clock = clock; this.flavors = flavors; @@ -160,13 +169,33 @@ public class NodeRepository extends AbstractComponent { this.applications = new Applications(db); this.canProvisionHosts = canProvisionHosts; this.spareCount = spareCount; - - // read and write all nodes to make sure they are stored in the latest version of the serialized format - for (State state : State.values()) - // TODO(mpolden): Add per-node locking. In its current state this may collide with other callers making - // node state changes. Example: A redeployment on another config server which moves a node - // to another state while this is constructed. - db.writeTo(state, db.readNodes(state), Agent.system, Optional.empty()); + rewriteNodes(); + } + + /** Read and write all nodes to make sure they are stored in the latest version of the serialized format */ + private void rewriteNodes() { + Instant start = clock.instant(); + int nodesWritten = 0; + if (useConfigServerLock) { + for (var state : State.values()) { + for (var node : db.readNodes(state)) { + try (var lock = lock(node)) { + var currentNode = db.readNode(node.hostname()); + if (currentNode.isEmpty()) continue; // Node disappeared while running this loop + db.writeTo(currentNode.get().state(), currentNode.get(), Agent.system, Optional.empty()); + nodesWritten++; + } + } + } + } else { + for (State state : State.values()) { + List<Node> nodes = db.readNodes(state); + db.writeTo(state, nodes, Agent.system, Optional.empty()); + nodesWritten += nodes.size(); + } + } + Instant end = clock.instant(); + log.log(Level.INFO, String.format("Rewrote %d nodes in %s", nodesWritten, Duration.between(start, end))); } /** Returns the curator database client used by this */ @@ -829,10 +858,14 @@ public class NodeRepository extends AbstractComponent { public Zone zone() { return zone; } /** Create a lock which provides exclusive rights to making changes to the given application */ - public Mutex lock(ApplicationId application) { return db.legacyLock(application); } + public Mutex lock(ApplicationId application) { + return useConfigServerLock ? db.lock(application) : db.legacyLock(application); + } /** Create a lock with a timeout which provides exclusive rights to making changes to the given application */ - public Mutex lock(ApplicationId application, Duration timeout) { return db.legacyLock(application, timeout); } + public Mutex lock(ApplicationId application, Duration timeout) { + return useConfigServerLock ? db.lock(application, timeout) : db.legacyLock(application, timeout); + } /** Create a lock which provides exclusive rights to modifying unallocated nodes */ public Mutex lockUnallocated() { return db.lockInactive(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java index 1431f21de47..3630cf4c1e4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerId.java @@ -50,6 +50,11 @@ public class LoadBalancerId { return Objects.hash(application, cluster); } + @Override + public String toString() { + return "load balancer " + serializedForm; + } + /** Create an instance from a serialized value on the form tenant:application:instance:cluster-id */ public static LoadBalancerId fromSerializedForm(String value) { int lastSeparator = value.lastIndexOf(":"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index bc4381573c6..559dbe63cba 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -43,7 +43,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { var firstProxyNode = proxyNodes.get(0); var networkNames = proxyNodes.stream() - .flatMap(node -> node.ipAddresses().stream()) + .flatMap(node -> node.ipConfig().primary().stream()) .map(SharedLoadBalancerService::withPrefixLength) .collect(Collectors.toSet()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index a762f718ab7..9980335bab0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -35,14 +35,15 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer { new DaemonThreadFactory("node repo application maintainer")); protected ApplicationMaintainer(Deployer deployer, Metric metric, NodeRepository nodeRepository, Duration interval) { - super(nodeRepository, interval); + super(nodeRepository, interval, metric); this.deployer = deployer; this.metric = metric; } @Override - protected final void maintain() { + protected final boolean maintain() { applicationsNeedingMaintenance().forEach(this::deploy); + return true; } /** Returns the number of deployments that are pending execution */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index c32b7854d4e..e2b98d8d000 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; -import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import java.time.Duration; import java.util.List; @@ -38,17 +37,19 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { Deployer deployer, Metric metric, Duration interval) { - super(nodeRepository, interval); + super(nodeRepository, interval, metric); this.autoscaler = new Autoscaler(metricsDb, nodeRepository); this.metric = metric; this.deployer = deployer; } @Override - protected void maintain() { - if ( ! nodeRepository().zone().environment().isProduction()) return; + protected boolean maintain() { + boolean success = true; + if ( ! nodeRepository().zone().environment().isProduction()) return success; activeNodesByApplication().forEach((applicationId, nodes) -> autoscale(applicationId, nodes)); + return success; } private void autoscale(ApplicationId application, List<Node> applicationNodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index f583728f9b8..92abae46f9a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -142,12 +142,12 @@ public class CapacityChecker { for (var host : hosts) { NodeResources hostResources = host.flavor().resources(); int occupiedIps = 0; - Set<String> ipPool = host.ipAddressPool().asSet(); + Set<String> ipPool = host.ipConfig().pool().asSet(); for (var child : nodeChildren.get(host)) { hostResources = hostResources.subtract(child.resources().justNumbers()); - occupiedIps += child.ipAddresses().stream().filter(ipPool::contains).count(); + occupiedIps += child.ipConfig().primary().stream().filter(ipPool::contains).count(); } - availableResources.put(host, new AllocationResources(hostResources, host.ipAddressPool().asSet().size() - occupiedIps)); + availableResources.put(host, new AllocationResources(hostResources, host.ipConfig().pool().asSet().size() - occupiedIps)); } return availableResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java index f428e276df8..eb5973f11a9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -25,8 +26,8 @@ public class DirtyExpirer extends Expirer { private final NodeRepository nodeRepository; - DirtyExpirer(NodeRepository nodeRepository, Clock clock, Duration dirtyTimeout) { - super(Node.State.dirty, History.Event.Type.deallocated, nodeRepository, clock, dirtyTimeout); + DirtyExpirer(NodeRepository nodeRepository, Clock clock, Duration dirtyTimeout, Metric metric) { + super(Node.State.dirty, History.Event.Type.deallocated, nodeRepository, clock, dirtyTimeout, metric); this.nodeRepository = nodeRepository; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 0a32970e056..b9005a028ff 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.jdisc.Metric; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; @@ -49,19 +50,21 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, HostProvisioner hostProvisioner, - FlagSource flagSource) { - super(nodeRepository, interval); + FlagSource flagSource, + Metric metric) { + super(nodeRepository, interval, metric); this.hostProvisioner = hostProvisioner; this.targetCapacityFlag = Flags.TARGET_CAPACITY.bindTo(flagSource); } @Override - protected void maintain() { + protected boolean maintain() { try (Mutex lock = nodeRepository().lockUnallocated()) { NodeList nodes = nodeRepository().list(); resumeProvisioning(nodes, lock); convergeToCapacity(nodes); } + return true; } /** Resume provisioning of already provisioned hosts and their children */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java index dc5155312e7..43f5210b233 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.History; @@ -32,8 +33,8 @@ public abstract class Expirer extends NodeRepositoryMaintainer { private final Duration expiryTime; Expirer(Node.State fromState, History.Event.Type eventType, NodeRepository nodeRepository, - Clock clock, Duration expiryTime) { - super(nodeRepository, min(Duration.ofMinutes(10), expiryTime)); + Clock clock, Duration expiryTime, Metric metric) { + super(nodeRepository, min(Duration.ofMinutes(10), expiryTime), metric); this.fromState = fromState; this.eventType = eventType; this.clock = clock; @@ -41,7 +42,7 @@ public abstract class Expirer extends NodeRepositoryMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { List<Node> expired = new ArrayList<>(); for (Node node : nodeRepository().getNodes(fromState)) { if (isExpired(node)) @@ -50,6 +51,7 @@ public abstract class Expirer extends NodeRepositoryMaintainer { if ( ! expired.isEmpty()) log.info(fromState + " expirer found " + expired.size() + " expired nodes: " + expired); expire(expired); + return true; } protected boolean isExpired(Node node) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index d65b4ce4248..2afbb0e6476 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -3,9 +3,9 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -22,24 +22,21 @@ import java.util.stream.Collectors; /** * This moves expired failed nodes: - * <ul> - * <li>To parked: If the node has known hardware failure, Docker hosts are moved to parked only when all of their - * children are already in parked - * <li>To dirty: If the node has failed less than 5 times OR the environment is dev, test or perf. - * Those environments have no protection against users running bogus applications, so - * we cannot use the node failure count to conclude the node has a failure. - * <li>Otherwise the node will remain in failed - * </ul> + * + * - To parked: If the node has known hardware failure, Docker hosts are moved to parked only when all of their + * children are already in parked. + * - To dirty: If the node is a host and has failed less than 5 times, or always if the node is a child. + * - Otherwise the node will remain in failed. + * * Failed content nodes are given a long expiry time to enable us to manually moved them back to * active to recover data in cases where the node was failed accidentally. - * <p> + * * Failed container (Vespa, not Docker) nodes are expired early as there's no data to potentially recover. - * </p> - * <p> + * * The purpose of the automatic recycling to dirty + fail count is that nodes which were moved * to failed due to some undetected hardware failure will end up being failed again. * When that has happened enough they will not be recycled. - * <p> + * * Nodes with detected hardware issues will not be recycled. * * @author bratseth @@ -56,8 +53,8 @@ public class FailedExpirer extends NodeRepositoryMaintainer { private final Duration defaultExpiry; // Grace period to allow recovery of data private final Duration containerExpiry; // Stateless nodes, no data to recover - FailedExpirer(NodeRepository nodeRepository, Zone zone, Clock clock, Duration interval) { - super(nodeRepository, interval); + FailedExpirer(NodeRepository nodeRepository, Zone zone, Clock clock, Duration interval, Metric metric) { + super(nodeRepository, interval, metric); this.nodeRepository = nodeRepository; this.zone = zone; this.clock = clock; @@ -74,7 +71,7 @@ public class FailedExpirer extends NodeRepositoryMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { List<Node> remainingNodes = new ArrayList<>(nodeRepository.list() .state(Node.State.failed) .nodeType(NodeType.tenant, NodeType.host) @@ -86,6 +83,7 @@ public class FailedExpirer extends NodeRepositoryMaintainer { node.history().hasEventBefore(History.Event.Type.failed, clock.instant().minus(containerExpiry))); recycleIf(remainingNodes, node -> node.history().hasEventBefore(History.Event.Type.failed, clock.instant().minus(defaultExpiry))); + return true; } /** Recycle the nodes matching condition, and remove those nodes from the nodes list. */ @@ -123,8 +121,7 @@ public class FailedExpirer extends NodeRepositoryMaintainer { /** Returns whether the current node fail count should be used as an indicator of hardware issue */ private boolean failCountIndicatesHardwareIssue(Node node) { - if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER) return false; - return (zone.environment() == Environment.prod || zone.environment() == Environment.staging) && - node.status().failCount() >= maxAllowedFailures; + return node.type().isHost() && node.status().failCount() >= maxAllowedFailures; } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java index 3cb7cc218a7..389fc0ee907 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -36,8 +37,8 @@ public class InactiveExpirer extends Expirer { private final NodeRepository nodeRepository; - InactiveExpirer(NodeRepository nodeRepository, Clock clock, Duration inactiveTimeout) { - super(Node.State.inactive, History.Event.Type.deactivated, nodeRepository, clock, inactiveTimeout); + InactiveExpirer(NodeRepository nodeRepository, Clock clock, Duration inactiveTimeout, Metric metric) { + super(Node.State.inactive, History.Event.Type.deactivated, nodeRepository, clock, inactiveTimeout, metric); this.nodeRepository = nodeRepository; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java index b933e549357..e317333135c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -2,10 +2,11 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.InfraDeployer; -import java.util.logging.Level; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -20,8 +21,8 @@ public class InfrastructureProvisioner extends NodeRepositoryMaintainer { private final InfraDeployer infraDeployer; - InfrastructureProvisioner(NodeRepository nodeRepository, InfraDeployer infraDeployer, Duration interval) { - super(nodeRepository, interval); + InfrastructureProvisioner(NodeRepository nodeRepository, InfraDeployer infraDeployer, Duration interval, Metric metric) { + super(nodeRepository, interval, metric); this.infraDeployer = infraDeployer; } @@ -38,7 +39,9 @@ public class InfrastructureProvisioner extends NodeRepositoryMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { infraDeployer.activateAllSupportedInfraApplications(false); + return true; } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 6edd57de1c1..90cf3ba8f54 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; @@ -39,17 +40,16 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { private final LoadBalancerService service; private final CuratorDatabaseClient db; - LoadBalancerExpirer(NodeRepository nodeRepository, Duration interval, LoadBalancerService service) { - super(nodeRepository, interval); + LoadBalancerExpirer(NodeRepository nodeRepository, Duration interval, LoadBalancerService service, Metric metric) { + super(nodeRepository, interval, metric); this.service = Objects.requireNonNull(service, "service must be non-null"); this.db = nodeRepository.database(); } @Override - protected void maintain() { + protected boolean maintain() { expireReserved(); - removeInactive(); - pruneReals(); + return removeInactive() & pruneReals(); } /** Move reserved load balancer that have expired to inactive */ @@ -63,7 +63,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { } /** Deprovision inactive load balancers that have expired */ - private void removeInactive() { + private boolean removeInactive() { var failed = new ArrayList<LoadBalancerId>(); var lastException = new AtomicReference<Exception>(); var now = nodeRepository().clock().instant(); @@ -88,10 +88,11 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { interval()), lastException.get()); } + return lastException.get() == null; } /** Remove reals from inactive load balancers */ - private void pruneReals() { + private boolean pruneReals() { var failed = new ArrayList<LoadBalancerId>(); var lastException = new AtomicReference<Exception>(); withLoadBalancersIn(State.inactive, lb -> { @@ -109,13 +110,14 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { }); if (!failed.isEmpty()) { log.log(Level.WARNING, String.format("Failed to remove reals from %d load balancers: %s, retrying in %s", - failed.size(), - failed.stream() - .map(LoadBalancerId::serializedForm) - .collect(Collectors.joining(", ")), - interval()), + failed.size(), + failed.stream() + .map(LoadBalancerId::serializedForm) + .collect(Collectors.joining(", ")), + interval()), lastException.get()); } + return lastException.get() == null; } /** Apply operation to all load balancers that exist in given state, while holding lock */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index c631de5f17b..e0d7dc5f19e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -50,7 +50,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { Supplier<Integer> pendingRedeploymentsSupplier, Duration interval, Clock clock) { - super(nodeRepository, interval); + super(nodeRepository, interval, metric); this.metric = metric; this.orchestrator = orchestrator; this.serviceMonitor = serviceMonitor; @@ -59,7 +59,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { } @Override - public void maintain() { + public boolean maintain() { NodeList nodes = nodeRepository().list(); ServiceModel serviceModel = serviceMonitor.getServiceModelSnapshot(); @@ -68,6 +68,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { updateMaintenanceMetrics(); updateDockerMetrics(nodes); updateTenantUsageMetrics(nodes); + return true; } private void updateMaintenanceMetrics() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 9c1892a1920..a2a189769bf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -78,7 +78,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { Duration downTimeLimit, Clock clock, Orchestrator orchestrator, ThrottlePolicy throttlePolicy, Metric metric) { // check ping status every five minutes, but at least twice as often as the down time limit - super(nodeRepository, min(downTimeLimit.dividedBy(2), Duration.ofMinutes(5))); + super(nodeRepository, min(downTimeLimit.dividedBy(2), Duration.ofMinutes(5)), metric); this.deployer = deployer; this.hostLivenessTracker = hostLivenessTracker; this.serviceMonitor = serviceMonitor; @@ -91,7 +91,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { int throttledHostFailures = 0; int throttledNodeFailures = 0; @@ -131,9 +131,11 @@ public class NodeFailer extends NodeRepositoryMaintainer { failActive(node, reason); } - metric.set(throttlingActiveMetric, Math.min( 1, throttledHostFailures + throttledNodeFailures), null); + int throttlingActive = Math.min(1, throttledHostFailures + throttledNodeFailures); + metric.set(throttlingActiveMetric, throttlingActive, null); metric.set(throttledHostFailuresMetric, throttledHostFailures, null); metric.set(throttledNodeFailuresMetric, throttledNodeFailures, null); + return throttlingActive == 0; } private void updateNodeLivenessEventsForReadyNodes(Mutex lock) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index eb2b46dd53e..222ee631968 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -2,8 +2,9 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; import com.yahoo.yolean.Exceptions; @@ -26,14 +27,15 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { public NodeMetricsDbMaintainer(NodeRepository nodeRepository, NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb, - Duration interval) { - super(nodeRepository, interval); + Duration interval, + Metric metric) { + super(nodeRepository, interval, metric); this.nodeMetrics = nodeMetrics; this.nodeMetricsDb = nodeMetricsDb; } @Override - protected void maintain() { + protected boolean maintain() { int warnings = 0; for (ApplicationId application : activeNodesByApplication().keySet()) { try { @@ -46,6 +48,7 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { } } nodeMetricsDb.gc(nodeRepository().clock()); + return warnings == 0; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java index c78ed72ff42..f64f27b1219 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.Flavor; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.IntFlag; @@ -32,15 +33,15 @@ public class NodeRebooter extends NodeRepositoryMaintainer { private final Clock clock; private final Random random; - NodeRebooter(NodeRepository nodeRepository, Clock clock, FlagSource flagSource) { - super(nodeRepository, Duration.ofMinutes(25)); + NodeRebooter(NodeRepository nodeRepository, Clock clock, FlagSource flagSource, Metric metric) { + super(nodeRepository, Duration.ofMinutes(25), metric); this.rebootIntervalInDays = Flags.REBOOT_INTERVAL_IN_DAYS.bindTo(flagSource); this.clock = clock; this.random = new Random(clock.millis()); // seed with clock for test determinism } @Override - protected void maintain() { + protected boolean maintain() { // Reboot candidates: Nodes in long-term states, where we know we can safely orchestrate a reboot List<Node> nodesToReboot = nodeRepository().getNodes(Node.State.active, Node.State.ready).stream() .filter(node -> node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) @@ -49,6 +50,7 @@ public class NodeRebooter extends NodeRepositoryMaintainer { if (!nodesToReboot.isEmpty()) nodeRepository().reboot(NodeListFilter.from(nodesToReboot)); + return true; } private boolean shouldReboot(Node node) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java index 8368569cda0..5f87cf9fd9b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java @@ -1,9 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.concurrent.maintenance.JobMetrics; import com.yahoo.concurrent.maintenance.Maintainer; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -21,8 +23,9 @@ public abstract class NodeRepositoryMaintainer extends Maintainer { private final NodeRepository nodeRepository; - public NodeRepositoryMaintainer(NodeRepository nodeRepository, Duration interval) { - super(null, interval, nodeRepository.clock().instant(), nodeRepository.jobControl(), nodeRepository.database().cluster()); + public NodeRepositoryMaintainer(NodeRepository nodeRepository, Duration interval, Metric metric) { + super(null, interval, nodeRepository.clock().instant(), nodeRepository.jobControl(), jobMetrics(metric), + nodeRepository.database().cluster()); this.nodeRepository = nodeRepository; } @@ -41,4 +44,10 @@ public abstract class NodeRepositoryMaintainer extends Maintainer { .collect(Collectors.groupingBy(node -> node.allocation().get().owner())); } + private static JobMetrics jobMetrics(Metric metric) { + return new JobMetrics((job, consecutiveFailures) -> { + metric.set("maintenance.consecutiveFailures", consecutiveFailures, metric.createContext(Map.of("job", job))); + }); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 4323622df8b..a5482281ef1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -75,25 +75,25 @@ public class NodeRepositoryMaintenance extends AbstractComponent { nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, defaults.failGrace, clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric); periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, metric, nodeRepository, defaults.redeployMaintainerInterval, defaults.periodicRedeployInterval); operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, metric, nodeRepository, defaults.operatorChangeRedeployInterval); - reservationExpirer = new ReservationExpirer(nodeRepository, clock, defaults.reservationExpiry); + reservationExpirer = new ReservationExpirer(nodeRepository, clock, defaults.reservationExpiry, metric); retiredExpirer = new RetiredExpirer(nodeRepository, orchestrator, deployer, metric, clock, defaults.retiredInterval, defaults.retiredExpiry); - inactiveExpirer = new InactiveExpirer(nodeRepository, clock, defaults.inactiveExpiry); - failedExpirer = new FailedExpirer(nodeRepository, zone, clock, defaults.failedExpirerInterval); - dirtyExpirer = new DirtyExpirer(nodeRepository, clock, defaults.dirtyExpiry); - provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, defaults.provisionedExpiry); - nodeRebooter = new NodeRebooter(nodeRepository, clock, flagSource); + inactiveExpirer = new InactiveExpirer(nodeRepository, clock, defaults.inactiveExpiry, metric); + failedExpirer = new FailedExpirer(nodeRepository, zone, clock, defaults.failedExpirerInterval, metric); + dirtyExpirer = new DirtyExpirer(nodeRepository, clock, defaults.dirtyExpiry, metric); + provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, defaults.provisionedExpiry, metric); + nodeRebooter = new NodeRebooter(nodeRepository, clock, flagSource, metric); metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval, clock); - infrastructureProvisioner = new InfrastructureProvisioner(nodeRepository, infraDeployer, defaults.infrastructureProvisionInterval); + infrastructureProvisioner = new InfrastructureProvisioner(nodeRepository, infraDeployer, defaults.infrastructureProvisionInterval, metric); loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService(nodeRepository).map(lbService -> - new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService)); + new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric)); dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner -> - new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource)); + new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource, metric)); spareCapacityMaintainer = new SpareCapacityMaintainer(deployer, nodeRepository, metric, defaults.spareCapacityMaintenanceInterval); - osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); + osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval, metric); rebalancer = new Rebalancer(deployer, nodeRepository, metric, clock, defaults.rebalancerInterval); - nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval); + nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval, metric); autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, nodeMetricsDb, deployer, metric, defaults.autoscalingInterval); - scalingSuggestionsMaintainer = new ScalingSuggestionsMaintainer(nodeRepository, nodeMetricsDb, defaults.scalingSuggestionsInterval); + scalingSuggestionsMaintainer = new ScalingSuggestionsMaintainer(nodeRepository, nodeMetricsDb, defaults.scalingSuggestionsInterval, metric); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintainButThrowOnException(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java index 11afbd785e8..be1190ccff4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.NodeType; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -17,17 +18,18 @@ import java.time.Duration; */ public class OsUpgradeActivator extends NodeRepositoryMaintainer { - public OsUpgradeActivator(NodeRepository nodeRepository, Duration interval) { - super(nodeRepository, interval); + public OsUpgradeActivator(NodeRepository nodeRepository, Duration interval, Metric metric) { + super(nodeRepository, interval, metric); } @Override - protected void maintain() { + protected boolean maintain() { for (var nodeType : NodeType.values()) { if (!nodeType.isHost()) continue; var active = canUpgradeOsOf(nodeType); nodeRepository().osVersions().resumeUpgradeOf(nodeType, active); } + return true; } /** Returns whether to allow OS upgrade of nodes of given type */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java index e1407f2a41d..d38bff091b0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -19,8 +20,8 @@ public class ProvisionedExpirer extends Expirer { private final NodeRepository nodeRepository; - ProvisionedExpirer(NodeRepository nodeRepository, Clock clock, Duration dirtyTimeout) { - super(Node.State.provisioned, History.Event.Type.provisioned, nodeRepository, clock, dirtyTimeout); + ProvisionedExpirer(NodeRepository nodeRepository, Clock clock, Duration dirtyTimeout, Metric metric) { + super(Node.State.provisioned, History.Event.Type.provisioned, nodeRepository, clock, dirtyTimeout, metric); this.nodeRepository = nodeRepository; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index 3df20fa9d08..9b9c7df5d0d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -31,22 +31,24 @@ public class Rebalancer extends NodeRepositoryMaintainer { Metric metric, Clock clock, Duration interval) { - super(nodeRepository, interval); + super(nodeRepository, interval, metric); this.deployer = deployer; this.metric = metric; this.clock = clock; } @Override - protected void maintain() { - if ( ! nodeRepository().zone().getCloud().allowHostSharing()) return; // Rebalancing not necessary - if (nodeRepository().zone().environment().isTest()) return; // Short lived deployments; no need to rebalance + protected boolean maintain() { + boolean success = true; + if ( ! nodeRepository().zone().getCloud().allowHostSharing()) return success; // Rebalancing not necessary + if (nodeRepository().zone().environment().isTest()) return success; // Short lived deployments; no need to rebalance // Work with an unlocked snapshot as this can take a long time and full consistency is not needed NodeList allNodes = nodeRepository().list(); updateSkewMetric(allNodes); - if ( ! zoneIsStable(allNodes)) return; + if ( ! zoneIsStable(allNodes)) return success; findBestMove(allNodes).execute(true, Agent.Rebalancer, deployer, metric, nodeRepository()); + return success; } /** We do this here rather than in MetricsReporter because it is expensive and frequent updates are unnecessary */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java index 03d466dbf09..27f77dd08a3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -22,8 +23,8 @@ public class ReservationExpirer extends Expirer { private final NodeRepository nodeRepository; - public ReservationExpirer(NodeRepository nodeRepository, Clock clock, Duration reservationPeriod) { - super(Node.State.reserved, History.Event.Type.reserved, nodeRepository, clock, reservationPeriod); + public ReservationExpirer(NodeRepository nodeRepository, Clock clock, Duration reservationPeriod, Metric metric) { + super(Node.State.reserved, History.Event.Type.reserved, nodeRepository, clock, reservationPeriod, metric); this.nodeRepository = nodeRepository; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java index a8566e24743..5b7f90102ba 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -39,7 +39,7 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { Clock clock, Duration maintenanceInterval, Duration retiredExpiry) { - super(nodeRepository, maintenanceInterval); + super(nodeRepository, maintenanceInterval, metric); this.deployer = deployer; this.metric = metric; this.orchestrator = orchestrator; @@ -48,7 +48,7 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { } @Override - protected void maintain() { + protected boolean maintain() { List<Node> activeNodes = nodeRepository().getNodes(Node.State.active); Map<ApplicationId, List<Node>> retiredNodesByApplication = activeNodes.stream() @@ -69,11 +69,12 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { nodeRepository().setRemovable(application, nodesToRemove); boolean success = deployment.activate(); - if ( ! success) return; + if ( ! success) return success; String nodeList = nodesToRemove.stream().map(Node::hostname).collect(Collectors.joining(", ")); log.info("Redeployed " + application + " to deactivate retired nodes: " + nodeList); } } + return true; } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index b68e8eacbaa..b0c52d10f7d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.jdisc.Metric; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -12,7 +13,6 @@ import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; -import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import java.time.Duration; import java.util.List; @@ -31,16 +31,19 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { public ScalingSuggestionsMaintainer(NodeRepository nodeRepository, NodeMetricsDb metricsDb, - Duration interval) { - super(nodeRepository, interval); + Duration interval, + Metric metric) { + super(nodeRepository, interval, metric); this.autoscaler = new Autoscaler(metricsDb, nodeRepository); } @Override - protected void maintain() { - if ( ! nodeRepository().zone().environment().isProduction()) return; + protected boolean maintain() { + boolean success = true; + if ( ! nodeRepository().zone().environment().isProduction()) return success; activeNodesByApplication().forEach((applicationId, nodes) -> suggest(applicationId, nodes)); + return success; } private void suggest(ApplicationId application, List<Node> applicationNodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java index 90c3a277080..20258e7947b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java @@ -56,15 +56,16 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer { Metric metric, Duration interval, int maxIterations) { - super(nodeRepository, interval); + super(nodeRepository, interval, metric); this.deployer = deployer; this.metric = metric; this.maxIterations = maxIterations; } @Override - protected void maintain() { - if ( ! nodeRepository().zone().getCloud().allowHostSharing()) return; + protected boolean maintain() { + boolean success = true; + if ( ! nodeRepository().zone().getCloud().allowHostSharing()) return success; CapacityChecker capacityChecker = new CapacityChecker(nodeRepository()); @@ -89,6 +90,7 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer { } metric.set("spareHostCapacity", spareHostCapacity, null); } + return success; } private boolean execute(List<Move> mitigation) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 5e2f0bd4761..cc62ae67e84 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -77,13 +77,15 @@ public class CuratorDatabaseClient { private final Clock clock; private final Zone zone; private final CuratorCounter provisionIndexCounter; + private final boolean logStackTracesOnLockTimeout; - public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, boolean useCache) { + public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, boolean useCache, boolean logStackTracesOnLockTimeout) { this.nodeSerializer = new NodeSerializer(flavors); this.zone = zone; this.db = new CuratorDatabase(curator, root, useCache); this.clock = clock; this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter").getAbsolute()); + this.logStackTracesOnLockTimeout = logStackTracesOnLockTimeout; initZK(); } @@ -394,6 +396,22 @@ public class CuratorDatabaseClient { try { return db.lock(lockPath(application), timeout); } catch (UncheckedTimeoutException e) { + if (logStackTracesOnLockTimeout) { + log.log(Level.WARNING, "Logging stack trace from all threads due to lock timeout"); + Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces(); + for (Map.Entry<Thread, StackTraceElement[]> kv : stackTraces.entrySet()) { + StringBuilder sb = new StringBuilder(); + sb.append("Thread '") + .append(kv.getKey().getName()) + .append("'\n"); + for (var stackTraceElement : kv.getValue()) { + sb.append("\tat ") + .append(stackTraceElement) + .append("\n"); + } + log.log(Level.WARNING, sb.toString()); + } + } throw new ApplicationLockException(e); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java index fd16e61417f..e8bd598626b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -75,7 +74,7 @@ public class HostCapacity { * Number of free (not allocated) IP addresses assigned to the dockerhost. */ int freeIPs(Node dockerHost) { - return dockerHost.ipAddressPool().findUnused(allNodes).size(); + return dockerHost.ipConfig().pool().findUnused(allNodes).size(); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index e710d70f20f..6403bbe2b0c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -165,33 +165,33 @@ public class LoadBalancerProvisioner { if (loadBalancer.isEmpty() && activate) return; // Nothing to activate as this load balancer was never prepared var force = loadBalancer.isPresent() && loadBalancer.get().state() != LoadBalancer.State.active; - var instance = provisionInstance(application, clusterId, nodes, force); + var instance = provisionInstance(id, nodes, force); LoadBalancer newLoadBalancer; if (loadBalancer.isEmpty()) { newLoadBalancer = new LoadBalancer(id, instance, LoadBalancer.State.reserved, now); } else { var newState = activate ? LoadBalancer.State.active : loadBalancer.get().state(); newLoadBalancer = loadBalancer.get().with(instance).with(newState, now); + if (loadBalancer.get().state() != newLoadBalancer.state()) { + log.log(logLevel(), "Moving " + newLoadBalancer.id() + " to state " + newLoadBalancer.state()); + } } db.writeLoadBalancer(newLoadBalancer); } - private LoadBalancerInstance provisionInstance(ApplicationId application, ClusterSpec.Id cluster, List<Node> nodes, - boolean force) { + private LoadBalancerInstance provisionInstance(LoadBalancerId id, List<Node> nodes, boolean force) { var reals = new LinkedHashSet<Real>(); for (var node : nodes) { for (var ip : reachableIpAddresses(node)) { reals.add(new Real(HostName.from(node.hostname()), ip)); } } - log.log(Level.FINE, "Creating load balancer for " + cluster + " in " + application.toShortString() + - ", targeting: " + reals); + log.log(logLevel(), "Creating " + id + ", targeting: " + reals); try { - return service.create(new LoadBalancerSpec(application, cluster, reals), force); + return service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals), force); } catch (Exception e) { - throw new LoadBalancerServiceException("Failed to (re)configure load balancer for " + cluster + " in " + - application + ", targeting: " + reals + ". The operation will be " + - "retried on next deployment", e); + throw new LoadBalancerServiceException("Failed to (re)configure " + id + ", targeting: " + + reals + ". The operation will be retried on next deployment", e); } } @@ -216,7 +216,7 @@ public class LoadBalancerProvisioner { /** Find IP addresses reachable by the load balancer service */ private Set<String> reachableIpAddresses(Node node) { - Set<String> reachable = new LinkedHashSet<>(node.ipAddresses()); + Set<String> reachable = new LinkedHashSet<>(node.ipConfig().primary()); // Remove addresses unreachable by the load balancer service switch (service.protocol()) { case ipv4: @@ -233,4 +233,8 @@ public class LoadBalancerProvisioner { return cluster.combinedId().orElse(cluster.id()); } + private Level logLevel() { + return nodeRepository.zone().system().isCd() ? Level.INFO : Level.FINE; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java index a7577392fe2..07e93111b6f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java @@ -55,7 +55,7 @@ public class NodeAclResponse extends HttpResponse { } private void toSlime(NodeAcl nodeAcl, Cursor array) { - nodeAcl.trustedNodes().forEach(node -> node.ipAddresses().forEach(ipAddress -> { + nodeAcl.trustedNodes().forEach(node -> node.ipConfig().primary().forEach(ipAddress -> { Cursor object = array.addObject(); object.setString("hostname", node.hostname()); object.setString("type", node.type().name()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 58e838018c6..3efd6e417cb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -180,8 +180,8 @@ class NodesResponse extends HttpResponse { object.setBool("wantToRetire", node.status().wantToRetire()); object.setBool("wantToDeprovision", node.status().wantToDeprovision()); toSlime(node.history(), object.setArray("history")); - ipAddressesToSlime(node.ipAddresses(), object.setArray("ipAddresses")); - ipAddressesToSlime(node.ipAddressPool().asSet(), object.setArray("additionalIpAddresses")); + ipAddressesToSlime(node.ipConfig().primary(), object.setArray("ipAddresses")); + ipAddressesToSlime(node.ipConfig().pool().asSet(), object.setArray("additionalIpAddresses")); node.reports().toSlime(object, "reports"); node.modelName().ifPresent(modelName -> object.setString("modelName", modelName)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index ba859655ab7..45da1f1d3ee 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -65,7 +65,7 @@ public class DynamicProvisioningMaintainerTest { Node host4 = tester.nodeRepository.getNode("host4").orElseThrow(); Node host41 = tester.nodeRepository.getNode("host4-1").orElseThrow(); assertTrue("No IP addresses assigned", - Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + Stream.of(host3, host4, host41).map(node -> node.ipConfig().primary()).allMatch(Set::isEmpty)); Node host3new = host3.with(host3.ipConfig().with(Set.of("::3:0"))); Node host4new = host4.with(host4.ipConfig().with(Set.of("::4:0"))); @@ -83,7 +83,7 @@ public class DynamicProvisioningMaintainerTest { tester.hostProvisioner.with(Behaviour.failProvisioning); Node host4 = tester.addNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned); Node host41 = tester.addNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, DynamicProvisioningTester.tenantApp); - assertTrue("No IP addresses assigned", Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + assertTrue("No IP addresses assigned", Stream.of(host4, host41).map(node -> node.ipConfig().primary()).allMatch(Set::isEmpty)); tester.maintainer.maintain(); assertEquals(Set.of("host4", "host4-1"), @@ -233,7 +233,8 @@ public class DynamicProvisioningMaintainerTest { this.maintainer = new DynamicProvisioningMaintainer(nodeRepository, Duration.ofDays(1), hostProvisioner, - flagSource); + flagSource, + new TestMetric()); } private DynamicProvisioningTester addInitialNodes() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index ed6f31984a5..f8e21ebbfce 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -263,7 +263,7 @@ public class FailedExpirerTest { false, 0); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); - this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30)); + this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30), new TestMetric()); } public ManualClock clock() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index 89e43f80479..3d17cbf0217 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -64,7 +64,7 @@ public class InactiveAndFailedExpirerTest { // Inactive times out tester.advanceTime(Duration.ofMinutes(14)); - new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10)).run(); + new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(0, tester.nodeRepository().getNodes(Node.State.inactive).size()); List<Node> dirty = tester.nodeRepository().getNodes(Node.State.dirty); assertEquals(2, dirty.size()); @@ -79,7 +79,7 @@ public class InactiveAndFailedExpirerTest { // Dirty times out for the other one tester.advanceTime(Duration.ofMinutes(14)); - new DirtyExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10)).run(); + new DirtyExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(0, tester.nodeRepository().getNodes(NodeType.tenant, Node.State.dirty).size()); List<Node> failed = tester.nodeRepository().getNodes(NodeType.tenant, Node.State.failed); assertEquals(1, failed.size()); @@ -107,7 +107,7 @@ public class InactiveAndFailedExpirerTest { // Inactive times out and node is moved to dirty tester.advanceTime(Duration.ofMinutes(14)); - new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10)).run(); + new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); List<Node> dirty = tester.nodeRepository().getNodes(Node.State.dirty); assertEquals(2, dirty.size()); @@ -158,7 +158,7 @@ public class InactiveAndFailedExpirerTest { // Inactive times out and one node is moved to parked tester.advanceTime(Duration.ofMinutes(11)); // Trigger InactiveExpirer - new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10)).run(); + new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(1, tester.nodeRepository().getNodes(Node.State.parked).size()); } @@ -180,7 +180,7 @@ public class InactiveAndFailedExpirerTest { assertEquals(1, inactiveNodes.size()); // See that nodes are moved to dirty immediately. - new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10)).run(); + new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(0, tester.nodeRepository().getNodes(Node.State.inactive).size()); List<Node> dirty = tester.nodeRepository().getNodes(Node.State.dirty); assertEquals(1, dirty.size()); @@ -207,7 +207,7 @@ public class InactiveAndFailedExpirerTest { .map(node -> node.withWantToRetire(true, true, Agent.system, tester.clock().instant())) .collect(Collectors.toList()), () -> {}); tester.advanceTime(Duration.ofMinutes(11)); - new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10)).run(); + new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(2, tester.nodeRepository().getNodes(Node.State.parked).size()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index a5e96369591..6c22f798fe0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -38,7 +38,8 @@ public class LoadBalancerExpirerTest { public void expire_inactive() { LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), Duration.ofDays(1), - tester.loadBalancerService()); + tester.loadBalancerService(), + new TestMetric()); Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true); // Deploy two applications with a total of three load balancers @@ -103,7 +104,8 @@ public class LoadBalancerExpirerTest { public void expire_reserved() { LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(), Duration.ofDays(1), - tester.loadBalancerService()); + tester.loadBalancerService(), + new TestMetric()); Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java index bae6de5a095..3ff81070516 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java @@ -26,7 +26,7 @@ public class NodeRebooterTest { var flagSource = new InMemoryFlagSource().withIntFlag(Flags.REBOOT_INTERVAL_IN_DAYS.id(), (int) rebootInterval.toDays()); var tester = new MaintenanceTester(); tester.createReadyHostNodes(15); - NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, flagSource); + NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, flagSource, new TestMetric()); assertReadyHosts(15, tester, 0L); @@ -69,7 +69,7 @@ public class NodeRebooterTest { var flagSource = new InMemoryFlagSource().withIntFlag(Flags.REBOOT_INTERVAL_IN_DAYS.id(), (int) rebootInterval.toDays()); var tester = new MaintenanceTester(); tester.createReadyHostNodes(2); - NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, flagSource); + NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, flagSource, new TestMetric()); assertReadyHosts(2, tester, 0L); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java index 65c7bf13b42..218812f9a3d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java @@ -34,7 +34,7 @@ public class OsUpgradeActivatorTest { @Test public void activates_upgrade() { var osVersions = tester.nodeRepository().osVersions(); - var osUpgradeActivator = new OsUpgradeActivator(tester.nodeRepository(), Duration.ofDays(1)); + var osUpgradeActivator = new OsUpgradeActivator(tester.nodeRepository(), Duration.ofDays(1), new TestMetric()); var version0 = Version.fromString("7.0"); // Create infrastructure nodes diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 6ca154f5f17..bd92c2a9aa2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -75,7 +75,7 @@ public class ReservationExpirerTest { // Reservation times out clock.advance(Duration.ofMinutes(14)); // Reserved but not used time out - new ReservationExpirer(nodeRepository, clock, Duration.ofMinutes(10)).run(); + new ReservationExpirer(nodeRepository, clock, Duration.ofMinutes(10), new TestMetric()).run(); // Assert nothing is reserved assertEquals(0, nodeRepository.getNodes(NodeType.tenant, Node.State.reserved).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index b7f21eb3114..be5c7f423c7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -25,7 +25,6 @@ import java.time.Duration; import java.util.List; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * Tests the scaling suggestions maintainer integration. @@ -66,7 +65,8 @@ public class ScalingSuggestionsMaintainerTest { ScalingSuggestionsMaintainer maintainer = new ScalingSuggestionsMaintainer(tester.nodeRepository(), nodeMetricsDb, - Duration.ofMinutes(1)); + Duration.ofMinutes(1), + new TestMetric()); maintainer.maintain(); assertEquals("14 nodes with [vcpu: 6.9, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps, storage type: remote]", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java index 880f5af5653..b3d567c8d58 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClientTest.java @@ -26,7 +26,7 @@ public class CuratorDatabaseClientTest { private final Curator curator = new MockCurator(); private final CuratorDatabaseClient zkClient = new CuratorDatabaseClient( - FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), Zone.defaultZone(), true); + FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), Zone.defaultZone(), true, false); @Test public void can_read_stored_host_information() throws Exception { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index f3fe1fc4915..5e4bfc2a7bc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -240,7 +240,7 @@ public class NodeSerializerTest { public void serializes_multiple_ip_addresses() { byte[] nodeWithMultipleIps = createNodeJson("node4.yahoo.tld", "127.0.0.4", "::4"); Node deserializedNode = nodeSerializer.fromJson(State.provisioned, nodeWithMultipleIps); - assertEquals(ImmutableSet.of("127.0.0.4", "::4"), deserializedNode.ipAddresses()); + assertEquals(ImmutableSet.of("127.0.0.4", "::4"), deserializedNode.ipConfig().primary()); } @Test @@ -250,12 +250,12 @@ public class NodeSerializerTest { // Test round-trip with IP address pool node = node.with(node.ipConfig().with(IP.Pool.of(Set.of("::1", "::2", "::3")))); Node copy = nodeSerializer.fromJson(node.state(), nodeSerializer.toJson(node)); - assertEquals(node.ipAddressPool().asSet(), copy.ipAddressPool().asSet()); + assertEquals(node.ipConfig().pool().asSet(), copy.ipConfig().pool().asSet()); // Test round-trip without IP address pool (handle empty pool) node = createNode(); copy = nodeSerializer.fromJson(node.state(), nodeSerializer.toJson(node)); - assertEquals(node.ipAddressPool().asSet(), copy.ipAddressPool().asSet()); + assertEquals(node.ipConfig().pool().asSet(), copy.ipConfig().pool().asSet()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index f27775db570..d165f865432 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -201,9 +201,9 @@ public class AclProvisioningTest { assertEquals(3, nodeAcls.get(0).trustedNodes().size()); Iterator<Node> trustedNodes = nodeAcls.get(0).trustedNodes().iterator(); - assertEquals(Set.of("127.0.1.1"), trustedNodes.next().ipAddresses()); - assertEquals(Set.of("127.0.1.2"), trustedNodes.next().ipAddresses()); - assertEquals(Set.of("127.0.1.3"), trustedNodes.next().ipAddresses()); + assertEquals(Set.of("127.0.1.1"), trustedNodes.next().ipConfig().primary()); + assertEquals(Set.of("127.0.1.2"), trustedNodes.next().ipConfig().primary()); + assertEquals(Set.of("127.0.1.3"), trustedNodes.next().ipConfig().primary()); } private List<Node> deploy(int nodeCount) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 100cf5704eb..29e371dd937 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -318,8 +318,8 @@ public class DynamicDockerAllocationTest { tester.activate(application, hosts); List<Node> activeNodes = tester.nodeRepository().getNodes(application); - assertEquals(ImmutableSet.of("127.0.127.13", "::13"), activeNodes.get(0).ipAddresses()); - assertEquals(ImmutableSet.of("127.0.127.2", "::2"), activeNodes.get(1).ipAddresses()); + assertEquals(ImmutableSet.of("127.0.127.13", "::13"), activeNodes.get(0).ipConfig().primary()); + assertEquals(ImmutableSet.of("127.0.127.2", "::2"), activeNodes.get(1).ipConfig().primary()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 3eb379b0914..845eeba972c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -22,6 +22,7 @@ import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.maintenance.ReservationExpirer; +import com.yahoo.vespa.hosted.provision.maintenance.TestMetric; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import org.junit.Test; @@ -791,7 +792,7 @@ public class ProvisioningTest { // Over 10 minutes pass since first reservation. First set of reserved nodes are not expired tester.clock().advance(Duration.ofMinutes(8).plus(Duration.ofSeconds(1))); ReservationExpirer expirer = new ReservationExpirer(tester.nodeRepository(), tester.clock(), - Duration.ofMinutes(10)); + Duration.ofMinutes(10), new TestMetric()); expirer.run(); assertEquals("Nodes remain reserved", 4, tester.getNodes(application, Node.State.reserved).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index b1ecd03aa13..81bf999a184 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -9,6 +9,7 @@ import com.yahoo.text.Utf8; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.OsUpgradeActivator; +import com.yahoo.vespa.hosted.provision.maintenance.TestMetric; import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import org.junit.After; @@ -763,7 +764,7 @@ public class NodesV2ApiTest { // Activate target var nodeRepository = (NodeRepository)tester.container().components().getComponent(MockNodeRepository.class.getName()); - var osUpgradeActivator = new OsUpgradeActivator(nodeRepository, Duration.ofDays(1)); + var osUpgradeActivator = new OsUpgradeActivator(nodeRepository, Duration.ofDays(1), new TestMetric()); osUpgradeActivator.run(); // Other node type does not return wanted OS version diff --git a/parent/pom.xml b/parent/pom.xml index a77750b9132..ce40eb464fc 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -255,7 +255,6 @@ <artifactId>bundle-plugin</artifactId> <version>${project.version}</version> <configuration> - <buildVespaPlatformBundle>true</buildVespaPlatformBundle> <configGenVersion>${project.version}</configGenVersion> <useCommonAssemblyIds>true</useCommonAssemblyIds> </configuration> diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index 8709b0d01b1..beff0f363fe 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -252,6 +252,9 @@ App::Main() } } } + // Ensure metric manager and state server are shut down before we start tearing + // down any service layer components that they may end up transitively using. + protonUP->shutdown_config_fetching_and_state_exposing_components_once(); if (spiProton) { spiProton->getNode().requestShutdown("controlled shutdown"); spiProton->shutdown(); diff --git a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp index 7488fc633fd..f034ccdd6d1 100644 --- a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp +++ b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp @@ -13,7 +13,7 @@ #include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/signalhandler.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/config/common/exceptions.h> @@ -207,7 +207,7 @@ App::Main() route.c_str(), feedFile.c_str())); fprintf(stderr, "running feed command: %s\n", feedCmd.c_str()); std::string feederOutput; - bool feedingOk = vespalib::SlaveProc::run(feedCmd.c_str(), feederOutput); + bool feedingOk = vespalib::ChildProcess::run(feedCmd.c_str(), feederOutput); if (!feedingOk) { fprintf(stderr, "error: feed command failed\n"); fprintf(stderr, "feed command output:\n-----\n%s\n-----\n", feederOutput.c_str()); diff --git a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp index b5bcd65cd33..dbfbbd16820 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp @@ -162,7 +162,7 @@ public: return wrappedTargets; } - // Called once by flush engine slave thread for each task done + // Called once by flush engine thread for each task done void taskDone() { std::lock_guard<std::mutex> guard(_lock); diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 24e1e886351..6fbd43eabbe 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -730,7 +730,8 @@ void checkQueryAddsLocation(Test &test, const string &loc_string) { SearchIterator::UP search = query.createSearch(*md); test.ASSERT_TRUE(search.get()); if (!test.EXPECT_NOT_EQUAL(string::npos, search->asString().find(loc_string))) { - fprintf(stderr, "search (missing loc_string): %s", search->asString().c_str()); + fprintf(stderr, "search (missing loc_string '%s'): %s", + loc_string.c_str(), search->asString().c_str()); } } diff --git a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp index 2570e64dbe2..36c34e38a04 100644 --- a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp +++ b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp @@ -83,7 +83,7 @@ Node::UP getQuery(const ViewResolver &resolver) query_builder.addStringTerm("bar", field, id[3], Weight(0)); } - query_builder.addLocationTerm(Location(Point(10, 10), 3, 0), + query_builder.addLocationTerm(Location(Point{10, 10}, 3, 0), field, id[7], Weight(0)); Node::UP node = query_builder.build(); diff --git a/searchcore/src/tests/proton/proton_configurer/CMakeLists.txt b/searchcore/src/tests/proton/proton_configurer/CMakeLists.txt index edc4060d067..83958954c29 100644 --- a/searchcore/src/tests/proton/proton_configurer/CMakeLists.txt +++ b/searchcore/src/tests/proton/proton_configurer/CMakeLists.txt @@ -5,5 +5,6 @@ vespa_add_executable(searchcore_proton_configurer_test_app TEST DEPENDS searchcore_server searchcore_fconfig + GTest::GTest ) vespa_add_test(NAME searchcore_proton_configurer_test_app COMMAND searchcore_proton_configurer_test_app) diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index c26b008f769..83706d966ae 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -19,7 +19,7 @@ #include <vespa/searchcore/proton/server/i_proton_disk_layout.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/searchcore/config/config-ranking-constants.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -119,12 +119,12 @@ struct ConfigFixture { _generation(1), _cachedConfigSnapshot() { - addDocType("_alwaysthere_"); + addDocType("_alwaysthere_", "default"); } ~ConfigFixture() { } - DBConfigFixture *addDocType(const std::string & name) { + DBConfigFixture *addDocType(const std::string & name, const std::string& bucket_space) { DocumenttypesConfigBuilder::Documenttype dt; dt.bodystruct = -1270491200; dt.headerstruct = 306916075; @@ -140,7 +140,7 @@ struct ConfigFixture { BucketspacesConfigBuilder::Documenttype bsdt; bsdt.name = name; - bsdt.bucketspace = "default"; + bsdt.bucketspace = bucket_space; _bucketspacesBuilder.documenttype.push_back(bsdt); DBConfigFixture::UP fixture = std::make_unique<DBConfigFixture>(); @@ -169,6 +169,12 @@ struct ConfigFixture { } } _dbConfig.erase(name); + for (auto it(_bucketspacesBuilder.documenttype.begin()), mt(_bucketspacesBuilder.documenttype.end()); it != mt; ++it) { + if (it->name == name) { + _bucketspacesBuilder.documenttype.erase(it); + break; + } + } } BootstrapConfig::SP getBootstrapConfig(int64_t generation) const { @@ -212,17 +218,21 @@ struct MyProtonConfigurerOwner; struct MyDocumentDBConfigOwner : public DocumentDBConfigOwner { vespalib::string _name; + document::BucketSpace _bucket_space; MyProtonConfigurerOwner &_owner; MyDocumentDBConfigOwner(const vespalib::string &name, + document::BucketSpace bucket_space, MyProtonConfigurerOwner &owner) : DocumentDBConfigOwner(), _name(name), + _bucket_space(bucket_space), _owner(owner) { } ~MyDocumentDBConfigOwner() { } void reconfigure(const DocumentDBConfig::SP & config) override; + document::BucketSpace getBucketSpace() const override { return _bucket_space; } }; struct MyLog @@ -266,8 +276,8 @@ struct MyProtonConfigurerOwner : public IProtonConfigurerOwner, (void) configId; (void) bootstrapConfig; (void) initializeThreads; - ASSERT_TRUE(_dbs.find(docTypeName) == _dbs.end()); - auto db = std::make_shared<MyDocumentDBConfigOwner>(docTypeName.getName(), *this); + EXPECT_TRUE(_dbs.find(docTypeName) == _dbs.end()); + auto db = std::make_shared<MyDocumentDBConfigOwner>(docTypeName.getName(), bucketSpace, *this); _dbs.insert(std::make_pair(docTypeName, db)); std::ostringstream os; os << "add db " << docTypeName.getName() << " " << documentDBConfig->getGeneration(); @@ -285,6 +295,7 @@ struct MyProtonConfigurerOwner : public IProtonConfigurerOwner, std::ostringstream os; os << "apply config " << bootstrapConfig->getGeneration(); _log.push_back(os.str()); + } void reconfigureDocumentDB(const vespalib::string &name, const DocumentDBConfig::SP &config) { @@ -329,14 +340,15 @@ struct MyProtonDiskLayout : public IProtonDiskLayout } }; -struct Fixture +class ProtonConfigurerTest : public ::testing::Test { MyProtonConfigurerOwner _owner; ConfigFixture _config; std::unique_ptr<IProtonDiskLayout> _diskLayout; ProtonConfigurer _configurer; - Fixture() +protected: + ProtonConfigurerTest() : _owner(), _config("test"), _diskLayout(), @@ -344,13 +356,13 @@ struct Fixture { _diskLayout = std::make_unique<MyProtonDiskLayout>(_owner); } - ~Fixture() { } + ~ProtonConfigurerTest() override; void assertLog(const std::vector<vespalib::string> &expLog) { - EXPECT_EQUAL(expLog, _owner._log); + EXPECT_EQ(expLog, _owner._log); } void sync() { _owner.sync(); } - void addDocType(const vespalib::string &name) { _config.addDocType(name); } + void addDocType(const vespalib::string &name, const std::string& bucket_space = "default") { _config.addDocType(name, bucket_space); } void removeDocType(const vespalib::string &name) { _config.removeDocType(name); } void applyConfig() { _configurer.reconfigure(_config.getConfigSnapshot()); @@ -375,91 +387,105 @@ struct Fixture } }; -TEST_F("require that nothing is applied before initial config", Fixture()) +ProtonConfigurerTest::~ProtonConfigurerTest() = default; + +TEST_F(ProtonConfigurerTest, require_that_nothing_is_applied_before_initial_config) { - f.applyConfig(); - TEST_DO(f1.assertLog({})); + applyConfig(); + assertLog({}); } -TEST_F("require that initial config is applied", Fixture()) +TEST_F(ProtonConfigurerTest, require_that_initial_config_is_applied) { - f.applyInitialConfig(); - TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); + applyInitialConfig(); + assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"}); } -TEST_F("require that new config is blocked", Fixture()) +TEST_F(ProtonConfigurerTest, require_that_new_config_is_blocked) { - f.applyInitialConfig(); - f.reconfigure(); - TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); + applyInitialConfig(); + reconfigure(); + assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"}); } -TEST_F("require that new config can be unblocked", Fixture()) +TEST_F(ProtonConfigurerTest, require_that_new_config_can_be_unblocked) { - f.applyInitialConfig(); - f.reconfigure(); - f.allowReconfig(); - TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); + applyInitialConfig(); + reconfigure(); + allowReconfig(); + assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"}); } -TEST_F("require that initial config is not reapplied due to config unblock", Fixture()) +TEST_F(ProtonConfigurerTest, require_that_initial_config_is_not_reapplied_due_to_config_unblock) { - f.applyInitialConfig(); - f.allowReconfig(); - TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); + applyInitialConfig(); + allowReconfig(); + assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"}); } -TEST_F("require that we can add document db", Fixture()) +TEST_F(ProtonConfigurerTest, require_that_we_can_add_document_db) { - f.applyInitialConfig(); - f.allowReconfig(); - f.addDocType("foobar"); - f.reconfigure(); - TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"})); + applyInitialConfig(); + allowReconfig(); + addDocType("foobar"); + reconfigure(); + assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"}); } -TEST_F("require that we can remove document db", Fixture()) +TEST_F(ProtonConfigurerTest, require_that_we_can_remove_document_db) { - f.addDocType("foobar"); - f.applyInitialConfig(); - f.allowReconfig(); - f.removeDocType("foobar"); - f.reconfigure(); - TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar", "remove dbdir foobar"})); + addDocType("foobar"); + applyInitialConfig(); + allowReconfig(); + removeDocType("foobar"); + reconfigure(); + assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar", "remove dbdir foobar"}); } -TEST_F("require that document db adds and reconfigs are intermingled", Fixture()) +TEST_F(ProtonConfigurerTest, require_that_document_db_adds_and_reconfigs_are_intermingled) { - f.addDocType("foobar"); - f.applyInitialConfig(); - f.allowReconfig(); - f.addDocType("abar"); - f.removeDocType("foobar"); - f.addDocType("foobar"); - f.addDocType("zbar"); - f.reconfigure(); - TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"})); + addDocType("foobar"); + applyInitialConfig(); + allowReconfig(); + addDocType("abar"); + removeDocType("foobar"); + addDocType("foobar"); + addDocType("zbar"); + reconfigure(); + assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"}); } -TEST_F("require that document db removes are applied at end", Fixture()) +TEST_F(ProtonConfigurerTest, require_that_document_db_removes_are_applied_at_end) { - f.addDocType("abar"); - f.addDocType("foobar"); - f.applyInitialConfig(); - f.allowReconfig(); - f.removeDocType("abar"); - f.reconfigure(); - TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,abar,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar", "remove dbdir abar"})); + addDocType("abar"); + addDocType("foobar"); + applyInitialConfig(); + allowReconfig(); + removeDocType("abar"); + reconfigure(); + assertLog({"initial dbs _alwaysthere_,abar,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar", "remove dbdir abar"}); } -TEST_F("require that new configs can be blocked again", Fixture()) +TEST_F(ProtonConfigurerTest, require_that_new_configs_can_be_blocked_again) { - f.applyInitialConfig(); - f.reconfigure(); - f.allowReconfig(); - f.disableReconfig(); - f.reconfigure(); - TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); + applyInitialConfig(); + reconfigure(); + allowReconfig(); + disableReconfig(); + reconfigure(); + assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"}); } -TEST_MAIN() { TEST_RUN_ALL(); } +TEST_F(ProtonConfigurerTest, require_that_bucket_space_for_document_type_change_exits) +{ + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + addDocType("globaldoc", "default"); + applyInitialConfig(); + removeDocType("globaldoc"); + addDocType("globaldoc", "global"); + allowReconfig(); + EXPECT_EXIT(reconfigure(), ::testing::ExitedWithCode(1), "Bucket space for document type globaldoc changed from default to global"); +} + + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index fd7069c618a..2f5ee2f22cb 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchcommon/common/schema.h> #include <vespa/searchlib/fef/indexproperties.h> @@ -154,7 +154,7 @@ struct Model { } bool verify() { generate(); - return vespalib::SlaveProc::run(vespalib::make_string("%s dir:%s", prog, gen_dir.c_str()).c_str()); + return vespalib::ChildProcess::run(vespalib::make_string("%s dir:%s", prog, gen_dir.c_str()).c_str()); } void verify_valid(std::initializer_list<std::string> features) { for (const std::string &f: features) { @@ -207,12 +207,12 @@ struct ShadowModel : Model { }; TEST_F("print usage", Model()) { - EXPECT_TRUE(!vespalib::SlaveProc::run(vespalib::make_string("%s", prog).c_str())); + EXPECT_TRUE(!vespalib::ChildProcess::run(vespalib::make_string("%s", prog).c_str())); } TEST_F("setup output directory", Model()) { - ASSERT_TRUE(vespalib::SlaveProc::run(vespalib::make_string("rm -rf %s", gen_dir.c_str()).c_str())); - ASSERT_TRUE(vespalib::SlaveProc::run(vespalib::make_string("mkdir %s", gen_dir.c_str()).c_str())); + ASSERT_TRUE(vespalib::ChildProcess::run(vespalib::make_string("rm -rf %s", gen_dir.c_str()).c_str())); + ASSERT_TRUE(vespalib::ChildProcess::run(vespalib::make_string("mkdir %s", gen_dir.c_str()).c_str())); } //----------------------------------------------------------------------------- @@ -317,7 +317,7 @@ TEST_F("require that imported attribute field can be used by rank feature", Simp //----------------------------------------------------------------------------- TEST_F("cleanup files", Model()) { - ASSERT_TRUE(vespalib::SlaveProc::run(vespalib::make_string("rm -rf %s", gen_dir.c_str()).c_str())); + ASSERT_TRUE(vespalib::ChildProcess::run(vespalib::make_string("rm -rf %s", gen_dir.c_str()).c_str())); } TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index 0869fc175a7..0dbffe2402a 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -181,38 +181,6 @@ DocsumContext::FillRankFeatures(search::docsummary::GetDocsumsState * state, sea state->_rankFeatures = _matcher->getRankFeatures(_request, _searchCtx, _attrCtx, _sessionMgr); } -namespace { -Location *getLocation(const string &loc_str, search::IAttributeManager &attrMgr) -{ - LOG(debug, "Filling document locations from location string: %s", loc_str.c_str()); - - Location *loc = new Location; - string location; - string::size_type pos = loc_str.find(':'); - if (pos != string::npos) { - string view = loc_str.substr(0, pos); - AttributeGuard::UP vec = attrMgr.getAttribute(view); - if (!vec->valid()) { - view = PositionDataType::getZCurveFieldName(view); - vec = attrMgr.getAttribute(view); - } - loc->setVecGuard(std::move(vec)); - location = loc_str.substr(pos + 1); - } else { - LOG(warning, "Location string lacks attribute vector specification. loc='%s'", loc_str.c_str()); - location = loc_str; - } - loc->parse(location); - return loc; -} -} // namespace - -void -DocsumContext::ParseLocation(search::docsummary::GetDocsumsState *state) -{ - state->_parsedLocation.reset(getLocation(_request.location, _attrMgr)); -} - std::unique_ptr<MatchingElements> DocsumContext::fill_matching_elements(const MatchingElementsFields &fields) { diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h index 1624048828f..d1b656915d9 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h @@ -52,7 +52,6 @@ public: // Implements GetDocsumsStateCallback void FillSummaryFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env) override; void FillRankFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env) override; - void ParseLocation(search::docsummary::GetDocsumsState * state) override; std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields &fields) override; }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt index 558914805d1..ffbab597118 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt @@ -27,6 +27,7 @@ vespa_add_library(searchcore_matching STATIC querynodes.cpp ranking_constants.cpp requestcontext.cpp + resolveviewvisitor.cpp result_processor.cpp same_element_builder.cpp sameelementmodifier.cpp diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 62a59ab7680..65959e6e6de 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -8,10 +8,9 @@ #include "sameelementmodifier.h" #include "unpacking_iterators_optimizer.h" #include <vespa/document/datatype/positiondatatype.h> -#include <vespa/searchlib/common/location.h> +#include <vespa/searchlib/common/geo_location_spec.h> +#include <vespa/searchlib/common/geo_location_parser.h> #include <vespa/searchlib/parsequery/stackdumpiterator.h> -#include <vespa/searchlib/query/tree/point.h> -#include <vespa/searchlib/query/tree/rectangle.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> #include <vespa/log/log.h> @@ -20,11 +19,13 @@ LOG_SETUP(".proton.matching.query"); using document::PositionDataType; using search::SimpleQueryStackDumpIterator; +using search::common::GeoLocation; +using search::common::GeoLocationParser; +using search::common::GeoLocationSpec; using search::fef::IIndexEnvironment; using search::fef::ITermData; using search::fef::MatchData; using search::fef::MatchDataLayout; -using search::fef::Location; using search::query::Node; using search::query::QueryTreeCreator; using search::query::Weight; @@ -58,37 +59,73 @@ inject(Node::UP query, Node::UP to_inject) { return query; } -void -addLocationNode(const string &location_str, Node::UP &query_tree, Location &fef_location) { - if (location_str.empty()) { - return; - } - string::size_type pos = location_str.find(':'); - if (pos == string::npos) { - LOG(warning, "Location string lacks attribute vector specification. loc='%s'", location_str.c_str()); - return; +std::vector<ProtonLocationTerm *> +find_location_terms(Node *tree) { + std::vector<ProtonLocationTerm *> retval; + std::vector<Node *> nodes; + nodes.push_back(tree); + for (size_t i = 0; i < nodes.size(); ++i) { + if (auto loc = dynamic_cast<ProtonLocationTerm *>(nodes[i])) { + retval.push_back(loc); + } + if (auto parent = dynamic_cast<const search::query::Intermediate *>(nodes[i])) { + for (Node * child : parent->getChildren()) { + nodes.push_back(child); + } + } } - const string view = PositionDataType::getZCurveFieldName(location_str.substr(0, pos)); - const string loc = location_str.substr(pos + 1); + return retval; +} - search::common::Location locationSpec; - if (!locationSpec.parse(loc)) { - LOG(warning, "Location parse error (location: '%s'): %s", location_str.c_str(), locationSpec.getParseError()); - return; +GeoLocationSpec parse_location_string(string str) { + GeoLocationSpec empty; + if (str.empty()) { + return empty; } + GeoLocationParser parser; + if (parser.parseOldFormatWithField(str)) { + auto attr_name = PositionDataType::getZCurveFieldName(parser.getFieldName()); + return GeoLocationSpec{attr_name, parser.getGeoLocation()}; + } else { + LOG(warning, "Location parse error (location: '%s'): %s", str.c_str(), parser.getParseError()); + } + return empty; +} + +GeoLocationSpec process_location_term(ProtonLocationTerm &pterm) { + auto old_view = pterm.getView(); + auto new_view = PositionDataType::getZCurveFieldName(old_view); + pterm.setView(new_view); + const GeoLocation &loc = pterm.getTerm(); + return GeoLocationSpec{new_view, loc}; +} + +void exchange_location_nodes(const string &location_str, + Node::UP &query_tree, + std::vector<GeoLocationSpec> &fef_locations) +{ + std::vector<GeoLocationSpec> locationSpecs; - int32_t id = -1; - Weight weight(100); - - if (locationSpec.getRankOnDistance()) { - query_tree = inject(std::move(query_tree), std::make_unique<ProtonLocationTerm>(loc, view, id, weight)); - fef_location.setAttribute(view); - fef_location.setXPosition(locationSpec.getX()); - fef_location.setYPosition(locationSpec.getY()); - fef_location.setXAspect(locationSpec.getXAspect()); - fef_location.setValid(true); - } else if (locationSpec.getPruneOnDistance()) { - query_tree = inject(std::move(query_tree), std::make_unique<ProtonLocationTerm>(loc, view, id, weight)); + auto parsed = parse_location_string(location_str); + if (parsed.location.valid()) { + locationSpecs.push_back(parsed); + } + for (ProtonLocationTerm * pterm : find_location_terms(query_tree.get())) { + auto spec = process_location_term(*pterm); + if (spec.location.valid()) { + locationSpecs.push_back(spec); + } + } + for (const GeoLocationSpec &spec : locationSpecs) { + if (spec.location.has_point) { + fef_locations.push_back(spec); + } + } + if (parsed.location.can_limit()) { + int32_t id = -1; + Weight weight(100); + query_tree = inject(std::move(query_tree), + std::make_unique<ProtonLocationTerm>(parsed.location, parsed.field_name, id, weight)); } } @@ -127,7 +164,7 @@ Query::buildTree(vespalib::stringref stack, const string &location, if (_query_tree) { SameElementModifier prefixSameElementSubIndexes; _query_tree->accept(prefixSameElementSubIndexes); - addLocationNode(location, _query_tree, _location); + exchange_location_nodes(location, _query_tree, _locations); _query_tree = UnpackingIteratorsOptimizer::optimize(std::move(_query_tree), bool(_whiteListBlueprint), split_unpacking_iterators, delay_unpacking_iterators); ResolveViewVisitor resolve_visitor(resolver, indexEnv); @@ -146,10 +183,12 @@ Query::extractTerms(vector<const ITermData *> &terms) } void -Query::extractLocations(vector<const Location *> &locations) +Query::extractLocations(vector<const GeoLocationSpec *> &locations) { locations.clear(); - locations.push_back(&_location); + for (const auto & loc : _locations) { + locations.push_back(&loc); + } } void diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index 60f40e24d1e..952b6260da1 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/searchlib/fef/location.h> +#include <vespa/searchlib/common/geo_location_spec.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/iindexenvironment.h> @@ -18,13 +18,16 @@ class ISearchContext; class Query { private: - using Blueprint=search::queryeval::Blueprint; + using Blueprint = search::queryeval::Blueprint; search::query::Node::UP _query_tree; Blueprint::UP _blueprint; - search::fef::Location _location; Blueprint::UP _whiteListBlueprint; + std::vector<search::common::GeoLocationSpec> _locations; public: + /** Convenience typedef. */ + using GeoLocationSpecPtrs = std::vector<const search::common::GeoLocationSpec *>; + Query(); ~Query(); /** @@ -65,7 +68,7 @@ public: * * @param locs where to collect locations **/ - void extractLocations(std::vector<const search::fef::Location *> &locs); + void extractLocations(GeoLocationSpecPtrs &locs); /** * Reserve room for terms in the query in the given match data diff --git a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp index fe0f6aaff91..448ce14dd51 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp @@ -5,7 +5,6 @@ using search::attribute::IAttributeContext; using search::fef::IIndexEnvironment; -using search::fef::Location; using search::fef::Properties; namespace proton::matching { @@ -17,7 +16,7 @@ QueryEnvironment::QueryEnvironment(const IIndexEnvironment &indexEnv, : _indexEnv(indexEnv), _attrContext(attrContext), _properties(properties), - _locations(1), + _locations(), _terms(), _field_length_inspector(field_length_inspector) { @@ -44,12 +43,6 @@ QueryEnvironment::getTerm(uint32_t idx) const return _terms[idx]; } -const search::fef::Location & -QueryEnvironment::getLocation() const -{ - return *_locations[0]; -} - const IAttributeContext & QueryEnvironment::getAttributeContext() const { diff --git a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h index 575694ae079..6daf488297d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h +++ b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h @@ -4,7 +4,6 @@ #include <vespa/searchlib/fef/iqueryenvironment.h> #include <vespa/searchlib/fef/properties.h> -#include <vespa/searchlib/fef/location.h> namespace search::index { class IFieldLengthInspector; } @@ -19,7 +18,7 @@ private: const search::fef::IIndexEnvironment &_indexEnv; const search::attribute::IAttributeContext &_attrContext; search::fef::Properties _properties; - std::vector<const search::fef::Location *> _locations; + GeoLocationSpecPtrs _locations; std::vector<const search::fef::ITermData *> _terms; const search::index::IFieldLengthInspector &_field_length_inspector; @@ -56,7 +55,7 @@ public: * * @return modifiable list of location data pointers **/ - std::vector<const search::fef::Location *> &locations() { + GeoLocationSpecPtrs &locations() { return _locations; } @@ -70,7 +69,9 @@ public: const search::fef::ITermData *getTerm(uint32_t idx) const override; // inherited from search::fef::IQueryEnvironment - const search::fef::Location & getLocation() const override; + GeoLocationSpecPtrs getAllLocations() const override { + return _locations; + } // inherited from search::fef::IQueryEnvironment const search::attribute::IAttributeContext & getAttributeContext() const override; diff --git a/searchcore/src/vespa/searchcore/proton/matching/resolveviewvisitor.cpp b/searchcore/src/vespa/searchcore/proton/matching/resolveviewvisitor.cpp new file mode 100644 index 00000000000..b12b48465d9 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/resolveviewvisitor.cpp @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "resolveviewvisitor.h" +#include <vespa/document/datatype/positiondatatype.h> +#include <vespa/log/log.h> + +LOG_SETUP(".proton.matching.resolveviewvisitor"); + +namespace proton::matching { + +void +ResolveViewVisitor::visit(ProtonLocationTerm &n) { + // if injected by query.cpp, this should work: + n.resolve(_resolver, _indexEnv); + if (n.numFields() == 0) { + // if received from QRS, this is needed: + auto oldView = n.getView(); + auto newView = document::PositionDataType::getZCurveFieldName(oldView); + n.setView(newView); + n.resolve(_resolver, _indexEnv); + LOG(debug, "ProtonLocationTerm found %zu field after view change %s -> %s", + n.numFields(), oldView.c_str(), newView.c_str()); + } +} + +} // namespace diff --git a/searchcore/src/vespa/searchcore/proton/matching/resolveviewvisitor.h b/searchcore/src/vespa/searchcore/proton/matching/resolveviewvisitor.h index 4a12e6adda9..f8c1a007c28 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/resolveviewvisitor.h +++ b/searchcore/src/vespa/searchcore/proton/matching/resolveviewvisitor.h @@ -22,6 +22,8 @@ public: template <class TermNode> void visitTerm(TermNode &n) { n.resolve(_resolver, _indexEnv); } + void visit(ProtonLocationTerm &n) override; + void visit(ProtonNodeTypes::Equiv &n) override { visitChildren(n); n.resolveFromChildren(n.getChildren()); diff --git a/searchcore/src/vespa/searchcore/proton/server/i_document_db_config_owner.h b/searchcore/src/vespa/searchcore/proton/server/i_document_db_config_owner.h index 10a47e7c6e4..feb33e11721 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_document_db_config_owner.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_document_db_config_owner.h @@ -4,6 +4,8 @@ #include <memory> +namespace document { class BucketSpace; } + namespace proton { class DocumentDBConfig; @@ -15,6 +17,7 @@ class IDocumentDBConfigOwner { public: virtual ~IDocumentDBConfigOwner() { } + virtual document::BucketSpace getBucketSpace() const = 0; virtual void reconfigure(const std::shared_ptr<DocumentDBConfig> & config) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 1e579739d85..b297dd860ea 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -223,6 +223,7 @@ Proton::Proton(const config::ConfigUri & configUri, _initStarted(false), _initComplete(false), _initDocumentDbsInSequence(false), + _has_shut_down_config_and_state_components(false), _documentDBReferenceRegistry(std::make_shared<DocumentDBReferenceRegistry>()), _nodeUpLock(), _nodeUp() @@ -400,16 +401,8 @@ Proton::~Proton() if ( ! _initComplete ) { LOG(warning, "Initialization of proton was halted. Shutdown sequence has been initiated."); } - _protonConfigFetcher.close(); - _protonConfigurer.setAllowReconfig(false); + shutdown_config_fetching_and_state_exposing_components_once(); _executor.sync(); - _customComponentRootToken.reset(); - _customComponentBindToken.reset(); - _stateServer.reset(); - if (_metricsEngine) { - _metricsEngine->removeMetricsHook(_metricsHook); - _metricsEngine->stop(); - } if (_matchEngine) { _matchEngine->close(); } @@ -463,6 +456,25 @@ Proton::~Proton() } void +Proton::shutdown_config_fetching_and_state_exposing_components_once() noexcept +{ + if (_has_shut_down_config_and_state_components) { + return; + } + _protonConfigFetcher.close(); + _protonConfigurer.setAllowReconfig(false); + _executor.sync(); + _customComponentRootToken.reset(); + _customComponentBindToken.reset(); + _stateServer.reset(); + if (_metricsEngine) { + _metricsEngine->removeMetricsHook(_metricsHook); + _metricsEngine->stop(); + } + _has_shut_down_config_and_state_components = true; +} + +void Proton::closeDocumentDBs(vespalib::ThreadStackExecutorBase & executor) { // Need to extract names first as _documentDBMap is modified while removing. std::vector<DocTypeName> docTypes; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index d5c1a8b7b78..6b561ad5deb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -124,6 +124,7 @@ private: bool _initStarted; bool _initComplete; bool _initDocumentDbsInSequence; + bool _has_shut_down_config_and_state_components; std::shared_ptr<IDocumentDBReferenceRegistry> _documentDBReferenceRegistry; std::mutex _nodeUpLock; std::set<BucketSpace> _nodeUp; // bucketspaces where node is up @@ -169,6 +170,15 @@ public: */ BootstrapConfig::SP init(); + /** + * Shuts down metric manager and state server functionality to avoid + * calls to these during service layer component tear-down. + * + * Explicitly noexcept to avoid consistency issues between this and the + * destructor if something throws during shutdown. + */ + void shutdown_config_fetching_and_state_exposing_components_once() noexcept; + // 2nd phase init: setup data structures. void init(const BootstrapConfig::SP & configSnapshot); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp index 45e3c978dd9..a7dada3047c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp @@ -15,6 +15,9 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <future> +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.proton_configurer"); + using vespalib::makeLambdaTask; using vespa::config::search::core::ProtonConfig; @@ -177,6 +180,13 @@ ProtonConfigurer::configureDocumentDB(const ProtonConfigSnapshot &configSnapshot } else { auto documentDB = dbitr->second.first.lock(); assert(documentDB); + auto old_bucket_space = documentDB->getBucketSpace(); + if (bucketSpace != old_bucket_space) { + vespalib::string old_bucket_space_name = document::FixedBucketSpaces::to_string(old_bucket_space); + vespalib::string bucket_space_name = document::FixedBucketSpaces::to_string(bucketSpace); + LOG(fatal, "Bucket space for document type %s changed from %s to %s. This triggers undefined behavior on a running system. Restarting process immediately to fix it.", docTypeName.getName().c_str(), old_bucket_space_name.c_str(), bucket_space_name.c_str()); + std::_Exit(1); + } documentDB->reconfigure(documentDBConfig); } } diff --git a/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp b/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp index a64704a08e9..cf1506a9118 100644 --- a/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp +++ b/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp @@ -17,7 +17,6 @@ #include <vespa/searchlib/attribute/singlestringattribute.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/query/tree/location.h> -#include <vespa/searchlib/query/tree/point.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/document_weight_search_iterator.h> #include <vespa/searchlib/test/searchiteratorverifier.h> diff --git a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp index 2eafeab20bd..87aea2e3e8c 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp @@ -339,29 +339,43 @@ TEST("requireThatPrefixTermsWork") { TEST("requireThatLocationTermsWork") { // 0xcc is z-curve for (10, 10). MyAttributeManager attribute_manager = makeAttributeManager(int64_t(0xcc)); - - SimpleLocationTerm node(Location(Point(10, 10), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(search(node, attribute_manager)); - node = SimpleLocationTerm(Location(Point(100, 100), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(!search(node, attribute_manager)); - node = SimpleLocationTerm(Location(Point(13, 13), 4, 0), field, 0, Weight(0)); - EXPECT_TRUE(!search(node, attribute_manager)); - node = SimpleLocationTerm(Location(Point(10, 13), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(search(node, attribute_manager)); + { + SimpleLocationTerm node(Location(Point{10, 10}, 3, 0), field, 0, Weight(0)); + EXPECT_TRUE(search(node, attribute_manager)); + } + { + SimpleLocationTerm node(Location(Point{100, 100}, 3, 0), field, 0, Weight(0)); + EXPECT_TRUE(!search(node, attribute_manager)); + } + { + SimpleLocationTerm node(Location(Point{13, 13}, 4, 0), field, 0, Weight(0)); + EXPECT_TRUE(!search(node, attribute_manager)); + } + { + SimpleLocationTerm node(Location(Point{10, 13}, 3, 0), field, 0, Weight(0)); + EXPECT_TRUE(search(node, attribute_manager)); + } } TEST("requireThatOptimizedLocationTermsWork") { // 0xcc is z-curve for (10, 10). MyAttributeManager attribute_manager = makeFastSearchLongAttributeManager(int64_t(0xcc)); - - SimpleLocationTerm node(Location(Point(10, 10), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(search(node, attribute_manager, true)); - node = SimpleLocationTerm(Location(Point(100, 100), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(!search(node, attribute_manager, true)); - node = SimpleLocationTerm(Location(Point(13, 13), 4, 0), field, 0, Weight(0)); - EXPECT_TRUE(!search(node, attribute_manager, true)); - node = SimpleLocationTerm(Location(Point(10, 13), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(search(node, attribute_manager, true)); + { + SimpleLocationTerm node(Location(Point{10, 10}, 3, 0), field, 0, Weight(0)); + EXPECT_TRUE(search(node, attribute_manager, true)); + } + { + SimpleLocationTerm node(Location(Point{100, 100}, 3, 0), field, 0, Weight(0)); + EXPECT_TRUE(!search(node, attribute_manager, true)); + } + { + SimpleLocationTerm node(Location(Point{13, 13}, 4, 0), field, 0, Weight(0)); + EXPECT_TRUE(!search(node, attribute_manager, true)); + } + { + SimpleLocationTerm node(Location(Point{10, 13}, 3, 0), field, 0, Weight(0)); + EXPECT_TRUE(search(node, attribute_manager, true)); + } } TEST("require that optimized location search works with wrapped bounding box (no hits)") { diff --git a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp index 5e633dcc97d..3098232b443 100644 --- a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp @@ -254,15 +254,22 @@ TEST(AttributeBlueprintTest, require_that_location_terms_work) { // 0xcc is z-curve for (10, 10). auto attribute_manager = makeAttributeManager(int64_t(0xcc)); - - SimpleLocationTerm node(Location(Point(10, 10), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(do_search(node, attribute_manager, false)); - node = SimpleLocationTerm(Location(Point(100, 100), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(!do_search(node, attribute_manager, false)); - node = SimpleLocationTerm(Location(Point(13, 13), 4, 0), field, 0, Weight(0)); - EXPECT_TRUE(!do_search(node, attribute_manager, false)); - node = SimpleLocationTerm(Location(Point(10, 13), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(do_search(node, attribute_manager, false)); + { + SimpleLocationTerm node(Location(Point{10, 10}, 3, 0), field, 0, Weight(0)); + EXPECT_TRUE(do_search(node, attribute_manager, false)); + } + { + SimpleLocationTerm node(Location(Point{100, 100}, 3, 0), field, 0, Weight(0)); + EXPECT_TRUE(!do_search(node, attribute_manager, false)); + } + { + SimpleLocationTerm node(Location(Point{13, 13}, 4, 0), field, 0, Weight(0)); + EXPECT_TRUE(!do_search(node, attribute_manager, false)); + } + { + SimpleLocationTerm node(Location(Point{10, 13}, 3, 0), field, 0, Weight(0)); + EXPECT_TRUE(do_search(node, attribute_manager, false)); + } } TEST(AttributeBlueprintTest, require_that_fast_search_location_terms_work) @@ -270,14 +277,14 @@ TEST(AttributeBlueprintTest, require_that_fast_search_location_terms_work) // 0xcc is z-curve for (10, 10). auto attribute_manager = makeFastSearchLongAttribute(int64_t(0xcc)); - SimpleLocationTerm node(Location(Point(10, 10), 3, 0), field, 0, Weight(0)); + SimpleLocationTerm node(Location(Point{10, 10}, 3, 0), field, 0, Weight(0)); #if 0 EXPECT_TRUE(search(node, attribute_manager)); - node = SimpleLocationTerm(Location(Point(100, 100), 3, 0),field, 0, Weight(0)); + node = SimpleLocationTerm(Location(Point{100, 100}, 3, 0),field, 0, Weight(0)); EXPECT_TRUE(!search(node, attribute_manager)); - node = SimpleLocationTerm(Location(Point(13, 13), 4, 0),field, 0, Weight(0)); + node = SimpleLocationTerm(Location(Point{13, 13}, 4, 0),field, 0, Weight(0)); EXPECT_TRUE(!search(node, attribute_manager)); - node = SimpleLocationTerm(Location(Point(10, 13), 3, 0),field, 0, Weight(0)); + node = SimpleLocationTerm(Location(Point{10, 13}, 3, 0),field, 0, Weight(0)); EXPECT_TRUE(search(node, attribute_manager)); #endif } diff --git a/searchlib/src/tests/common/location/CMakeLists.txt b/searchlib/src/tests/common/location/CMakeLists.txt index 64a894096d5..ea0d96529e1 100644 --- a/searchlib/src/tests/common/location/CMakeLists.txt +++ b/searchlib/src/tests/common/location/CMakeLists.txt @@ -1,8 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_location_test_app TEST +vespa_add_executable(searchlib_geo_location_test_app TEST SOURCES - location_test.cpp + geo_location_test.cpp DEPENDS searchlib + GTest::GTest ) -vespa_add_test(NAME searchlib_location_test_app COMMAND searchlib_location_test_app) +vespa_add_test(NAME searchlib_geo_location_test_app COMMAND searchlib_geo_location_test_app) diff --git a/searchlib/src/tests/common/location/geo_location_test.cpp b/searchlib/src/tests/common/location/geo_location_test.cpp new file mode 100644 index 00000000000..31b844d0fc8 --- /dev/null +++ b/searchlib/src/tests/common/location/geo_location_test.cpp @@ -0,0 +1,394 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <stdio.h> +#include <vespa/searchlib/common/geo_location.h> +#include <vespa/searchlib/common/geo_location_spec.h> +#include <vespa/searchlib/common/geo_location_parser.h> +#include <vespa/vespalib/gtest/gtest.h> + +using search::common::GeoLocation; +using search::common::GeoLocationParser; + +using Box = search::common::GeoLocation::Box; +using Point = search::common::GeoLocation::Point; +using Range = search::common::GeoLocation::Range; +using Aspect = search::common::GeoLocation::Aspect; + +constexpr int32_t plus_inf = std::numeric_limits<int32_t>::max(); +constexpr int32_t minus_inf = std::numeric_limits<int32_t>::min(); +constexpr uint32_t u32_inf = std::numeric_limits<uint32_t>::max(); + +bool is_parseable(const char *str) { + GeoLocationParser parser; + return parser.parseOldFormat(str); +} + +GeoLocation parse(const char *str) { + GeoLocationParser parser; + EXPECT_TRUE(parser.parseOldFormat(str)); + return parser.getGeoLocation(); +} + +TEST(GeoLocationParserTest, malformed_bounding_boxes_are_not_parseable) { + EXPECT_TRUE(is_parseable("[2,10,20,30,40]")); + EXPECT_FALSE(is_parseable("[2,10,20,30,40][2,10,20,30,40]")); + EXPECT_FALSE(is_parseable("[1,10,20,30,40]")); + EXPECT_FALSE(is_parseable("[3,10,20,30,40]")); + EXPECT_FALSE(is_parseable("[2, 10, 20, 30, 40]")); + EXPECT_FALSE(is_parseable("[2,10,20,30,40")); + EXPECT_FALSE(is_parseable("[2,10,20,30]")); + EXPECT_FALSE(is_parseable("[10,20,30,40]")); +} + +TEST(GeoLocationParserTest, malformed_circles_are_not_parseable) { + EXPECT_TRUE(is_parseable("(2,10,20,5,0,0,0)")); + EXPECT_FALSE(is_parseable("(2,10,20,5,0,0,0)(2,10,20,5,0,0,0)")); + EXPECT_FALSE(is_parseable("(1,10,20,5,0,0,0)")); + EXPECT_FALSE(is_parseable("(3,10,20,5,0,0,0)")); + EXPECT_FALSE(is_parseable("(2, 10, 20, 5, 0, 0, 0)")); + EXPECT_FALSE(is_parseable("(2,10,20,5)")); + EXPECT_FALSE(is_parseable("(2,10,20,5,0,0,0")); + EXPECT_FALSE(is_parseable("(2,10,20,5,0,0,0,1000")); + EXPECT_FALSE(is_parseable("(10,20,5)")); +} + +TEST(GeoLocationParserTest, bounding_boxes_can_be_parsed) { + auto loc = parse("[2,10,20,30,40]"); + EXPECT_EQ(false, loc.has_point); + EXPECT_EQ(true, loc.bounding_box.active()); + EXPECT_EQ(0u, loc.x_aspect.multiplier); + EXPECT_EQ(0, loc.point.x); + EXPECT_EQ(0, loc.point.y); + EXPECT_EQ(std::numeric_limits<uint32_t>::max(), loc.radius); + EXPECT_EQ(10, loc.bounding_box.x.low); + EXPECT_EQ(20, loc.bounding_box.y.low); + EXPECT_EQ(30, loc.bounding_box.x.high); + EXPECT_EQ(40, loc.bounding_box.y.high); +} + +TEST(GeoLocationParserTest, circles_can_be_parsed) { + auto loc = parse("(2,10,20,5,0,0,0)"); + EXPECT_EQ(true, loc.has_point); + EXPECT_EQ(true, loc.bounding_box.active()); + EXPECT_EQ(0u, loc.x_aspect.multiplier); + EXPECT_EQ(10, loc.point.x); + EXPECT_EQ(20, loc.point.y); + EXPECT_EQ(5u, loc.radius); + EXPECT_EQ(5, loc.bounding_box.x.low); + EXPECT_EQ(15, loc.bounding_box.y.low); + EXPECT_EQ(15, loc.bounding_box.x.high); + EXPECT_EQ(25, loc.bounding_box.y.high); +} + +TEST(GeoLocationParserTest, circles_can_have_aspect_ratio) { + auto loc = parse("(2,10,20,5,0,0,0,2147483648)"); + EXPECT_EQ(true, loc.has_point); + EXPECT_EQ(true, loc.bounding_box.active()); + EXPECT_EQ(2147483648u, loc.x_aspect.multiplier); + EXPECT_EQ(10, loc.point.x); + EXPECT_EQ(20, loc.point.y); + EXPECT_EQ(5u, loc.radius); + EXPECT_EQ(-1, loc.bounding_box.x.low); + EXPECT_EQ(15, loc.bounding_box.y.low); + EXPECT_EQ(21, loc.bounding_box.x.high); + EXPECT_EQ(25, loc.bounding_box.y.high); +} + +TEST(GeoLocationParserTest, bounding_box_can_be_specified_after_circle) { + auto loc = parse("(2,10,20,5,0,0,0)[2,10,20,30,40]"); + EXPECT_EQ(true, loc.has_point); + EXPECT_EQ(true, loc.bounding_box.active()); + EXPECT_EQ(0u, loc.x_aspect.multiplier); + EXPECT_EQ(10, loc.point.x); + EXPECT_EQ(20, loc.point.y); + EXPECT_EQ(5u, loc.radius); + EXPECT_EQ(10, loc.bounding_box.x.low); + EXPECT_EQ(20, loc.bounding_box.y.low); + EXPECT_EQ(15, loc.bounding_box.x.high); + EXPECT_EQ(25, loc.bounding_box.y.high); +} + +TEST(GeoLocationParserTest, circles_can_be_specified_after_bounding_box) { + auto loc = parse("[2,10,20,30,40](2,10,20,5,0,0,0)"); + EXPECT_EQ(true, loc.has_point); + EXPECT_EQ(true, loc.bounding_box.active()); + EXPECT_EQ(0u, loc.x_aspect.multiplier); + EXPECT_EQ(10, loc.point.x); + EXPECT_EQ(20, loc.point.y); + EXPECT_EQ(5u, loc.radius); + EXPECT_EQ(10, loc.bounding_box.x.low); + EXPECT_EQ(20, loc.bounding_box.y.low); + EXPECT_EQ(15, loc.bounding_box.x.high); + EXPECT_EQ(25, loc.bounding_box.y.high); +} + +TEST(GeoLocationParserTest, santa_search_gives_non_wrapped_bounding_box) { + auto loc = parse("(2,122163600,89998536,290112,4,2000,0,109704)"); + EXPECT_GE(loc.bounding_box.x.high, loc.bounding_box.x.low); + EXPECT_GE(loc.bounding_box.y.high, loc.bounding_box.y.low); +} + +TEST(GeoLocationParserTest, near_boundary_search_gives_non_wrapped_bounding_box) { + auto loc1 = parse("(2,2000000000,2000000000,3000000000,0,1,0)"); + EXPECT_GE(loc1.bounding_box.x.high, loc1.bounding_box.x.low); + EXPECT_GE(loc1.bounding_box.y.high, loc1.bounding_box.y.low); + EXPECT_EQ(std::numeric_limits<int32_t>::max(), loc1.bounding_box.y.high); + EXPECT_EQ(std::numeric_limits<int32_t>::max(), loc1.bounding_box.y.high); + + auto loc2 = parse("(2,-2000000000,-2000000000,3000000000,0,1,0)"); + EXPECT_GE(loc2.bounding_box.x.high, loc2.bounding_box.x.low); + EXPECT_GE(loc2.bounding_box.y.high, loc2.bounding_box.y.low); + EXPECT_EQ(std::numeric_limits<int32_t>::min(), loc2.bounding_box.x.low); + EXPECT_EQ(std::numeric_limits<int32_t>::min(), loc2.bounding_box.y.low); +} + +void check_box(const GeoLocation &location, Box expected) +{ + int32_t lx = expected.x.low; + int32_t hx = expected.x.high; + int32_t ly = expected.y.low; + int32_t hy = expected.y.high; + EXPECT_TRUE(location.inside_limit(Point{lx,ly})); + EXPECT_TRUE(location.inside_limit(Point{lx,hy})); + EXPECT_TRUE(location.inside_limit(Point{hx,ly})); + EXPECT_TRUE(location.inside_limit(Point{hx,hy})); + + EXPECT_FALSE(location.inside_limit(Point{lx,ly-1})); + EXPECT_FALSE(location.inside_limit(Point{lx,hy+1})); + EXPECT_FALSE(location.inside_limit(Point{lx-1,ly})); + EXPECT_FALSE(location.inside_limit(Point{lx-1,hy})); + EXPECT_FALSE(location.inside_limit(Point{hx,ly-1})); + EXPECT_FALSE(location.inside_limit(Point{hx,hy+1})); + EXPECT_FALSE(location.inside_limit(Point{hx+1,ly})); + EXPECT_FALSE(location.inside_limit(Point{hx+1,hy})); + + EXPECT_FALSE(location.inside_limit(Point{plus_inf,plus_inf})); + EXPECT_FALSE(location.inside_limit(Point{minus_inf,minus_inf})); +} + +TEST(GeoLocationTest, invalid_location) { + GeoLocation invalid; + EXPECT_FALSE(invalid.valid()); + EXPECT_FALSE(invalid.has_radius()); + EXPECT_FALSE(invalid.can_limit()); + EXPECT_FALSE(invalid.has_point); + EXPECT_FALSE(invalid.bounding_box.active()); + EXPECT_FALSE(invalid.x_aspect.active()); + + EXPECT_EQ(invalid.sq_distance_to(Point{0,0}), 0); + EXPECT_EQ(invalid.sq_distance_to(Point{999999,999999}), 0); + EXPECT_EQ(invalid.sq_distance_to(Point{-999999,-999999}), 0); + EXPECT_EQ(invalid.sq_distance_to(Point{plus_inf,plus_inf}), 0); + EXPECT_EQ(invalid.sq_distance_to(Point{minus_inf,minus_inf}), 0); + + EXPECT_TRUE(invalid.inside_limit(Point{0,0})); + EXPECT_TRUE(invalid.inside_limit(Point{999999,999999})); + EXPECT_TRUE(invalid.inside_limit(Point{-999999,-999999})); + EXPECT_TRUE(invalid.inside_limit(Point{plus_inf,plus_inf})); + EXPECT_TRUE(invalid.inside_limit(Point{minus_inf,minus_inf})); +} + +TEST(GeoLocationTest, point_location) { + GeoLocation location(Point{300,-400}); + EXPECT_TRUE(location.valid()); + EXPECT_FALSE(location.has_radius()); + EXPECT_FALSE(location.can_limit()); + EXPECT_TRUE(location.has_point); + EXPECT_FALSE(location.bounding_box.active()); + EXPECT_FALSE(location.x_aspect.active()); + + EXPECT_EQ(location.sq_distance_to(Point{0,0}), 500*500); + EXPECT_EQ(location.sq_distance_to(Point{300,-400}), 0); + EXPECT_EQ(location.sq_distance_to(Point{300,400}), 640000); + + EXPECT_TRUE(location.inside_limit(Point{0,0})); + EXPECT_TRUE(location.inside_limit(Point{999999,999999})); + EXPECT_TRUE(location.inside_limit(Point{-999999,-999999})); + EXPECT_TRUE(location.inside_limit(Point{plus_inf,plus_inf})); + EXPECT_TRUE(location.inside_limit(Point{minus_inf,minus_inf})); +} + +TEST(GeoLocationTest, point_and_radius) { + GeoLocation location(Point{300,-400}, 500); + EXPECT_TRUE(location.valid()); + EXPECT_TRUE(location.has_radius()); + EXPECT_TRUE(location.can_limit()); + EXPECT_TRUE(location.has_point); + EXPECT_TRUE(location.bounding_box.active()); + EXPECT_FALSE(location.x_aspect.active()); + + EXPECT_EQ(location.radius, 500); + + EXPECT_EQ(location.sq_distance_to(Point{0,0}), 500*500); + EXPECT_EQ(location.sq_distance_to(Point{300,-400}), 0); + EXPECT_EQ(location.sq_distance_to(Point{300,400}), 640000); + + EXPECT_TRUE(location.inside_limit(Point{0,0})); + EXPECT_TRUE(location.inside_limit(Point{-200,-400})); + EXPECT_TRUE(location.inside_limit(Point{800,-400})); + EXPECT_TRUE(location.inside_limit(Point{300,-400})); + EXPECT_TRUE(location.inside_limit(Point{300,100})); + EXPECT_TRUE(location.inside_limit(Point{300,-900})); + + check_box(location, Box{Range{0,600},{-800,0}}); +} + +TEST(GeoLocationTest, point_and_aspect) { + GeoLocation location(Point{600,400}, Aspect{0.5}); + // same: GeoLocation location(Point{600,400}, Aspect{1ul << 31}); + EXPECT_TRUE(location.valid()); + EXPECT_FALSE(location.has_radius()); + EXPECT_FALSE(location.can_limit()); + EXPECT_TRUE(location.has_point); + EXPECT_FALSE(location.bounding_box.active()); + EXPECT_TRUE(location.x_aspect.active()); + EXPECT_EQ(location.x_aspect.multiplier, 1ul << 31); + + EXPECT_EQ(location.sq_distance_to(Point{0,0}), 500*500); + EXPECT_EQ(location.sq_distance_to(Point{600,400}), 0); + EXPECT_EQ(location.sq_distance_to(Point{1200,800}), 500*500); + + EXPECT_TRUE(location.inside_limit(Point{0,0})); + EXPECT_TRUE(location.inside_limit(Point{999999,999999})); + EXPECT_TRUE(location.inside_limit(Point{-999999,-999999})); + EXPECT_TRUE(location.inside_limit(Point{plus_inf,plus_inf})); + EXPECT_TRUE(location.inside_limit(Point{minus_inf,minus_inf})); +} + +TEST(GeoLocationTest, point_radius_and_aspect) { + GeoLocation location(Point{1200,400}, 500, Aspect{0.25}); + EXPECT_TRUE(location.valid()); + EXPECT_TRUE(location.has_radius()); + EXPECT_TRUE(location.can_limit()); + EXPECT_TRUE(location.has_point); + EXPECT_TRUE(location.bounding_box.active()); + EXPECT_TRUE(location.x_aspect.active()); + EXPECT_EQ(location.x_aspect.multiplier, 1ul << 30); + + EXPECT_EQ(location.radius, 500); + + EXPECT_EQ(location.sq_distance_to(Point{0,0}), 500*500); + EXPECT_EQ(location.sq_distance_to(Point{1200,400}), 0); + EXPECT_EQ(location.sq_distance_to(Point{1240,400}), 100); + + EXPECT_TRUE(location.inside_limit(Point{1200,400})); + EXPECT_TRUE(location.inside_limit(Point{0,0})); + EXPECT_TRUE(location.inside_limit(Point{2400,0})); + EXPECT_TRUE(location.inside_limit(Point{2400,800})); + EXPECT_TRUE(location.inside_limit(Point{0,800})); + // note: must be 4 outside since 3*0.25 may be truncated to 0 + EXPECT_FALSE(location.inside_limit(Point{-4,0})); + EXPECT_FALSE(location.inside_limit(Point{-4,800})); + EXPECT_FALSE(location.inside_limit(Point{2404,0})); + EXPECT_FALSE(location.inside_limit(Point{2404,800})); + EXPECT_FALSE(location.inside_limit(Point{2400,-1})); + EXPECT_FALSE(location.inside_limit(Point{2400,801})); + EXPECT_FALSE(location.inside_limit(Point{0,-1})); + EXPECT_FALSE(location.inside_limit(Point{0,801})); + EXPECT_FALSE(location.inside_limit(Point{plus_inf,plus_inf})); + EXPECT_FALSE(location.inside_limit(Point{minus_inf,minus_inf})); +} + +TEST(GeoLocationTest, box_location) { + Box mybox{Range{300,350},Range{400,450}}; + GeoLocation location(mybox); + EXPECT_TRUE(location.valid()); + EXPECT_FALSE(location.has_radius()); + EXPECT_TRUE(location.can_limit()); + EXPECT_FALSE(location.has_point); + EXPECT_TRUE(location.bounding_box.active()); + EXPECT_FALSE(location.x_aspect.active()); + + // currently does not measure distance outside box: + EXPECT_EQ(location.sq_distance_to(Point{0,0}), 0); + EXPECT_EQ(location.sq_distance_to(Point{300,400}), 0); + EXPECT_EQ(location.sq_distance_to(Point{350,450}), 0); + EXPECT_EQ(location.sq_distance_to(Point{450,550}), 0); + + EXPECT_TRUE(location.inside_limit(Point{333,444})); + EXPECT_FALSE(location.inside_limit(Point{0,0})); + check_box(location, mybox); +} + +TEST(GeoLocationTest, box_and_point) { + Box mybox{Range{287,343},Range{366,401}}; + GeoLocation location(mybox, Point{300,400}); + EXPECT_TRUE(location.valid()); + EXPECT_FALSE(location.has_radius()); + EXPECT_TRUE(location.can_limit()); + EXPECT_TRUE(location.has_point); + EXPECT_TRUE(location.bounding_box.active()); + EXPECT_FALSE(location.x_aspect.active()); + + EXPECT_EQ(location.sq_distance_to(Point{0,0}), 500*500); + EXPECT_EQ(location.sq_distance_to(Point{300,400}), 0); + EXPECT_EQ(location.sq_distance_to(Point{300,423}), 23*23); + + check_box(location, mybox); +} + +TEST(GeoLocationTest, box_point_and_aspect) { + Box mybox{Range{-1000,350},Range{-1000,600}}; + GeoLocation location(mybox, Point{600,400}, Aspect{0.5}); + EXPECT_TRUE(location.valid()); + EXPECT_FALSE(location.has_radius()); + EXPECT_TRUE(location.can_limit()); + EXPECT_TRUE(location.has_point); + EXPECT_TRUE(location.bounding_box.active()); + EXPECT_TRUE(location.x_aspect.active()); + + EXPECT_EQ(location.sq_distance_to(Point{0,0}), 500*500); + EXPECT_EQ(location.sq_distance_to(Point{600,400}), 0); + EXPECT_EQ(location.sq_distance_to(Point{600,407}), 7*7); + EXPECT_EQ(location.sq_distance_to(Point{614,400}), 7*7); + + check_box(location, mybox); +} + +TEST(GeoLocationTest, box_point_and_radius) { + Box mybox{Range{-1000,350},Range{-1000,600}}; + GeoLocation location(mybox, Point{300,400}, 500); + EXPECT_TRUE(location.valid()); + EXPECT_TRUE(location.has_radius()); + EXPECT_TRUE(location.can_limit()); + EXPECT_TRUE(location.has_point); + EXPECT_TRUE(location.bounding_box.active()); + EXPECT_FALSE(location.x_aspect.active()); + + EXPECT_EQ(location.radius, 500); + + EXPECT_EQ(location.sq_distance_to(Point{0,0}), 500*500); + EXPECT_EQ(location.sq_distance_to(Point{300,400}), 0); + EXPECT_EQ(location.sq_distance_to(Point{300,423}), 23*23); + + EXPECT_EQ(location.bounding_box.x.low, -200); + EXPECT_EQ(location.bounding_box.y.low, -100); + EXPECT_EQ(location.bounding_box.x.high, 350); + EXPECT_EQ(location.bounding_box.y.high, 600); +} + +TEST(GeoLocationTest, box_point_radius_and_aspect) { + Box mybox{Range{-1000,650},Range{-1000,700}}; + GeoLocation location(mybox, Point{600,400}, 500, Aspect{0.5}); + EXPECT_TRUE(location.valid()); + EXPECT_TRUE(location.has_radius()); + EXPECT_TRUE(location.can_limit()); + EXPECT_TRUE(location.has_point); + EXPECT_TRUE(location.bounding_box.active()); + EXPECT_TRUE(location.x_aspect.active()); + + EXPECT_EQ(location.radius, 500); + + EXPECT_EQ(location.sq_distance_to(Point{0,0}), 500*500); + EXPECT_EQ(location.sq_distance_to(Point{600,400}), 0); + EXPECT_EQ(location.sq_distance_to(Point{600,407}), 7*7); + EXPECT_EQ(location.sq_distance_to(Point{614,400}), 7*7); + + EXPECT_GE(location.bounding_box.x.low, -402); + EXPECT_LE(location.bounding_box.x.low, -400); + EXPECT_EQ(location.bounding_box.y.low, -100); + EXPECT_EQ(location.bounding_box.x.high, 650); + EXPECT_EQ(location.bounding_box.y.high, 700); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/common/location/location_test.cpp b/searchlib/src/tests/common/location/location_test.cpp deleted file mode 100644 index d781e5b7275..00000000000 --- a/searchlib/src/tests/common/location/location_test.cpp +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/searchlib/common/location.h> -#include <vespa/searchlib/attribute/attributeguard.h> - - -using search::common::Location; - -bool is_parseable(const char *str) { - Location loc; - return loc.parse(str); -} - -Location parse(const char *str) { - Location loc; - if (!EXPECT_TRUE(loc.parse(str))) { - fprintf(stderr, " parse error: %s\n", loc.getParseError()); - } - return loc; -} - -TEST("require that malformed bounding boxes are not parseable") { - EXPECT_TRUE(is_parseable("[2,10,20,30,40]")); - EXPECT_FALSE(is_parseable("[2,10,20,30,40][2,10,20,30,40]")); - EXPECT_FALSE(is_parseable("[1,10,20,30,40]")); - EXPECT_FALSE(is_parseable("[3,10,20,30,40]")); - EXPECT_FALSE(is_parseable("[2, 10, 20, 30, 40]")); - EXPECT_FALSE(is_parseable("[2,10,20,30,40")); - EXPECT_FALSE(is_parseable("[2,10,20,30]")); - EXPECT_FALSE(is_parseable("[10,20,30,40]")); -} - -TEST("require that malformed circles are not parseable") { - EXPECT_TRUE(is_parseable("(2,10,20,5,0,0,0)")); - EXPECT_FALSE(is_parseable("(2,10,20,5,0,0,0)(2,10,20,5,0,0,0)")); - EXPECT_FALSE(is_parseable("(1,10,20,5,0,0,0)")); - EXPECT_FALSE(is_parseable("(3,10,20,5,0,0,0)")); - EXPECT_FALSE(is_parseable("(2, 10, 20, 5, 0, 0, 0)")); - EXPECT_FALSE(is_parseable("(2,10,20,5)")); - EXPECT_FALSE(is_parseable("(2,10,20,5,0,0,0")); - EXPECT_FALSE(is_parseable("(2,10,20,5,0,0,0,1000")); - EXPECT_FALSE(is_parseable("(10,20,5)")); -} - -TEST("require that bounding boxes can be parsed") { - Location loc = parse("[2,10,20,30,40]"); - EXPECT_EQUAL(false, loc.getRankOnDistance()); - EXPECT_EQUAL(true, loc.getPruneOnDistance()); - EXPECT_EQUAL(0u, loc.getXAspect()); - EXPECT_EQUAL(0, loc.getX()); - EXPECT_EQUAL(0, loc.getY()); - EXPECT_EQUAL(std::numeric_limits<uint32_t>::max(), loc.getRadius()); - EXPECT_EQUAL(10, loc.getMinX()); - EXPECT_EQUAL(20, loc.getMinY()); - EXPECT_EQUAL(30, loc.getMaxX()); - EXPECT_EQUAL(40, loc.getMaxY()); -} - -TEST("require that circles can be parsed") { - Location loc = parse("(2,10,20,5,0,0,0)"); - EXPECT_EQUAL(true, loc.getRankOnDistance()); - EXPECT_EQUAL(true, loc.getPruneOnDistance()); - EXPECT_EQUAL(0u, loc.getXAspect()); - EXPECT_EQUAL(10, loc.getX()); - EXPECT_EQUAL(20, loc.getY()); - EXPECT_EQUAL(5u, loc.getRadius()); - EXPECT_EQUAL(5, loc.getMinX()); - EXPECT_EQUAL(15, loc.getMinY()); - EXPECT_EQUAL(15, loc.getMaxX()); - EXPECT_EQUAL(25, loc.getMaxY()); -} - -TEST("require that circles can have aspect ratio") { - Location loc = parse("(2,10,20,5,0,0,0,2147483648)"); - EXPECT_EQUAL(true, loc.getRankOnDistance()); - EXPECT_EQUAL(true, loc.getPruneOnDistance()); - EXPECT_EQUAL(2147483648u, loc.getXAspect()); - EXPECT_EQUAL(10, loc.getX()); - EXPECT_EQUAL(20, loc.getY()); - EXPECT_EQUAL(5u, loc.getRadius()); - EXPECT_EQUAL(-1, loc.getMinX()); - EXPECT_EQUAL(15, loc.getMinY()); - EXPECT_EQUAL(21, loc.getMaxX()); - EXPECT_EQUAL(25, loc.getMaxY()); -} - -TEST("require that bounding box can be specified after circle") { - Location loc = parse("(2,10,20,5,0,0,0)[2,10,20,30,40]"); - EXPECT_EQUAL(true, loc.getRankOnDistance()); - EXPECT_EQUAL(true, loc.getPruneOnDistance()); - EXPECT_EQUAL(0u, loc.getXAspect()); - EXPECT_EQUAL(10, loc.getX()); - EXPECT_EQUAL(20, loc.getY()); - EXPECT_EQUAL(5u, loc.getRadius()); - EXPECT_EQUAL(10, loc.getMinX()); - EXPECT_EQUAL(20, loc.getMinY()); - EXPECT_EQUAL(15, loc.getMaxX()); - EXPECT_EQUAL(25, loc.getMaxY()); -} - -TEST("require that circles can be specified after bounding box") { - Location loc = parse("[2,10,20,30,40](2,10,20,5,0,0,0)"); - EXPECT_EQUAL(true, loc.getRankOnDistance()); - EXPECT_EQUAL(true, loc.getPruneOnDistance()); - EXPECT_EQUAL(0u, loc.getXAspect()); - EXPECT_EQUAL(10, loc.getX()); - EXPECT_EQUAL(20, loc.getY()); - EXPECT_EQUAL(5u, loc.getRadius()); - EXPECT_EQUAL(10, loc.getMinX()); - EXPECT_EQUAL(20, loc.getMinY()); - EXPECT_EQUAL(15, loc.getMaxX()); - EXPECT_EQUAL(25, loc.getMaxY()); -} - -TEST("require that santa search gives non-wrapped bounding box") { - Location loc = parse("(2,122163600,89998536,290112,4,2000,0,109704)"); - EXPECT_GREATER_EQUAL(loc.getMaxX(), loc.getMinX()); - EXPECT_GREATER_EQUAL(loc.getMaxY(), loc.getMinY()); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index f886ba59c1c..25b5ba20d26 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -62,6 +62,8 @@ using search::StringAttribute; using search::SingleBoolAttribute; using search::WeightedSetStringExtAttribute; using search::attribute::WeightedEnumContent; +using search::common::GeoLocation; +using search::common::GeoLocationSpec; using AttributePtr = AttributeVector::SP; using AVC = search::attribute::Config; @@ -507,8 +509,8 @@ Test::assertCloseness(feature_t exp, const vespalib::string & attr, double dista int32_t x = 0; positions.emplace_back(x, x); setupForDistanceTest(ft, "pos", positions, false); - ft.getQueryEnv().getLocation().setXPosition((int)distance); - ft.getQueryEnv().getLocation().setValid(true); + GeoLocation::Point p{int32_t(distance), 0}; + ft.getQueryEnv().addLocation(GeoLocationSpec{attr, p}); if (maxDistance > 0) { ft.getIndexEnv().getProperties().add(feature + ".maxDistance", vespalib::make_string("%u", (unsigned int)maxDistance)); @@ -857,13 +859,16 @@ Test::testDistance() { // non-existing attribute FtFeatureTest ft(_factory, "distance(pos)"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos"); - ft.getQueryEnv().getLocation().setValid(true); + GeoLocation::Point p{0, 0}; + ft.getQueryEnv().addLocation(GeoLocationSpec{"pos", p}); + ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0))); } { // label FtFeatureTest ft(_factory, "distance(label,foo)"); - ft.getQueryEnv().getLocation().setValid(true); + GeoLocation::Point p{0, 0}; + ft.getQueryEnv().addLocation(GeoLocationSpec{"pos", p}); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().addScore("distance(label,foo)", std::numeric_limits<feature_t>::max()))); } @@ -873,7 +878,8 @@ Test::testDistance() pos->commit(); ft.getIndexEnv().getAttributeMap().add(pos); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos"); - ft.getQueryEnv().getLocation().setValid(true); + GeoLocation::Point p{0, 0}; + ft.getQueryEnv().addLocation(GeoLocationSpec{"pos", p}); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0))); } @@ -883,7 +889,8 @@ Test::testDistance() pos->commit(); ft.getIndexEnv().getAttributeMap().add(pos); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos"); - ft.getQueryEnv().getLocation().setValid(true); + GeoLocation::Point p{0, 0}; + ft.getQueryEnv().addLocation(GeoLocationSpec{"pos", p}); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0))); } @@ -893,7 +900,8 @@ Test::testDistance() pos->commit(); ft.getIndexEnv().getAttributeMap().add(pos); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::WEIGHTEDSET, DataType::INT64, "pos"); - ft.getQueryEnv().getLocation().setValid(true); + GeoLocation::Point p{0, 0}; + ft.getQueryEnv().addLocation(GeoLocationSpec{"pos", p}); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0))); } @@ -939,10 +947,9 @@ Test::assert2DZDistance(feature_t exp, const vespalib::string & positions, pos.emplace_back(x, y); } setupForDistanceTest(ft, "pos", pos, true); - ft.getQueryEnv().getLocation().setXPosition(xquery); - ft.getQueryEnv().getLocation().setYPosition(yquery); - ft.getQueryEnv().getLocation().setXAspect(xAspect); - ft.getQueryEnv().getLocation().setValid(true); + GeoLocation::Point p{xquery, yquery}; + GeoLocation::Aspect aspect{xAspect}; + ft.getQueryEnv().addLocation(GeoLocationSpec{"pos", {p, aspect}}); ASSERT_TRUE(ft.setup()); ASSERT_TRUE(ft.execute(RankResult().setEpsilon(1e-4). addScore("distance(pos)", exp))); diff --git a/searchlib/src/tests/query/query_visitor_test.cpp b/searchlib/src/tests/query/query_visitor_test.cpp index edbc29be784..8441dc2227f 100644 --- a/searchlib/src/tests/query/query_visitor_test.cpp +++ b/searchlib/src/tests/query/query_visitor_test.cpp @@ -90,7 +90,7 @@ void Test::requireThatAllNodesCanBeVisited() { checkVisit<WandTerm>(new SimpleWandTerm("field", 0, Weight(42), 57, 67, 77.7)); checkVisit<Rank>(new SimpleRank); checkVisit<NumberTerm>(new SimpleNumberTerm("0.42", "field", 0, Weight(0))); - const Location location(Point(10, 10), 20, 0); + const Location location(Point{10, 10}, 20, 0); checkVisit<LocationTerm>(new SimpleLocationTerm(location, "field", 0, Weight(0))); checkVisit<PrefixTerm>(new SimplePrefixTerm("t", "field", 0, Weight(0))); checkVisit<RangeTerm>(new SimpleRangeTerm(Range(0, 1), "field", 0, Weight(0))); diff --git a/searchlib/src/tests/query/querybuilder_test.cpp b/searchlib/src/tests/query/querybuilder_test.cpp index d093bc4242e..269600d26d4 100644 --- a/searchlib/src/tests/query/querybuilder_test.cpp +++ b/searchlib/src/tests/query/querybuilder_test.cpp @@ -32,7 +32,7 @@ const size_t distance = 4; const string int1 = "42"; const string float1 = "3.14"; const Range range(32, 64); -const Point position(100, 100); +const Point position{100, 100}; const int max_distance = 20; const uint32_t x_aspect = 0; const Location location(position, max_distance, x_aspect); diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 4a34a07a773..cb587d77133 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -248,11 +248,14 @@ public: LocationPostFilterBlueprint(const FieldSpec &field, const IAttributeVector &attribute, const Location &loc) : ComplexLeafBlueprint(field), _attribute(attribute), - _location() + _location(loc) { - _location.setVec(attribute); - _location.parse(loc.getLocationString()); - uint32_t estHits = _attribute.getNumDocs(); + uint32_t estHits = 0; + if (loc.valid()) { + _location.setVec(attribute); + estHits = _attribute.getNumDocs(); + } + LOG(debug, "location %s in attribute with numdocs %u", loc.getOldFormatString().c_str(), estHits); HitEstimate estimate(estHits, estHits == 0); setEstimate(estimate); } @@ -272,14 +275,16 @@ Blueprint::UP make_location_blueprint(const FieldSpec &field, const IAttributeVector &attribute, const Location &loc) { auto post_filter = std::make_unique<LocationPostFilterBlueprint>(field, attribute, loc); const common::Location &location = post_filter->location(); - if (location.getMinX() > location.getMaxX() || - location.getMinY() > location.getMaxY()) + if (location.bounding_box.x.low > location.bounding_box.x.high || + location.bounding_box.y.low > location.bounding_box.y.high) { return std::make_unique<queryeval::EmptyBlueprint>(field); } ZCurve::RangeVector rangeVector = ZCurve::find_ranges( - location.getMinX(), location.getMinY(), - location.getMaxX(), location.getMaxY()); + location.bounding_box.x.low, + location.bounding_box.y.low, + location.bounding_box.x.high, + location.bounding_box.y.high); auto pre_filter = std::make_unique<LocationPreFilterBlueprint>(field, attribute, rangeVector); if (!pre_filter->should_use()) { return post_filter; diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt index 5d30260a169..7e1cfd8fec5 100644 --- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt @@ -12,6 +12,9 @@ vespa_add_library(searchlib_common OBJECT featureset.cpp fileheadercontext.cpp gatecallback.cpp + geo_location.cpp + geo_location_spec.cpp + geo_location_parser.cpp growablebitvector.cpp indexmetainfo.cpp location.cpp diff --git a/searchlib/src/vespa/searchlib/common/documentlocations.cpp b/searchlib/src/vespa/searchlib/common/documentlocations.cpp index b03176f0ad2..b8f05581b41 100644 --- a/searchlib/src/vespa/searchlib/common/documentlocations.cpp +++ b/searchlib/src/vespa/searchlib/common/documentlocations.cpp @@ -21,5 +21,9 @@ DocumentLocations::setVecGuard(std::unique_ptr<search::AttributeGuard> guard) { setVec(*_vec_guard.get()->get()); } +DocumentLocations::DocumentLocations(DocumentLocations &&) = default; +DocumentLocations & DocumentLocations::operator = (DocumentLocations &&) = default; + + } // namespace common } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/documentlocations.h b/searchlib/src/vespa/searchlib/common/documentlocations.h index 1dab68ca11f..51d5be76e65 100644 --- a/searchlib/src/vespa/searchlib/common/documentlocations.h +++ b/searchlib/src/vespa/searchlib/common/documentlocations.h @@ -25,8 +25,8 @@ private: const search::attribute::IAttributeVector *_vec; public: - DocumentLocations(DocumentLocations &&) = default; - DocumentLocations & operator = (DocumentLocations &&) = default; + DocumentLocations(DocumentLocations &&); + DocumentLocations & operator = (DocumentLocations &&); DocumentLocations(); virtual ~DocumentLocations(); diff --git a/searchlib/src/vespa/searchlib/common/geo_location.cpp b/searchlib/src/vespa/searchlib/common/geo_location.cpp new file mode 100644 index 00000000000..6dd7b83ae37 --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/geo_location.cpp @@ -0,0 +1,184 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "geo_location.h" + +using vespalib::geo::ZCurve; + +namespace search::common { + +namespace { + +ZCurve::BoundingBox to_z(GeoLocation::Box box) { + return ZCurve::BoundingBox(box.x.low, box.x.high, + box.y.low, box.y.high); +} + +GeoLocation::Box +adjust_bounding_box(GeoLocation::Box orig, GeoLocation::Point point, uint32_t radius, GeoLocation::Aspect x_aspect) +{ + if (radius == GeoLocation::radius_inf) { + // only happens if GeoLocation is explicitly constructed with "infinite" radius + return orig; + } + uint32_t maxdx = radius; + if (x_aspect.active()) { + // x_aspect is a 32-bit fixed-point number in range [0,1] + // so this implements maxdx = ceil(radius/x_aspect) + uint64_t maxdx2 = ((static_cast<uint64_t>(radius) << 32) + 0xffffffffu) / x_aspect.multiplier; + if (maxdx2 >= 0xffffffffu) { + maxdx = 0xffffffffu; + } else { + maxdx = static_cast<uint32_t>(maxdx2); + } + } + // implied limits from radius and point: + int64_t implied_max_x = int64_t(point.x) + int64_t(maxdx); + int64_t implied_min_x = int64_t(point.x) - int64_t(maxdx); + + int64_t implied_max_y = int64_t(point.y) + int64_t(radius); + int64_t implied_min_y = int64_t(point.y) - int64_t(radius); + + int32_t max_x = orig.x.high; + int32_t min_x = orig.x.low; + + int32_t max_y = orig.y.high; + int32_t min_y = orig.y.low; + + if (implied_max_x < max_x) max_x = implied_max_x; + if (implied_min_x > min_x) min_x = implied_min_x; + + if (implied_max_y < max_y) max_y = implied_max_y; + if (implied_min_y > min_y) min_y = implied_min_y; + + return GeoLocation::Box{GeoLocation::Range{min_x, max_x}, + GeoLocation::Range{min_y, max_y}}; +} + +} // namespace <unnamed> + +GeoLocation::GeoLocation() + : has_point(false), + point{0, 0}, + radius(radius_inf), + x_aspect(), + bounding_box(no_box), + _sq_radius(sq_radius_inf), + _z_bounding_box(0,0,0,0) +{} + +GeoLocation::GeoLocation(Point p) + : has_point(true), + point(p), + radius(radius_inf), + x_aspect(), + bounding_box(no_box), + _sq_radius(sq_radius_inf), + _z_bounding_box(0,0,0,0) +{} + +GeoLocation::GeoLocation(Point p, Aspect xa) + : has_point(true), + point(p), + radius(radius_inf), + x_aspect(xa), + bounding_box(no_box), + _sq_radius(sq_radius_inf), + _z_bounding_box(0,0,0,0) +{} + +GeoLocation::GeoLocation(Point p, uint32_t r) + : has_point(true), + point(p), + radius(r), + x_aspect(), + bounding_box(adjust_bounding_box(no_box, p, r, Aspect())), + _sq_radius(uint64_t(r) * uint64_t(r)), + _z_bounding_box(to_z(bounding_box)) +{} + +GeoLocation::GeoLocation(Point p, uint32_t r, Aspect xa) + : has_point(true), + point(p), + radius(r), + x_aspect(xa), + bounding_box(adjust_bounding_box(no_box, p, r, xa)), + _sq_radius(uint64_t(r) * uint64_t(r)), + _z_bounding_box(to_z(bounding_box)) +{} + +GeoLocation::GeoLocation(Box b) + : has_point(false), + point{0, 0}, + radius(radius_inf), + x_aspect(), + bounding_box(b), + _sq_radius(sq_radius_inf), + _z_bounding_box(to_z(bounding_box)) +{} + +GeoLocation::GeoLocation(Box b, Point p) + : has_point(true), + point(p), + radius(radius_inf), + x_aspect(), + bounding_box(b), + _sq_radius(sq_radius_inf), + _z_bounding_box(to_z(bounding_box)) +{} + +GeoLocation::GeoLocation(Box b, Point p, Aspect xa) + : has_point(true), + point(p), + radius(radius_inf), + x_aspect(xa), + bounding_box(b), + _sq_radius(sq_radius_inf), + _z_bounding_box(to_z(bounding_box)) +{} + +GeoLocation::GeoLocation(Box b, Point p, uint32_t r) + : has_point(true), + point(p), + radius(r), + x_aspect(), + bounding_box(adjust_bounding_box(b, p, r, Aspect())), + _sq_radius(uint64_t(r) * uint64_t(r)), + _z_bounding_box(to_z(bounding_box)) +{} + +GeoLocation::GeoLocation(Box b, Point p, uint32_t r, Aspect xa) + : has_point(true), + point(p), + radius(r), + x_aspect(xa), + bounding_box(adjust_bounding_box(b, p, r, xa)), + _sq_radius(uint64_t(r) * uint64_t(r)), + _z_bounding_box(to_z(bounding_box)) +{} + +uint64_t GeoLocation::sq_distance_to(Point p) const { + if (has_point) { + uint64_t dx = (p.x > point.x) ? (p.x - point.x) : (point.x - p.x); + if (x_aspect.active()) { + // x_aspect is a 32-bit fixed-point number in range [0,1] + // this implements dx = (dx * x_aspect) + dx = (dx * x_aspect.multiplier) >> 32; + } + uint64_t dy = (p.y > point.y) ? (p.y - point.y) : (point.y - p.y); + return dx*dx + dy*dy; + } + return 0; +} + +bool GeoLocation::inside_limit(Point p) const { + if (p.x < bounding_box.x.low) return false; + if (p.x > bounding_box.x.high) return false; + + if (p.y < bounding_box.y.low) return false; + if (p.y > bounding_box.y.high) return false; + + uint64_t sq_dist = sq_distance_to(p); + return sq_dist <= _sq_radius; +} + +} // namespace search::common diff --git a/searchlib/src/vespa/searchlib/common/geo_location.h b/searchlib/src/vespa/searchlib/common/geo_location.h new file mode 100644 index 00000000000..261951caf3e --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/geo_location.h @@ -0,0 +1,92 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <string> +#include <cstdint> +#include <limits> +#include <vespa/vespalib/geo/zcurve.h> + +namespace search::common { + +/** + * An immutable struct for a (geo) location. + * Contains a point with optional radius, a bounding box, or both. + **/ +struct GeoLocation +{ + // contained structs and helper constants: + static constexpr int32_t range_low = std::numeric_limits<int32_t>::min(); + static constexpr int32_t range_high = std::numeric_limits<int32_t>::max(); + static constexpr uint32_t radius_inf = std::numeric_limits<uint32_t>::max(); + struct Point { + const int32_t x; + const int32_t y; + Point() = delete; + }; + struct Aspect { + uint32_t multiplier; + Aspect() : multiplier(0) {} + Aspect(uint32_t multiplier_in) : multiplier(multiplier_in) {} + // for unit tests: + Aspect(double multiplier_in) : multiplier(multiplier_in*4294967296.0) {} + bool active() const { return multiplier != 0; } + }; + struct Range { + const int32_t low; + const int32_t high; + bool active() const { + return (low != range_low) || (high != range_high); + } + }; + static constexpr Range no_range = {range_low, range_high}; + struct Box { + const Range x; + const Range y; + bool active() const { return x.active() || y.active(); } + }; + static constexpr Box no_box = {no_range, no_range}; + + // actual content of struct: + const bool has_point; + Point point; + uint32_t radius; + Aspect x_aspect; + Box bounding_box; + GeoLocation(); + + // constructors: + GeoLocation(Point p); + GeoLocation(Point p, Aspect xa); + GeoLocation(Point p, uint32_t r); + GeoLocation(Point p, uint32_t r, Aspect xa); + GeoLocation(Box b); + GeoLocation(Box b, Point p); + GeoLocation(Box b, Point p, Aspect xa); + GeoLocation(Box b, Point p, uint32_t r); + GeoLocation(Box b, Point p, uint32_t r, Aspect xa); + + // helper methods: + bool has_radius() const { return radius != radius_inf; } + bool valid() const { return has_point || bounding_box.active(); } + bool can_limit() const { return bounding_box.active(); } + + uint64_t sq_distance_to(Point p) const; + bool inside_limit(Point p) const; + + bool inside_limit(int64_t zcurve_encoded_xy) const { + if (_z_bounding_box.getzFailBoundingBoxTest(zcurve_encoded_xy)) return false; + int32_t x = 0; + int32_t y = 0; + vespalib::geo::ZCurve::decode(zcurve_encoded_xy, &x, &y); + return inside_limit(Point{x, y}); + } + +private: + // constants for implementation of helper methods: + static constexpr uint64_t sq_radius_inf = std::numeric_limits<uint64_t>::max(); + const uint64_t _sq_radius; + const vespalib::geo::ZCurve::BoundingBox _z_bounding_box; +}; + +} // namespace diff --git a/searchlib/src/vespa/searchlib/common/geo_location_parser.cpp b/searchlib/src/vespa/searchlib/common/geo_location_parser.cpp new file mode 100644 index 00000000000..05c53348699 --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/geo_location_parser.cpp @@ -0,0 +1,209 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "geo_location_parser.h" +#include <limits> +#include <vespa/vespalib/stllike/asciistream.h> + +namespace { + +int getInt(const char * &p) { + int val; + bool isminus; + val = 0; + isminus = false; + if (*p == '-') { + isminus = true; + p++; + } + while (*p >= '0' && *p <= '9') { + val *= 10; + val += (*p++ - '0'); + } + return isminus ? - val : val; +} + +} // namespace <unnamed> + +namespace search::common { + +GeoLocationParser::GeoLocationParser() + : _valid(false), + _has_point(false), + _has_bounding_box(false), + _field_name(), + _x(0), + _y(0), + _x_aspect(0u), + _radius(std::numeric_limits<uint32_t>::max()), + _min_x(std::numeric_limits<int32_t>::min()), + _max_x(std::numeric_limits<int32_t>::max()), + _min_y(std::numeric_limits<int32_t>::min()), + _max_y(std::numeric_limits<int32_t>::max()), + _parseError(NULL) +{} + +bool +GeoLocationParser::correctDimensionalitySkip(const char * &p) { + if (*p == '2') { + p++; + if (*p != ',') { + _parseError = "Missing comma after 2D dimensionality"; + return false; + } + p++; + return true; + } + _parseError = "Bad dimensionality spec, not 2D"; + return false; +} + +bool +GeoLocationParser::parseOldFormatWithField(const std::string &str) +{ + auto sep = str.find(':'); + if (sep == std::string::npos) { + _parseError = "Location string lacks field specification."; + return false; + } + _field_name = str.substr(0, sep); + std::string only_loc = str.substr(sep + 1); + return parseOldFormat(only_loc); +} + +bool +GeoLocationParser::parseOldFormat(const std::string &locStr) +{ + bool foundBoundingBox = false; + bool foundLoc = false; + const char *p = locStr.c_str(); + while (*p != '\0') { + if (*p == '[') { + p++; + if (foundBoundingBox) { + _parseError = "Duplicate bounding box"; + return false; + } + foundBoundingBox = true; + if (!correctDimensionalitySkip(p)) { + return false; + } + _min_x = getInt(p); + if (*p != ',') { + _parseError = "Missing ',' after minx"; + return false; + } + p++; + _min_y = getInt(p); + if (*p != ',') { + _parseError = "Missing ',' after miny"; + return false; + } + p++; + _max_x = getInt(p); + if (*p != ',') { + _parseError = "Missing ',' after maxx"; + return false; + } + p++; + _max_y = getInt(p); + if (*p != ']') { + _parseError = "Missing ']' after maxy"; + return false; + } + p++; + } else if (*p == '(') { + p++; + if (foundLoc) { + _parseError = "Duplicate location"; + return false; + } + foundLoc = true; + if (!correctDimensionalitySkip(p)) { + return false; + } + _x = getInt(p); + if (*p != ',') { + _parseError = "Missing ',' after x position"; + return false; + } + p++; + _y = getInt(p); + if (*p != ',') { + _parseError = "Missing ',' after y position"; + return false; + } + p++; + _radius = getInt(p); + if (*p != ',') { + _parseError = "Missing ',' after radius"; + return false; + } + p++; + /* _tableID = */ (void) getInt(p); + if (*p != ',') { + _parseError = "Missing ',' after tableID"; + return false; + } + p++; + /* _rankMultiplier = */ (void) getInt(p); + if (*p != ',') { + _parseError = "Missing ',' after rank multiplier"; + return false; + } + p++; + /* _rankOnlyOnDistance = */ (void) getInt(p); + if (*p == ',') { + p++; + _x_aspect = getInt(p); + if (*p != ')') { + _parseError = "Missing ')' after xAspect"; + return false; + } + } else { + if (*p != ')') { + _parseError = "Missing ')' after rankOnlyOnDistance flag"; + return false; + } + } + p++; + } else if (*p == ' ') { + p++; + } else { + _parseError = "Unexpected char in location spec"; + return false; + } + } + _has_point = foundLoc; + _has_bounding_box = foundBoundingBox; + _valid = (_has_point || _has_bounding_box); + return _valid; +} + +GeoLocation +GeoLocationParser::getGeoLocation() const +{ + GeoLocation::Aspect aspect(_x_aspect); + if (_has_bounding_box) { + GeoLocation::Range x_range{_min_x, _max_x}; + GeoLocation::Range y_range{_min_y, _max_y}; + GeoLocation::Box bounding_box{x_range, y_range}; + if (_has_point) { + GeoLocation::Point point{_x, _y}; + if (_radius == GeoLocation::radius_inf) { + return GeoLocation(bounding_box, point, aspect); + } + return GeoLocation(bounding_box, point, _radius, aspect); + } + return GeoLocation(bounding_box); + } + if (_has_point) { + GeoLocation::Point point{_x, _y}; + if (_radius == GeoLocation::radius_inf) { + return GeoLocation(point, aspect); + } + return GeoLocation(point, _radius, aspect); + } + return GeoLocation(); +} + +} // namespace diff --git a/searchlib/src/vespa/searchlib/common/geo_location_parser.h b/searchlib/src/vespa/searchlib/common/geo_location_parser.h new file mode 100644 index 00000000000..8936a620d21 --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/geo_location_parser.h @@ -0,0 +1,47 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <string> +#include <cstdint> +#include "geo_location.h" +#include "geo_location_spec.h" + +namespace search::common { + +/** + * Parser for a geo-location string representation. + **/ +class GeoLocationParser +{ +public: + GeoLocationParser(); + + bool parseOldFormat(const std::string &locStr); + bool parseOldFormatWithField(const std::string &str); + + std::string getFieldName() const { return _field_name; } + GeoLocation getGeoLocation() const; + + const char * getParseError() const { return _parseError; } +private: + bool _valid; + bool _has_point; + bool _has_bounding_box; + + std::string _field_name; + + int32_t _x; /* Query X position */ + int32_t _y; /* Query Y position */ + uint32_t _x_aspect; /* X distance multiplier fraction */ + uint32_t _radius; /* Radius for euclidean distance */ + int32_t _min_x; /* Min X coordinate */ + int32_t _max_x; /* Max X coordinate */ + int32_t _min_y; /* Min Y coordinate */ + int32_t _max_y; /* Max Y coordinate */ + + const char *_parseError; + bool correctDimensionalitySkip(const char * &p); +}; + +} // namespace diff --git a/searchlib/src/vespa/searchlib/common/geo_location_spec.cpp b/searchlib/src/vespa/searchlib/common/geo_location_spec.cpp new file mode 100644 index 00000000000..271946e2df6 --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/geo_location_spec.cpp @@ -0,0 +1,3 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "geo_location_spec.h" diff --git a/searchlib/src/vespa/searchlib/common/geo_location_spec.h b/searchlib/src/vespa/searchlib/common/geo_location_spec.h new file mode 100644 index 00000000000..42c2b8e6c8c --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/geo_location_spec.h @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <string> +#include <cstdint> +#include "geo_location.h" + +namespace search::common { + +/** + * Immutable specification of a geo-location query item. + **/ +struct GeoLocationSpec +{ +public: + const std::string field_name; + const GeoLocation location; +}; + +} // namespace diff --git a/searchlib/src/vespa/searchlib/common/location.cpp b/searchlib/src/vespa/searchlib/common/location.cpp index 6927d9ab6cb..171dcecaa33 100644 --- a/searchlib/src/vespa/searchlib/common/location.cpp +++ b/searchlib/src/vespa/searchlib/common/location.cpp @@ -5,198 +5,6 @@ namespace search::common { -Location::Location() : - _zBoundingBox(0,0,0,0), - _x(0), - _y(0), - _xAspect(0u), - _radius(std::numeric_limits<uint32_t>::max()), - _minx(std::numeric_limits<int32_t>::min()), - _maxx(std::numeric_limits<int32_t>::max()), - _miny(std::numeric_limits<int32_t>::min()), - _maxy(std::numeric_limits<int32_t>::max()), - _rankOnDistance(false), - _pruneOnDistance(false), - _parseError(NULL) -{ -} +Location::Location(const GeoLocation &from) : GeoLocation(from) {} - -bool -Location::getDimensionality(const char **pp) -{ - if (**pp == '2') { - (*pp)++; - if (**pp != ',') { - _parseError = "Missing comma after 2D dimensionality"; - return false; - } - (*pp)++; - return true; - } - _parseError = "Bad dimensionality spec, not 2D"; - return false; -} - - -int -Location::getInt(const char **pp) -{ - const char *p = *pp; - int val; - bool isminus; - - val = 0; - isminus = false; - if (*p == '-') { - isminus = true; - p++; - } - while (*p >= '0' && *p <= '9') - val = val * 10 + *p++ - '0'; - *pp = p; - return isminus ? - val : val; -} - -bool Location::parse(const vespalib::string &locStr) -{ - bool hadCutoff = false; - bool hadLoc = false; - const char *p = locStr.c_str(); - while (*p != '\0') { - if (*p == '[') { - p++; - if (hadCutoff) { - _parseError = "Duplicate square cutoff"; - return false; - } - hadCutoff = true; - if (!getDimensionality(&p)) - return false; - _minx = getInt(&p); - if (*p != ',') { - _parseError = "Missing ',' after minx"; - return false; - } - p++; - _miny = getInt(&p); - if (*p != ',') { - _parseError = "Missing ',' after miny"; - return false; - } - p++; - _maxx = getInt(&p); - if (*p != ',') { - _parseError = "Missing ',' after maxx"; - return false; - } - p++; - _maxy = getInt(&p); - if (*p != ']') { - _parseError = "Missing ']' after maxy"; - return false; - } - p++; - } else if (*p == '(') { - p++; - if (hadLoc) { - _parseError = "Duplicate location"; - return false; - } - hadLoc = true; - if (!getDimensionality(&p)) - return false; - _x = getInt(&p); - if (*p != ',') { - _parseError = "Missing ',' after x position"; - return false; - } - p++; - _y = getInt(&p); - if (*p != ',') { - _parseError = "Missing ',' after y position"; - return false; - } - p++; - _radius = getInt(&p); - if (*p != ',') { - _parseError = "Missing ',' after radius"; - return false; - } - p++; - /* _tableID = */ (void) getInt(&p); - if (*p != ',') { - _parseError = "Missing ',' after tableID"; - return false; - } - p++; - /* _rankMultiplier = */ (void) getInt(&p); - if (*p != ',') { - _parseError = "Missing ',' after rank multiplier"; - return false; - } - p++; - /* _rankOnlyOnDistance = */ (void) (getInt(&p) != 0); - if (*p == ',') { - p++; - _xAspect = getInt(&p); - if (*p != ')') { - _parseError = "Missing ')' after xAspect"; - return false; - } - } else { - if (*p != ')') { - _parseError = "Missing ')' after rankOnlyOnDistance flag"; - return false; - } - } - p++; - } else if (*p == ' ') - p++; - else { - _parseError = "Unexpected char in location spec"; - return false; - } - } - - if (hadLoc) { - _rankOnDistance = true; - uint32_t maxdx = _radius; - if (_xAspect != 0) { - uint64_t maxdx2 = ((static_cast<uint64_t>(_radius) << 32) + 0xffffffffu) / - _xAspect; - if (maxdx2 >= 0xffffffffu) - maxdx = 0xffffffffu; - else - maxdx = static_cast<uint32_t>(maxdx2); - } - if (static_cast<int32_t>(_x - maxdx) > _minx && - static_cast<int64_t>(_x) - static_cast<int64_t>(maxdx) > - static_cast<int64_t>(_minx)) - _minx = _x - maxdx; - if (static_cast<int32_t>(_x + maxdx) < _maxx && - static_cast<int64_t>(_x) + static_cast<int64_t>(maxdx) < - static_cast<int64_t>(_maxx)) - _maxx = _x + maxdx; - if (static_cast<int32_t>(_y - _radius) > _miny && - static_cast<int64_t>(_y) - static_cast<int64_t>(_radius) > - static_cast<int64_t>(_miny)) - _miny = _y - _radius; - if (static_cast<int32_t>(_y + _radius) < _maxy && - static_cast<int64_t>(_y) + static_cast<int64_t>(_radius) < - static_cast<int64_t>(_maxy)) - _maxy = _y + _radius; - } - if (_minx != std::numeric_limits<int32_t>::min() || - _maxx != std::numeric_limits<int32_t>::max() || - _miny != std::numeric_limits<int32_t>::min() || - _maxy != std::numeric_limits<int32_t>::max()) - { - _pruneOnDistance = true; - } - _zBoundingBox = vespalib::geo::ZCurve::BoundingBox(_minx, _maxx, _miny, _maxy); - - return true; -} - -} +} // namespace diff --git a/searchlib/src/vespa/searchlib/common/location.h b/searchlib/src/vespa/searchlib/common/location.h index a00bb83648a..197f92326cd 100644 --- a/searchlib/src/vespa/searchlib/common/location.h +++ b/searchlib/src/vespa/searchlib/common/location.h @@ -3,51 +3,19 @@ #pragma once #include "documentlocations.h" -#include <vespa/vespalib/geo/zcurve.h> - -#include <vespa/vespalib/stllike/string.h> +#include "geo_location.h" namespace search::common { -class Location : public DocumentLocations +class Location : public DocumentLocations, + public GeoLocation { -private: - static int getInt(const char **pp); - bool getDimensionality(const char **pp); - public: - Location(); - bool getRankOnDistance() const { return _rankOnDistance; } - bool getPruneOnDistance() const { return _pruneOnDistance; } - uint32_t getXAspect() const { return _xAspect; } - int32_t getX() const { return _x; } - int32_t getY() const { return _y; } - uint32_t getRadius() const { return _radius; } - const char * getParseError() const { return _parseError; } - int32_t getMinX() const { return _minx; } - int32_t getMinY() const { return _miny; } - int32_t getMaxX() const { return _maxx; } - int32_t getMaxY() const { return _maxy; } - bool getzFailBoundingBoxTest(int64_t docxy) const { - return _zBoundingBox.getzFailBoundingBoxTest(docxy); - } - - bool parse(const vespalib::string &locStr); - -private: - vespalib::geo::ZCurve::BoundingBox _zBoundingBox; - int32_t _x; /* Query X position */ - int32_t _y; /* Query Y position */ - uint32_t _xAspect; /* X distance multiplier fraction */ - uint32_t _radius; /* Radius for euclidean distance */ - int32_t _minx; /* Min X coordinate */ - int32_t _maxx; /* Max X coordinate */ - int32_t _miny; /* Min Y coordinate */ - int32_t _maxy; /* Max Y coordinate */ - - bool _rankOnDistance; - bool _pruneOnDistance; - const char *_parseError; + Location(const GeoLocation& from); + ~Location() {} + Location(Location &&) = default; + bool getRankOnDistance() const { return has_point; } + bool getPruneOnDistance() const { return can_limit(); } }; } diff --git a/searchlib/src/vespa/searchlib/common/locationiterators.cpp b/searchlib/src/vespa/searchlib/common/locationiterators.cpp index 16e465bcd05..d90ed3b41f3 100644 --- a/searchlib/src/vespa/searchlib/common/locationiterators.cpp +++ b/searchlib/src/vespa/searchlib/common/locationiterators.cpp @@ -4,6 +4,9 @@ #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.common.locationiterators"); + using namespace search::common; class FastS_2DZLocationIterator : public search::queryeval::SearchIterator @@ -11,7 +14,6 @@ class FastS_2DZLocationIterator : public search::queryeval::SearchIterator private: const unsigned int _numDocs; const bool _strict; - const uint64_t _radius2; const Location & _location; std::vector<search::AttributeVector::largeint_t> _pos; @@ -31,7 +33,6 @@ FastS_2DZLocationIterator(unsigned int numDocs, : SearchIterator(), _numDocs(numDocs), _strict(strict), - _radius2(static_cast<uint64_t>(location.getRadius()) * location.getRadius()), _location(location), _pos() { @@ -45,6 +46,8 @@ FastS_2DZLocationIterator::~FastS_2DZLocationIterator() = default; void FastS_2DZLocationIterator::doSeek(uint32_t docId) { + LOG(debug, "FastS_2DZLocationIterator: seek(%u) with numDocs=%u endId=%u", + docId, _numDocs, getEndId()); if (__builtin_expect(docId >= _numDocs, false)) { setAtEnd(); return; @@ -62,24 +65,9 @@ FastS_2DZLocationIterator::doSeek(uint32_t docId) } for (uint32_t i = 0; i < numValues; i++) { int64_t docxy(pos[i]); - if ( ! location.getzFailBoundingBoxTest(docxy)) { - int32_t docx = 0; - int32_t docy = 0; - vespalib::geo::ZCurve::decode(docxy, &docx, &docy); - uint32_t dx = (location.getX() > docx) - ? location.getX() - docx - : docx - location.getX(); - if (location.getXAspect() != 0) - dx = ((uint64_t) dx * location.getXAspect()) >> 32; - - uint32_t dy = (location.getY() > docy) - ? location.getY() - docy - : docy - location.getY(); - uint64_t dist2 = (uint64_t) dx * dx + (uint64_t) dy * dy; - if (dist2 <= _radius2) { - setDocId(docId); - return; - } + if (location.inside_limit(docxy)) { + setDocId(docId); + return; } } diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index 7a624e64d67..6582bcae92a 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -2,7 +2,7 @@ #include "distancefeature.h" #include <vespa/searchcommon/common/schema.h> -#include <vespa/searchlib/fef/location.h> +#include <vespa/searchlib/common/geo_location_spec.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/vespalib/geo/zcurve.h> @@ -82,7 +82,8 @@ ConvertRawscoreToDistance::execute(uint32_t docId) feature_t DistanceExecutor::calculateDistance(uint32_t docId) { - if (_location.isValid() && _pos != nullptr) { + if ((! _locations.empty()) && (_pos != nullptr)) { + LOG(debug, "calculate 2D Z-distance from %zu locations", _locations.size()); return calculate2DZDistance(docId); } return DEFAULT_DISTANCE; @@ -97,35 +98,24 @@ DistanceExecutor::calculate2DZDistance(uint32_t docId) uint64_t sqabsdist = std::numeric_limits<uint64_t>::max(); int32_t docx = 0; int32_t docy = 0; - for (uint32_t i = 0; i < numValues; ++i) { - vespalib::geo::ZCurve::decode(_intBuf[i], &docx, &docy); - uint32_t dx; - uint32_t dy; - if (_location.getXPosition() > docx) { - dx = _location.getXPosition() - docx; - } else { - dx = docx - _location.getXPosition(); - } - if (_location.getXAspect() != 0) { - dx = ((uint64_t) dx * _location.getXAspect()) >> 32; - } - if (_location.getYPosition() > docy) { - dy = _location.getYPosition() - docy; - } else { - dy = docy - _location.getYPosition(); - } - uint64_t sqdist = (uint64_t) dx * dx + (uint64_t) dy * dy; - if (sqdist < sqabsdist) { - sqabsdist = sqdist; + for (auto loc : _locations) { + assert(loc); + assert(loc->location.valid()); + for (uint32_t i = 0; i < numValues; ++i) { + vespalib::geo::ZCurve::decode(_intBuf[i], &docx, &docy); + uint64_t sqdist = loc->location.sq_distance_to({docx, docy}); + if (sqdist < sqabsdist) { + sqabsdist = sqdist; + } } } return static_cast<feature_t>(std::sqrt(static_cast<feature_t>(sqabsdist))); } -DistanceExecutor::DistanceExecutor(const Location & location, +DistanceExecutor::DistanceExecutor(GeoLocationSpecPtrs locations, const search::attribute::IAttributeVector * pos) : FeatureExecutor(), - _location(location), + _locations(locations), _pos(pos), _intBuf() { @@ -231,6 +221,7 @@ DistanceBlueprint::setup(const IIndexEnvironment & env, return setup_geopos(env, z); } if (allow_bad_field) { + // TODO remove on Vespa 8 // backwards compatibility fallback: return setup_geopos(env, arg); } @@ -251,11 +242,30 @@ DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash if (_use_item_label) { return stash.create<ConvertRawscoreToDistance>(env, _arg_string); } + // expect geo pos: const search::attribute::IAttributeVector * pos = nullptr; - const Location & location = env.getLocation(); - LOG(debug, "DistanceBlueprint::createExecutor location.valid='%s', attribute='%s'", - location.isValid() ? "true" : "false", _arg_string.c_str()); - if (_use_geo_pos && location.isValid()) { + GeoLocationSpecPtrs matching_locs; + GeoLocationSpecPtrs other_locs; + + for (auto loc_ptr : env.getAllLocations()) { + if (_use_geo_pos && loc_ptr && loc_ptr->location.valid()) { + if (loc_ptr->field_name == _arg_string) { + LOG(debug, "found loc from query env matching '%s'", _arg_string.c_str()); + matching_locs.push_back(loc_ptr); + } else { + LOG(debug, "found loc(%s) from query env not matching arg(%s)", + loc_ptr->field_name.c_str(), _arg_string.c_str()); + other_locs.push_back(loc_ptr); + } + } + } + if (matching_locs.empty() && other_locs.empty()) { + LOG(debug, "createExecutor: no valid locations"); + return stash.create<DistanceExecutor>(matching_locs, nullptr); + } + LOG(debug, "createExecutor: valid location, attribute='%s'", _arg_string.c_str()); + + if (_use_geo_pos) { pos = env.getAttributeContext().getAttribute(_arg_string); if (pos != nullptr) { if (!pos->isIntegerType()) { @@ -271,8 +281,8 @@ DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _arg_string.c_str()); } } - - return stash.create<DistanceExecutor>(location, pos); + LOG(debug, "use '%s' locations with pos=%p", matching_locs.empty() ? "other" : "matching", pos); + return stash.create<DistanceExecutor>(matching_locs.empty() ? other_locs : matching_locs, pos); } } diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.h b/searchlib/src/vespa/searchlib/features/distancefeature.h index 3a8edd5ee94..ece139c6546 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.h +++ b/searchlib/src/vespa/searchlib/features/distancefeature.h @@ -7,12 +7,15 @@ namespace search::features { +/** Convenience typedef. */ +using GeoLocationSpecPtrs = std::vector<const search::common::GeoLocationSpec *>; + /** * Implements the executor for the distance feature. */ class DistanceExecutor : public fef::FeatureExecutor { private: - const fef::Location & _location; + GeoLocationSpecPtrs _locations; const attribute::IAttributeVector * _pos; attribute::IntegerContent _intBuf; @@ -23,10 +26,11 @@ public: /** * Constructs an executor for the distance feature. * - * @param location the location object associated with the query environment. + * @param locations location objects associated with the query environment. * @param pos the attribute to use for positions (expects zcurve encoding). */ - DistanceExecutor(const fef::Location & location, const attribute::IAttributeVector * pos); + DistanceExecutor(GeoLocationSpecPtrs locations, + const attribute::IAttributeVector * pos); void execute(uint32_t docId) override; static const feature_t DEFAULT_DISTANCE; diff --git a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt index 396775b20c5..178de1b8b87 100644 --- a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt @@ -16,7 +16,6 @@ vespa_add_library(searchlib_fef OBJECT filetablefactory.cpp functiontablefactory.cpp indexproperties.cpp - location.cpp matchdata.cpp matchdatalayout.cpp objectstore.cpp diff --git a/searchlib/src/vespa/searchlib/fef/fef.h b/searchlib/src/vespa/searchlib/fef/fef.h index 7dd24bd17ae..b677ba128c9 100644 --- a/searchlib/src/vespa/searchlib/fef/fef.h +++ b/searchlib/src/vespa/searchlib/fef/fef.h @@ -35,7 +35,6 @@ #include "itablemanager.h" #include "itermdata.h" #include "itermfielddata.h" -#include "location.h" #include "matchdata.h" #include "matchdatalayout.h" #include "parameter.h" diff --git a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h index 041e9ec67bc..7c6a84916f4 100644 --- a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h @@ -6,9 +6,10 @@ #include "objectstore.h" #include <vespa/searchcommon/attribute/iattributecontext.h> +namespace search::common { class GeoLocationSpec; } + namespace search::fef { -class Location; class Properties; class ITermData; @@ -24,6 +25,9 @@ public: **/ typedef std::shared_ptr<IQueryEnvironment> SP; + /** Convenience typedef. */ + using GeoLocationSpecPtrs = std::vector<const search::common::GeoLocationSpec *>; + /** * Obtain the set of properties associated with this query * environment. This set of properties is known through the system @@ -58,9 +62,9 @@ public: /** * Obtain the location information associated with this query environment. * - * @return location object. + * @return pointers to location objects. **/ - virtual const Location & getLocation() const = 0; + virtual GeoLocationSpecPtrs getAllLocations() const = 0; /** * Returns the attribute context for this query. diff --git a/searchlib/src/vespa/searchlib/fef/location.cpp b/searchlib/src/vespa/searchlib/fef/location.cpp deleted file mode 100644 index 978daa0f930..00000000000 --- a/searchlib/src/vespa/searchlib/fef/location.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "location.h" - -namespace search { -namespace fef { - -Location::Location() : - _attr(), - _xPos(0), - _yPos(0), - _xAspect(0), - _valid(false) -{ -} - -} // namespace fef -} // namespace search diff --git a/searchlib/src/vespa/searchlib/fef/location.h b/searchlib/src/vespa/searchlib/fef/location.h deleted file mode 100644 index 5be7d1ce822..00000000000 --- a/searchlib/src/vespa/searchlib/fef/location.h +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> - -namespace search { -namespace fef { - -/** - * This class contains location data that is associated with a query. - **/ -class Location -{ -private: - vespalib::string _attr; - int32_t _xPos; - int32_t _yPos; - uint32_t _xAspect; - bool _valid; - -public: - /** - * Creates an empty object. - **/ - Location(); - - /** - * Sets the name of the attribute to use for x positions. - * - * @param xAttr the attribute name. - * @return this to allow chaining. - **/ - Location & - setAttribute(const vespalib::string & attr) - { - _attr = attr; - return *this; - } - - /** - * Returns the name of the attribute to use for x positions. - * - * @return the attribute name. - **/ - const vespalib::string & getAttribute() const { return _attr; } - - /** - * Sets the x position of this location. - * - * @param xPos the x position. - * @return this to allow chaining. - **/ - Location & setXPosition(int32_t xPos) { _xPos = xPos; return *this; } - - /** - * Returns the x position of this location. - * - * @return the x position. - **/ - int32_t getXPosition() const { return _xPos; } - - /** - * Sets the y position of this location. - * - * @param yPos the y position. - * @return this to allow chaining. - **/ - Location & setYPosition(int32_t yPos) { _yPos = yPos; return *this; } - - /** - * Returns the y position of this location. - * - * @return the y position. - **/ - int32_t getYPosition() const { return _yPos; } - - /** - * Sets the x distance multiplier fraction. - * - * @param xAspect the x aspect. - * @return this to allow chaining. - **/ - Location & setXAspect(uint32_t xAspect) { _xAspect = xAspect; return *this; } - - /** - * Returns the x distance multiplier fraction. - * - * @return the x aspect. - **/ - uint32_t getXAspect() const { return _xAspect; } - - /** - * Sets whether this is a valid location object. - * - * @param valid true if this is valid. - * @return this to allow chaining. - **/ - Location & setValid(bool valid) { _valid = valid; return *this; } - - /** - * Returns whether this is a valid location object. - * - * @param true if this is a valid. - **/ - bool isValid() const { return _valid; } -}; - -} // namespace fef -} // namespace search - diff --git a/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.h b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.h index b2a3d416f5a..90bf4e8955f 100644 --- a/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.h +++ b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.h @@ -73,7 +73,9 @@ public: } const Properties & getProperties() const override { return _queryEnv.getProperties(); } - const Location & getLocation() const override { return _queryEnv.getLocation(); } + GeoLocationSpecPtrs getAllLocations() const override { + return _queryEnv.getAllLocations(); + } const attribute::IAttributeContext & getAttributeContext() const override { return _queryEnv.getAttributeContext(); } double get_average_field_length(const vespalib::string &field_name) const override { return _queryEnv.get_average_field_length(field_name); } const IIndexEnvironment & getIndexEnvironment() const override { return _queryEnv.getIndexEnvironment(); } diff --git a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp index 4697675c071..d602e74ddfb 100644 --- a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp @@ -8,7 +8,7 @@ QueryEnvironment::QueryEnvironment(IndexEnvironment *env) : _indexEnv(env), _terms(), _properties(), - _location(), + _locations(), _attrCtx((env == nullptr) ? attribute::IAttributeContext::UP() : env->getAttributeMap().createContext()) { } diff --git a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h index 40898281794..d8d5c802360 100644 --- a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h @@ -4,12 +4,14 @@ #include "indexenvironment.h" #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/fef/iqueryenvironment.h> -#include <vespa/searchlib/fef/location.h> #include <vespa/searchlib/fef/simpletermdata.h> +#include <vespa/searchlib/common/geo_location_spec.h> #include <unordered_map> namespace search::fef::test { +using search::common::GeoLocationSpec; + /** * Implementation of the IQueryEnvironment interface used for testing. */ @@ -22,7 +24,7 @@ private: IndexEnvironment *_indexEnv; std::vector<SimpleTermData> _terms; Properties _properties; - Location _location; + std::vector<GeoLocationSpec> _locations; search::attribute::IAttributeContext::UP _attrCtx; std::unordered_map<std::string, double> _avg_field_lengths; @@ -38,7 +40,13 @@ public: const Properties &getProperties() const override { return _properties; } uint32_t getNumTerms() const override { return _terms.size(); } const ITermData *getTerm(uint32_t idx) const override { return idx < _terms.size() ? &_terms[idx] : NULL; } - const Location & getLocation() const override { return _location; } + GeoLocationSpecPtrs getAllLocations() const override { + GeoLocationSpecPtrs locations; + for (const auto & loc : _locations) { + locations.push_back(&loc); + } + return locations; + } const search::attribute::IAttributeContext &getAttributeContext() const override { return *_attrCtx; } double get_average_field_length(const vespalib::string& field_name) const override { auto itr = _avg_field_lengths.find(field_name); @@ -82,7 +90,7 @@ public: Properties & getProperties() { return _properties; } /** Returns a reference to the location of this. */ - Location & getLocation() { return _location; } + void addLocation(const GeoLocationSpec &spec) { _locations.push_back(spec); } std::unordered_map<std::string, double>& get_avg_field_lengths() { return _avg_field_lengths; } }; diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.h b/searchlib/src/vespa/searchlib/parsequery/parse.h index b4dd9826b84..68e259b92e8 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.h +++ b/searchlib/src/vespa/searchlib/parsequery/parse.h @@ -53,7 +53,7 @@ public: ITEM_REGEXP = 24, ITEM_WORD_ALTERNATIVES = 25, ITEM_NEAREST_NEIGHBOR = 26, - ITEM_LOCATION_TERM = 27, + ITEM_GEO_LOCATION_TERM = 27, ITEM_MAX = 28, // Indicates how long tables must be. ITEM_UNDEF = 31, }; diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp index 17cbd6dce1b..1820fb0e969 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp @@ -207,7 +207,7 @@ SimpleQueryStackDumpIterator::next() } break; case ParseItem::ITEM_NUMTERM: - case ParseItem::ITEM_LOCATION_TERM: + case ParseItem::ITEM_GEO_LOCATION_TERM: case ParseItem::ITEM_TERM: case ParseItem::ITEM_PREFIXTERM: case ParseItem::ITEM_SUBSTRINGTERM: diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp index f1599e820ef..66466b030d0 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp @@ -63,8 +63,13 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor } } break; + case ParseItem::ITEM_GEO_LOCATION_TERM: + // TODO implement this: + // vespalib::string field = queryRep.getIndexName(); + // vespalib::stringref location_term = queryRep.getTerm(); + // qn = std::make_unique<LocationQueryNode> ...something .... + // break; case ParseItem::ITEM_NUMTERM: - case ParseItem::ITEM_LOCATION_TERM: case ParseItem::ITEM_TERM: case ParseItem::ITEM_PREFIXTERM: case ParseItem::ITEM_REGEXP: diff --git a/searchlib/src/vespa/searchlib/query/tree/location.cpp b/searchlib/src/vespa/searchlib/query/tree/location.cpp index 216c5ec5ad0..6e678f9e682 100644 --- a/searchlib/src/vespa/searchlib/query/tree/location.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/location.cpp @@ -6,58 +6,71 @@ #include <vespa/vespalib/stllike/asciistream.h> using vespalib::asciistream; +using search::common::GeoLocation; namespace search::query { -Location::Location(const Point &point, uint32_t max_dist, uint32_t x_aspect) { - asciistream loc; - loc << "(2" // dimensionality - << "," << point.x - << "," << point.y - << "," << max_dist - << "," << "0" // table id. - << "," << "1" // rank multiplier. - << "," << "0" // rank only on distance. - << "," << x_aspect // x aspect. - << ")"; - _location_string = loc.str(); +static GeoLocation::Box convert(const Rectangle &rect) { + GeoLocation::Range x_range{rect.left, rect.right}; + GeoLocation::Range y_range{rect.top, rect.bottom}; + return GeoLocation::Box{x_range, y_range}; } +Location::Location(const Point &p, uint32_t max_dist, uint32_t aspect) + : Parent(p, max_dist, GeoLocation::Aspect(aspect)) +{} + Location::Location(const Rectangle &rect, - const Point &point, uint32_t max_dist, uint32_t x_aspect) -{ - asciistream loc; - loc << "(2" // dimensionality - << "," << point.x - << "," << point.y - << "," << max_dist - << "," << "0" // table id. - << "," << "1" // rank multiplier. - << "," << "0" // rank only on distance. - << "," << x_aspect // x aspect. - << ")"; - loc << "[2," << rect.left - << "," << rect.top - << "," << rect.right - << "," << rect.bottom - << "]" ; - _location_string = loc.str(); + const Point &p, uint32_t max_dist, uint32_t aspect) + : Parent(convert(rect), p, max_dist, GeoLocation::Aspect(aspect)) +{} -} +Location::Location(const Rectangle &rect) + : Parent(convert(rect)) +{} -Location::Location(const Rectangle &rect) { - asciistream loc; - loc << "[2," << rect.left - << "," << rect.top - << "," << rect.right - << "," << rect.bottom - << "]" ; - _location_string = loc.str(); +bool +Location::operator==(const Location &other) const +{ + auto me = getOldFormatString(); + auto it = other.getOldFormatString(); + if (me == it) { + return true; + } else { + // dump 'me' and 'it' here if unit tests fail + return false; + } +} + +std::string +Location::getOldFormatString() const +{ + // we need to product what search::common::GeoLocationParser can parse + vespalib::asciistream buf; + if (has_point) { + buf << "(2" // dimensionality + << "," << point.x + << "," << point.y + << "," << radius + << "," << "0" // table id. + << "," << "1" // rank multiplier. + << "," << "0" // rank only on distance. + << "," << x_aspect.multiplier // aspect multiplier + << ")"; + } + if (bounding_box.active()) { + buf << "[2," << bounding_box.x.low + << "," << bounding_box.y.low + << "," << bounding_box.x.high + << "," << bounding_box.y.high + << "]" ; + } + return buf.str(); } vespalib::asciistream &operator<<(vespalib::asciistream &out, const Location &loc) { - return out << loc.getLocationString(); + return out << loc.getOldFormatString(); } } diff --git a/searchlib/src/vespa/searchlib/query/tree/location.h b/searchlib/src/vespa/searchlib/query/tree/location.h index e1826c7184a..6b8090f45e1 100644 --- a/searchlib/src/vespa/searchlib/query/tree/location.h +++ b/searchlib/src/vespa/searchlib/query/tree/location.h @@ -2,29 +2,26 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> +#include <string> +#include <vespa/searchlib/common/geo_location_spec.h> +#include "point.h" +#include "rectangle.h" namespace vespalib { class asciistream; } namespace search::query { -struct Point; -struct Rectangle; - -class Location { - vespalib::string _location_string; +class Location : public search::common::GeoLocation { + using Parent = search::common::GeoLocation; public: - Location() : _location_string() {} + Location() {} + Location(const Parent &spec) : Parent(spec) {} + ~Location() {} Location(const Point &p, uint32_t dist, uint32_t x_asp); Location(const Rectangle &rect); Location(const Rectangle &rect, const Point &p, uint32_t dist, uint32_t x_asp); - Location(const vespalib::string &s) : _location_string(s) {} - bool operator==(const Location &other) const { - return _location_string == other._location_string; - } - const vespalib::string &getLocationString() const { - return _location_string; - } + bool operator==(const Location &other) const; + std::string getOldFormatString() const; }; vespalib::asciistream &operator<<(vespalib::asciistream &out, const Location &loc); diff --git a/searchlib/src/vespa/searchlib/query/tree/point.h b/searchlib/src/vespa/searchlib/query/tree/point.h index 89d0bc1db44..48700681158 100644 --- a/searchlib/src/vespa/searchlib/query/tree/point.h +++ b/searchlib/src/vespa/searchlib/query/tree/point.h @@ -3,18 +3,10 @@ #pragma once #include <cstdint> +#include <vespa/searchlib/common/geo_location.h> namespace search::query { -struct Point { - int64_t x; - int64_t y; - Point() : x(0), y(0) {} - Point(int64_t x_in, int64_t y_in) : x(x_in), y(y_in) {} -}; - -inline bool operator==(const Point &p1, const Point &p2) { - return p1.x == p2.x && p1.y == p2.y; -} +using Point = search::common::GeoLocation::Point; } diff --git a/searchlib/src/vespa/searchlib/query/tree/rectangle.h b/searchlib/src/vespa/searchlib/query/tree/rectangle.h index 97be9ddeb32..358e994aacd 100644 --- a/searchlib/src/vespa/searchlib/query/tree/rectangle.h +++ b/searchlib/src/vespa/searchlib/query/tree/rectangle.h @@ -5,20 +5,14 @@ namespace search::query { struct Rectangle { - int64_t left; - int64_t top; - int64_t right; - int64_t bottom; + int32_t left; + int32_t top; + int32_t right; + int32_t bottom; Rectangle() : left(0), top(0), right(0), bottom(0) {} - Rectangle(int64_t l, int64_t t, int64_t r, int64_t b) + Rectangle(int32_t l, int32_t t, int32_t r, int32_t b) : left(l), top(t), right(r), bottom(b) {} }; -inline bool operator==(const Rectangle &r1, const Rectangle &r2) { - return r1.left == r2.left && r1.right == r2.right - && r1.top == r2.top && r1.bottom == r2.bottom; } - -} - diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp index f33520d8b0e..82302e4ab48 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp @@ -228,7 +228,7 @@ class QueryNodeConverter : public QueryVisitor { } void visit(LocationTerm &node) override { - createTerm(node, ParseItem::ITEM_LOCATION_TERM); + createTerm(node, ParseItem::ITEM_GEO_LOCATION_TERM); } void visit(PrefixTerm &node) override { diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h index 898db9785f6..a7e00d41555 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h @@ -6,6 +6,7 @@ #include "querybuilder.h" #include "term.h" #include <vespa/searchlib/parsequery/stackdumpiterator.h> +#include <vespa/searchlib/common/geo_location_parser.h> #include <vespa/vespalib/objects/hexdump.h> namespace search::query { @@ -141,17 +142,15 @@ private: t = &builder.addStringTerm(term, view, id, weight); } else if (type == ParseItem::ITEM_SUFFIXTERM) { t = &builder.addSuffixTerm(term, view, id, weight); - } else if (type == ParseItem::ITEM_LOCATION_TERM) { - Location loc(term); + } else if (type == ParseItem::ITEM_GEO_LOCATION_TERM) { + search::common::GeoLocationParser parser; + parser.parseOldFormat(term); + Location loc(parser.getGeoLocation()); t = &builder.addLocationTerm(loc, view, id, weight); } else if (type == ParseItem::ITEM_NUMTERM) { if (term[0] == '[' || term[0] == '<' || term[0] == '>') { Range range(term); t = &builder.addRangeTerm(range, view, id, weight); - } else if (term[0] == '(') { - // TODO: handled above, should remove this block - Location loc(term); - t = &builder.addLocationTerm(loc, view, id, weight); } else { t = &builder.addNumberTerm(term, view, id, weight); } diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 0ac2f09e1b0..55c363a12c1 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -167,7 +167,6 @@ public: ~StateCallback() {} void FillSummaryFeatures(GetDocsumsState*, IDocsumEnvironment*) override {} void FillRankFeatures(GetDocsumsState*, IDocsumEnvironment*) override {} - void ParseLocation(GetDocsumsState*) override {} std::unique_ptr<MatchingElements> fill_matching_elements(const MatchingElementsFields&) override { auto result = std::make_unique<MatchingElements>(); result->add_matching_elements(doc_id, _field_name, _matching_elements); diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index 6fd0c39f06f..683bab49353 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -108,7 +108,6 @@ public: struct MyGetDocsumsStateCallback : GetDocsumsStateCallback { virtual void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override {} virtual void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override {} - virtual void ParseLocation(GetDocsumsState *) override {} std::unique_ptr<MatchingElements> fill_matching_elements(const MatchingElementsFields &) override { abort(); } }; diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index 6fceef37f09..69249056c17 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -77,7 +77,6 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { uint32_t getSummaryClassId() const override { return 0; } void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } - void ParseLocation(GetDocsumsState *) override { } std::unique_ptr<MatchingElements> fill_matching_elements(const search::MatchingElementsFields &) override { abort(); } }; diff --git a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp b/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp index 87317234a27..cd695984e03 100644 --- a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp +++ b/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp @@ -179,7 +179,7 @@ ExtractKeywordsTest::RunTest(int testno, bool verify) case 1: { // check that skipping these works also: - stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_LOCATION_TERM, "no")); + stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_GEO_LOCATION_TERM, "no")); stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_NEAREST_NEIGHBOR, "no")); // multi term query stack.Push(new search::SimpleQueryStackItem(search::ParseItem::ITEM_TERM, "foobar")); diff --git a/searchsummary/src/tests/extractkeywords/simplequerystackitem.cpp b/searchsummary/src/tests/extractkeywords/simplequerystackitem.cpp index 5a4b6d76b8f..f12822949f9 100644 --- a/searchsummary/src/tests/extractkeywords/simplequerystackitem.cpp +++ b/searchsummary/src/tests/extractkeywords/simplequerystackitem.cpp @@ -24,7 +24,7 @@ void assert_term_type(ParseItem::ItemType type) { assert(type == ParseItem::ITEM_TERM || type == ParseItem::ITEM_NUMTERM || type == ParseItem::ITEM_NEAREST_NEIGHBOR || - type == ParseItem::ITEM_LOCATION_TERM || + type == ParseItem::ITEM_GEO_LOCATION_TERM || type == ParseItem::ITEM_PREFIXTERM || type == ParseItem::ITEM_SUBSTRINGTERM || type == ParseItem::ITEM_SUFFIXTERM || @@ -152,7 +152,7 @@ SimpleQueryStackItem::AppendBuffer(RawBuf *buf) const break; case ITEM_TERM: case ITEM_NUMTERM: - case ITEM_LOCATION_TERM: + case ITEM_GEO_LOCATION_TERM: case ITEM_PREFIXTERM: case ITEM_SUBSTRINGTERM: case ITEM_EXACTSTRINGTERM: diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index ebbf97e9f55..1f9b553b56a 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -2,11 +2,22 @@ #include "docsumstate.h" #include <vespa/juniper/rpinterface.h> +#include <vespa/document/datatype/positiondatatype.h> #include <vespa/searchcommon/attribute/iattributecontext.h> -#include <vespa/searchlib/common/location.h> +#include <vespa/searchlib/common/geo_location.h> +#include <vespa/searchlib/common/geo_location_parser.h> +#include <vespa/searchlib/common/geo_location_spec.h> #include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchlib/parsequery/parse.h> +#include <vespa/searchlib/parsequery/stackdumpiterator.h> #include "docsum_field_writer_state.h" +#include <vespa/log/log.h> +LOG_SETUP(".searchsummary.docsummary.docsumstate"); + +using search::common::GeoLocationParser; +using search::common::GeoLocationSpec; + namespace search::docsummary { GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) @@ -22,7 +33,7 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) _attributes(), _fieldWriterStates(), _jsonStringer(), - _parsedLocation(), + _parsedLocations(), _summaryFeatures(NULL), _summaryFeaturesCached(false), _rankFeatures(NULL), @@ -58,4 +69,43 @@ GetDocsumsState::get_matching_elements(const MatchingElementsFields &matching_el return *_matching_elements; } +void +GetDocsumsState::parse_locations() +{ + using document::PositionDataType; + assert(_parsedLocations.empty()); // only allowed to call this once + if (! _args.getLocation().empty()) { + GeoLocationParser parser; + if (parser.parseOldFormatWithField(_args.getLocation())) { + auto view = parser.getFieldName(); + auto attr_name = PositionDataType::getZCurveFieldName(view); + GeoLocationSpec spec{attr_name, parser.getGeoLocation()}; + _parsedLocations.push_back(spec); + } else { + LOG(warning, "could not parse location string '%s' from request", + _args.getLocation().c_str()); + } + } + auto stackdump = _args.getStackDump(); + if (! stackdump.empty()) { + search::SimpleQueryStackDumpIterator iterator(stackdump); + while (iterator.next()) { + if (iterator.getType() == search::ParseItem::ITEM_GEO_LOCATION_TERM) { + vespalib::string view = iterator.getIndexName(); + vespalib::string term = iterator.getTerm(); + GeoLocationParser parser; + if (parser.parseOldFormat(term)) { + auto attr_name = PositionDataType::getZCurveFieldName(view); + GeoLocationSpec spec{attr_name, parser.getGeoLocation()}; + _parsedLocations.push_back(spec); + } else { + LOG(warning, "could not parse location string '%s' from stack dump", + term.c_str()); + } + } + } + } +} + + } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index 57cae341682..46f8e52dd37 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -5,6 +5,7 @@ #include <vespa/searchlib/util/rawbuf.h> #include <vespa/searchsummary/docsummary/getdocsumargs.h> #include <vespa/searchlib/common/featureset.h> +#include <vespa/searchlib/common/geo_location_spec.h> #include <vespa/vespalib/util/jsonwriter.h> namespace juniper { @@ -34,7 +35,6 @@ class GetDocsumsStateCallback public: virtual void FillSummaryFeatures(GetDocsumsState * state, IDocsumEnvironment * env) = 0; virtual void FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) = 0; - virtual void ParseLocation(GetDocsumsState * state) = 0; virtual std::unique_ptr<MatchingElements> fill_matching_elements(const MatchingElementsFields &matching_elems_fields) = 0; virtual ~GetDocsumsStateCallback(void) { } GetDocsumsStateCallback(const GetDocsumsStateCallback &) = delete; @@ -80,7 +80,8 @@ public: vespalib::JSONStringer _jsonStringer; // used by AbsDistanceDFW - std::unique_ptr<search::common::Location> _parsedLocation; + std::vector<search::common::GeoLocationSpec> _parsedLocations; + void parse_locations(); // used by SummaryFeaturesDFW FeatureSet::SP _summaryFeatures; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index 37239fe9da6..8cc577355cf 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -270,7 +270,7 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const case search::ParseItem::ITEM_PREDICATE_QUERY: case search::ParseItem::ITEM_SAME_ELEMENT: case search::ParseItem::ITEM_NEAREST_NEIGHBOR: - case search::ParseItem::ITEM_LOCATION_TERM: + case search::ParseItem::ITEM_GEO_LOCATION_TERM: if (!v->VisitOther(&item, iterator.getArity())) { rc = SkipItem(&iterator); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp index 4e1544ee5d7..0af92adf2d2 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp @@ -8,6 +8,7 @@ GetDocsumArgs::GetDocsumArgs() : _ranking(), _resultClassName(), _dumpFeatures(false), + _locations_possible(true), _stackItems(0), _stackDump(), _location(), @@ -27,6 +28,7 @@ GetDocsumArgs::initFromDocsumRequest(const search::engine::DocsumRequest &req) _stackItems = req.stackItems; _stackDump = req.stackDump; _location = req.location; + _locations_possible = true; _timeout = req.getTimeLeft(); _propertiesMap = req.propertiesMap; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h index c17f44baec9..0231b004674 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h @@ -17,6 +17,7 @@ private: vespalib::string _ranking; vespalib::string _resultClassName; bool _dumpFeatures; + bool _locations_possible; uint32_t _stackItems; std::vector<char> _stackDump; vespalib::string _location; @@ -31,15 +32,14 @@ public: void SetRankProfile(const vespalib::string &ranking) { _ranking = ranking; } void setResultClassName(vespalib::stringref name) { _resultClassName = name; } void SetStackDump(uint32_t stackItems, uint32_t stackDumpLen, const char *stackDump); - void setLocation(vespalib::stringref location) { - _location = location; - } - + void locations_possible(bool value) { _locations_possible = value; } + bool locations_possible() const { return _locations_possible; } + const vespalib::string &getLocation() const { return _location; } + void setLocation(const vespalib::string & location) { _location = location; } void setTimeout(vespalib::duration timeout) { _timeout = timeout; } vespalib::duration getTimeout() const { return _timeout; } const vespalib::string & getResultClassName() const { return _resultClassName; } - const vespalib::string & getLocation() const { return _location; } const vespalib::stringref getStackDump() const { return vespalib::stringref(&_stackDump[0], _stackDump.size()); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h index 087ddfd8d40..353f0df8bee 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h @@ -9,8 +9,9 @@ namespace search::attribute { class IAttributeContext; } namespace search::docsummary { /** - * Field writer that filters matched elements (according to the query) from a complex field - * (map of primitives, map of struct, array of struct) that is retrieved from the document store. + * Field writer that filters matched elements (according to the query) from a multi-value or complex field + * (array of primitive, weighted set of primitive, map of primitives, map of struct, array of struct) + * that is retrieved from the document store. */ class MatchedElementsFilterDFW : public IDocsumFieldWriter { private: diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index ecdde13b919..4fc2b1f4221 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -22,44 +22,57 @@ using search::attribute::BasicType; using search::attribute::IntegerContent; using search::common::Location; +LocationAttrDFW::AllLocations +LocationAttrDFW::getAllLocations(GetDocsumsState *state) +{ + AllLocations retval; + if (! state->_args.locations_possible()) { + return retval; + } + if (state->_parsedLocations.empty()) { + state->parse_locations(); + } + for (const auto & loc : state->_parsedLocations) { + if (loc.location.valid()) { + LOG(debug, "found location(field %s) for DFW(field %s)\n", + loc.field_name.c_str(), getAttributeName().c_str()); + if (getAttributeName() == loc.field_name) { + retval.matching.push_back(&loc.location); + } else { + retval.other.push_back(&loc.location); + } + } + } + if (retval.empty()) { + // avoid doing things twice + state->_args.locations_possible(false); + } + return retval; +} + AbsDistanceDFW::AbsDistanceDFW(const vespalib::string & attrName) : - AttrDFW(attrName) + LocationAttrDFW(attrName) { } uint64_t -AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState *state) +AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState *state, + const std::vector<const GeoLoc *> &locations) { - search::common::Location &location = *state->_parsedLocation; - const auto& attribute = get_attribute(*state); - uint64_t absdist = std::numeric_limits<int64_t>::max(); - int32_t docx = 0; - int32_t docy = 0; - IntegerContent pos; - pos.fill(attribute, docid); - uint32_t numValues = pos.size(); - for (uint32_t i = 0; i < numValues; i++) { - int64_t docxy(pos[i]); - vespalib::geo::ZCurve::decode(docxy, &docx, &docy); - uint32_t dx; - if (location.getX() > docx) { - dx = location.getX() - docx; - } else { - dx = docx - location.getX(); - } - if (location.getXAspect() != 0) { - dx = ((uint64_t) dx * location.getXAspect()) >> 32; - } - uint32_t dy; - if (location.getY() > docy) { - dy = location.getY() - docy; - } else { - dy = docy - location.getY(); - } - uint64_t dist2 = dx * (uint64_t) dx + - dy * (uint64_t) dy; - if (dist2 < absdist) { - absdist = dist2; + const auto& attribute = get_attribute(*state); + for (auto location : locations) { + int32_t docx = 0; + int32_t docy = 0; + IntegerContent pos; + pos.fill(attribute, docid); + uint32_t numValues = pos.size(); + for (uint32_t i = 0; i < numValues; i++) { + int64_t docxy(pos[i]); + vespalib::geo::ZCurve::decode(docxy, &docx, &docy); + uint64_t dist2 = location->sq_distance_to(GeoLoc::Point{docx, docy}); + if (dist2 < absdist) { + absdist = dist2; + } } } return (uint64_t) std::sqrt((double) absdist); @@ -68,22 +81,11 @@ AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState *state) void AbsDistanceDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) { - bool forceEmpty = true; - - const vespalib::string &locationStr = state->_args.getLocation(); - if (locationStr.size() > 0) { - if (!state->_parsedLocation) { - state->_callback.ParseLocation(state); - } - assert(state->_parsedLocation); - if (state->_parsedLocation->getParseError() == nullptr) { - forceEmpty = false; - } + const auto & all_locations = getAllLocations(state); + if (all_locations.empty()) { + return; } - if (forceEmpty) return; - - uint64_t absdist = findMinDistance(docid, state); - + uint64_t absdist = findMinDistance(docid, state, all_locations.best()); if (type == RES_INT) { target.insertLong(absdist); } else { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h index 999da6f1860..c135737e44c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h @@ -3,13 +3,40 @@ #pragma once #include "attributedfw.h" +#include <vespa/searchlib/common/geo_location_spec.h> namespace search::docsummary { -class AbsDistanceDFW : public AttrDFW +class LocationAttrDFW : public AttrDFW +{ +public: + using GeoLoc = search::common::GeoLocation; + + LocationAttrDFW(const vespalib::string & attrName) + : AttrDFW(attrName) + {} + + struct AllLocations { + std::vector<const GeoLoc *> matching; + std::vector<const GeoLoc *> other; + + ~AllLocations() {} + + bool empty() const { + return matching.empty() && other.empty(); + } + const std::vector<const GeoLoc *> & best() const { + return matching.empty() ? other : matching; + } + }; + AllLocations getAllLocations(GetDocsumsState *state); +}; + +class AbsDistanceDFW : public LocationAttrDFW { private: - uint64_t findMinDistance(uint32_t docid, GetDocsumsState *state); + uint64_t findMinDistance(uint32_t docid, GetDocsumsState *state, + const std::vector<const GeoLoc *> &locations); public: AbsDistanceDFW(const vespalib::string & attrName); diff --git a/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h b/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h index b3ee405c856..f8b51ca14d0 100644 --- a/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h +++ b/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h @@ -18,7 +18,6 @@ public: ~MockStateCallback() override { } void FillSummaryFeatures(GetDocsumsState*, IDocsumEnvironment*) override { } void FillRankFeatures(GetDocsumsState*, IDocsumEnvironment*) override { } - void ParseLocation(GetDocsumsState*) override { } std::unique_ptr<MatchingElements> fill_matching_elements(const search::MatchingElementsFields&) override { return std::make_unique<MatchingElements>(_matching_elems); } diff --git a/staging_vespalib/src/tests/state_server/state_server_test.cpp b/staging_vespalib/src/tests/state_server/state_server_test.cpp index e61d3d216cd..2a07de10968 100644 --- a/staging_vespalib/src/tests/state_server/state_server_test.cpp +++ b/staging_vespalib/src/tests/state_server/state_server_test.cpp @@ -12,7 +12,7 @@ #include <vespa/vespalib/net/simple_metrics_producer.h> #include <vespa/vespalib/net/simple_component_config_producer.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> #include <vespa/vespalib/net/state_explorer.h> #include <vespa/vespalib/net/slime_explorer.h> #include <vespa/vespalib/net/generic_state_handler.h> @@ -40,7 +40,7 @@ std::map<vespalib::string,vespalib::string> empty_params; vespalib::string run_cmd(const vespalib::string &cmd) { std::string out; - ASSERT_TRUE(SlaveProc::run(cmd.c_str(), out)); + ASSERT_TRUE(ChildProcess::run(cmd.c_str(), out)); return out; } diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index b9062b9284c..479a76e2fc6 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -81,6 +81,7 @@ <extensions>true</extensions> <configuration> <discApplicationClass>com.yahoo.container.standalone.StandaloneContainerApplication</discApplicationClass> + <buildLegacyVespaPlatformBundle>true</buildLegacyVespaPlatformBundle> <discPreInstallBundle> configdefinitions-jar-with-dependencies.jar, config-provisioning-jar-with-dependencies.jar, diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java index dd755e8e6dd..8ca20fdf7cd 100644 --- a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneSubscriberTest.java @@ -2,8 +2,9 @@ package com.yahoo.container.standalone; import com.yahoo.config.ConfigInstance; -import com.yahoo.container.BundlesConfig; import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.di.config.ApplicationBundlesConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.di.config.Subscriber; import com.yahoo.vespa.config.ConfigKey; import org.junit.Ignore; @@ -23,7 +24,8 @@ import static org.junit.Assert.assertThat; * @author ollivir */ public class StandaloneSubscriberTest { - private static ConfigKey<ConfigInstance> bundlesKey = key("bundles"); + private static ConfigKey<ConfigInstance> platformBundlesKey = key("platform-bundles"); + private static ConfigKey<ConfigInstance> applicationBundlesKey = key("application-bundles"); private static ConfigKey<ConfigInstance> componentsKey = key("components"); private static ConfigKey<ConfigInstance> key(String name) { @@ -35,16 +37,19 @@ public class StandaloneSubscriberTest { public void standalone_subscriber() throws Exception { withContainerModel("<container version=\"1.0\"></container>", root -> { Set<ConfigKey<ConfigInstance>> keys = new HashSet<>(); - keys.add(bundlesKey); + keys.add(platformBundlesKey); + keys.add(applicationBundlesKey); keys.add(componentsKey); Subscriber subscriber = new StandaloneSubscriberFactory(root).getSubscriber(keys); Map<ConfigKey<ConfigInstance>, ConfigInstance> config = subscriber.config(); assertThat(config.size(), is(2)); - BundlesConfig bundlesConfig = (BundlesConfig) config.get(bundlesKey); + PlatformBundlesConfig platformBundlesConfig = (PlatformBundlesConfig) config.get(platformBundlesKey); + ApplicationBundlesConfig applicationBundlesConfig = (ApplicationBundlesConfig) config.get(applicationBundlesKey); ComponentsConfig componentsConfig = (ComponentsConfig) config.get(componentsKey); - assertThat(bundlesConfig.bundle().size(), is(0)); + assertThat(platformBundlesConfig.bundlePaths().size(), is(0)); + assertThat(applicationBundlesConfig.bundles().size(), is(0)); assertThat(componentsConfig.components().size(), greaterThan(10)); return null; }); diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index d698cbb5e05..1e0144e9efb 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -175,16 +175,16 @@ void MetricsTest::createFakeLoad() thread.internalJoin.count.inc(3 * n); thread.mergeBuckets.count.inc(2 * n); - thread.bytesMerged.inc(1000 * n); thread.getBucketDiff.count.inc(4 * n); thread.getBucketDiffReply.inc(4 * n); thread.applyBucketDiff.count.inc(4 * n); thread.applyBucketDiffReply.inc(4 * n); - thread.mergeLatencyTotal.addValue(300 * n); - thread.mergeMetadataReadLatency.addValue(20 * n); - thread.mergeDataReadLatency.addValue(40 * n); - thread.mergeDataWriteLatency.addValue(50 * n); - thread.mergeAverageDataReceivedNeeded.addValue(0.8); + thread.merge_handler_metrics.bytesMerged.inc(1000 * n); + thread.merge_handler_metrics.mergeLatencyTotal.addValue(300 * n); + thread.merge_handler_metrics.mergeMetadataReadLatency.addValue(20 * n); + thread.merge_handler_metrics.mergeDataReadLatency.addValue(40 * n); + thread.merge_handler_metrics.mergeDataWriteLatency.addValue(50 * n); + thread.merge_handler_metrics.mergeAverageDataReceivedNeeded.addValue(0.8); } } for (uint32_t i=0; i<_visitorMetrics->threads.size(); ++i) { diff --git a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp index e9f878bfe1e..93c484368de 100644 --- a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp +++ b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp @@ -31,9 +31,9 @@ class BlockingMockProvider : public PersistenceProviderWrapper public: typedef std::unique_ptr<BlockingMockProvider> UP; - mutable uint32_t _bucketInfoInvocations; - uint32_t _createBucketInvocations; - uint32_t _deleteBucketInvocations; + mutable std::atomic<uint32_t> _bucketInfoInvocations; + std::atomic<uint32_t> _createBucketInvocations; + std::atomic<uint32_t> _deleteBucketInvocations; BlockingMockProvider(spi::PersistenceProvider& wrappedProvider, vespalib::Barrier& queueBarrier, diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index 262906a4baf..df0ea3e6680 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -901,7 +901,7 @@ TEST_F(MergeHandlerTest, apply_bucket_diff_spi_failures) { EXPECT_EQ("", doTestSPIException(handler, providerWrapper, invoker, *it)); // Casual, in-place testing of bug 6752085. // This will fail if we give NaN to the metric in question. - EXPECT_TRUE(std::isfinite(getEnv()._metrics.mergeAverageDataReceivedNeeded.getLast())); + EXPECT_TRUE(std::isfinite(getEnv()._metrics.merge_handler_metrics.mergeAverageDataReceivedNeeded.getLast())); } } diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 43cf43b02b6..73788d0affe 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -277,8 +277,8 @@ TwoPhaseUpdateOperation::schedulePutsWithUpdatedDocument(std::shared_ptr<documen op.start(intermediate, _manager.getClock().getTimeInMillis()); transitionTo(SendState::PUTS_SENT); - LOG(debug, "Update(%s): sending Put commands with doc %s", - update_doc_id().c_str(), doc->toString(true).c_str()); + LOG(debug, "Update(%s): sending Puts at timestamp %" PRIu64, update_doc_id().c_str(), putTimestamp); + LOG(spam, "Update(%s): Put document is: %s", update_doc_id().c_str(), doc->toString(true).c_str()); if (intermediate._reply.get()) { sendReplyWithResult(sender, intermediate._reply->getResult()); diff --git a/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.h b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.h index 695f80750aa..232f1186879 100644 --- a/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.h +++ b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.h @@ -50,6 +50,7 @@ private: uint16_t _distributorIndex; bool _bucketOwnershipTransfer; std::unordered_map<uint16_t, size_t> _rejectedRequests; + std::unordered_map<uint16_t, size_t> _failed_requests; // Also includes rejections BucketDatabase::MergingProcessor::Result merge(BucketDatabase::Merger&) override; void insert_remaining_at_end(BucketDatabase::TrailingInserter&) override; @@ -122,6 +123,13 @@ public: auto iter = _rejectedRequests.find(node); return ((iter != _rejectedRequests.end()) ? iter->second : 0); } + void increment_request_failures(uint16_t node) { + _failed_requests[node]++; + } + [[nodiscard]] size_t request_failures(uint16_t node) const noexcept { + auto iter = _failed_requests.find(node); + return ((iter != _failed_requests.end()) ? iter->second : 0); + } }; } diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp index 62a520abc87..3dda989ff74 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp @@ -12,7 +12,7 @@ #include <vespa/vespalib/util/xmlstream.hpp> #include <climits> -#include <vespa/log/log.h> +#include <vespa/log/bufferedlogger.h> LOG_SETUP(".pendingclusterstate"); using document::BucketSpace; @@ -248,7 +248,22 @@ PendingClusterState::Summary::Summary(const std::string& prevClusterState, PendingClusterState::Summary::Summary(const Summary &) = default; PendingClusterState::Summary & PendingClusterState::Summary::operator = (const Summary &) = default; -PendingClusterState::Summary::~Summary() { } +PendingClusterState::Summary::~Summary() = default; + +void PendingClusterState::update_reply_failure_statistics(const api::ReturnCode& result, const BucketSpaceAndNode& source) { + auto transition_iter = _pendingTransitions.find(source.bucketSpace); + assert(transition_iter != _pendingTransitions.end()); + auto& transition = *transition_iter->second; + transition.increment_request_failures(source.node); + // Edge triggered (rate limited) warning for content node bucket fetching failures + if (transition.request_failures(source.node) == RequestFailureWarningEdgeTriggerThreshold) { + LOGBP(warning, "Have failed multiple bucket info fetch requests towards node %u. Last received error is: %s", + source.node, result.toString().c_str()); + } + if (result.getResult() == api::ReturnCode::REJECTED) { + transition.incrementRequestRejections(source.node); + } +} bool PendingClusterState::onRequestBucketInfoReply(const std::shared_ptr<api::RequestBucketInfoReply>& reply) @@ -266,11 +281,7 @@ PendingClusterState::onRequestBucketInfoReply(const std::shared_ptr<api::Request resendTime += framework::MilliSecTime(100); _delayedRequests.emplace_back(resendTime, bucketSpaceAndNode); _sentMessages.erase(iter); - if (result.getResult() == api::ReturnCode::REJECTED) { - auto transitionIter = _pendingTransitions.find(bucketSpaceAndNode.bucketSpace); - assert(transitionIter != _pendingTransitions.end()); - transitionIter->second->incrementRequestRejections(bucketSpaceAndNode.node); - } + update_reply_failure_statistics(result, bucketSpaceAndNode); return true; } diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.h b/storage/src/vespa/storage/distributor/pendingclusterstate.h index 7aa35b32b8e..f79a3185c67 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.h +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.h @@ -155,6 +155,10 @@ public: std::string requestNodesToString() const; private: + // With 100ms resend timeout, this requires a particular node to have failed + // for _at least_ threshold/10 seconds before a log warning is emitted. + constexpr static size_t RequestFailureWarningEdgeTriggerThreshold = 20; + /** * Creates a pending cluster state that represents * a set system state command from the fleet controller. @@ -211,6 +215,7 @@ private: std::string getPrevClusterStateBundleString() const { return _prevClusterStateBundle.getBaselineClusterState()->toString(); } + void update_reply_failure_statistics(const api::ReturnCode& result, const BucketSpaceAndNode& source); std::shared_ptr<api::SetSystemStateCommand> _cmd; diff --git a/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt b/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt index d394ec3a5cf..182bf04b1cf 100644 --- a/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt +++ b/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(storage_filestorpersistence OBJECT filestorhandlerimpl.cpp filestormanager.cpp filestormetrics.cpp + merge_handler_metrics.cpp mergestatus.cpp modifiedbucketchecker.cpp DEPENDS diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 65ee78f7642..85604299e85 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -130,9 +130,9 @@ FileStorManager::configure(std::unique_ptr<vespa::config::content::StorFilestorC _metrics->initDiskMetrics(_disks.size(), _component.getLoadTypes()->getMetricLoadTypes(), numStripes, numThreads); _filestorHandler = std::make_unique<FileStorHandler>(numThreads, numStripes, *this, *_metrics, _partitions, _compReg); - uint32_t numResposeThreads = computeNumResponseThreads(_config->numResponseThreads); - if (numResposeThreads > 0) { - _sequencedExecutor = vespalib::SequencedTaskExecutor::create(numResposeThreads, 10000, selectSequencer(_config->responseSequencerType)); + uint32_t numResponseThreads = computeNumResponseThreads(_config->numResponseThreads); + if (numResponseThreads > 0) { + _sequencedExecutor = vespalib::SequencedTaskExecutor::create(numResponseThreads, 10000, selectSequencer(_config->responseSequencerType)); } for (uint32_t i=0; i<_component.getDiskCount(); ++i) { if (_partitions[i].isUp()) { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp index 0be046b2e9e..a80b17d4f19 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp @@ -152,23 +152,9 @@ FileStorThreadMetrics::FileStorThreadMetrics(const std::string& name, const std: mergeBuckets("mergebuckets", "Number of times buckets have been merged.", this), getBucketDiff("getbucketdiff", "Number of getbucketdiff commands that have been processed.", this), applyBucketDiff("applybucketdiff", "Number of applybucketdiff commands that have been processed.", this), - bytesMerged("bytesmerged", {}, "Total number of bytes merged into this node.", this), getBucketDiffReply("getbucketdiffreply", {}, "Number of getbucketdiff replies that have been processed.", this), applyBucketDiffReply("applybucketdiffreply", {}, "Number of applybucketdiff replies that have been processed.", this), - mergeLatencyTotal("mergelatencytotal", {}, - "Latency of total merge operation, from master node receives " - "it, until merge is complete and master node replies.", this), - mergeMetadataReadLatency("mergemetadatareadlatency", {}, - "Latency of time used in a merge step to check metadata of " - "current node to see what data it has.", this), - mergeDataReadLatency("mergedatareadlatency", {}, - "Latency of time used in a merge step to read data other " - "nodes need.", this), - mergeDataWriteLatency("mergedatawritelatency", {}, - "Latency of time used in a merge step to write data needed to " - "current node.", this), - mergeAverageDataReceivedNeeded("mergeavgdatareceivedneeded", {}, "Amount of data transferred from previous node " - "in chain that we needed to apply locally.", this), + merge_handler_metrics(this), batchingSize("batchingsize", {}, "Number of operations batched per bucket (only counts " "batches of size > 1)", this) { } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h index 3bc7fa33660..be1c5c48213 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h @@ -10,6 +10,7 @@ #pragma once +#include "merge_handler_metrics.h" #include <vespa/metrics/metrics.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> @@ -99,15 +100,9 @@ struct FileStorThreadMetrics : public metrics::MetricSet Op mergeBuckets; Op getBucketDiff; Op applyBucketDiff; - - metrics::LongCountMetric bytesMerged; metrics::LongCountMetric getBucketDiffReply; metrics::LongCountMetric applyBucketDiffReply; - metrics::DoubleAverageMetric mergeLatencyTotal; - metrics::DoubleAverageMetric mergeMetadataReadLatency; - metrics::DoubleAverageMetric mergeDataReadLatency; - metrics::DoubleAverageMetric mergeDataWriteLatency; - metrics::DoubleAverageMetric mergeAverageDataReceivedNeeded; + MergeHandlerMetrics merge_handler_metrics; metrics::LongAverageMetric batchingSize; FileStorThreadMetrics(const std::string& name, const std::string& desc, const metrics::LoadTypeSet& lt); diff --git a/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.cpp b/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.cpp new file mode 100644 index 00000000000..d4e82b8a64f --- /dev/null +++ b/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.cpp @@ -0,0 +1,28 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "merge_handler_metrics.h" + +namespace storage { + +MergeHandlerMetrics::MergeHandlerMetrics(metrics::MetricSet* owner) + : bytesMerged("bytesmerged", {}, "Total number of bytes merged into this node.", owner), + mergeLatencyTotal("mergelatencytotal", {}, + "Latency of total merge operation, from master node receives " + "it, until merge is complete and master node replies.", owner), + mergeMetadataReadLatency("mergemetadatareadlatency", {}, + "Latency of time used in a merge step to check metadata of " + "current node to see what data it has.", owner), + mergeDataReadLatency("mergedatareadlatency", {}, + "Latency of time used in a merge step to read data other " + "nodes need.", owner), + mergeDataWriteLatency("mergedatawritelatency", {}, + "Latency of time used in a merge step to write data needed to " + "current node.", owner), + mergeAverageDataReceivedNeeded("mergeavgdatareceivedneeded", {}, "Amount of data transferred from previous node " + "in chain that we needed to apply locally.", owner), + put_latency("put_latency", {}, "Latency of individual puts that are part of merge operations", owner), + remove_latency("remove_latency", {}, "Latency of individual removes that are part of merge operations", owner) +{} + +MergeHandlerMetrics::~MergeHandlerMetrics() = default; + +} diff --git a/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.h b/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.h new file mode 100644 index 00000000000..ffa3cf204a3 --- /dev/null +++ b/storage/src/vespa/storage/persistence/filestorage/merge_handler_metrics.h @@ -0,0 +1,32 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/metrics/metrics.h> + +namespace storage { + +// Provides a convenient wrapper for all MergeHandler-related metrics. +// This is _not_ its own MetricSet; metrics are owned by an explicitly provided +// parent. This is to prevent metric paths from changing, as external aggregation +// depends on the existing paths. +struct MergeHandlerMetrics { + metrics::LongCountMetric bytesMerged; + // Aggregate metrics: + metrics::DoubleAverageMetric mergeLatencyTotal; + metrics::DoubleAverageMetric mergeMetadataReadLatency; + metrics::DoubleAverageMetric mergeDataReadLatency; + metrics::DoubleAverageMetric mergeDataWriteLatency; + metrics::DoubleAverageMetric mergeAverageDataReceivedNeeded; + // Individual operation metrics. These capture both count and latency sum, so + // no need for explicit count metric on the side. + metrics::DoubleAverageMetric put_latency; + metrics::DoubleAverageMetric remove_latency; + // Iteration over metadata and document payload data is already covered by + // the merge[Meta]Data(Read|Write)Latency metrics, so not repeated here. Can be + // explicitly added if deemed required. + + explicit MergeHandlerMetrics(metrics::MetricSet* owner); + ~MergeHandlerMetrics(); +}; + +} diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 3483b15dd0e..70894858887 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -5,7 +5,6 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/document/fieldset/fieldsets.h> -#include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <algorithm> @@ -34,7 +33,7 @@ MergeHandler::MergeHandler(spi::PersistenceProvider& spi, namespace { -int getDeleteFlag() { +constexpr int getDeleteFlag() { // Referred into old slotfile code before. Where should this number come from? return 2; } @@ -71,8 +70,7 @@ checkResult(const spi::Result& result, } -class IteratorGuard -{ +class IteratorGuard { spi::PersistenceProvider& _spi; spi::IteratorId _iteratorId; spi::Context& _context; @@ -84,15 +82,13 @@ public: _iteratorId(iteratorId), _context(context) {} - ~IteratorGuard() - { + ~IteratorGuard() { assert(_iteratorId != 0); _spi.destroyIterator(_iteratorId, _context); } }; -struct IndirectDocEntryTimestampPredicate -{ +struct IndirectDocEntryTimestampPredicate { bool operator()(const spi::DocEntry::UP& e1, const spi::DocEntry::UP& e2) const { @@ -106,8 +102,7 @@ struct IndirectDocEntryTimestampPredicate } }; -struct DiffEntryTimestampPredicate -{ +struct DiffEntryTimestampPredicate { bool operator()(const api::ApplyBucketDiffCommand::Entry& e, const api::Timestamp timestamp) const { @@ -221,7 +216,6 @@ MergeHandler::buildBucketInfoList( bucket.toString().c_str(), providerInfo.toString().c_str(), dbInfo.toString().c_str()); - DUMP_LOGGED_BUCKET_OPERATIONS(bucket.getBucketId()); } entry->setBucketInfo(providerInfo); @@ -538,16 +532,14 @@ MergeHandler::applyDiffEntry(const spi::Bucket& bucket, // Regular put entry Document::SP doc(deserializeDiffDocument(e, repo)); DocumentId docId = doc->getId(); - checkResult(_spi.put(bucket, timestamp, std::move(doc), context), - bucket, - docId, - "put"); + framework::MilliSecTimer start_time(_env._component.getClock()); + checkResult(_spi.put(bucket, timestamp, std::move(doc), context), bucket, docId, "put"); + _env._metrics.merge_handler_metrics.put_latency.addValue(start_time.getElapsedTimeAsDouble()); } else { DocumentId docId(e._docName); - checkResult(_spi.remove(bucket, timestamp, docId, context), - bucket, - docId, - "remove"); + framework::MilliSecTimer start_time(_env._component.getClock()); + checkResult(_spi.remove(bucket, timestamp, docId, context), bucket, docId, "remove"); + _env._metrics.merge_handler_metrics.remove_latency.addValue(start_time.getElapsedTimeAsDouble()); } } @@ -660,10 +652,10 @@ MergeHandler::applyDiffLocally( } if (byteCount + notNeededByteCount != 0) { - _env._metrics.mergeAverageDataReceivedNeeded.addValue( + _env._metrics.merge_handler_metrics.mergeAverageDataReceivedNeeded.addValue( static_cast<double>(byteCount) / (byteCount + notNeededByteCount)); } - _env._metrics.bytesMerged.inc(byteCount); + _env._metrics.merge_handler_metrics.bytesMerged.inc(byteCount); LOG(debug, "Merge(%s): Applied %u entries locally from ApplyBucketDiff.", bucket.toString().c_str(), addedCount); @@ -870,7 +862,7 @@ MergeHandler::processBucketMerge(const spi::Bucket& bucket, MergeStatus& status, if (applyDiffNeedLocalData(cmd->getDiff(), 0, true)) { framework::MilliSecTimer startTime(_env._component.getClock()); fetchLocalData(bucket, cmd->getLoadType(), cmd->getDiff(), 0, context); - _env._metrics.mergeDataReadLatency.addValue( + _env._metrics.merge_handler_metrics.mergeDataReadLatency.addValue( startTime.getElapsedTimeAsDouble()); } status.pendingId = cmd->getMsgId(); @@ -966,7 +958,7 @@ MergeHandler::handleMergeBucket(api::MergeBucketCommand& cmd, MessageTracker::UP "Bucket not found in buildBucketInfo step"); return tracker; } - _env._metrics.mergeMetadataReadLatency.addValue(s->startTime.getElapsedTimeAsDouble()); + _env._metrics.merge_handler_metrics.mergeMetadataReadLatency.addValue(s->startTime.getElapsedTimeAsDouble()); LOG(spam, "Sending GetBucketDiff %" PRIu64 " for %s to next node %u " "with diff of %u entries.", cmd2->getMsgId(), @@ -1039,19 +1031,19 @@ namespace { result.push_back(b); ++j; } else { - // If we find equal timestamped entries that are not the - // same.. Flag an error. But there is nothing we can do - // about it. Note it as if it is the same entry so we - // dont try to merge them. + // If we find equal timestamped entries that are not the + // same.. Flag an error. But there is nothing we can do + // about it. Note it as if it is the same entry so we + // dont try to merge them. if (!(a == b)) { if (a._gid == b._gid && a._flags == b._flags) { if ((a._flags & getDeleteFlag()) != 0 && (b._flags & getDeleteFlag()) != 0) { - // Unfortunately this can happen, for instance - // if a remove comes to a bucket out of sync - // and reuses different headers in the two - // versions. + // Unfortunately this can happen, for instance + // if a remove comes to a bucket out of sync + // and reuses different headers in the two + // versions. LOG(debug, "Found entries with equal timestamps of " "the same gid who both are remove " "entries: %s <-> %s.", @@ -1071,9 +1063,9 @@ namespace { } else if ((a._flags & getDeleteFlag()) != (b._flags & getDeleteFlag())) { - // If we find one remove and one put entry on the - // same timestamp we are going to keep the remove - // entry to make the copies consistent. + // If we find one remove and one put entry on the + // same timestamp we are going to keep the remove + // entry to make the copies consistent. const api::GetBucketDiffCommand::Entry& deletedEntry( (a._flags & getDeleteFlag()) != 0 ? a : b); result.push_back(deletedEntry); @@ -1146,7 +1138,7 @@ MergeHandler::handleGetBucketDiff(api::GetBucketDiffCommand& cmd, MessageTracker LOG(error, "Diffing %s found suspect entries.", bucket.toString().c_str()); } - _env._metrics.mergeMetadataReadLatency.addValue( + _env._metrics.merge_handler_metrics.mergeMetadataReadLatency.addValue( startTime.getElapsedTimeAsDouble()); // If last node in merge chain, we can send reply straight away @@ -1243,7 +1235,6 @@ MergeHandler::handleGetBucketDiffReply(api::GetBucketDiffReply& reply, LOG(warning, "Got GetBucketDiffReply for %s which we have no " "merge state for.", bucket.toString().c_str()); - DUMP_LOGGED_BUCKET_OPERATIONS(bucket.getBucketId()); return; } @@ -1252,7 +1243,6 @@ MergeHandler::handleGetBucketDiffReply(api::GetBucketDiffReply& reply, LOG(warning, "Got GetBucketDiffReply for %s which had message " "id %" PRIu64 " when we expected %" PRIu64 ". Ignoring reply.", bucket.toString().c_str(), reply.getMsgId(), s.pendingId); - DUMP_LOGGED_BUCKET_OPERATIONS(bucket.getBucketId()); return; } api::StorageReply::SP replyToSend; @@ -1280,7 +1270,7 @@ MergeHandler::handleGetBucketDiffReply(api::GetBucketDiffReply& reply, // We have sent something on, and shouldn't reply now. clearState = false; } else { - _env._metrics.mergeLatencyTotal.addValue( + _env._metrics.merge_handler_metrics.mergeLatencyTotal.addValue( s.startTime.getElapsedTimeAsDouble()); } } @@ -1327,7 +1317,7 @@ MergeHandler::handleApplyBucketDiff(api::ApplyBucketDiffCommand& cmd, MessageTra if (applyDiffNeedLocalData(cmd.getDiff(), index, !lastInChain)) { framework::MilliSecTimer startTime(_env._component.getClock()); fetchLocalData(bucket, cmd.getLoadType(), cmd.getDiff(), index, tracker->context()); - _env._metrics.mergeDataReadLatency.addValue(startTime.getElapsedTimeAsDouble()); + _env._metrics.merge_handler_metrics.mergeDataReadLatency.addValue(startTime.getElapsedTimeAsDouble()); } else { LOG(spam, "Merge(%s): Moving %zu entries, didn't need " "local data on node %u (%u).", @@ -1340,7 +1330,7 @@ MergeHandler::handleApplyBucketDiff(api::ApplyBucketDiffCommand& cmd, MessageTra framework::MilliSecTimer startTime(_env._component.getClock()); api::BucketInfo info(applyDiffLocally(bucket, cmd.getLoadType(), cmd.getDiff(), index, tracker->context())); - _env._metrics.mergeDataWriteLatency.addValue( + _env._metrics.merge_handler_metrics.mergeDataWriteLatency.addValue( startTime.getElapsedTimeAsDouble()); } else { LOG(spam, "Merge(%s): Didn't need fetched data on node %u (%u).", @@ -1409,7 +1399,6 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply, LOG(warning, "Got ApplyBucketDiffReply for %s which we have no " "merge state for.", bucket.toString().c_str()); - DUMP_LOGGED_BUCKET_OPERATIONS(bucket.getBucketId()); return; } @@ -1418,7 +1407,6 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply, LOG(warning, "Got ApplyBucketDiffReply for %s which had message " "id %" PRIu64 " when we expected %" PRIu64 ". Ignoring reply.", bucket.toString().c_str(), reply.getMsgId(), s.pendingId); - DUMP_LOGGED_BUCKET_OPERATIONS(bucket.getBucketId()); return; } bool clearState = true; @@ -1436,7 +1424,7 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply, framework::MilliSecTimer startTime(_env._component.getClock()); fetchLocalData(bucket, reply.getLoadType(), diff, index, s.context); - _env._metrics.mergeDataReadLatency.addValue( + _env._metrics.merge_handler_metrics.mergeDataReadLatency.addValue( startTime.getElapsedTimeAsDouble()); } if (applyDiffHasLocallyNeededData(diff, index)) { @@ -1444,7 +1432,7 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply, api::BucketInfo info( applyDiffLocally(bucket, reply.getLoadType(), diff, index, s.context)); - _env._metrics.mergeDataWriteLatency.addValue( + _env._metrics.merge_handler_metrics.mergeDataWriteLatency.addValue( startTime.getElapsedTimeAsDouble()); } else { LOG(spam, "Merge(%s): Didn't need fetched data on node %u (%u)", @@ -1491,7 +1479,7 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply, // We have sent something on and shouldn't reply now. clearState = false; } else { - _env._metrics.mergeLatencyTotal.addValue( + _env._metrics.merge_handler_metrics.mergeLatencyTotal.addValue( s.startTime.getElapsedTimeAsDouble()); } } diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 76c241d0627..a96e9870d36 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -919,7 +919,7 @@ PersistenceThread::processLockedMessage(FileStorHandler::LockedMessage lock) { void PersistenceThread::run(framework::ThreadHandle& thread) { - LOG(debug, "Started persistence thread with pid %d", getpid()); + LOG(debug, "Started persistence thread"); while (!thread.interrupted() && !_env._fileStorHandler.closed(_env._partition)) { thread.registerTick(); @@ -933,7 +933,7 @@ PersistenceThread::run(framework::ThreadHandle& thread) vespalib::MonitorGuard flushMonitorGuard(_flushMonitor); flushMonitorGuard.broadcast(); } - LOG(debug, "Closing down persistence thread %d", getpid()); + LOG(debug, "Closing down persistence thread"); vespalib::MonitorGuard flushMonitorGuard(_flushMonitor); _closed = true; flushMonitorGuard.broadcast(); diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index d0d1056d4fd..c0adb01ad47 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -46,7 +46,7 @@ void CommunicationManager::receiveStorageReply(const std::shared_ptr<api::StorageReply>& reply) { assert(reply); - enque_or_process(reply); + enqueue_or_process(reply); } namespace { @@ -101,7 +101,7 @@ CommunicationManager::handleMessage(std::unique_ptr<mbus::Message> msg) cmd->setTrace(docMsgPtr->getTrace()); cmd->setTransportContext(std::make_unique<StorageTransportContext>(std::move(docMsgPtr))); - enque_or_process(std::move(cmd)); + enqueue_or_process(std::move(cmd)); } else if (protocolName == mbusprot::StorageProtocol::NAME) { std::unique_ptr<mbusprot::StorageCommand> storMsgPtr(static_cast<mbusprot::StorageCommand*>(msg.release())); @@ -113,7 +113,7 @@ CommunicationManager::handleMessage(std::unique_ptr<mbus::Message> msg) cmd->setTrace(storMsgPtr->getTrace()); cmd->setTransportContext(std::make_unique<StorageTransportContext>(std::move(storMsgPtr))); - enque_or_process(std::move(cmd)); + enqueue_or_process(std::move(cmd)); } else { LOGBM(warning, "Received unsupported message type %d for protocol '%s'", msg->getType(), msg->getProtocol().c_str()); @@ -443,7 +443,7 @@ CommunicationManager::process(const std::shared_ptr<api::StorageMessage>& msg) } void -CommunicationManager::enque_or_process(std::shared_ptr<api::StorageMessage> msg) +CommunicationManager::enqueue_or_process(std::shared_ptr<api::StorageMessage> msg) { assert(msg); if (_skip_thread) { diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index c2d5ea7039d..23b59f5a42a 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -114,7 +114,7 @@ private: bool _skip_thread; void updateMetrics(const MetricLockGuard &) override; - void enque_or_process(std::shared_ptr<api::StorageMessage> msg); + void enqueue_or_process(std::shared_ptr<api::StorageMessage> msg); // Test needs access to configure() for live reconfig testing. friend struct CommunicationManagerTest; diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp index fd16f43050a..2595514e6f4 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp @@ -19,16 +19,22 @@ ComponentRegisterImpl::ComponentRegisterImpl() _shutdownListener(nullptr) { } -ComponentRegisterImpl::~ComponentRegisterImpl() { } +ComponentRegisterImpl::~ComponentRegisterImpl() = default; void ComponentRegisterImpl::registerComponent(ManagedComponent& mc) { vespalib::LockGuard lock(_componentLock); _components.push_back(&mc); - if (_clock != 0) mc.setClock(*_clock); - if (_threadPool != 0) mc.setThreadPool(*_threadPool); - if (_metricManager != 0) mc.setMetricRegistrator(*this); + if (_clock) { + mc.setClock(*_clock); + } + if (_threadPool) { + mc.setThreadPool(*_threadPool); + } + if (_metricManager) { + mc.setMetricRegistrator(*this); + } mc.setUpgradeFlag(_upgradeFlag); } @@ -36,7 +42,7 @@ void ComponentRegisterImpl::requestShutdown(vespalib::stringref reason) { vespalib::LockGuard lock(_componentLock); - if (_shutdownListener != 0) { + if (_shutdownListener) { _shutdownListener->requestShutdown(reason); } } @@ -47,7 +53,7 @@ ComponentRegisterImpl::setMetricManager(metrics::MetricManager& mm) std::vector<ManagedComponent*> components; { vespalib::LockGuard lock(_componentLock); - assert(_metricManager == 0); + assert(_metricManager == nullptr); components = _components; _metricManager = &mm; } @@ -55,8 +61,8 @@ ComponentRegisterImpl::setMetricManager(metrics::MetricManager& mm) metrics::MetricLockGuard lock(mm.getMetricLock()); mm.registerMetric(lock, _topMetricSet); } - for (uint32_t i=0; i<components.size(); ++i) { - components[i]->setMetricRegistrator(*this); + for (auto* component : _components) { + component->setMetricRegistrator(*this); } } @@ -64,10 +70,10 @@ void ComponentRegisterImpl::setClock(Clock& c) { vespalib::LockGuard lock(_componentLock); - assert(_clock == 0); + assert(_clock == nullptr); _clock = &c; - for (uint32_t i=0; i<_components.size(); ++i) { - _components[i]->setClock(c); + for (auto* component : _components) { + component->setClock(c); } } @@ -75,10 +81,10 @@ void ComponentRegisterImpl::setThreadPool(ThreadPool& tp) { vespalib::LockGuard lock(_componentLock); - assert(_threadPool == 0); + assert(_threadPool == nullptr); _threadPool = &tp; - for (uint32_t i=0; i<_components.size(); ++i) { - _components[i]->setThreadPool(tp); + for (auto* component : _components) { + component->setThreadPool(tp); } } @@ -87,8 +93,8 @@ ComponentRegisterImpl::setUpgradeFlag(UpgradeFlags flag) { vespalib::LockGuard lock(_componentLock); _upgradeFlag = flag; - for (uint32_t i=0; i<_components.size(); ++i) { - _components[i]->setUpgradeFlag(_upgradeFlag); + for (auto* component : _components) { + component->setUpgradeFlag(_upgradeFlag); } } @@ -96,14 +102,14 @@ const StatusReporter* ComponentRegisterImpl::getStatusReporter(vespalib::stringref id) { vespalib::LockGuard lock(_componentLock); - for (uint32_t i=0; i<_components.size(); ++i) { - if (_components[i]->getStatusReporter() != 0 - && _components[i]->getStatusReporter()->getId() == id) + for (auto* component : _components) { + if ((component->getStatusReporter() != nullptr) + && (component->getStatusReporter()->getId() == id)) { - return _components[i]->getStatusReporter(); + return component->getStatusReporter(); } } - return 0; + return nullptr; } std::vector<const StatusReporter*> @@ -111,9 +117,9 @@ ComponentRegisterImpl::getStatusReporters() { std::vector<const StatusReporter*> reporters; vespalib::LockGuard lock(_componentLock); - for (uint32_t i=0; i<_components.size(); ++i) { - if (_components[i]->getStatusReporter() != 0) { - reporters.push_back(_components[i]->getStatusReporter()); + for (auto* component : _components) { + if (component->getStatusReporter() != nullptr) { + reporters.emplace_back(component->getStatusReporter()); } } return reporters; @@ -147,9 +153,9 @@ ComponentRegisterImpl::registerUpdateHook(vespalib::stringref name, SecondTime period) { vespalib::LockGuard lock(_componentLock); - metrics::UpdateHook::UP hookPtr(new MetricHookWrapper(name, hook)); + auto hookPtr = std::make_unique<MetricHookWrapper>(name, hook); _metricManager->addMetricUpdateHook(*hookPtr, period.getTime()); - _hooks.push_back(std::move(hookPtr)); + _hooks.emplace_back(std::move(hookPtr)); } metrics::MetricLockGuard @@ -162,11 +168,7 @@ void ComponentRegisterImpl::registerShutdownListener(ShutdownListener& listener) { vespalib::LockGuard lock(_componentLock); - if (_shutdownListener != 0) { - throw vespalib::IllegalStateException( - "A shutdown listener is already registered. Add functionality " - "for having multiple if we need multiple.", VESPA_STRLOC); - } + assert(_shutdownListener == nullptr); _shutdownListener = &listener; } diff --git a/storageframework/src/vespa/storageframework/generic/metric/metricregistrator.h b/storageframework/src/vespa/storageframework/generic/metric/metricregistrator.h index f8ab6699d3c..6cdbf58d555 100644 --- a/storageframework/src/vespa/storageframework/generic/metric/metricregistrator.h +++ b/storageframework/src/vespa/storageframework/generic/metric/metricregistrator.h @@ -22,7 +22,7 @@ namespace storage::framework { struct MetricUpdateHook; struct MetricRegistrator { - virtual ~MetricRegistrator() {} + virtual ~MetricRegistrator() = default; virtual void registerMetric(metrics::Metric&) = 0; virtual void registerUpdateHook(vespalib::stringref name, MetricUpdateHook& hook, SecondTime period) = 0; diff --git a/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp b/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp index fe717313ca4..d51bd57e942 100644 --- a/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp @@ -83,6 +83,9 @@ Matcher::select_query_nodes(const MatchingElementsFields& fields, const QueryNod if (fields.has_struct_field(query_term->getIndex())) { _sub_field_terms.emplace_back(fields.get_enclosing_field(query_term->getIndex()), query_term); } + if (fields.has_field(query_term->getIndex())) { + _sub_field_terms.emplace_back(query_term->getIndex(), query_term); + } } else if (auto and_not = as<AndNotQueryNode>(query_node)) { select_query_nodes(fields, *(*and_not)[0]); } else if (auto intermediate = as<QueryConnector>(query_node)) { diff --git a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp index 37809b207ad..06bdb9f69bb 100644 --- a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp @@ -1,12 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "queryenvironment.h" -#include <vespa/searchlib/common/location.h> +#include <vespa/searchlib/common/geo_location.h> +#include <vespa/searchlib/common/geo_location_spec.h> +#include <vespa/searchlib/common/geo_location_parser.h> #include <vespa/log/log.h> LOG_SETUP(".searchvisitor.queryenvironment"); using search::IAttributeManager; +using search::common::GeoLocationParser; +using search::common::GeoLocationSpec; using search::fef::Properties; using vespalib::string; @@ -14,34 +18,24 @@ namespace streaming { namespace { -search::fef::Location +std::vector<GeoLocationSpec> parseLocation(const string & location_str) { - search::fef::Location fefLocation; + std::vector<GeoLocationSpec> fefLocations; if (location_str.empty()) { - return fefLocation; + return fefLocations; } - string::size_type pos = location_str.find(':'); - if (pos == string::npos) { - LOG(warning, "Location string lacks attribute vector specification. loc='%s'. Location ignored.", - location_str.c_str()); - return fefLocation; - } - string attr = location_str.substr(0, pos); - const string location = location_str.substr(pos + 1); - - search::common::Location locationSpec; - if (!locationSpec.parse(location)) { + GeoLocationParser locationParser; + if (!locationParser.parseOldFormatWithField(location_str)) { LOG(warning, "Location parse error (location: '%s'): %s. Location ignored.", - location.c_str(), locationSpec.getParseError()); - return fefLocation; + location_str.c_str(), locationParser.getParseError()); + return fefLocations; + } + auto loc = locationParser.getGeoLocation(); + if (loc.has_point) { + fefLocations.push_back(GeoLocationSpec{locationParser.getFieldName(), loc}); } - fefLocation.setAttribute(attr); - fefLocation.setXPosition(locationSpec.getX()); - fefLocation.setYPosition(locationSpec.getY()); - fefLocation.setXAspect(locationSpec.getXAspect()); - fefLocation.setValid(true); - return fefLocation; + return fefLocations; } } @@ -54,7 +48,7 @@ QueryEnvironment::QueryEnvironment(const string & location_str, _properties(properties), _attrCtx(attrMgr->createContext()), _queryTerms(), - _location(parseLocation(location_str)) + _locations(parseLocation(location_str)) { } diff --git a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h index f580cec8870..df354d578e6 100644 --- a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h @@ -6,7 +6,6 @@ #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/fef/iindexenvironment.h> #include <vespa/searchlib/fef/iqueryenvironment.h> -#include <vespa/searchlib/fef/location.h> #include <vespa/searchlib/fef/properties.h> #include "indexenvironment.h" @@ -23,7 +22,7 @@ private: const search::fef::Properties &_properties; search::attribute::IAttributeContext::UP _attrCtx; std::vector<const search::fef::ITermData *> _queryTerms; - search::fef::Location _location; + std::vector<search::common::GeoLocationSpec> _locations; public: typedef std::unique_ptr<QueryEnvironment> UP; @@ -49,7 +48,13 @@ public: } // inherit documentation - virtual const search::fef::Location & getLocation() const override { return _location; } + GeoLocationSpecPtrs getAllLocations() const override { + GeoLocationSpecPtrs retval; + for (const auto & loc : _locations) { + retval.push_back(&loc); + } + return retval; + } // inherit documentation virtual const search::attribute::IAttributeContext & getAttributeContext() const override { return *_attrCtx; } diff --git a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp index 0b4a83bf714..d19a5a323f2 100644 --- a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp +++ b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp @@ -1,11 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> #include <vbench/test/all.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> #include <vespa/vespalib/net/crypto_engine.h> using namespace vbench; -using vespalib::SlaveProc; +using vespalib::ChildProcess; using InputReader = vespalib::InputReader; using OutputWriter = vespalib::OutputWriter; @@ -32,7 +32,7 @@ void readUntil(Input &input, SimpleBuffer &buffer, const string &end) { TEST("dumpurl usage") { std::string out; - EXPECT_FALSE(SlaveProc::run("../../apps/dumpurl/vbench_dumpurl_app", out)); + EXPECT_FALSE(ChildProcess::run("../../apps/dumpurl/vbench_dumpurl_app", out)); fprintf(stderr, "%s\n", out.c_str()); } @@ -48,7 +48,7 @@ TEST_MT_F("run dumpurl", 2, ServerSocket()) { out.write("data"); } else { std::string out; - EXPECT_TRUE(SlaveProc::run(strfmt("../../apps/dumpurl/vbench_dumpurl_app localhost %d /foo", + EXPECT_TRUE(ChildProcess::run(strfmt("../../apps/dumpurl/vbench_dumpurl_app localhost %d /foo", f1.port()).c_str(), out)); fprintf(stderr, "%s\n", out.c_str()); } diff --git a/vbench/src/tests/app_vbench/app_vbench_test.cpp b/vbench/src/tests/app_vbench/app_vbench_test.cpp index 8b5bb71425d..0e89d30e853 100644 --- a/vbench/src/tests/app_vbench/app_vbench_test.cpp +++ b/vbench/src/tests/app_vbench/app_vbench_test.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> #include <vbench/test/all.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> #include <vespa/vespalib/test/make_tls_options_for_testing.h> @@ -11,7 +11,7 @@ #include <fcntl.h> using namespace vbench; -using vespalib::SlaveProc; +using vespalib::ChildProcess; using InputReader = vespalib::InputReader; using OutputWriter = vespalib::OutputWriter; @@ -32,7 +32,7 @@ void write_file(const vespalib::string &file_name, const vespalib::string &conte TEST("vbench usage") { std::string out; - EXPECT_FALSE(SlaveProc::run("../../apps/vbench/vbench_app", out)); + EXPECT_FALSE(ChildProcess::run("../../apps/vbench/vbench_app", out)); fprintf(stderr, "%s\n", out.c_str()); } @@ -69,14 +69,14 @@ struct Servers { TEST_MT_F("run vbench", 2, Servers()) { if (thread_id == 0) { std::string out; - EXPECT_TRUE(SlaveProc::run(strfmt("sed 's/_LOCAL_PORT_/%d/' vbench.cfg.template > vbench.cfg", f1.portal->listen_port()).c_str())); - EXPECT_TRUE(SlaveProc::run("../../apps/vbench/vbench_app run vbench.cfg 2> vbench.out", out)); + EXPECT_TRUE(ChildProcess::run(strfmt("sed 's/_LOCAL_PORT_/%d/' vbench.cfg.template > vbench.cfg", f1.portal->listen_port()).c_str())); + EXPECT_TRUE(ChildProcess::run("../../apps/vbench/vbench_app run vbench.cfg 2> vbench.out", out)); fprintf(stderr, "null crypto: %s\n", out.c_str()); EXPECT_GREATER(f1.my_get.cnt, 10u); } else { std::string tls_out; - EXPECT_TRUE(SlaveProc::run(strfmt("sed 's/_LOCAL_PORT_/%d/' vbench.tls.cfg.template > vbench.tls.cfg", f1.tls_portal->listen_port()).c_str())); - EXPECT_TRUE(SlaveProc::run("../../apps/vbench/vbench_app run vbench.tls.cfg 2> vbench.tls.out", tls_out)); + EXPECT_TRUE(ChildProcess::run(strfmt("sed 's/_LOCAL_PORT_/%d/' vbench.tls.cfg.template > vbench.tls.cfg", f1.tls_portal->listen_port()).c_str())); + EXPECT_TRUE(ChildProcess::run("../../apps/vbench/vbench_app run vbench.tls.cfg 2> vbench.tls.out", tls_out)); fprintf(stderr, "tls crypto: %s\n", tls_out.c_str()); EXPECT_GREATER(f1.my_tls_get.cnt, 10u); } diff --git a/vespa-osgi-testrunner/CMakeLists.txt b/vespa-osgi-testrunner/CMakeLists.txt index 58aba186710..4b097a2f7bf 100644 --- a/vespa-osgi-testrunner/CMakeLists.txt +++ b/vespa-osgi-testrunner/CMakeLists.txt @@ -1,2 +1,3 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_fat_java_artifact(vespa-osgi-testrunner) +install_config_definition(src/main/resources/configdefinitions/junit-test-runner.def com.yahoo.vespa.testrunner.junit-test-runner.def) diff --git a/vespajlib/src/main/java/com/yahoo/collections/PredicateSplit.java b/vespajlib/src/main/java/com/yahoo/collections/PredicateSplit.java deleted file mode 100644 index 1b3941df7bf..00000000000 --- a/vespajlib/src/main/java/com/yahoo/collections/PredicateSplit.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.collections; - -import java.util.List; -import java.util.ArrayList; -import java.util.function.Predicate; - -/** - * Class holding the result of a partition-by-predicate operation. - **/ -public class PredicateSplit<E> { - public final List<E> falseValues; /// list of values where the predicate returned false - public final List<E> trueValues; /// list of values where the predicate returned true - - private PredicateSplit() { - falseValues = new ArrayList<E>(); - trueValues = new ArrayList<E>(); - } - - /** - * Perform a partition-by-predicate operation. - * Each value in the input is tested by the predicate and - * added to either the falseValues list or the trueValues list. - * @param collection The input collection. - * @param predicate A test for selecting the target list. - * @return Two lists bundled in an object. - **/ - public static <V> PredicateSplit<V> partition(Iterable<V> collection, Predicate<? super V> predicate) - { - PredicateSplit<V> r = new PredicateSplit<V>(); - for (V value : collection) { - if (predicate.test(value)) { - r.trueValues.add(value); - } else { - r.falseValues.add(value); - } - } - return r; - } -} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java new file mode 100644 index 00000000000..a43e2156025 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/JobMetrics.java @@ -0,0 +1,41 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent.maintenance; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiConsumer; + +/** + * Tracks and forwards maintenance job metrics. + * + * @author mpolden + */ +public class JobMetrics { + + private final BiConsumer<String, Long> metricConsumer; + + private final Map<String, Long> incompleteRuns = new ConcurrentHashMap<>(); + + public JobMetrics(BiConsumer<String, Long> metricConsumer) { + this.metricConsumer = metricConsumer; + } + + /** Record a run for given job */ + public void recordRunOf(String job) { + incompleteRuns.compute(job, (ignored, run) -> run == null ? 1 : ++run); + } + + /** Record successful run of given job */ + public void recordSuccessOf(String job) { + incompleteRuns.put(job, 0L); + } + + /** Forward metrics for given job to metric consumer */ + public void forward(String job) { + Long incompleteRuns = this.incompleteRuns.get(job); + if (incompleteRuns != null) { + metricConsumer.accept(job, incompleteRuns); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 9c40e5ec54f..eb9b91c812c 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -26,17 +26,19 @@ public abstract class Maintainer implements Runnable, AutoCloseable { private final String name; private final JobControl jobControl; + private final JobMetrics jobMetrics; private final Duration interval; private final ScheduledExecutorService service; - public Maintainer(String name, Duration interval, Instant startedAt, JobControl jobControl, List<String> clusterHostnames) { - this(name, interval, staggeredDelay(interval, startedAt, HostName.getLocalhost(), clusterHostnames), jobControl); + public Maintainer(String name, Duration interval, Instant startedAt, JobControl jobControl, JobMetrics jobMetrics, List<String> clusterHostnames) { + this(name, interval, staggeredDelay(interval, startedAt, HostName.getLocalhost(), clusterHostnames), jobControl, jobMetrics); } - public Maintainer(String name, Duration interval, Duration initialDelay, JobControl jobControl) { + public Maintainer(String name, Duration interval, Duration initialDelay, JobControl jobControl, JobMetrics jobMetrics) { this.name = name; this.interval = requireInterval(interval); this.jobControl = Objects.requireNonNull(jobControl); + this.jobMetrics = Objects.requireNonNull(jobMetrics); service = new ScheduledThreadPoolExecutor(1, r -> new Thread(r, name() + "-worker")); service.scheduleAtFixedRate(this, initialDelay.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS); jobControl.started(name(), this); @@ -72,8 +74,8 @@ public abstract class Maintainer implements Runnable, AutoCloseable { @Override public final String toString() { return name(); } - /** Called once each time this maintenance job should run */ - protected abstract void maintain(); + /** Called once each time this maintenance job should run. Returns whether the maintenance run was succesful */ + protected abstract boolean maintain(); /** Returns the interval at which this job is set to run */ protected Duration interval() { return interval; } @@ -82,7 +84,13 @@ public abstract class Maintainer implements Runnable, AutoCloseable { @SuppressWarnings("unused") public final void lockAndMaintain() { try (var lock = jobControl.lockJob(name())) { - maintain(); + try { + jobMetrics.recordRunOf(name()); + if (maintain()) jobMetrics.recordSuccessOf(name()); + } finally { + // Always forward metrics + jobMetrics.forward(name()); + } } } diff --git a/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java b/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java index 9f3d3b837f8..58164801c7c 100644 --- a/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java +++ b/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java @@ -2,7 +2,7 @@ package com.yahoo.geo; /** - * utility for parsing geographical coordinates + * Utility for parsing geographical coordinates * * @author arnej27959 **/ diff --git a/vespajlib/src/main/java/com/yahoo/geo/DistanceParser.java b/vespajlib/src/main/java/com/yahoo/geo/DistanceParser.java new file mode 100644 index 00000000000..1ae68afa4ac --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/geo/DistanceParser.java @@ -0,0 +1,79 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.geo; + +import com.google.common.annotations.Beta; + +/** + * Utility for parsing a geographical distance with unit. + **/ +@Beta +public class DistanceParser { + // according to wikipedia: + // Earth's equatorial radius = 6378137 meter - not used + // meters per mile = 1609.344 + // 180 degrees equals one half diameter equals PI*r + // Earth's polar radius = 6356752 meter + + public final static double m2deg = 180.0 / (Math.PI * 6356752.0); + public final static double km2deg = 1000.000 * 180.0 / (Math.PI * 6356752.0); + public final static double mi2deg = 1609.344 * 180.0 / (Math.PI * 6356752.0); + + private final double degrees; + + public double getDegrees() { return degrees; } + + /** + * Parse a distance in some kind of units, converting to geographical degrees. + * Note that the number and the unit should be separated by a single space only, + * or not separated at all. + * Supported units are "m", "km", "miles", and "deg", + * the last one meaning degrees with no conversion. + * For brevity "mi" = "miles" and "d" = "deg". + **/ + static public double parse(String distance) { + var parser = new DistanceParser(distance, false); + return parser.degrees; + } + + DistanceParser(String distance, boolean assumeMicroDegrees) { + if (distance.endsWith(" km")) { + double km = Double.valueOf(distance.substring(0, distance.length()-3)); + degrees = km * km2deg; + } else if (distance.endsWith(" m")) { + double meters = Double.valueOf(distance.substring(0, distance.length()-2)); + degrees = meters * m2deg; + } else if (distance.endsWith(" miles")) { + double miles = Double.valueOf(distance.substring(0, distance.length()-6)); + degrees = miles * mi2deg; + } else if (distance.endsWith(" mi")) { + double miles = Double.valueOf(distance.substring(0, distance.length()-3)); + degrees = miles * mi2deg; + } else if (distance.endsWith(" deg")) { + degrees = Double.valueOf(distance.substring(0, distance.length()-4)); + } else if (distance.endsWith(" d")) { + degrees = Double.valueOf(distance.substring(0, distance.length()-2)); + } else if (distance.endsWith("km")) { + double km = Double.valueOf(distance.substring(0, distance.length()-2)); + degrees = km * km2deg; + } else if (distance.endsWith("m")) { + double meters = Double.valueOf(distance.substring(0, distance.length()-1)); + degrees = meters * m2deg; + } else if (distance.endsWith("miles")) { + double miles = Double.valueOf(distance.substring(0, distance.length()-5)); + degrees = miles * mi2deg; + } else if (distance.endsWith("mi")) { + double miles = Double.valueOf(distance.substring(0, distance.length()-2)); + degrees = miles * mi2deg; + } else if (distance.endsWith("deg")) { + degrees = Double.valueOf(distance.substring(0, distance.length()-3)); + } else if (distance.endsWith("d")) { + degrees = Double.valueOf(distance.substring(0, distance.length()-1)); + } else if (assumeMicroDegrees) { + degrees = Integer.parseInt(distance) * 0.000001; + } else { + throw new IllegalArgumentException("missing unit for distance: "+distance); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/geo/OneDegreeParser.java b/vespajlib/src/main/java/com/yahoo/geo/OneDegreeParser.java new file mode 100644 index 00000000000..cf23a24e702 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/geo/OneDegreeParser.java @@ -0,0 +1,282 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.geo; + +/** + * Utility for parsing one geographical coordinate + * + * @author arnej27959 + **/ +class OneDegreeParser { + /** + * the parsed latitude (degrees north if positive) + **/ + public double latitude = 0; + public boolean foundLatitude = false; + + /** + * the parsed longitude (degrees east if positive) + **/ + public double longitude = 0; + public boolean foundLongitude = false; + + public static boolean isDigit(char ch) { + return (ch >= '0' && ch <= '9'); + } + public static boolean isCompassDirection(char ch) { + return (ch == 'N' || ch == 'S' || ch == 'E' || ch == 'W'); + } + + private String parseString = null; + private int len = 0; + private int pos = 0; + + public String toString() { + if (foundLatitude) { + return parseString + " -> latitude(" + latitude + ")"; + } else { + return parseString + " -> longitude(" + longitude + ")"; + } + } + + private char getNextChar() throws IllegalArgumentException { + if (pos == len) { + pos++; + return 0; + } else if (pos > len) { + throw new IllegalArgumentException("position after end of string when parsing <"+parseString+">"); + } else { + return parseString.charAt(pos++); + } + } + + /** + * Parse the given string. + * + * The string must contain either a latitude or a longitude. + * A latitude should contain "N" or "S" and a number signifying + * degrees north or south, or a signed number. + * A longitude should contain "E" or "W" and a number + * signifying degrees east or west, or a signed number. + * <br> + * Fractional degrees are recommended as the main input format, + * but degrees plus fractional minutes may be used for testing. + * You can use the degree sign (U+00B0 as seen in unicode at + * http://www.unicode.org/charts/PDF/U0080.pdf) to separate + * degrees from minutes, put the direction (NSEW) between as a + * separator, or use a small letter 'o' as a replacement for the + * degrees sign. + * <br> + * Some valid input formats: <br> + * "37.416383" and "-122.024683" → Sunnyvale <br> + * "N37.416383" and "W122.024683" → Sunnyvale <br> + * "37N24.983" and "122W01.481" → same <br> + * "N37\u00B024.983" and "W122\u00B001.481" → same <br> + * "63.418417" and "10.433033" → Trondheim <br> + * "N63.418417" and "E10.433033" → same <br> + * "N63o25.105" and "E10o25.982" → same <br> + * "E10o25.982" and "N63o25.105" → same <br> + * "N63.418417" and "E10.433033" → same <br> + * "63N25.105" and "10E25.982" → same <br> + * @param assumeNorthSouth Latitude assumed, otherwise longitude + * @param toParse Latitude or longitude string to parse + * + **/ + public OneDegreeParser(boolean assumeNorthSouth, String toParse) throws IllegalArgumentException { + this.parseString = toParse; + this.len = parseString.length(); + consumeString(assumeNorthSouth); + } + + private void consumeString(boolean assumeNorthSouth) throws IllegalArgumentException { + char ch = getNextChar(); + + double degrees = 0.0; + double minutes = 0.0; + double seconds = 0.0; + boolean degSet = false; + boolean minSet = false; + boolean secSet = false; + boolean dirSet = false; + boolean foundDot = false; + boolean foundDigits = false; + + boolean findingLatitude = false; + boolean findingLongitude = false; + + double sign = +1.0; + + int lastpos = -1; + + // sign must be first character in string if present: + if (ch == '+') { + // unary plus is a nop + ch = getNextChar(); + } else if (ch == '-') { + sign = -1.0; + ch = getNextChar(); + } + do { + // did we find a valid char? + boolean valid = false; + if (pos == lastpos) { + throw new IllegalArgumentException("internal logic error at <"+parseString+"> pos:"+pos); + } else { + lastpos = pos; + } + + // first, see if we can find some number + double accum = 0.0; + + if (isDigit(ch) || ch == '.') { + valid = true; + if (foundDigits) { + throw new IllegalArgumentException("found digits after not consuming previous digits when parsing <"+parseString+">"); + } + double divider = 1.0; + foundDot = false; + while (isDigit(ch)) { + foundDigits = true; + accum *= 10; + accum += (ch - '0'); + ch = getNextChar(); + } + if (ch == '.') { + foundDot = true; + ch = getNextChar(); + while (isDigit(ch)) { + foundDigits = true; + accum *= 10; + accum += (ch - '0'); + divider *= 10; + ch = getNextChar(); + } + } + if (!foundDigits) { + throw new IllegalArgumentException("just a . is not a valid number when parsing <"+parseString+">"); + } + accum /= divider; + } + + // next, did we find a separator after the number? + // degree sign is a separator after degrees, before minutes + if (ch == '\u00B0' || ch == 'o') { + valid = true; + if (degSet) { + throw new IllegalArgumentException("degrees sign only valid just after degrees when parsing <"+parseString+">"); + } + if (!foundDigits) { + throw new IllegalArgumentException("must have number before degrees sign when parsing <"+parseString+">"); + } + if (foundDot) { + throw new IllegalArgumentException("cannot have fractional degrees before degrees sign when parsing <"+parseString+">"); + } + ch = getNextChar(); + } + // apostrophe is a separator after minutes, before seconds + if (ch == '\'') { + if (minSet || !degSet || !foundDigits) { + throw new IllegalArgumentException("minutes sign only valid just after minutes when parsing <"+parseString+">"); + } + if (foundDot) { + throw new IllegalArgumentException("cannot have fractional minutes before minutes sign when parsing <"+parseString+">"); + } + ch = getNextChar(); + } + + // if we found some number, assign it into the next unset variable + if (foundDigits) { + valid = true; + if (degSet) { + if (minSet) { + if (secSet) { + throw new IllegalArgumentException("extra number after full field when parsing <"+parseString+">"); + } else { + seconds = accum; + secSet = true; + } + } else { + minutes = accum; + minSet = true; + if (foundDot) { + secSet = true; + } + } + } else { + degrees = accum; + degSet = true; + if (foundDot) { + minSet = true; + secSet = true; + } + } + foundDot = false; + foundDigits = false; + } + + // there may to be a direction (NSEW) somewhere, too + if (isCompassDirection(ch)) { + valid = true; + if (dirSet) { + throw new IllegalArgumentException("already set direction once, cannot add direction: "+ch+" when parsing <"+parseString+">"); + } + dirSet = true; + if (ch == 'S' || ch == 'W') { + sign = -1; + } else { + sign = 1; + } + if (ch == 'E' || ch == 'W') { + findingLongitude = true; + } else { + findingLatitude = true; + } + ch = getNextChar(); + } + + // lastly, did we find the end-of-string? + if (ch == 0) { + valid = true; + if (!dirSet) { + if (assumeNorthSouth) { + findingLatitude = true; + } else { + findingLongitude = true; + } + } + if (!degSet) { + throw new IllegalArgumentException("end of field without any number seen when parsing <"+parseString+">"); + } + degrees += minutes / 60.0; + degrees += seconds / 3600.0; + degrees *= sign; + + if (findingLatitude) { + if (degrees < -90.0 || degrees > 90.0) { + throw new IllegalArgumentException("out of range [-90,+90]: "+degrees+" when parsing <"+parseString+">"); + } + latitude = degrees; + foundLatitude = true; + } else if (findingLongitude) { + if (degrees < -180.0 || degrees > 180.0) { + throw new IllegalArgumentException("out of range [-180,+180]: "+degrees+" when parsing <"+parseString+">"); + } + longitude = degrees; + foundLongitude = true; + } + break; + } + if (!valid) { + throw new IllegalArgumentException("invalid character: "+ch+" when parsing <"+parseString+">"); + } + } while (ch != 0); + // everything parsed OK + if (foundLatitude && foundLongitude) { + throw new IllegalArgumentException("found both latitude and longitude from: "+parseString); + } + if (foundLatitude || foundLongitude) { + return; + } + throw new IllegalArgumentException("found neither latitude nor longitude from: "+parseString); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/geo/ParsedDegree.java b/vespajlib/src/main/java/com/yahoo/geo/ParsedDegree.java new file mode 100644 index 00000000000..84b87614182 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/geo/ParsedDegree.java @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.geo; + +/** + * Utility for holding one geographical coordinate + * + * @author arnej27959 + **/ +public class ParsedDegree { + /** + * the parsed latitude or longitude + * Degrees north or east if positive + * Degrees south or west if negative + **/ + public final double degrees; + + // one of these two flag will be true: + public final boolean isLatitude; + public final boolean isLongitude; + + public ParsedDegree(double value, boolean isLat, boolean isLon) { + this.degrees = value; + this.isLatitude = isLat; + this.isLongitude = isLon; + if (isLat && isLon) { + throw new IllegalArgumentException("value cannot be both latitude and longitude at once"); + } + if (isLat || isLon) { + return; + } + throw new IllegalArgumentException("value must be either latitude or longitude"); + } + + static public ParsedDegree fromString(String toParse, boolean assumeLatitude, boolean assumeLongitude) { + if (assumeLatitude && assumeLongitude) { + throw new IllegalArgumentException("value cannot be both latitude and longitude at once"); + } + var parser = new OneDegreeParser(assumeLatitude, toParse); + if (parser.foundLatitude) { + return new ParsedDegree(parser.latitude, true, false); + } + if (parser.foundLongitude) { + return new ParsedDegree(parser.longitude, false, true); + } + throw new IllegalArgumentException("could not parse: "+toParse); + } + + public String toString() { + if (isLatitude) { + return "Latitude: "+degrees+" degrees"; + } else { + return "Longitude: "+degrees+" degrees"; + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8.java b/vespajlib/src/main/java/com/yahoo/text/Utf8.java index 6f40b590a64..cb8ca244fe2 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Utf8.java +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8.java @@ -13,10 +13,10 @@ import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; /** - * utility class with functions for handling UTF-8 + * Utility class with functions for handling UTF-8 * * @author arnej27959 - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen * @author baldersheim * */ diff --git a/vespajlib/src/test/java/com/yahoo/collections/PredicateSplitTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/PredicateSplitTestCase.java deleted file mode 100644 index 05a719d6a4d..00000000000 --- a/vespajlib/src/test/java/com/yahoo/collections/PredicateSplitTestCase.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.collections; - -import org.junit.Test; - -import java.util.List; -import java.util.ArrayList; - -import static org.junit.Assert.assertEquals; - -public class PredicateSplitTestCase { - @Test - public void requireThatSplitWorks() { - List<Integer> l = new ArrayList<Integer>(); - l.add(1); - l.add(6); - l.add(2); - l.add(4); - l.add(5); - PredicateSplit<Integer> result = PredicateSplit.partition(l, x -> (x % 2 == 0)); - assertEquals((long) result.falseValues.size(), 2L); - assertEquals((long) result.falseValues.get(0), 1L); - assertEquals((long) result.falseValues.get(1), 5L); - - assertEquals((long) result.trueValues.size(), 3L); - assertEquals((long) result.trueValues.get(0), 6L); - assertEquals((long) result.trueValues.get(1), 2L); - assertEquals((long) result.trueValues.get(2), 4L); - } -} diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlStateMock.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlStateMock.java new file mode 100644 index 00000000000..28c701a67db --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlStateMock.java @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent.maintenance; + +import com.yahoo.transaction.Mutex; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * @author mpolden + */ +class JobControlStateMock implements JobControlState { + + private final Set<String> inactiveJobs = new HashSet<>(); + + @Override + public Set<String> readInactiveJobs() { + return Collections.unmodifiableSet(inactiveJobs); + } + + @Override + public Mutex lockMaintenanceJob(String job) { + return () -> {}; + } + + public void setActive(String job, boolean active) { + if (active) { + inactiveJobs.remove(job); + } else { + inactiveJobs.add(job); + } + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java index 0640ab2835a..a0ca9b529c5 100644 --- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java +++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/JobControlTest.java @@ -1,15 +1,8 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent.maintenance; -import com.yahoo.transaction.Mutex; import org.junit.Test; -import java.time.Duration; -import java.time.Instant; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -21,18 +14,13 @@ public class JobControlTest { @Test public void testJobControl() { - MockJobControlState state = new MockJobControlState(); + JobControlStateMock state = new JobControlStateMock(); JobControl jobControl = new JobControl(state); - MockMaintainer maintainer1 = new MockMaintainer(); - MockMaintainer maintainer2 = new MockMaintainer(); - assertTrue(jobControl.jobs().isEmpty()); - String job1 = "Job1"; String job2 = "Job2"; - - jobControl.started(job1, maintainer1); - jobControl.started(job2, maintainer2); + TestMaintainer maintainer1 = new TestMaintainer(job1, jobControl); + TestMaintainer maintainer2 = new TestMaintainer(job2, jobControl); assertEquals(2, jobControl.jobs().size()); assertTrue(jobControl.jobs().contains(job1)); assertTrue(jobControl.jobs().contains(job2)); @@ -59,79 +47,36 @@ public class JobControlTest { // Run jobs on-demand jobControl.run(job1); jobControl.run(job1); - assertEquals(2, maintainer1.maintenanceInvocations); + assertEquals(2, maintainer1.totalRuns()); jobControl.run(job2); - assertEquals(1, maintainer2.maintenanceInvocations); + assertEquals(1, maintainer2.totalRuns()); // Running jobs on-demand ignores inactive flag state.setActive(job1, false); jobControl.run(job1); - assertEquals(3, maintainer1.maintenanceInvocations); + assertEquals(3, maintainer1.totalRuns()); } @Test public void testJobControlMayDeactivateJobs() { - MockJobControlState state = new MockJobControlState(); + JobControlStateMock state = new JobControlStateMock(); JobControl jobControl = new JobControl(state); - MockMaintainer mockMaintainer = new MockMaintainer(jobControl); + TestMaintainer mockMaintainer = new TestMaintainer(null, jobControl); - assertTrue(jobControl.jobs().contains("MockMaintainer")); + assertTrue(jobControl.jobs().contains("TestMaintainer")); - assertEquals(0, mockMaintainer.maintenanceInvocations); + assertEquals(0, mockMaintainer.totalRuns()); mockMaintainer.run(); - assertEquals(1, mockMaintainer.maintenanceInvocations); + assertEquals(1, mockMaintainer.totalRuns()); - state.setActive("MockMaintainer", false); + state.setActive("TestMaintainer", false); mockMaintainer.run(); - assertEquals(1, mockMaintainer.maintenanceInvocations); + assertEquals(1, mockMaintainer.totalRuns()); - state.setActive("MockMaintainer", true); + state.setActive("TestMaintainer", true); mockMaintainer.run(); - assertEquals(2, mockMaintainer.maintenanceInvocations); - } - - private static class MockJobControlState implements JobControlState { - - private final Set<String> inactiveJobs = new HashSet<>(); - - @Override - public Set<String> readInactiveJobs() { - return new HashSet<>(inactiveJobs); - } - - @Override - public Mutex lockMaintenanceJob(String job) { - return () -> {}; - } - - public void setActive(String job, boolean active) { - if (active) { - inactiveJobs.remove(job); - } else { - inactiveJobs.add(job); - } - } - - } - - private static class MockMaintainer extends Maintainer { - - int maintenanceInvocations = 0; - - private MockMaintainer(JobControl jobControl) { - super(null, Duration.ofHours(1), Instant.now(), jobControl, List.of()); - } - - private MockMaintainer() { - this(new JobControl(new MockJobControlState())); - } - - @Override - protected void maintain() { - maintenanceInvocations++; - } - + assertEquals(2, mockMaintainer.totalRuns()); } } diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java index 820d1fc3d1d..2bfaad894a5 100644 --- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java +++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; import static org.junit.Assert.assertEquals; @@ -36,4 +37,33 @@ public class MaintainerTest { assertEquals(300, Maintainer.staggeredDelay(interval, now, "cfg0", cluster).toMillis()); } + @Test + public void success_metric() { + AtomicLong consecutiveFailures = new AtomicLong(); + JobMetrics jobMetrics = new JobMetrics((job, count) -> consecutiveFailures.set(count)); + TestMaintainer maintainer = new TestMaintainer(jobMetrics); + + // Maintainer fails twice in a row + maintainer.successOnNextRun(false).run(); + assertEquals(1, consecutiveFailures.get()); + maintainer.successOnNextRun(false).run(); + assertEquals(2, consecutiveFailures.get()); + + // Maintainer runs successfully + maintainer.successOnNextRun(true).run(); + assertEquals(0, consecutiveFailures.get()); + + // Maintainer runs successfully again + maintainer.run(); + assertEquals(0, consecutiveFailures.get()); + + // Maintainer throws + maintainer.throwOnNextRun(true).run(); + assertEquals(1, consecutiveFailures.get()); + + // Maintainer recovers + maintainer.throwOnNextRun(false).run(); + assertEquals(0, consecutiveFailures.get()); + } + } diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java new file mode 100644 index 00000000000..5eae643fe40 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/TestMaintainer.java @@ -0,0 +1,48 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent.maintenance; + +import java.time.Duration; + +/** + * @author mpolden + */ +class TestMaintainer extends Maintainer { + + private int totalRuns = 0; + private boolean success = true; + private boolean throwing = false; + + public TestMaintainer(String name, JobControl jobControl, JobMetrics jobMetrics) { + super(name, Duration.ofDays(1), Duration.ofDays(1), jobControl, jobMetrics); + } + + public TestMaintainer(JobMetrics jobMetrics) { + this(null, new JobControl(new JobControlStateMock()), jobMetrics); + } + + public TestMaintainer(String name, JobControl jobControl) { + this(name, jobControl, new JobMetrics((job, instant) -> {})); + } + + public int totalRuns() { + return totalRuns; + } + + public TestMaintainer successOnNextRun(boolean success) { + this.success = success; + return this; + } + + public TestMaintainer throwOnNextRun(boolean throwing) { + this.throwing = throwing; + return this; + } + + @Override + protected boolean maintain() { + if (throwing) throw new RuntimeException("Maintenance run failed"); + totalRuns++; + return success; + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/geo/OneDegreeParserTestCase.java b/vespajlib/src/test/java/com/yahoo/geo/OneDegreeParserTestCase.java new file mode 100644 index 00000000000..b0da6a0a131 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/geo/OneDegreeParserTestCase.java @@ -0,0 +1,204 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.geo; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests for the OneDegreeParser class. + * + * @author arnej27959 + */ +public class OneDegreeParserTestCase { + + private static final double delta = 0.000000000001; + + private OneDegreeParser parser; + + private void checkLat(boolean assumeLatitude, String toParse, double expected) { + parser = new OneDegreeParser(assumeLatitude, toParse); + assertEquals(expected, parser.latitude, delta); + assertTrue(parser.foundLatitude); + assertFalse(parser.foundLongitude); + } + private void checkLon(boolean assumeLatitude, String toParse, double expected) { + parser = new OneDegreeParser(assumeLatitude, toParse); + assertEquals(expected, parser.longitude, delta); + assertFalse(parser.foundLatitude); + assertTrue(parser.foundLongitude); + } + private void checkLat(String toParse, double expected) { + checkLat(true, toParse, expected); + checkLat(false, toParse, expected); + } + private void checkLon(String toParse, double expected) { + checkLon(true, toParse, expected); + checkLon(false, toParse, expected); + } + + private void checkZeroLat(boolean assumeLatitude, String toParse) { + checkLat(assumeLatitude, toParse, 0d); + } + + private void checkZeroLon(boolean assumeLatitude, String toParse) { + checkLon(assumeLatitude, toParse, 0d); + } + + /** + * Tests different inputs that should all produce 0 or -0. + */ + @Test + public void testZero() { + checkZeroLat(true, "0"); + checkZeroLat(true, "0.0"); + checkZeroLat(true, "0o0.0"); + checkZeroLat(true, "0o0'0"); + checkZeroLat(true, "0\u00B00'0"); + + checkZeroLon(false, "0"); + checkZeroLon(false, "0.0"); + checkZeroLon(false, "0o0.0"); + checkZeroLon(false, "0o0'0"); + checkZeroLon(false, "0\u00B00'0"); + + checkZeroLat(false, "N0"); + checkZeroLat(false, "N0.0"); + checkZeroLat(false, "N0\u00B00'0"); + checkZeroLat(false, "S0"); + checkZeroLat(false, "S0.0"); + checkZeroLat(false, "S0o0'0"); + checkZeroLat(false, "S0\u00B00'0"); + + checkZeroLon(true, "E0"); + checkZeroLon(true, "E0.0"); + checkZeroLon(true, "E0\u00B00'0"); + checkZeroLon(true, "W0"); + checkZeroLon(true, "W0.0"); + checkZeroLon(true, "W0o0'0"); + checkZeroLon(true, "W0\u00B00'0"); + } + + /** + * Tests inputs that are close to 0. + */ + @Test + public void testNearZero() { + checkLat("N0.0001", 0.0001); + checkLat("S0.0001", -0.0001); + checkLon("E0.0001", 0.0001); + checkLon("W0.0001", -0.0001); + + checkLat("N0.000001", 0.000001); + checkLat("S0.000001", -0.000001); + checkLon("E0.000001", 0.000001); + checkLon("W0.000001", -0.000001); + + checkLat("N0\u00B00'1", 1/3600d); + checkLat("S0\u00B00'1", -1/3600d); + checkLon("E0\u00B00'1", 1/3600d); + checkLon("W0\u00B00'1", -1/3600d); + } + + /** + * Tests inputs that are close to latitude 90/-90 degrees and longitude 180/-180 degrees. + */ + @Test + public void testNearBoundary() { + checkLat("N89.9999", 89.9999); + checkLat("S89.9999", -89.9999); + checkLon("E179.9999", 179.9999); + checkLon("W179.9999", -179.9999); + + checkLat("N89.999999", 89.999999); + checkLat("S89.999999", -89.999999); + checkLon("E179.999999", 179.999999); + checkLon("W179.999999", -179.999999); + + checkLat("N89\u00B059'59", 89+59/60d+59/3600d); + checkLat("S89\u00B059'59", -(89+59/60d+59/3600d)); + checkLon("E179\u00B059'59", 179+59/60d+59/3600d); + checkLon("W179\u00B059'59", -(179+59/60d+59/3600d)); + } + + /** + * Tests inputs that are on latitude 90/-90 degrees and longitude 180/-180 degrees. + */ + @Test + public void testOnBoundary() { + checkLat("N90", 90d); + checkLat("N90\u00B00'0", 90d); + checkLat("S90", -90d); + checkLat("S90\u00B00'0", -90d); + + checkLon("E180", 180d); + checkLon("E180\u00B00'0", 180d); + checkLon("W180", -180d); + checkLon("W180\u00B00'0", -180d); + } + + private String parseException(boolean assumeLatitude, String toParse) { + String message = ""; + try { + parser = new OneDegreeParser(assumeLatitude, toParse); + assertTrue(false); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + return message; + } + + /** + * Tests inputs that are above latitude 90/-90 degrees and longitude 180/-180 degrees. + */ + @Test + public void testAboveBoundary() { + String message = parseException(false, "N90.0001"); + assertEquals("out of range [-90,+90]: 90.0001 when parsing <N90.0001>", message); + message = parseException(false, "S90.0001"); + assertEquals("out of range [-90,+90]: -90.0001 when parsing <S90.0001>", message); + message = parseException(true, "E180.0001"); + assertEquals("out of range [-180,+180]: 180.0001 when parsing <E180.0001>", message); + message = parseException(true, "W180.0001"); + assertEquals("out of range [-180,+180]: -180.0001 when parsing <W180.0001>", message); + message = parseException(false, "N90.000001"); + assertEquals("out of range [-90,+90]: 90.000001 when parsing <N90.000001>", message); + message = parseException(false, "S90.000001"); + assertEquals("out of range [-90,+90]: -90.000001 when parsing <S90.000001>", message); + message = parseException(true, "E180.000001"); + assertEquals("out of range [-180,+180]: 180.000001 when parsing <E180.000001>", message); + message = parseException(true, "W180.000001"); + assertEquals("out of range [-180,+180]: -180.000001 when parsing <W180.000001>", message); + } + + /** + * Tests various inputs that contain syntax errors. + */ + @Test + public void testInputErrors() { + String message = parseException(false, "N90S90"); + assertEquals("already set direction once, cannot add direction: S when parsing <N90S90>", message); + message = parseException(false, "E120W120"); + assertEquals("already set direction once, cannot add direction: W when parsing <E120W120>", message); + message = parseException(false, "E"); + assertEquals("end of field without any number seen when parsing <E>", message); + message = parseException(false, ""); + assertEquals("end of field without any number seen when parsing <>", message); + message = parseException(false, "NW25"); + assertEquals("already set direction once, cannot add direction: W when parsing <NW25>", message); + message = parseException(false, "N16.25\u00B0"); + assertEquals("cannot have fractional degrees before degrees sign when parsing <N16.25\u00B0>", message); + message = parseException(false, "N16\u00B022.40'"); + assertEquals("cannot have fractional minutes before minutes sign when parsing <N16\u00B022.40'>", message); + message = parseException(false, ""); + assertEquals("end of field without any number seen when parsing <>", message); + message = parseException(false, "Yahoo!"); + assertEquals("invalid character: Y when parsing <Yahoo!>", message); + message = parseException(false, "N63O025.105"); + assertEquals("invalid character: O when parsing <N63O025.105>", message); + } + +} diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 1ca9816a921..c894a798b91 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -97,7 +97,7 @@ vespa_define_module( src/tests/sharedptr src/tests/signalhandler src/tests/simple_thread_bundle - src/tests/slaveproc + src/tests/child_process src/tests/slime src/tests/slime/external_data_value src/tests/slime/summary-feature-benchmark diff --git a/vespalib/src/tests/assert/assert_test.cpp b/vespalib/src/tests/assert/assert_test.cpp index 860f56304ff..8b770f0f3eb 100644 --- a/vespalib/src/tests/assert/assert_test.cpp +++ b/vespalib/src/tests/assert/assert_test.cpp @@ -1,6 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/assert.h> #include <vespa/vespalib/io/fileutil.h> @@ -16,19 +16,19 @@ TEST("that it borks the first time.") { vespalib::rmdir("var", true); ASSERT_TRUE(vespalib::mkdir(assertDir, true)); { - SlaveProc proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); + ChildProcess proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); proc.wait(); ASSERT_EQUAL(proc.getExitCode() & 0x7f, 6); } { - SlaveProc proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); + ChildProcess proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); proc.readLine(assertName); proc.wait(); ASSERT_EQUAL(proc.getExitCode() & 0x7f, 0); } ASSERT_EQUAL(0, unlink(assertName.c_str())); { - SlaveProc proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); + ChildProcess proc("ulimit -c 0 && exec env VESPA_HOME=./ ./vespalib_asserter_app myassert 10000"); proc.wait(); ASSERT_EQUAL(proc.getExitCode() & 0x7f, 6); } diff --git a/vespalib/src/tests/child_process/.gitignore b/vespalib/src/tests/child_process/.gitignore new file mode 100644 index 00000000000..7e094c772a6 --- /dev/null +++ b/vespalib/src/tests/child_process/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +child_process_test +vespalib_child_process_test_app diff --git a/vespalib/src/tests/child_process/CMakeLists.txt b/vespalib/src/tests/child_process/CMakeLists.txt new file mode 100644 index 00000000000..1c9af730510 --- /dev/null +++ b/vespalib/src/tests/child_process/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_child_process_test_app TEST + SOURCES + child_process_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_child_process_test_app COMMAND vespalib_child_process_test_app) diff --git a/vespalib/src/tests/slaveproc/slaveproc_test.cpp b/vespalib/src/tests/child_process/child_process_test.cpp index 547da991211..ada29cd370d 100644 --- a/vespalib/src/tests/slaveproc/slaveproc_test.cpp +++ b/vespalib/src/tests/child_process/child_process_test.cpp @@ -1,57 +1,57 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> -using vespalib::SlaveProc; +using vespalib::ChildProcess; TEST("simple run, ignore output") { - EXPECT_TRUE(SlaveProc::run("echo foo")); + EXPECT_TRUE(ChildProcess::run("echo foo")); } TEST("simple run, ignore output, failure") { - EXPECT_TRUE(!SlaveProc::run("false")); + EXPECT_TRUE(!ChildProcess::run("false")); } TEST("simple run, ignore output, timeout") { - EXPECT_TRUE(!SlaveProc::run("exec sleep 60", 10)); + EXPECT_TRUE(!ChildProcess::run("exec sleep 60", 10)); } TEST("simple run") { std::string out; - EXPECT_TRUE(SlaveProc::run("/bin/echo -n foo", out)); + EXPECT_TRUE(ChildProcess::run("/bin/echo -n foo", out)); EXPECT_EQUAL(out, "foo"); } TEST("simple run, strip single-line trailing newline") { std::string out; - EXPECT_TRUE(SlaveProc::run("echo foo", out)); + EXPECT_TRUE(ChildProcess::run("echo foo", out)); EXPECT_EQUAL(out, "foo"); } TEST("simple run, don't strip multi-line output") { std::string out; - EXPECT_TRUE(SlaveProc::run("perl -e 'print \"foo\\n\\n\"'", out)); + EXPECT_TRUE(ChildProcess::run("perl -e 'print \"foo\\n\\n\"'", out)); EXPECT_EQUAL(out, "foo\n\n"); } TEST("simple run with input") { std::string in = "bar"; std::string out; - EXPECT_TRUE(SlaveProc::run(in, "cat", out)); + EXPECT_TRUE(ChildProcess::run(in, "cat", out)); EXPECT_EQUAL(out, "bar"); } TEST("simple run with input, strip single-line trailing newline") { std::string in = "bar\n"; std::string out; - EXPECT_TRUE(SlaveProc::run(in, "cat", out)); + EXPECT_TRUE(ChildProcess::run(in, "cat", out)); EXPECT_EQUAL(out, "bar"); } TEST("simple run with input, don't strip multi-line output") { std::string in = "bar\n\n"; std::string out; - EXPECT_TRUE(SlaveProc::run(in, "cat", out)); + EXPECT_TRUE(ChildProcess::run(in, "cat", out)); EXPECT_EQUAL("bar\n\n", out); } @@ -64,11 +64,11 @@ TEST_MT("simple run, partial output due to timeout", 2) { (thread_id == 0) ? "out" : "", timeout); if (thread_id == 0) { out.clear(); - EXPECT_TRUE(!SlaveProc::run(my_cmd, out, timeout)); + EXPECT_TRUE(!ChildProcess::run(my_cmd, out, timeout)); } else { out.clear(); std::string in = "ignored\n"; - EXPECT_TRUE(!SlaveProc::run(in, my_cmd, out, timeout)); + EXPECT_TRUE(!ChildProcess::run(in, my_cmd, out, timeout)); } if (out == "foo") { break; @@ -78,7 +78,7 @@ TEST_MT("simple run, partial output due to timeout", 2) { } TEST("proc failure") { - SlaveProc proc("false"); + ChildProcess proc("false"); // read with length 0 will wait for output EXPECT_TRUE(proc.read(NULL, 0) == 0); EXPECT_TRUE(proc.wait(60000)); @@ -90,7 +90,7 @@ TEST("basic read/write") { int x; int read; char buf[64]; - SlaveProc proc("cat"); + ChildProcess proc("cat"); EXPECT_TRUE(proc.running()); EXPECT_TRUE(!proc.failed()); @@ -117,7 +117,7 @@ TEST("basic read/write") { TEST("continuos run, readLine") { std::string str; - SlaveProc proc("cat"); + ChildProcess proc("cat"); EXPECT_TRUE(proc.running()); EXPECT_TRUE(!proc.failed()); @@ -142,7 +142,7 @@ TEST("continuos run, readLine") { TEST("readLine, eof flushes last line") { std::string str; - SlaveProc proc("cat"); + ChildProcess proc("cat"); EXPECT_TRUE(proc.running()); EXPECT_TRUE(!proc.failed()); @@ -166,7 +166,7 @@ TEST("readLine, eof flushes last line") { TEST("long continuos run, readLine") { std::string in; std::string out; - SlaveProc proc("cat"); + ChildProcess proc("cat"); EXPECT_TRUE(proc.running()); EXPECT_TRUE(!proc.failed()); diff --git a/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp b/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp index 63defa58c41..787e46cfd91 100644 --- a/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp +++ b/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp @@ -1,23 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> -using vespalib::SlaveProc; +using vespalib::ChildProcess; TEST("no arguments") { - SlaveProc drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache"); + ChildProcess drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache"); drop.wait(); EXPECT_EQUAL(1, drop.getExitCode()); } TEST("file does not exist") { - SlaveProc drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache not_exist"); + ChildProcess drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache not_exist"); drop.wait(); EXPECT_EQUAL(2, drop.getExitCode()); } TEST("All is well") { - SlaveProc drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache vespalib_drop_file_from_cache_test_app"); + ChildProcess drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache vespalib_drop_file_from_cache_test_app"); drop.wait(); EXPECT_EQUAL(0, drop.getExitCode()); } diff --git a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp index 8d81d9e0821..89030a19cd6 100644 --- a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp +++ b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/exception.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> using namespace vespalib; @@ -14,32 +14,32 @@ using namespace vespalib; #endif TEST("that uncaught exception causes negative exitcode.") { - SlaveProc proc("ulimit -c 0 && exec ./vespalib_caught_uncaught_app uncaught"); + ChildProcess proc("ulimit -c 0 && exec ./vespalib_caught_uncaught_app uncaught"); proc.wait(); EXPECT_LESS(proc.getExitCode(), 0); } TEST("that uncaught silenced exception causes exitcode 66") { - SlaveProc proc("exec ./vespalib_caught_uncaught_app silenced_and_uncaught"); + ChildProcess proc("exec ./vespalib_caught_uncaught_app silenced_and_uncaught"); proc.wait(); EXPECT_EQUAL(proc.getExitCode(), 66); } TEST("that caught silenced exception followed by an uncaught causes negative exitcode.") { - SlaveProc proc("ulimit -c 0 && exec ./vespalib_caught_uncaught_app uncaught_after_silenced_and_caught"); + ChildProcess proc("ulimit -c 0 && exec ./vespalib_caught_uncaught_app uncaught_after_silenced_and_caught"); proc.wait(); EXPECT_LESS(proc.getExitCode(), 0); } TEST("that caught silenced exception causes exitcode 0") { - SlaveProc proc("exec ./vespalib_caught_uncaught_app silenced_and_caught"); + ChildProcess proc("exec ./vespalib_caught_uncaught_app silenced_and_caught"); proc.wait(); EXPECT_EQUAL(proc.getExitCode(), 0); } #ifndef __SANITIZE_ADDRESS__ TEST("that mmap within limits are fine cause exitcode 0") { - SlaveProc proc("exec ./vespalib_mmap_app 150000000 10485760 1"); + ChildProcess proc("exec ./vespalib_mmap_app 150000000 10485760 1"); proc.wait(); EXPECT_EQUAL(proc.getExitCode(), 0); } @@ -48,13 +48,13 @@ TEST("that mmap within limits are fine cause exitcode 0") { // setrlimit with RLIMIT_AS is broken on Darwin #else TEST("that mmap beyond limits cause negative exitcode.") { - SlaveProc proc("ulimit -c 0 && exec ./vespalib_mmap_app 100000000 10485760 10"); + ChildProcess proc("ulimit -c 0 && exec ./vespalib_mmap_app 100000000 10485760 10"); proc.wait(); EXPECT_LESS(proc.getExitCode(), 0); } TEST("that mmap beyond limits with set VESPA_SILENCE_CORE_ON_OOM cause exitcode 66.") { - SlaveProc proc("VESPA_SILENCE_CORE_ON_OOM=1 exec ./vespalib_mmap_app 100000000 10485760 10"); + ChildProcess proc("VESPA_SILENCE_CORE_ON_OOM=1 exec ./vespalib_mmap_app 100000000 10485760 10"); proc.wait(); EXPECT_EQUAL(proc.getExitCode(), 66); } diff --git a/vespalib/src/tests/host_name/host_name_test.cpp b/vespalib/src/tests/host_name/host_name_test.cpp index 4e0c59d836d..442da2ac2e8 100644 --- a/vespalib/src/tests/host_name/host_name_test.cpp +++ b/vespalib/src/tests/host_name/host_name_test.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/host_name.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> using namespace vespalib; diff --git a/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp b/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp index a7509014b00..db7a51222b7 100644 --- a/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp +++ b/vespalib/src/tests/make_fixture_macros/make_fixture_macros_test.cpp @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> using namespace vespalib; bool runPrint(const char *cmd) { std::string out; - bool res = SlaveProc::run(cmd, out); + bool res = ChildProcess::run(cmd, out); fprintf(stderr, "%s", out.c_str()); return res; } diff --git a/vespalib/src/tests/slaveproc/.gitignore b/vespalib/src/tests/slaveproc/.gitignore deleted file mode 100644 index 8851e99df13..00000000000 --- a/vespalib/src/tests/slaveproc/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -slaveproc_test -vespalib_slaveproc_test_app diff --git a/vespalib/src/tests/slaveproc/CMakeLists.txt b/vespalib/src/tests/slaveproc/CMakeLists.txt deleted file mode 100644 index 58211d671bc..00000000000 --- a/vespalib/src/tests/slaveproc/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespalib_slaveproc_test_app TEST - SOURCES - slaveproc_test.cpp - DEPENDS - vespalib -) -vespa_add_test(NAME vespalib_slaveproc_test_app COMMAND vespalib_slaveproc_test_app) diff --git a/vespalib/src/tests/tutorial/make_tutorial.cpp b/vespalib/src/tests/tutorial/make_tutorial.cpp index 4890b0db62a..de0eb585e3c 100644 --- a/vespalib/src/tests/tutorial/make_tutorial.cpp +++ b/vespalib/src/tests/tutorial/make_tutorial.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> #include <vespa/vespalib/util/stringfmt.h> #include <sys/mman.h> #include <sys/stat.h> @@ -20,7 +20,7 @@ std::string readFile(const std::string &filename) { std::string runCommand(const std::string &cmd) { std::string out; - ASSERT_TRUE(SlaveProc::run(cmd.c_str(), out)); + ASSERT_TRUE(ChildProcess::run(cmd.c_str(), out)); return out; } diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 4029c4881c4..0973653861f 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -46,7 +46,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT sig_catch.cpp signalhandler.cpp simple_thread_bundle.cpp - slaveproc.cpp + child_process.cpp stash.cpp string_hash.cpp stringfmt.cpp diff --git a/vespalib/src/vespa/vespalib/util/slaveproc.cpp b/vespalib/src/vespa/vespalib/util/child_process.cpp index 6f40aa9a4d7..ce0c2eb1779 100644 --- a/vespalib/src/vespa/vespalib/util/slaveproc.cpp +++ b/vespalib/src/vespa/vespalib/util/child_process.cpp @@ -1,17 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "guard.h" -#include "slaveproc.h" +#include "child_process.h" #include <cstring> namespace vespalib { -namespace slaveproc { +namespace child_process { using namespace std::chrono; /** - * @brief SlaveProc internal timeout management. + * @brief ChildProcess internal timeout management. **/ class Timer { @@ -54,14 +54,14 @@ public: } }; -} // namespace slaveproc +} // namespace child_process -using slaveproc::Timer; +using child_process::Timer; //----------------------------------------------------------------------------- void -SlaveProc::Reader::OnReceiveData(const void *data, size_t length) +ChildProcess::Reader::OnReceiveData(const void *data, size_t length) { const char *buf = (const char *) data; MonitorGuard lock(_cond); @@ -80,7 +80,7 @@ SlaveProc::Reader::OnReceiveData(const void *data, size_t length) bool -SlaveProc::Reader::hasData() +ChildProcess::Reader::hasData() { // NB: caller has lock on _cond return (!_data.empty() || !_queue.empty()); @@ -88,7 +88,7 @@ SlaveProc::Reader::hasData() bool -SlaveProc::Reader::waitForData(Timer &timer, MonitorGuard &lock) +ChildProcess::Reader::waitForData(Timer &timer, MonitorGuard &lock) { // NB: caller has lock on _cond CounterGuard count(_waitCnt); @@ -100,7 +100,7 @@ SlaveProc::Reader::waitForData(Timer &timer, MonitorGuard &lock) void -SlaveProc::Reader::updateEOF() +ChildProcess::Reader::updateEOF() { // NB: caller has lock on _cond if (_data.empty() && _queue.empty() && _gotEOF) { @@ -109,7 +109,7 @@ SlaveProc::Reader::updateEOF() } -SlaveProc::Reader::Reader() +ChildProcess::Reader::Reader() : _cond(), _queue(), _data(), @@ -120,13 +120,13 @@ SlaveProc::Reader::Reader() } -SlaveProc::Reader::~Reader() +ChildProcess::Reader::~Reader() { } uint32_t -SlaveProc::Reader::read(char *buf, uint32_t len, int msTimeout) +ChildProcess::Reader::read(char *buf, uint32_t len, int msTimeout) { if (eof()) { return 0; @@ -156,7 +156,7 @@ SlaveProc::Reader::read(char *buf, uint32_t len, int msTimeout) bool -SlaveProc::Reader::readLine(std::string &line, int msTimeout) +ChildProcess::Reader::readLine(std::string &line, int msTimeout) { line.clear(); if (eof()) { @@ -193,7 +193,7 @@ SlaveProc::Reader::readLine(std::string &line, int msTimeout) //----------------------------------------------------------------------------- void -SlaveProc::checkProc() +ChildProcess::checkProc() { if (_running) { bool stillRunning; @@ -205,7 +205,7 @@ SlaveProc::checkProc() } -SlaveProc::SlaveProc(const char *cmd) +ChildProcess::ChildProcess(const char *cmd) : _reader(), _proc(cmd, true, &_reader), _running(false), @@ -217,11 +217,11 @@ SlaveProc::SlaveProc(const char *cmd) } -SlaveProc::~SlaveProc() = default; +ChildProcess::~ChildProcess() = default; bool -SlaveProc::write(const char *buf, uint32_t len) +ChildProcess::write(const char *buf, uint32_t len) { if (len == 0) { return true; @@ -231,28 +231,28 @@ SlaveProc::write(const char *buf, uint32_t len) bool -SlaveProc::close() +ChildProcess::close() { return _proc.WriteStdin(nullptr, 0); } uint32_t -SlaveProc::read(char *buf, uint32_t len, int msTimeout) +ChildProcess::read(char *buf, uint32_t len, int msTimeout) { return _reader.read(buf, len, msTimeout); } bool -SlaveProc::readLine(std::string &line, int msTimeout) +ChildProcess::readLine(std::string &line, int msTimeout) { return _reader.readLine(line, msTimeout); } bool -SlaveProc::wait(int msTimeout) +ChildProcess::wait(int msTimeout) { bool done = true; checkProc(); @@ -273,7 +273,7 @@ SlaveProc::wait(int msTimeout) bool -SlaveProc::running() +ChildProcess::running() { checkProc(); return _running; @@ -281,24 +281,24 @@ SlaveProc::running() bool -SlaveProc::failed() +ChildProcess::failed() { checkProc(); return _failed; } int -SlaveProc::getExitCode() +ChildProcess::getExitCode() { return _exitCode; } bool -SlaveProc::run(const std::string &input, const char *cmd, +ChildProcess::run(const std::string &input, const char *cmd, std::string &output, int msTimeout) { - SlaveProc proc(cmd); + ChildProcess proc(cmd); Timer timer(msTimeout); char buf[4096]; proc.write(input.data(), input.length()); @@ -317,7 +317,7 @@ SlaveProc::run(const std::string &input, const char *cmd, bool -SlaveProc::run(const char *cmd, std::string &output, int msTimeout) +ChildProcess::run(const char *cmd, std::string &output, int msTimeout) { std::string input; // empty input return run(input, cmd, output, msTimeout); @@ -325,7 +325,7 @@ SlaveProc::run(const char *cmd, std::string &output, int msTimeout) bool -SlaveProc::run(const char *cmd, int msTimeout) +ChildProcess::run(const char *cmd, int msTimeout) { std::string input; // empty input std::string output; // ignore output diff --git a/vespalib/src/vespa/vespalib/util/slaveproc.h b/vespalib/src/vespa/vespalib/util/child_process.h index c08a13f0b1d..0ae7206ac48 100644 --- a/vespalib/src/vespa/vespalib/util/slaveproc.h +++ b/vespalib/src/vespa/vespalib/util/child_process.h @@ -8,17 +8,17 @@ #include <queue> #include "sync.h" -namespace vespalib::slaveproc { class Timer; } +namespace vespalib::child_process { class Timer; } namespace vespalib { /** - * @brief Slave Process utility class for running external programs + * @brief Child Process utility class for running external programs * * Designed for use in unit tests and other places * where you need to run, control and communicate with * some external program. **/ -class SlaveProc +class ChildProcess { private: class Reader : public FastOS_ProcessRedirectListener @@ -33,7 +33,7 @@ private: void OnReceiveData(const void *data, size_t length) override; bool hasData(); - bool waitForData(slaveproc::Timer &timer, MonitorGuard &lock); + bool waitForData(child_process::Timer &timer, MonitorGuard &lock); void updateEOF(); public: @@ -54,19 +54,19 @@ private: void checkProc(); public: - SlaveProc(const SlaveProc &) = delete; - SlaveProc &operator=(const SlaveProc &) = delete; + ChildProcess(const ChildProcess &) = delete; + ChildProcess &operator=(const ChildProcess &) = delete; /** - * @brief Run a slave process + * @brief Run a child process * * Starts a process running the given command * @param cmd A shell command line to run **/ - explicit SlaveProc(const char *cmd); + explicit ChildProcess(const char *cmd); /** @brief destructor doing cleanup if needed */ - ~SlaveProc(); + ~ChildProcess(); /** * @return process id diff --git a/vespamalloc/src/tests/doubledelete/expectsignal.cpp b/vespamalloc/src/tests/doubledelete/expectsignal.cpp index 9ac93da014d..a0f6561889a 100644 --- a/vespamalloc/src/tests/doubledelete/expectsignal.cpp +++ b/vespamalloc/src/tests/doubledelete/expectsignal.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> #include <sys/wait.h> using namespace vespalib; @@ -25,7 +25,7 @@ int Test::Main() fprintf(stderr, "argc=%d : Running '%s' expecting signal %d\n", _argc, _argv[2], retval); - SlaveProc cmd(_argv[2]); + ChildProcess cmd(_argv[2]); for(std::string line; cmd.readLine(line, 60000);) { fprintf(stdout, "%s\n", line.c_str()); } diff --git a/vespamalloc/src/tests/overwrite/expectsignal.cpp b/vespamalloc/src/tests/overwrite/expectsignal.cpp index e78f1b7b181..2b37499f85d 100644 --- a/vespamalloc/src/tests/overwrite/expectsignal.cpp +++ b/vespamalloc/src/tests/overwrite/expectsignal.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/util/child_process.h> #include <sys/wait.h> using namespace vespalib; @@ -24,7 +24,7 @@ int Test::Main() fprintf(stderr, "argc=%d : Running '%s' expecting signal %d\n", _argc, _argv[2], retval); - SlaveProc cmd(_argv[2]); + ChildProcess cmd(_argv[2]); for(std::string line; cmd.readLine(line, 60000);) { fprintf(stdout, "%s\n", line.c_str()); } diff --git a/vsm/src/vespa/vsm/vsm/docsumconfig.cpp b/vsm/src/vespa/vsm/vsm/docsumconfig.cpp index 2512bea26df..fb25df83ded 100644 --- a/vsm/src/vespa/vsm/vsm/docsumconfig.cpp +++ b/vsm/src/vespa/vsm/vsm/docsumconfig.cpp @@ -26,6 +26,9 @@ void populate_fields(MatchingElementsFields& fields, VsmfieldsConfig& fields_con if (spec.name.substr(0, prefix.size()) == prefix) { fields.add_mapping(field_name, spec.name); } + if (spec.name == field_name) { + fields.add_field(field_name); + } } } diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp index 5d8c7735c0e..8307954faae 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -38,11 +38,6 @@ void GetDocsumsStateCallback::FillRankFeatures(GetDocsumsState * state, IDocsumE } } -void GetDocsumsStateCallback::ParseLocation(GetDocsumsState *state) -{ - (void) state; -} - void GetDocsumsStateCallback::FillDocumentLocations(GetDocsumsState *state, IDocsumEnvironment * env) { (void) state; diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.h b/vsm/src/vespa/vsm/vsm/vsm-adapter.h index cffae318586..31e472713de 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.h +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.h @@ -40,7 +40,6 @@ public: GetDocsumsStateCallback(); void FillSummaryFeatures(GetDocsumsState * state, IDocsumEnvironment * env) override; void FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) override; - void ParseLocation(GetDocsumsState * state) override; virtual void FillDocumentLocations(GetDocsumsState * state, IDocsumEnvironment * env); virtual std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields& fields) override; void setSummaryFeatures(const search::FeatureSet::SP & sf) { _summaryFeatures = sf; } |