diff options
author | Harald Musum <musum@verizonmedia.com> | 2020-07-29 13:00:27 +0200 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2020-07-29 13:00:27 +0200 |
commit | 8c5b3ff1f9345c73a3763d5adefe672859e39ef4 (patch) | |
tree | 6c076ab26dbd882e779c22a085a7aa6d6820a0d3 /config-model | |
parent | ce2d05cb3c104a4d7147b72720aae317b0cc88b5 (diff) | |
parent | fb30d3667ab506976c776817aa3451f97ceed83a (diff) |
Merge branch 'master' into hmusum/less-hardcoding-in-mock
Diffstat (limited to 'config-model')
32 files changed, 474 insertions, 233 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java index 0d0da71bd0f..7f1a9933188 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java @@ -6,6 +6,7 @@ import com.yahoo.document.ReferenceDataType; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -30,10 +31,24 @@ public class DocumentReferenceResolver { } public void resolveReferences(SDDocumentType documentType) { - DocumentReferences references = new DocumentReferences(createFieldToDocumentReferenceMapping(documentType)); + var references = new DocumentReferences(createFieldToDocumentReferenceMapping(documentType)); documentType.setDocumentReferences(references); } + public void resolveInheritedReferences(SDDocumentType documentType) { + resolveInheritedReferencesRecursive(documentType, documentType.getInheritedTypes()); + } + + private void resolveInheritedReferencesRecursive(SDDocumentType documentType, + Collection<SDDocumentType> inheritedTypes) { + for (var inheritedType : inheritedTypes) { + documentType.getDocumentReferences().get().mergeFrom(inheritedType.getDocumentReferences().get()); + } + for (var inheritedType : inheritedTypes) { + resolveInheritedReferencesRecursive(documentType, inheritedType.getInheritedTypes()); + } + } + private Map<String, DocumentReference> createFieldToDocumentReferenceMapping(SDDocumentType documentType) { return fieldStream(documentType) .filter(field -> field.getDataType() instanceof ReferenceDataType) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java index 37f5ab1bbde..4a3995b2d40 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java @@ -18,6 +18,10 @@ public class DocumentReferences implements Iterable<Map.Entry<String, DocumentRe this.references = references; } + public void mergeFrom(DocumentReferences other) { + references.putAll(other.references); + } + @Override public Iterator<Map.Entry<String, DocumentReference>> iterator() { return Collections.unmodifiableSet(references.entrySet()).iterator(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index eb68e6af203..1fab30f9ea4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -239,6 +239,7 @@ public class SearchBuilder { var resolver = new DocumentReferenceResolver(searchList); sdocs.forEach(resolver::resolveReferences); + sdocs.forEach(resolver::resolveInheritedReferences); var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList); sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java index d86ed265b77..77064038053 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java @@ -48,11 +48,13 @@ public class MatchedElementsOnlyResolver extends Processor { if (isComplexFieldWithOnlyStructFieldAttributes(sourceField)) { field.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); } - } else if (isSupportedAttributeField(sourceField)) { - field.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } else if (isSupportedMultiValueField(sourceField)) { + if (sourceField.doesAttributing()) { + field.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER); + } } else if (validate) { fail(summary, field, "'matched-elements-only' is not supported for this field type. " + - "Supported field types are: array attribute, weighted set attribute, " + + "Supported field types are: array of primitive, weighted set of primitive, " + "array of simple struct, map of primitive type to simple struct, " + "and map of primitive type to primitive type"); } @@ -60,10 +62,9 @@ public class MatchedElementsOnlyResolver extends Processor { // else case is handled in SummaryFieldsMustHaveValidSource } - private boolean isSupportedAttributeField(ImmutableSDField sourceField) { + private boolean isSupportedMultiValueField(ImmutableSDField sourceField) { var type = sourceField.getDataType(); - return sourceField.doesAttributing() && - (isArrayOfPrimitiveType(type) || isWeightedsetOfPrimitiveType(type)); + return (isArrayOfPrimitiveType(type) || isWeightedsetOfPrimitiveType(type)); } private boolean isArrayOfPrimitiveType(DataType type) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index f7a04c36da0..0192e9d42c6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -66,12 +66,12 @@ public class HostSystem extends AbstractConfigProducer<Host> { * @return the host with the given hostname, or null if no such host */ public HostResource getHostByHostname(String name) { - System.out.println("Getting name=" + name + " all hosts: " + hostname2host); + String localhost = "localhost"; HostResource hostResource = hostname2host.get(name); if (hostResource == null) { - // Create a new HostResource if this is the host this code is running on (as when running tests) - if (HostName.getLocalhost().equals(name)) { - String localhost = "localhost"; + // Create a new HostResource if this is the host this code is running on (as it is when running tests) + // TODO: please eliminate the ugly hack using "localhost.fortestingpurposesonly" + if (HostName.getLocalhost().equals(name) || "localhost.fortestingpurposesonly".equals(name)) { if (! getChildren().containsKey(localhost)) { new Host(this, localhost); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 7484e0cd9a0..1b5be1c2f97 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -264,7 +264,8 @@ public class Admin extends AbstractConfigProducer implements Serializable { FileDistributor fileDistributor = fileDistribution.getFileDistributor(); HostResource hostResource = hostSystem().getHostByHostname(fileDistributor.fileSourceHost()); if (hostResource == null && ! multitenant) - throw new IllegalArgumentException("Could not find " + host + " in the application's " + hostSystem()); + throw new IllegalArgumentException("Could not find " + fileDistributor.fileSourceHost() + + " in the application's " + hostSystem()); FileDistributionConfigProvider configProvider = new FileDistributionConfigProvider(fileDistribution, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 24e88d7ef7d..f9338f9cb35 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -31,7 +31,8 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain @Override public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); - builder.jvm.heapsize(384); + builder.jvm.heapsize(384) + .verbosegc(true); } protected boolean messageBusEnabled() { return false; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 547c05d2c9b..08f4e2fa12f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -5,8 +5,8 @@ import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.container.BundlesConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.model.application.validation.RestartConfigs; @@ -14,18 +14,17 @@ import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.xml.PlatformBundles; import java.util.Set; import java.util.TreeSet; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - /** * Container implementation for cluster-controllers */ @RestartConfigs({FleetcontrollerConfig.class, ZookeeperServerConfig.class}) public class ClusterControllerContainer extends Container implements - BundlesConfig.Producer, + PlatformBundlesConfig.Producer, ZookeeperServerConfig.Producer { private static final ComponentSpecification CLUSTERCONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-apps"); @@ -55,11 +54,12 @@ public class ClusterControllerContainer extends Container implements } addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, "controller", isHosted)); - addFileBundle("lib/jars/clustercontroller-apps-jar-with-dependencies.jar"); - addFileBundle("lib/jars/clustercontroller-apputil-jar-with-dependencies.jar"); - addFileBundle("lib/jars/clustercontroller-core-jar-with-dependencies.jar"); - addFileBundle("lib/jars/clustercontroller-utils-jar-with-dependencies.jar"); - addFileBundle("lib/jars/zookeeper-server-jar-with-dependencies.jar"); + // TODO: Why are bundles added here instead of in the cluster? + addFileBundle("clustercontroller-apps"); + addFileBundle("clustercontroller-apputil"); + addFileBundle("clustercontroller-core"); + addFileBundle("clustercontroller-utils"); + addFileBundle("zookeeper-server"); } @Override @@ -82,8 +82,8 @@ public class ClusterControllerContainer extends Container implements super.addHandler(h); } - private void addFileBundle(String bundlePath) { - bundles.add("file:" + getDefaults().underVespaHome(bundlePath)); + private void addFileBundle(String bundleName) { + bundles.add(PlatformBundles.absoluteBundlePath(bundleName).toString()); } private ComponentModel createComponentModel(String id, String className, ComponentSpecification bundle) { @@ -102,10 +102,8 @@ public class ClusterControllerContainer extends Container implements } @Override - public void getConfig(BundlesConfig.Builder builder) { - for (String bundle : bundles) { - builder.bundle(bundle); - } + public void getConfig(PlatformBundlesConfig.Builder builder) { + bundles.forEach(builder::bundlePaths); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 86b6ab8a25c..4dc9811a024 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -38,9 +38,9 @@ import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.xml.PlatformBundles; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -58,8 +58,6 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.getDefaultPublicConsumer; import static com.yahoo.vespa.model.admin.monitoring.MetricSet.emptyMetricSet; import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.getVespaMetricsConsumer; -import static com.yahoo.vespa.model.container.xml.BundleMapper.JarSuffix.JAR_WITH_DEPS; -import static com.yahoo.vespa.model.container.xml.BundleMapper.absoluteBundlePath; /** * Container cluster for metrics proxy containers. @@ -76,7 +74,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC public static final Logger log = Logger.getLogger(MetricsProxyContainerCluster.class.getName()); private static final String METRICS_PROXY_NAME = "metrics-proxy"; - static final Path METRICS_PROXY_BUNDLE_FILE = absoluteBundlePath((Paths.get(METRICS_PROXY_NAME + JAR_WITH_DEPS.suffix))); + static final Path METRICS_PROXY_BUNDLE_FILE = PlatformBundles.absoluteBundlePath(METRICS_PROXY_NAME); static final String METRICS_PROXY_BUNDLE_NAME = "com.yahoo.vespa." + METRICS_PROXY_NAME; static final class AppDimensionNames { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 941870e980b..20a7a6cc856 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -642,6 +642,7 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.puts.sum.failures.total.rate")); metrics.add(new Metric("vds.distributor.puts.sum.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.puts.sum.failures.test_and_set_failed.rate")); + metrics.add(new Metric("vds.distributor.puts.sum.failures.concurrent_mutations.rate")); metrics.add(new Metric("vds.distributor.removes.sum.latency.max")); metrics.add(new Metric("vds.distributor.removes.sum.latency.sum")); metrics.add(new Metric("vds.distributor.removes.sum.latency.count")); @@ -650,6 +651,7 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.removes.sum.failures.total.rate")); metrics.add(new Metric("vds.distributor.removes.sum.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.removes.sum.failures.test_and_set_failed.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.failures.concurrent_mutations.rate")); metrics.add(new Metric("vds.distributor.updates.sum.latency.max")); metrics.add(new Metric("vds.distributor.updates.sum.latency.sum")); metrics.add(new Metric("vds.distributor.updates.sum.latency.count")); @@ -658,6 +660,7 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.updates.sum.failures.total.rate")); metrics.add(new Metric("vds.distributor.updates.sum.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.updates.sum.failures.test_and_set_failed.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.failures.concurrent_mutations.rate")); metrics.add(new Metric("vds.distributor.updates.sum.diverging_timestamp_updates.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.ok.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.failures.total.rate")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 04fe77d9e05..b0ac02d0fe8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -8,8 +8,8 @@ import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.container.BundlesConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.di.config.ApplicationBundlesConfig; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.handler.metrics.MetricsProxyApiConfig; import com.yahoo.container.handler.metrics.MetricsV2Handler; @@ -21,7 +21,6 @@ import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ConfigProducerGroup; @@ -29,9 +28,9 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.Servlet; import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; import com.yahoo.vespa.model.container.jersey.RestApi; +import com.yahoo.vespa.model.container.xml.PlatformBundles; import com.yahoo.vespa.model.utils.FileSender; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; @@ -47,7 +46,7 @@ import java.util.stream.Stream; * @author gjoranv */ public final class ApplicationContainerCluster extends ContainerCluster<ApplicationContainer> implements - BundlesConfig.Producer, + ApplicationBundlesConfig.Producer, QrStartConfig.Producer, RankProfilesConfig.Producer, RankingConstantsConfig.Producer, @@ -135,11 +134,11 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private void addTestrunnerComponentsIfTester(DeployState deployState) { if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) { - addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-testrunner-components-jar-with-dependencies.jar"))); - addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-osgi-testrunner-jar-with-dependencies.jar"))); - addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/tenant-cd-api-jar-with-dependencies.jar"))); + addPlatformBundle(PlatformBundles.absoluteBundlePath("vespa-testrunner-components")); + addPlatformBundle(PlatformBundles.absoluteBundlePath("vespa-osgi-testrunner")); + addPlatformBundle(PlatformBundles.absoluteBundlePath("tenant-cd-api")); if(deployState.zone().system().isPublic()) { - addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/cloud-tenant-cd-jar-with-dependencies.jar"))); + addPlatformBundle(PlatformBundles.absoluteBundlePath("cloud-tenant-cd")); } } } @@ -189,10 +188,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public Optional<Integer> getMemoryPercentage() { return Optional.ofNullable(memoryPercentage); } @Override - public void getConfig(BundlesConfig.Builder builder) { + public void getConfig(ApplicationBundlesConfig.Builder builder) { applicationBundles.stream().map(FileReference::value) - .forEach(builder::bundle); - super.getConfig(builder); + .forEach(builder::bundles); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 5127616ad5e..240157fb7aa 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -12,12 +12,12 @@ import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.Zone; -import com.yahoo.container.BundlesConfig; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.core.document.ContainerDocumentConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; @@ -69,8 +69,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import static com.yahoo.container.core.BundleLoaderProperties.DISK_BUNDLE_PREFIX; - /** * Parent class for all container cluster types. * @@ -87,7 +85,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> ContainerDocumentConfig.Producer, HealthMonitorConfig.Producer, ApplicationMetadataConfig.Producer, - BundlesConfig.Producer, + PlatformBundlesConfig.Producer, IndexInfoConfig.Producer, IlscriptsConfig.Producer, SchemamappingConfig.Producer, @@ -464,6 +462,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> /** * Adds a bundle present at a known location at the target container nodes. + * Note that the set of platform bundles cannot change during the jdisc container's lifetime. * * @param bundlePath usually an absolute path, e.g. '$VESPA_HOME/lib/jars/foo.jar' */ @@ -472,13 +471,10 @@ public abstract class ContainerCluster<CONTAINER extends Container> } @Override - public void getConfig(BundlesConfig.Builder builder) { - platformBundles.stream() .map(ContainerCluster::toFileReferenceString) - .forEach(builder::bundle); - } - - private static String toFileReferenceString(Path path) { - return DISK_BUNDLE_PREFIX + path.toString(); + public void getConfig(PlatformBundlesConfig.Builder builder) { + platformBundles.stream() + .map(Path::toString) + .forEach(builder::bundlePaths); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 58d12663c86..51f526d5efd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -1,9 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.QrSearchersConfig; -import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; @@ -25,7 +23,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import static com.yahoo.vespa.model.container.xml.BundleMapper.searchAndDocprocBundle; +import static com.yahoo.vespa.model.container.xml.PlatformBundles.searchAndDocprocBundle; /** * @author gjoranv diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java index 284aa3b46c0..232e8fcbd1a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.xml.BundleMapper; +import com.yahoo.vespa.model.container.xml.PlatformBundles; import com.yahoo.vespa.model.search.IndexedSearchCluster; /** @@ -32,7 +32,7 @@ public class DispatcherComponent extends Component<AbstractConfigProducer<?>, Co String dispatcherComponentId = "dispatcher." + indexedSearchCluster.getClusterName(); // used by ClusterSearcher return new ComponentModel(dispatcherComponentId, com.yahoo.search.dispatch.Dispatcher.class.getName(), - BundleMapper.searchAndDocprocBundle); + PlatformBundles.searchAndDocprocBundle); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java index 2689c2ce71b..248b30eafa7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.container.search; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.xml.BundleMapper; +import com.yahoo.vespa.model.container.xml.PlatformBundles; public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent, ComponentModel> { @@ -13,6 +13,6 @@ public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent private static ComponentModel toComponentModel(String clusterName) { String componentId = "rpcresourcepool." + clusterName; - return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), BundleMapper.searchAndDocprocBundle); + return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), PlatformBundles.searchAndDocprocBundle); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index 19d1b6546a6..4e0bff1c1fc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -26,13 +26,14 @@ public class BundleInstantiationSpecificationBuilder { BundleInstantiationSpecification instSpec = new BundleInstantiationSpecification(id, classId, bundle); validate(instSpec); - return bundle == null ? setBundleForKnownClass(instSpec) : instSpec; + return bundle == null ? setBundleForSearchAndDocprocComponents(instSpec) : instSpec; } - private static BundleInstantiationSpecification setBundleForKnownClass(BundleInstantiationSpecification spec) { - return BundleMapper.getBundle(spec.getClassName()). - map(spec::inBundle). - orElse(spec); + private static BundleInstantiationSpecification setBundleForSearchAndDocprocComponents(BundleInstantiationSpecification spec) { + if (PlatformBundles.isSearchAndDocprocClass(spec.getClassName())) + return spec.inBundle(PlatformBundles.searchAndDocprocBundle); + else + return spec; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java deleted file mode 100644 index aa7cb5eb539..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.xml; - -import com.yahoo.vespa.defaults.Defaults; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -/** - * @author gjoranv - * @author Ulf Lilleengen - */ -public class BundleMapper { - - public enum JarSuffix { - JAR_WITH_DEPS("-jar-with-dependencies.jar"), - DEPLOY("-deploy.jar"); - - public final String suffix; - - JarSuffix(String suffix) { - this.suffix = suffix; - } - } - - public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); - - public static final String searchAndDocprocBundle = "container-search-and-docproc"; - - private static final Map<String, String> bundleFromClass; - private static final Map<String, Path> bundleFileFromClass; - - public static Optional<String> getBundle(String className) { - return Optional.ofNullable(bundleFromClass.get(className)); - } - - public static Optional<Path> getBundlePath(String className) { - return Optional.ofNullable(absoluteBundlePath(bundleFileFromClass.get(className))); - } - - public static Path absoluteBundlePath(Path fileName) { - if (fileName == null) return null; - return LIBRARY_PATH.resolve(fileName); - } - - /** - * TODO: This is a temporary hack to ensure that users can use our internal components without - * specifying the bundle in which the components reside. Ideally, this information - * should be generated during vespamodel build time. - * - * The container_maven_plugin has much of the logic in place, but needs to be extended. - */ - static { - bundleFromClass = new HashMap<>(); - bundleFileFromClass = new HashMap<>(); - - bundleFromClass.put("com.yahoo.docproc.AbstractConcreteDocumentFactory", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.docproc.DocumentProcessor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.docproc.SimpleDocumentProcessor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.docproc.util.JoinerDocumentProcessor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.docproc.util.SplitterDocumentProcessor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.example.TimingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.language.simple.SimpleLinguistics", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.cluster.ClusterSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.fastsearch.FastSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.fastsearch.VespaBackEndSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.CJKSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.CollapsePhraseSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.LiteralBoostSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.NoRankingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.NonPhrasingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.NormalizingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.PhrasingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.RecallSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.StemmingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.BlendingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.FieldCollapsingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.FillSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.JSONDebugSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.JuniperSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.MultipleResultsSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.PosSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.QuotingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.ValidateSortingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.semantics.SemanticSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.statistics.StatisticsSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.templates.SearchRendererAdaptor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.Searcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.cluster.ClusterSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.cluster.PingableSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.FederationSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.ForwardingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.http.ConfiguredHTTPClientSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.http.HTTPClientSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.http.HTTPProviderSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.http.HTTPSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.news.NewsSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.federation.vespa.VespaSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.grouping.GroupingQueryParser", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.grouping.GroupingValidator", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.grouping.vespa.GroupingExecutor", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.handler.SearchWithRendererHandler", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.PageTemplate", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.PageTemplateSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.engine.Resolver", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.pagetemplates.model.Renderer", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.query.rewrite.QueryRewriteSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.query.rewrite.SearchChainDispatcherSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.GenericExpansionRewriter", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.MisspellRewriter", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.NameRewriter", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.AllLowercasingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.DefaultPositionSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.LowercasingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.NGramSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.VespaLowercasingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.rendering.Renderer", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.rendering.SectionedRenderer", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.searchchain.ForkingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.searchchain.example.ExampleSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.searchers.CacheControlSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.statistics.PeakQpsSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.statistics.TimingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.vespa.streamingvisitors.MetricsSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher", searchAndDocprocBundle); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index b83632a58a0..41e092c7ea5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -154,21 +154,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ApplicationContainerCluster cluster = createContainerCluster(spec, modelContext); addClusterContent(cluster, spec, modelContext); - addBundlesForPlatformComponents(cluster); cluster.setMessageBusEnabled(rpcServerEnabled); cluster.setRpcServerEnabled(rpcServerEnabled); cluster.setHttpServerEnabled(httpServerEnabled); model.setCluster(cluster); } - private void addBundlesForPlatformComponents(ApplicationContainerCluster cluster) { - for (Component<?, ?> component : cluster.getAllComponents()) { - String componentClass = component.model.bundleInstantiationSpec.getClassName(); - BundleMapper.getBundlePath(componentClass). - ifPresent(cluster::addPlatformBundle); - } - } - private ApplicationContainerCluster createContainerCluster(Element spec, ConfigModelContext modelContext) { return new VespaDomBuilder.DomConfigProducerBuilder<ApplicationContainerCluster>() { @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/PlatformBundles.java new file mode 100644 index 00000000000..dc2437c1834 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/PlatformBundles.java @@ -0,0 +1,119 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.vespa.defaults.Defaults; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Set; + +/** + * @author gjoranv + * @author Ulf Lilleengen + */ +public class PlatformBundles { + + private enum JarSuffix { + JAR_WITH_DEPS("-jar-with-dependencies.jar"), + DEPLOY("-deploy.jar"); + + public final String suffix; + + JarSuffix(String suffix) { + this.suffix = suffix; + } + } + + public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); + public static final String searchAndDocprocBundle = "container-search-and-docproc"; + + private static final Set<String> searchAndDocprocComponents; + + public static boolean isSearchAndDocprocClass(String className) { + return searchAndDocprocComponents.contains(className); + } + + public static Path absoluteBundlePath(String fileName) { + if (fileName == null) return null; + return LIBRARY_PATH.resolve(Paths.get(fileName + JarSuffix.JAR_WITH_DEPS.suffix)); + } + + // This is a hack to allow users to declare components from the search-and-docproc bundle without naming the bundle. + static { + searchAndDocprocComponents = Set.of( + "com.yahoo.docproc.AbstractConcreteDocumentFactory", + "com.yahoo.docproc.DocumentProcessor", + "com.yahoo.docproc.SimpleDocumentProcessor", + "com.yahoo.docproc.util.JoinerDocumentProcessor", + "com.yahoo.docproc.util.SplitterDocumentProcessor", + "com.yahoo.example.TimingSearcher", + "com.yahoo.language.simple.SimpleLinguistics", + "com.yahoo.prelude.cluster.ClusterSearcher", + "com.yahoo.prelude.fastsearch.FastSearcher", + "com.yahoo.prelude.fastsearch.VespaBackEndSearcher", + "com.yahoo.prelude.querytransform.CJKSearcher", + "com.yahoo.prelude.querytransform.CollapsePhraseSearcher", + "com.yahoo.prelude.querytransform.LiteralBoostSearcher", + "com.yahoo.prelude.querytransform.NoRankingSearcher", + "com.yahoo.prelude.querytransform.NonPhrasingSearcher", + "com.yahoo.prelude.querytransform.NormalizingSearcher", + "com.yahoo.prelude.querytransform.PhrasingSearcher", + "com.yahoo.prelude.querytransform.RecallSearcher", + "com.yahoo.prelude.querytransform.StemmingSearcher", + "com.yahoo.prelude.searcher.BlendingSearcher", + "com.yahoo.prelude.searcher.FieldCollapsingSearcher", + "com.yahoo.prelude.searcher.FillSearcher", + "com.yahoo.prelude.searcher.JSONDebugSearcher", + "com.yahoo.prelude.searcher.JuniperSearcher", + "com.yahoo.prelude.searcher.MultipleResultsSearcher", + "com.yahoo.prelude.searcher.PosSearcher", + "com.yahoo.prelude.searcher.QuotingSearcher", + "com.yahoo.prelude.searcher.ValidateSortingSearcher", + "com.yahoo.prelude.semantics.SemanticSearcher", + "com.yahoo.prelude.statistics.StatisticsSearcher", + "com.yahoo.prelude.templates.SearchRendererAdaptor", + "com.yahoo.search.Searcher", + "com.yahoo.search.cluster.ClusterSearcher", + "com.yahoo.search.cluster.PingableSearcher", + "com.yahoo.search.federation.FederationSearcher", + "com.yahoo.search.federation.ForwardingSearcher", + "com.yahoo.search.federation.http.ConfiguredHTTPClientSearcher", + "com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher", + "com.yahoo.search.federation.http.HTTPClientSearcher", + "com.yahoo.search.federation.http.HTTPProviderSearcher", + "com.yahoo.search.federation.http.HTTPSearcher", + "com.yahoo.search.federation.news.NewsSearcher", + "com.yahoo.search.federation.vespa.VespaSearcher", + "com.yahoo.search.grouping.GroupingQueryParser", + "com.yahoo.search.grouping.GroupingValidator", + "com.yahoo.search.grouping.vespa.GroupingExecutor", + "com.yahoo.search.handler.SearchWithRendererHandler", + "com.yahoo.search.pagetemplates.PageTemplate", + "com.yahoo.search.pagetemplates.PageTemplateSearcher", + "com.yahoo.search.pagetemplates.engine.Resolver", + "com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver", + "com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver", + "com.yahoo.search.pagetemplates.model.Renderer", + "com.yahoo.search.query.rewrite.QueryRewriteSearcher", + "com.yahoo.search.query.rewrite.SearchChainDispatcherSearcher", + "com.yahoo.search.query.rewrite.rewriters.GenericExpansionRewriter", + "com.yahoo.search.query.rewrite.rewriters.MisspellRewriter", + "com.yahoo.search.query.rewrite.rewriters.NameRewriter", + "com.yahoo.search.querytransform.AllLowercasingSearcher", + "com.yahoo.search.querytransform.DefaultPositionSearcher", + "com.yahoo.search.querytransform.LowercasingSearcher", + "com.yahoo.search.querytransform.NGramSearcher", + "com.yahoo.search.querytransform.VespaLowercasingSearcher", + "com.yahoo.search.rendering.Renderer", + "com.yahoo.search.rendering.SectionedRenderer", + "com.yahoo.search.searchchain.ForkingSearcher", + "com.yahoo.search.searchchain.example.ExampleSearcher", + "com.yahoo.search.searchers.CacheControlSearcher", + "com.yahoo.search.statistics.PeakQpsSearcher", + "com.yahoo.search.statistics.TimingSearcher", + "com.yahoo.vespa.streamingvisitors.MetricsSearcher", + "com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher" + ); + } + +} diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg new file mode 100644 index 00000000000..1f4d5619248 --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg @@ -0,0 +1,108 @@ +attribute[].name "ref_from_a" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "ref_from_b" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "from_a_int_field" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true +attribute[].name "from_b_int_field" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].distancemetric EUCLIDEAN +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/child_a.sd b/config-model/src/test/derived/imported_fields_inherited_reference/child_a.sd new file mode 100644 index 00000000000..75c16d1eefe --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/child_a.sd @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +schema child_a { + document child_a { + field ref_from_a type reference<parent> { + indexing: attribute + } + } +} diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/child_b.sd b/config-model/src/test/derived/imported_fields_inherited_reference/child_b.sd new file mode 100644 index 00000000000..b4349fcc65d --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/child_b.sd @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +schema child_b { + document child_b inherits child_a { + field ref_from_b type reference<parent> { + indexing: attribute + } + } +} diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/child_c.sd b/config-model/src/test/derived/imported_fields_inherited_reference/child_c.sd new file mode 100644 index 00000000000..f798b8c3446 --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/child_c.sd @@ -0,0 +1,7 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +schema child_c { + document child_c inherits child_b { + } + import field ref_from_a.int_field as from_a_int_field {} + import field ref_from_b.int_field as from_b_int_field {} +} diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg new file mode 100644 index 00000000000..ca490b053f7 --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg @@ -0,0 +1,110 @@ +enablecompression false +documenttype[].id -94853056 +documenttype[].name "child_a" +documenttype[].version 0 +documenttype[].headerstruct 867409663 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].datatype[].id 867409663 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "child_a.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "ref_from_a" +documenttype[].datatype[].sstruct.field[].id 300427062 +documenttype[].datatype[].sstruct.field[].datatype 427398467 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].fieldsets{[]}.fields[] "ref_from_a" +documenttype[].referencetype[].id 427398467 +documenttype[].referencetype[].target_type_id 1175161836 +documenttype[].id -94852095 +documenttype[].name "child_b" +documenttype[].version 0 +documenttype[].headerstruct 670896158 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].inherits[].id -94853056 +documenttype[].datatype[].id 670896158 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "child_b.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "ref_from_b" +documenttype[].datatype[].sstruct.field[].id 185778735 +documenttype[].datatype[].sstruct.field[].datatype 427398467 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].fieldsets{[]}.fields[] "ref_from_a" +documenttype[].fieldsets{[]}.fields[] "ref_from_b" +documenttype[].id -94851134 +documenttype[].name "child_c" +documenttype[].version 0 +documenttype[].headerstruct 474382653 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].inherits[].id -94852095 +documenttype[].datatype[].id 474382653 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "child_c.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].fieldsets{[]}.fields[] "ref_from_a" +documenttype[].fieldsets{[]}.fields[] "ref_from_b" +documenttype[].importedfield[].name "from_a_int_field" +documenttype[].importedfield[].name "from_b_int_field" +documenttype[].id 1175161836 +documenttype[].name "parent" +documenttype[].version 0 +documenttype[].headerstruct 836075987 +documenttype[].bodystruct 0 +documenttype[].inherits[].id 8 +documenttype[].datatype[].id 836075987 +documenttype[].datatype[].type STRUCT +documenttype[].datatype[].array.element.id 0 +documenttype[].datatype[].map.key.id 0 +documenttype[].datatype[].map.value.id 0 +documenttype[].datatype[].wset.key.id 0 +documenttype[].datatype[].wset.createifnonexistent false +documenttype[].datatype[].wset.removeifzero false +documenttype[].datatype[].annotationref.annotation.id 0 +documenttype[].datatype[].sstruct.name "parent.header" +documenttype[].datatype[].sstruct.version 0 +documenttype[].datatype[].sstruct.compression.type NONE +documenttype[].datatype[].sstruct.compression.level 0 +documenttype[].datatype[].sstruct.compression.threshold 95 +documenttype[].datatype[].sstruct.compression.minsize 200 +documenttype[].datatype[].sstruct.field[].name "int_field" +documenttype[].datatype[].sstruct.field[].id 2128822283 +documenttype[].datatype[].sstruct.field[].datatype 0 +documenttype[].datatype[].sstruct.field[].detailedtype "" +documenttype[].fieldsets{[]}.fields[] "int_field" + diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/imported-fields.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/imported-fields.cfg new file mode 100644 index 00000000000..e574bfa2393 --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/imported-fields.cfg @@ -0,0 +1,6 @@ +attribute[].name "from_a_int_field" +attribute[].referencefield "ref_from_a" +attribute[].targetfield "int_field" +attribute[].name "from_b_int_field" +attribute[].referencefield "ref_from_b" +attribute[].targetfield "int_field" diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/parent.sd b/config-model/src/test/derived/imported_fields_inherited_reference/parent.sd new file mode 100644 index 00000000000..0509f1a8565 --- /dev/null +++ b/config-model/src/test/derived/imported_fields_inherited_reference/parent.sd @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +schema parent { + document parent { + field int_field type int { + indexing: attribute + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java index b06745f9a75..a0dd89229dd 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java @@ -30,4 +30,9 @@ public class ImportedFieldsTestCase extends AbstractExportingTestCase { public void configs_for_imported_position_field_summary_are_derived() throws IOException, ParseException { assertCorrectDeriving("imported_position_field_summary", "child"); } + + @Test + public void derives_configs_for_imported_fields_when_reference_fields_are_inherited() throws IOException, ParseException { + assertCorrectDeriving("imported_fields_inherited_reference", "child_c"); + } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java index 0ef696df6cf..ab98706fd44 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java @@ -149,10 +149,10 @@ public class MatchedElementsOnlyResolverTestCase { exceptionRule.expect(IllegalArgumentException.class); exceptionRule.expectMessage("For search 'test', document summary 'default', summary field 'my_field': " + "'matched-elements-only' is not supported for this field type. " + - "Supported field types are: array attribute, weighted set attribute, " + + "Supported field types are: array of primitive, weighted set of primitive, " + "array of simple struct, map of primitive type to simple struct, " + "and map of primitive type to primitive type"); - buildSearch(joinLines("field my_field type array<string> {", + buildSearch(joinLines("field my_field type string {", " indexing: summary", " summary: matched-elements-only", "}")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index bed77bd5c77..34f06519ac9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -15,8 +15,8 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.Zone; -import com.yahoo.container.BundlesConfig; import com.yahoo.container.core.ApplicationMetadataConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames; @@ -44,7 +44,6 @@ import static org.hamcrest.CoreMatchers.endsWith; import static org.hamcrest.CoreMatchers.hasItem; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -56,11 +55,11 @@ public class MetricsProxyContainerClusterTest { @Test public void metrics_proxy_bundle_is_included_in_bundles_config() { VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); - var builder = new BundlesConfig.Builder(); + var builder = new PlatformBundlesConfig.Builder(); model.getConfig(builder, CLUSTER_CONFIG_ID); - BundlesConfig config = builder.build(); - assertEquals(1, config.bundle().size()); - assertThat(config.bundle(0).value(), endsWith(METRICS_PROXY_BUNDLE_FILE.toString())); + PlatformBundlesConfig config = builder.build(); + assertEquals(1, config.bundlePaths().size()); + assertThat(config.bundlePaths(0), endsWith(METRICS_PROXY_BUNDLE_FILE.toString())); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 03c05af1145..97359b392a5 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.container; import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.RoutingProviderConfig; -import com.yahoo.config.FileReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -16,7 +15,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; -import com.yahoo.container.BundlesConfig; +import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.Host; @@ -292,9 +291,9 @@ public class ContainerClusterTest { .zone(zone).build(); MockRoot root = new MockRoot("foo", state); ApplicationContainerCluster cluster = new ApplicationContainerCluster(root, "container0", "container1", state); - BundlesConfig.Builder bundleBuilder = new BundlesConfig.Builder(); + var bundleBuilder = new PlatformBundlesConfig.Builder(); cluster.getConfig(bundleBuilder); - List<String> installedBundles = bundleBuilder.build().bundle().stream().map(FileReference::value).collect(Collectors.toList()); + List<String> installedBundles = bundleBuilder.build().bundlePaths(); assertEquals(expectedBundleNames.size(), installedBundles.size()); assertThat(installedBundles, containsInAnyOrder( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java index d2a840d1fbc..70ae6a27324 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -33,7 +33,7 @@ public class BundleInstantiationSpecificationBuilderTest { @Test public void bundle_is_replaced_for_internal_class() throws IOException, SAXException { String internalClass = GroupingValidator.class.getName(); - verifyExpectedBundle(internalClass, null, BundleMapper.searchAndDocprocBundle); + verifyExpectedBundle(internalClass, null, PlatformBundles.searchAndDocprocBundle); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index eda90b03147..e9048cf7863 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -3,17 +3,16 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.docproc.DocprocConfig; import com.yahoo.config.docproc.SchemamappingConfig; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; -import com.yahoo.container.BundlesConfig; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.HostPorts; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ApplicationContainer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.docproc.DocprocChain; import com.yahoo.vespa.model.container.docproc.DocumentProcessor; @@ -30,8 +29,8 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.core.IsNull.notNullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** @@ -46,7 +45,6 @@ public class DocprocBuilderTest extends DomBuilderTest { private ContainerMbusConfig containerMbusConfig; private ComponentsConfig componentsConfig; private ChainsConfig chainsConfig; - private BundlesConfig bundlesConfig; private SchemamappingConfig schemamappingConfig; private DocprocConfig docprocConfig; private QrStartConfig qrStartConfig; @@ -64,7 +62,6 @@ public class DocprocBuilderTest extends DomBuilderTest { cluster.getConfigId() + "/component/com.yahoo.docproc.jdisc.DocumentProcessingHandler"); documentmanagerConfig = root.getConfig(DocumentmanagerConfig.class, cluster.getConfigId()); - bundlesConfig = root.getConfig(BundlesConfig.class, cluster.getConfigId()); schemamappingConfig = root.getConfig(SchemamappingConfig.class, cluster.getContainers().get(0).getConfigId()); qrStartConfig = root.getConfig(QrStartConfig.class, cluster.getConfigId()); docprocConfig = root.getConfig(DocprocConfig.class, cluster.getConfigId()); @@ -207,11 +204,6 @@ public class DocprocBuilderTest extends DomBuilderTest { } @Test - public void testBundlesConfig() { - assertTrue(bundlesConfig.bundle().isEmpty()); - } - - @Test public void testSchemaMappingConfig() { assertTrue(schemamappingConfig.fieldmapping().isEmpty()); } |