diff options
author | HÃ¥kon Hallingstad <hakon@verizonmedia.com> | 2020-04-20 10:36:33 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-04-20 10:36:33 +0200 |
commit | b105eead1fbbcefbb85bc962749f2a12fa660bbe (patch) | |
tree | 7a3c996c00b854066d32608a002335715fb98c96 /config-model | |
parent | f61f6c701dc91e839b865f158a6da56ff166def7 (diff) | |
parent | 9ab0ef70e9ed4f422df67603f26bcb0c7918fdc4 (diff) |
Merge branch 'master' into hakonhall/remove-use-bucket-space-metric-feature-flag
Diffstat (limited to 'config-model')
308 files changed, 5969 insertions, 2149 deletions
diff --git a/config-model/pom.xml b/config-model/pom.xml index 25b733985f5..33f6657561c 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -105,6 +105,10 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java index a9677d4b34c..4cd0c1815dd 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java @@ -133,6 +133,11 @@ public final class XmlHelper { return Optional.ofNullable(element.getAttribute(name)).filter(s -> !s.isEmpty()); } + public static Optional<Element> getOptionalChild(Element parent, String childName) { + return Optional.ofNullable(XML.getChild(parent, childName)); + + } + public static Optional<String> getOptionalChildValue(Element parent, String childName) { Element child = XML.getChild(parent, childName); if (child == null) return Optional.empty(); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java index 140cb3001a0..b9ad830090c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java @@ -8,7 +8,6 @@ import java.util.Optional; /** * @author Ulf Lilleengen - * @since 5.1 */ public interface ConfigDefinitionStore { diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 7c9e930bb4f..3fb7ba6bc3a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -12,10 +12,11 @@ import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -36,9 +37,8 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.container.search.QueryProfilesBuilder; import com.yahoo.vespa.model.container.search.SemanticRuleBuilder; import com.yahoo.vespa.model.container.search.SemanticRules; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.Reader; @@ -62,11 +62,12 @@ public class DeployState implements ConfigDefinitionStore { private final DeployLogger logger; private final FileRegistry fileRegistry; private final DocumentModel documentModel; - private final List<SearchDefinition> searchDefinitions; + private final List<NamedSchema> schemas; private final ApplicationPackage applicationPackage; private final Optional<ConfigDefinitionRepo> configDefinitionRepo; private final Optional<ApplicationPackage> permanentApplicationPackage; private final Optional<Model> previousModel; + private final boolean accessLoggingEnabledByDefault; private final ModelContext.Properties properties; private final Version vespaVersion; private final Set<ContainerEndpoint> endpoints; @@ -76,8 +77,10 @@ public class DeployState implements ConfigDefinitionStore { private final ImportedMlModels importedModels; private final ValidationOverrides validationOverrides; private final Version wantedNodeVespaVersion; + private final Optional<String> wantedDockerImageRepo; private final Instant now; private final HostProvisioner provisioner; + private final Provisioned provisioned; public static DeployState createTestState() { return new Builder().build(); @@ -97,18 +100,21 @@ public class DeployState implements ConfigDefinitionStore { FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, + Provisioned provisioned, ModelContext.Properties properties, Version vespaVersion, Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo, - java.util.Optional<Model> previousModel, + Optional<Model> previousModel, Set<ContainerEndpoint> endpoints, Collection<MlModelImporter> modelImporters, Zone zone, QueryProfiles queryProfiles, SemanticRules semanticRules, Instant now, - Version wantedNodeVespaVersion) { + Version wantedNodeVespaVersion, + boolean accessLoggingEnabledByDefault, + Optional<String> wantedDockerImageRepo) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.rankProfileRegistry = rankProfileRegistry; @@ -116,8 +122,10 @@ public class DeployState implements ConfigDefinitionStore { this.properties = properties; this.vespaVersion = vespaVersion; this.previousModel = previousModel; + this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault; this.provisioner = hostProvisioner.orElse(getDefaultModelHostProvisioner(applicationPackage)); - this.searchDefinitions = searchDocumentModel.getSearchDefinitions(); + this.provisioned = provisioned; + this.schemas = searchDocumentModel.getSchemas(); this.documentModel = searchDocumentModel.getDocumentModel(); this.permanentApplicationPackage = permanentApplicationPackage; this.configDefinitionRepo = configDefinitionRepo; @@ -137,6 +145,7 @@ public class DeployState implements ConfigDefinitionStore { this.wantedNodeVespaVersion = wantedNodeVespaVersion; this.now = now; + this.wantedDockerImageRepo = wantedDockerImageRepo; } public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { @@ -147,6 +156,8 @@ public class DeployState implements ConfigDefinitionStore { } } + public Provisioned provisioned() { return provisioned; } + /** Get the global rank profile registry for this application. */ public final RankProfileRegistry rankProfileRegistry() { return rankProfileRegistry; } @@ -157,9 +168,7 @@ public class DeployState implements ConfigDefinitionStore { public final Optional<ConfigDefinition> getConfigDefinition(ConfigDefinitionKey defKey) { if (existingConfigDefs == null) { existingConfigDefs = new LinkedHashMap<>(); - if (configDefinitionRepo.isPresent()) { - existingConfigDefs.putAll(createLazyMapping(configDefinitionRepo.get())); - } + configDefinitionRepo.ifPresent(definitionRepo -> existingConfigDefs.putAll(createLazyMapping(definitionRepo))); existingConfigDefs.putAll(applicationPackage.getAllExistingConfigDefs()); } if ( ! existingConfigDefs.containsKey(defKey)) return Optional.empty(); @@ -205,8 +214,8 @@ public class DeployState implements ConfigDefinitionStore { return applicationPackage; } - public List<SearchDefinition> getSearchDefinitions() { - return searchDefinitions; + public List<NamedSchema> getSchemas() { + return schemas; } public DocumentModel getDocumentModel() { @@ -217,6 +226,10 @@ public class DeployState implements ConfigDefinitionStore { return logger; } + public boolean getAccessLoggingEnabledByDefault() { + return accessLoggingEnabledByDefault; + } + public FileRegistry getFileRegistry() { return fileRegistry; } @@ -253,6 +266,8 @@ public class DeployState implements ConfigDefinitionStore { public Version getWantedNodeVespaVersion() { return wantedNodeVespaVersion; } + public Optional<String> getWantedDockerImageRepo() { return wantedDockerImageRepo; } + public Instant now() { return now; } public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return properties.endpointCertificateSecrets(); } @@ -279,6 +294,7 @@ public class DeployState implements ConfigDefinitionStore { private FileRegistry fileRegistry = new MockFileRegistry(); private DeployLogger logger = new BaseDeployLogger(); private Optional<HostProvisioner> hostProvisioner = Optional.empty(); + private Provisioned provisioned = new Provisioned(); private Optional<ApplicationPackage> permanentApplicationPackage = Optional.empty(); private ModelContext.Properties properties = new TestProperties(); private Version version = new Version(1, 0, 0); @@ -289,6 +305,8 @@ public class DeployState implements ConfigDefinitionStore { private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; + private boolean accessLoggingEnabledByDefault = true; + private Optional<String> wantedDockerImageRepo = Optional.empty(); public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; @@ -310,6 +328,11 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder provisioned(Provisioned provisioned) { + this.provisioned = provisioned; + return this; + } + public Builder permanentApplicationPackage(Optional<ApplicationPackage> permanentApplicationPackage) { this.permanentApplicationPackage = permanentApplicationPackage; return this; @@ -360,6 +383,20 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder wantedDockerImageRepo(Optional<String> dockerImageRepo) { + this.wantedDockerImageRepo = dockerImageRepo; + return this; + } + + /** + * Whether access logging is enabled for an application without an accesslog element in services.xml. + * True by default. + */ + public Builder accessLoggingEnabledByDefault(boolean accessLoggingEnabledByDefault) { + this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault; + return this; + } + public DeployState build() { return build(new ValidationParameters()); } @@ -375,6 +412,7 @@ public class DeployState implements ConfigDefinitionStore { fileRegistry, logger, hostProvisioner, + provisioned, properties, version, permanentApplicationPackage, @@ -386,7 +424,9 @@ public class DeployState implements ConfigDefinitionStore { queryProfiles, semanticRules, now, - wantedNodeVespaVersion); + wantedNodeVespaVersion, + accessLoggingEnabledByDefault, + wantedDockerImageRepo); } private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, @@ -399,20 +439,18 @@ public class DeployState implements ConfigDefinitionStore { for (NamedReader reader : readers) { try { String readerName = reader.getName(); - String searchName = builder.importReader(reader, readerName, logger); + String topLevelName = builder.importReader(reader, readerName, logger); String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX); - names.put(searchName, sdName); - if ( ! sdName.equals(searchName)) { - throw new IllegalArgumentException("Search definition file name ('" + sdName + "') and name of " + - "search element ('" + searchName + + names.put(topLevelName, sdName); + if ( ! sdName.equals(topLevelName)) { + throw new IllegalArgumentException("Schema definition file name ('" + sdName + "') and name of " + + "top level element ('" + topLevelName + "') are not equal for file '" + readerName + "'"); } } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse search definition file '" + - getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e); + throw new IllegalArgumentException("Could not parse sd file '" + reader.getName() + "'", e); } catch (IOException e) { - throw new IllegalArgumentException("Could not read search definition file '" + - getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e); + throw new IllegalArgumentException("Could not read sd file '" + reader.getName() + "'", e); } finally { closeIgnoreException(reader.getReader()); } @@ -421,10 +459,6 @@ public class DeployState implements ConfigDefinitionStore { return SearchDocumentModel.fromBuilderAndNames(builder, names); } - private String getSearchDefinitionRelativePath(String name) { - return ApplicationPackage.SEARCH_DEFINITIONS_DIR + File.separator + name; - } - private static String stripSuffix(String nodeName, String postfix) { assert (nodeName.endsWith(postfix)); return nodeName.substring(0, nodeName.length() - postfix.length()); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java index cdd4f6f8e8a..9b9729dddb3 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java @@ -3,7 +3,7 @@ package com.yahoo.config.model.deploy; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.vespa.documentmodel.DocumentModel; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; import java.util.ArrayList; import java.util.List; @@ -13,16 +13,15 @@ import java.util.Map; * Internal helper class to retrieve document model and search definitions. * * @author Ulf Lilleengen - * @since 5.1 */ public class SearchDocumentModel { private final DocumentModel documentModel; - private final List<SearchDefinition> searchDefinitions; + private final List<NamedSchema> schemas; - public SearchDocumentModel(DocumentModel documentModel, List<SearchDefinition> searchDefinitions) { + public SearchDocumentModel(DocumentModel documentModel, List<NamedSchema> schemas) { this.documentModel = documentModel; - this.searchDefinitions = searchDefinitions; + this.schemas = schemas; } @@ -30,22 +29,22 @@ public class SearchDocumentModel { return documentModel; } - public List<SearchDefinition> getSearchDefinitions() { - return searchDefinitions; + public List<NamedSchema> getSchemas() { + return schemas; } public static SearchDocumentModel fromBuilderAndNames(SearchBuilder builder, Map<String, String> names) { - List<SearchDefinition> ret = new ArrayList<>(); + List<NamedSchema> ret = new ArrayList<>(); for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) { - ret.add(new SearchDefinition(names.get(search.getName()), search)); + ret.add(new NamedSchema(names.get(search.getName()), search)); } return new SearchDocumentModel(builder.getModel(), ret); } public static SearchDocumentModel fromBuilder(SearchBuilder builder) { - List<SearchDefinition> ret = new ArrayList<>(); + List<NamedSchema> ret = new ArrayList<>(); for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) { - ret.add(new SearchDefinition(search.getName(), search)); + ret.add(new NamedSchema(search.getName(), search)); } return new SearchDocumentModel(builder.getModel(), ret); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 9f4d1b09f91..99225beba4f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -4,10 +4,11 @@ package com.yahoo.config.model.deploy; import com.google.common.collect.ImmutableList; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; @@ -39,9 +40,11 @@ public class TestProperties implements ModelContext.Properties { private boolean isFirstTimeDeployment = false; private boolean useDedicatedNodeForLogserver = false; private boolean useAdaptiveDispatch = false; + private double topKProbability = 1.0; private double defaultTermwiseLimit = 1.0; + private double softStartSeconds = 0.0; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); - + private AthenzDomain athenzDomain; @Override public boolean multitenant() { return multitenant; } @Override public ApplicationId applicationId() { return applicationId; } @@ -60,13 +63,30 @@ public class TestProperties implements ModelContext.Properties { @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } @Override public Optional<TlsSecrets> tlsSecrets() { return endpointCertificateSecrets.map(TlsSecrets::new); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } + + @Override + public double defaultSoftStartSeconds() { + return softStartSeconds; + } + + @Override public double defaultTopKProbability() { return topKProbability; } @Override public boolean useBucketSpaceMetric() { return true; } + @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); } public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; } + public TestProperties setTopKProbability(double probability) { + topKProbability = probability; + return this; + } + public TestProperties setSoftStartSeconds(double softStartSeconds) { + this.softStartSeconds = softStartSeconds; + return this; + } + public TestProperties setApplicationId(ApplicationId applicationId) { this.applicationId = applicationId; return this; @@ -102,6 +122,16 @@ public class TestProperties implements ModelContext.Properties { return this; } + public TestProperties setZone(Zone zone) { + this.zone = zone; + return this; + } + + public TestProperties setAthenzDomain(AthenzDomain domain) { + this.athenzDomain = domain; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 48c21f370f4..cce5a7850a0 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.subscription.ConfigInstanceUtil; import com.yahoo.log.LogLevel; -import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; @@ -21,14 +20,8 @@ import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.utils.FreezableMap; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; import java.io.PrintStream; import java.io.Serializable; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -291,60 +284,6 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce public AbstractConfigProducer getParent() { return parent; } - /** - * Writes files that need to be written. The files will usually only be - * written when the Vespa model is generated through the deploy-application - * script. - * - * TODO: Make sure all implemented ConfigProducers call createConfig() - * instead of getConfig() when implementing this method. - */ - public void writeFiles(File directory) throws java.io.IOException { - if (!directory.isDirectory() && !directory.mkdirs()) { - throw new java.io.IOException("Cannot create directory: "+ directory); - } - for (Method m : getClass().getMethods()) { - try { - ConfigInstance.Builder builder = getBuilderIfIsGetConfig(m); - if (builder!=null) { - writeBuilder(directory, m, builder); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - private void writeBuilder(File directory, Method m, - ConfigInstance.Builder builder) throws IllegalAccessException, - InvocationTargetException, InstantiationException, - NoSuchMethodException, IOException { - m.invoke(this, builder); - Class<?> configInstClass = builder.getClass().getEnclosingClass(); - ConfigInstance inst = (ConfigInstance) configInstClass.getConstructor(builder.getClass()).newInstance(builder); - List<String> payloadList = ConfigInstance.serialize(inst); - File outfn = new File(directory, ConfigInstance.getDefName(inst.getClass()) + ".MODEL.cfg"); - FileOutputStream out = new FileOutputStream(outfn); - for (String s : payloadList) { - out.write(Utf8.toBytes(s)); - out.write('\n'); - } - } - - /** - * New Builder instance if m is getConfig(SomeConfig.Builder), or null - */ - private ConfigInstance.Builder getBuilderIfIsGetConfig(Method m) throws ReflectiveOperationException { - if (!"getConfig".equals(m.getName())) return null; - Type[] params = m.getParameterTypes(); - if (params.length!=1) return null; - Type param = params[0]; - if (!(param instanceof Class)) return null; - Class<?> paramClass = (Class<?>) param; - if (!(ConfigInstance.Builder.class.isAssignableFrom(paramClass))) return null; - return (ConfigInstance.Builder) paramClass.getDeclaredConstructor().newInstance(); - } - public void dump(PrintStream out) { for (ConfigProducer c : getChildren().values()) { out.println("id: " + c.getConfigId()); diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java index f909f3864da..201b69c1aae 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java @@ -45,10 +45,16 @@ public class HostsXmlProvisioner implements HostProvisioner { } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, int groups, ProvisionLogger logger) { throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported"); } + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, ProvisionLogger logger) { + throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported"); + } + private HostSpec host2HostSpec(Host host) { return new HostSpec(host.hostname(), host.aliases()); } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index bfbe2eaddb3..8706bb44ded 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -4,8 +4,10 @@ package com.yahoo.config.model.provision; import com.yahoo.collections.ListMap; import com.yahoo.collections.Pair; import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; @@ -49,7 +51,6 @@ public class InMemoryProvisioner implements HostProvisioner { /** Free hosts of each resource size */ private final ListMap<NodeResources, Host> freeNodes = new ListMap<>(); - private final Map<String, HostSpec> legacyMapping = new LinkedHashMap<>(); private final Map<ClusterSpec, List<HostSpec>> allocations = new LinkedHashMap<>(); /** Indexes must be unique across all groups in a cluster */ @@ -58,6 +59,10 @@ public class InMemoryProvisioner implements HostProvisioner { /** Use this index as start index for all clusters */ private final int startIndexForClusters; + private final boolean useMaxResources; + + private Provisioned provisioned = new Provisioned(); + /** Creates this with a number of nodes with resources 1, 3, 9, 1 */ public InMemoryProvisioner(int nodeCount) { this(nodeCount, defaultResources); @@ -65,27 +70,31 @@ public class InMemoryProvisioner implements HostProvisioner { /** Creates this with a number of nodes with given resources */ public InMemoryProvisioner(int nodeCount, NodeResources resources) { - this(Map.of(resources, createHostInstances(nodeCount)), true, 0); + this(Map.of(resources, createHostInstances(nodeCount)), true, false, 0); } /** Creates this with a set of host names of the flavor 'default' */ public InMemoryProvisioner(boolean failOnOutOfCapacity, String... hosts) { - this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, 0); + this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, false, 0); } /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) { - this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, 0, retiredHostNames); + this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, 0, retiredHostNames); } /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { - this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, startIndexForClusters, retiredHostNames); + this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, startIndexForClusters, retiredHostNames); } - public InMemoryProvisioner(Map<NodeResources, Collection<Host>> hosts, boolean failOnOutOfCapacity, - int startIndexForClusters, String ... retiredHostNames) { + public InMemoryProvisioner(Map<NodeResources, Collection<Host>> hosts, + boolean failOnOutOfCapacity, + boolean useMaxResources, + int startIndexForClusters, + String ... retiredHostNames) { this.failOnOutOfCapacity = failOnOutOfCapacity; + this.useMaxResources = useMaxResources; for (Map.Entry<NodeResources, Collection<Host>> hostsWithResources : hosts.entrySet()) for (Host host : hostsWithResources.getValue()) freeNodes.put(hostsWithResources.getKey(), host); @@ -106,44 +115,52 @@ public class InMemoryProvisioner implements HostProvisioner { @Override public HostSpec allocateHost(String alias) { - if (legacyMapping.containsKey(alias)) return legacyMapping.get(alias); List<Host> defaultHosts = freeNodes.get(defaultResources); if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts with default resources available"); Host newHost = freeNodes.removeValue(defaultResources, 0); - HostSpec hostSpec = new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version()); - legacyMapping.put(alias, hostSpec); - return hostSpec; + // Note: Always returns HostSpec with empty dockerImageRepo, which is OK since this method is never used when docker image repo is set + return new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version(), Optional.empty()); } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) { - if (cluster.group().isPresent() && groups > 1) + return prepare(cluster, requestedCapacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { + provisioned.add(cluster.id(), requested); + if (useMaxResources) + return prepare(cluster, requested.maxResources(), requested.isRequired(), requested.canFail()); + else + return prepare(cluster, requested.minResources(), requested.isRequired(), requested.canFail()); + } + + public List<HostSpec> prepare(ClusterSpec cluster, ClusterResources requested, boolean required, boolean canFail) { + if (cluster.group().isPresent() && requested.groups() > 1) throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created"); - if (requestedCapacity.nodeCount() % groups != 0) - throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " + - groups + " groups, but the node count is not divisible into this number of groups"); - int capacity = failOnOutOfCapacity || requestedCapacity.isRequired() - ? requestedCapacity.nodeCount() - : Math.min(requestedCapacity.nodeCount(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster)); - if (groups > capacity) - groups = capacity; + int capacity = failOnOutOfCapacity || required + ? requested.nodes() + : Math.min(requested.nodes(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster)); + int groups = requested.groups() > capacity ? capacity : requested.groups(); List<HostSpec> allocation = new ArrayList<>(); if (groups == 1) { allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(0))), - requestedCapacity.nodeResources(), + requested.nodeResources(), capacity, startIndexForClusters, - requestedCapacity.canFail())); + canFail)); } else { for (int i = 0; i < groups; i++) { allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(i))), - requestedCapacity.nodeResources(), + requested.nodeResources(), capacity / groups, allocation.size(), - requestedCapacity.canFail())); + canFail)); } } for (ListIterator<HostSpec> i = allocation.listIterator(); i.hasNext(); ) { @@ -154,15 +171,24 @@ public class InMemoryProvisioner implements HostProvisioner { return allocation; } + /** Create a new provisioned instance to record provision requests to this and returns it */ + public Provisioned startProvisionedRecording() { + provisioned = new Provisioned(); + return provisioned; + } + private HostSpec retire(HostSpec host) { return new HostSpec(host.hostname(), host.aliases(), host.flavor(), Optional.of(host.membership().get().retire()), - host.version()); + host.version(), + Optional.empty(), + Optional.empty(), + host.dockerImageRepo()); } - private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, Optional<NodeResources> requestedResources, + private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, NodeResources requestedResources, int nodesInGroup, int startIndex, boolean canFail) { List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>()); allocations.put(clusterGroup, allocation); @@ -170,8 +196,8 @@ public class InMemoryProvisioner implements HostProvisioner { // Check if the current allocations are compatible with the new request for (int i = allocation.size() - 1; i >= 0; i--) { Optional<NodeResources> currentResources = allocation.get(0).flavor().map(Flavor::resources); - if (currentResources.isEmpty() || requestedResources.isEmpty()) continue; - if (!currentResources.get().compatibleWith(requestedResources.get())) { + if (currentResources.isEmpty() || requestedResources == NodeResources.unspecified) continue; + if (!currentResources.get().compatibleWith(requestedResources)) { HostSpec removed = allocation.remove(i); freeNodes.put(currentResources.get(), new Host(removed.hostname())); // Return the node back to free pool } @@ -182,7 +208,7 @@ public class InMemoryProvisioner implements HostProvisioner { // Find the smallest host that can fit the requested requested Optional<NodeResources> hostResources = freeNodes.keySet().stream() .sorted(new MemoryDiskCpu()) - .filter(resources -> requestedResources.isEmpty() || resources.satisfies(requestedResources.get())) + .filter(resources -> requestedResources == NodeResources.unspecified || resources.satisfies(requestedResources)) .findFirst(); if (hostResources.isEmpty()) { if (canFail) @@ -195,8 +221,9 @@ public class InMemoryProvisioner implements HostProvisioner { if (freeNodes.get(hostResources.get()).isEmpty()) freeNodes.removeAll(hostResources.get()); ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++); allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(), - hostResources.map(Flavor::new), Optional.of(membership), - newHost.version(), Optional.empty(), requestedResources)); + hostResources.map(Flavor::new), Optional.of(membership), + newHost.version(), Optional.empty(), + requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources))); } nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex); diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index 180a16f3c8f..8945223447f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -30,6 +30,7 @@ public class SingleNodeProvisioner implements HostProvisioner { host = new Host(HostName.getLocalhost()); this.hostSpec = new HostSpec(host.hostname(), host.aliases()); } + public SingleNodeProvisioner(Flavor flavor) { host = new Host(HostName.getLocalhost()); this.hostSpec = new HostSpec(host.hostname(), host.aliases(), flavor); @@ -41,7 +42,14 @@ public class SingleNodeProvisioner implements HostProvisioner { } @Override - public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1 + @Deprecated // TODO: Remove after April 2020 + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + return prepare(cluster, capacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + // TODO: This should fail if capacity requested is more than 1 List<HostSpec> hosts = new ArrayList<>(); hosts.add(new HostSpec(host.hostname(), host.aliases(), ClusterMembership.from(cluster, counter++))); return hosts; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index eb61bda83a6..10649df88e1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -55,23 +55,23 @@ public class MockApplicationPackage implements ApplicationPackage { private final File root; private final String hostsS; private final String servicesS; - private final List<String> searchDefinitions; - private final String searchDefinitionDir; + private final List<String> schemas; + private final String schemaDir; private final Optional<String> deploymentSpec; private final Optional<String> validationOverrides; private final boolean failOnValidateXml; private final QueryProfileRegistry queryProfileRegistry; private final ApplicationMetaData applicationMetaData; - protected MockApplicationPackage(File root, String hosts, String services, List<String> searchDefinitions, - String searchDefinitionDir, + protected MockApplicationPackage(File root, String hosts, String services, List<String> schemas, + String schemaDir, String deploymentSpec, String validationOverrides, boolean failOnValidateXml, String queryProfile, String queryProfileType) { this.root = root; this.hostsS = hosts; this.servicesS = services; - this.searchDefinitions = searchDefinitions; - this.searchDefinitionDir = searchDefinitionDir; + this.schemas = schemas; + this.schemaDir = schemaDir; this.deploymentSpec = Optional.ofNullable(deploymentSpec); this.validationOverrides = Optional.ofNullable(validationOverrides); this.failOnValidateXml = failOnValidateXml; @@ -108,7 +108,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public Reader getHosts() { - if (hostsS==null) return null; + if (hostsS == null) return null; return new StringReader(hostsS); } @@ -118,7 +118,7 @@ public class MockApplicationPackage implements ApplicationPackage { SearchBuilder searchBuilder = new SearchBuilder(this, new RankProfileRegistry(), queryProfileRegistry); - for (String sd : searchDefinitions) { + for (String sd : schemas) { try { String name = searchBuilder.importString(sd); readers.add(new NamedReader(name + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd))); @@ -184,7 +184,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public Reader getRankingExpression(String name) { - File expressionFile = new File(searchDefinitionDir, name); + File expressionFile = new File(schemaDir, name); try { return IOUtils.createReader(expressionFile, "utf-8"); } @@ -200,9 +200,9 @@ public class MockApplicationPackage implements ApplicationPackage { public static ApplicationPackage fromSearchDefinitionDirectory(String dir) { return new MockApplicationPackage.Builder() - .withEmptyHosts() - .withEmptyServices() - .withSearchDefinitionDir(dir).build(); + .withEmptyHosts() + .withEmptyServices() + .withSchemaDir(dir).build(); } public static class Builder { @@ -210,8 +210,8 @@ public class MockApplicationPackage implements ApplicationPackage { private File root = new File("nonexisting"); private String hosts = null; private String services = null; - private List<String> searchDefinitions = Collections.emptyList(); - private String searchDefinitionDir = null; + private List<String> schemas = Collections.emptyList(); + private String schemaDir = null; private String deploymentSpec = null; private String validationOverrides = null; private boolean failOnValidateXml = false; @@ -245,17 +245,17 @@ public class MockApplicationPackage implements ApplicationPackage { } public Builder withSearchDefinition(String searchDefinition) { - this.searchDefinitions = Collections.singletonList(searchDefinition); + this.schemas = Collections.singletonList(searchDefinition); return this; } - public Builder withSearchDefinitions(List<String> searchDefinition) { - this.searchDefinitions = Collections.unmodifiableList(searchDefinition); + public Builder withSchemas(List<String> searchDefinition) { + this.schemas = Collections.unmodifiableList(searchDefinition); return this; } - public Builder withSearchDefinitionDir(String searchDefinitionDir) { - this.searchDefinitionDir = searchDefinitionDir; + public Builder withSchemaDir(String schemaDir) { + this.schemaDir = schemaDir; return this; } @@ -285,7 +285,7 @@ public class MockApplicationPackage implements ApplicationPackage { } public ApplicationPackage build() { - return new MockApplicationPackage(root, hosts, services, searchDefinitions, searchDefinitionDir, + return new MockApplicationPackage(root, hosts, services, schemas, schemaDir, deploymentSpec, validationOverrides, failOnValidateXml, queryProfile, queryProfileType); } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index 13f271ebe9d..87d6554f691 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Set; @@ -66,7 +67,7 @@ public class MockRoot extends AbstractConfigProducerRoot { super(rootConfigId); hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner(), deployState.getDeployLogger()); this.deployState = deployState; - fileDistributor = new FileDistributor(deployState.getFileRegistry(), null); + fileDistributor = new FileDistributor(deployState.getFileRegistry(), List.of(), deployState.isHosted()); } public FileDistributionConfigProducer getFileDistributionConfigProducer() { diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index fc42864f1d0..41a30c4553d 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -69,25 +69,37 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private final StructDataType body; private final Set<FieldSet> fieldSets = new LinkedHashSet<>(); private final Set<Name> documentReferences; + // Imported fields are virtual and therefore exist outside of the SD's document field definition + // block itself. But for features like imported fields in a non-search context (e.g. GC selections) + // it is necessary to know that certain identifiers refer to imported fields instead of being unknown + // document fields. To achieve this, we track the names of imported fields as part of the document + // config itself. + private final Set<String> importedFieldNames; public NewDocumentType(Name name) { this(name, emptySet()); } - public NewDocumentType(Name name, Set<Name> documentReferences) { + public NewDocumentType(Name name, Set<Name> documentReferences, Set<String> importedFieldNames) { this( name, new StructDataType(name.getName() + ".header"), new StructDataType(name.getName() + ".body"), new FieldSets(), - documentReferences); + documentReferences, + importedFieldNames); + } + + public NewDocumentType(Name name, Set<Name> documentReferences) { + this(name, documentReferences, emptySet()); } public NewDocumentType(Name name, StructDataType header, StructDataType body, FieldSets fs, - Set<Name> documentReferences) { + Set<Name> documentReferences, + Set<String> importedFieldNames) { super(name.getName()); this.name = name; this.header = header; @@ -102,6 +114,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } } this.documentReferences = documentReferences; + this.importedFieldNames = importedFieldNames; } public Name getFullName() { @@ -389,4 +402,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp return documentReferences; } + public Set<String> getImportedFieldNames() { + return importedFieldNames; + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index f0b1b427531..d3a78321106 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -18,6 +18,7 @@ import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedFields; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -27,6 +28,7 @@ import com.yahoo.vespa.documentmodel.SearchField; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -338,7 +340,8 @@ public class DocumentModelBuilder { sdoc.getDocumentType().contentStruct(), sdoc.getDocumentType().getBodyType(), sdoc.getFieldSets(), - convertDocumentReferencesToNames(sdoc.getDocumentReferences())); + convertDocumentReferencesToNames(sdoc.getDocumentReferences()), + convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); for (SDDocumentType n : sdoc.getInheritedTypes()) { NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); @@ -404,6 +407,13 @@ public class DocumentModelBuilder { .collect(toSet()); } + private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) { + if (importedFields == null) { + return emptySet(); + } + return Collections.unmodifiableSet(importedFields.fields().keySet()); + } + private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) { for (Field f : fields) { DataType type = f.getDataType(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java index 14f8a0a9d37..0d0da71bd0f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java @@ -25,8 +25,8 @@ public class DocumentReferenceResolver { private final Map<String, Search> searchMapping; - public DocumentReferenceResolver(List<Search> searchDefinitions) { - this.searchMapping = createDocumentNameToSearchMapping(searchDefinitions); + public DocumentReferenceResolver(List<Search> schemas) { + this.searchMapping = createDocumentNameToSearchMapping(schemas); } public void resolveReferences(SDDocumentType documentType) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java index 9ff749a994c..ade8ae21870 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java @@ -41,7 +41,7 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier { } if (structUsedByField.getName().equals(structType.getName())) { //this field is using this type!! - field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), field.isHeader(), 0); + field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); field.populateWithStructMatching(sdoc, field.getName(), field.getDataType(), field.getMatching()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java new file mode 100644 index 00000000000..25cdd1e08cd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java @@ -0,0 +1,31 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDDocumentType; + +import java.util.List; + +/** + * Enumerates and emplaces a set of all imported fields into a SDDocumentType from + * its corresponding Search instance. + */ +public class ImportedFieldsEnumerator { + + private final List<Search> schemas; + + public ImportedFieldsEnumerator(List<Search> schemas) { + this.schemas = schemas; + } + + public void enumerateImportedFields(SDDocumentType documentType) { + var search = this.schemas.stream() + .filter(s -> s.getDocument() != null) + .filter(s -> s.getDocument().getName().equals(documentType.getName())) + .findFirst(); + if (search.isEmpty()) { + return; // No imported fields present. + } + search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index e46db1d1b5f..aba6cf9a233 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.HnswIndexParams; import com.yahoo.searchdefinition.document.RankType; import com.yahoo.searchdefinition.document.Stemming; @@ -9,6 +10,9 @@ import java.io.Serializable; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; import java.util.Set; /** @@ -20,6 +24,8 @@ import java.util.Set; */ public class Index implements Cloneable, Serializable { + public static enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES } + public enum Type { VESPA("vespa"); @@ -57,6 +63,10 @@ public class Index implements Cloneable, Serializable { /** The boolean index definition, if set */ private BooleanIndexDefinition boolIndex; + private Optional<HnswIndexParams> hnswIndexParams = Optional.empty(); + + private Optional<DistanceMetric> distanceMetric = Optional.empty(); + /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */ private boolean interleavedFeatures = false; @@ -115,20 +125,26 @@ public class Index implements Cloneable, Serializable { } @Override - public int hashCode() { - return name.hashCode() + ( prefix ? 17 : 0 ); + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Index index = (Index) o; + return prefix == index.prefix && + normalized == index.normalized && + interleavedFeatures == index.interleavedFeatures && + Objects.equals(name, index.name) && + rankType == index.rankType && + Objects.equals(aliases, index.aliases) && + stemming == index.stemming && + type == index.type && + Objects.equals(boolIndex, index.boolIndex) && + Objects.equals(distanceMetric, index.distanceMetric) && + Objects.equals(hnswIndexParams, index.hnswIndexParams); } @Override - public boolean equals(Object object) { - if ( ! (object instanceof Index)) return false; - - Index other=(Index)object; - return - this.name.equals(other.name) && - this.prefix==other.prefix && - this.stemming==other.stemming && - this.normalized==other.normalized; + public int hashCode() { + return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, distanceMetric, hnswIndexParams, interleavedFeatures); } public String toString() { @@ -176,6 +192,24 @@ public class Index implements Cloneable, Serializable { boolIndex = def; } + public Optional<DistanceMetric> getDistanceMetric() { + return distanceMetric; + } + + public void setDistanceMetric(String value) { + String upper = value.toUpperCase(Locale.ENGLISH); + DistanceMetric dm = DistanceMetric.valueOf(upper); + distanceMetric = Optional.of(dm); + } + + public Optional<HnswIndexParams> getHnswIndexParams() { + return hnswIndexParams; + } + + public void setHnswIndexParams(HnswIndexParams params) { + hnswIndexParams = Optional.of(params); + } + public void setInterleavedFeatures(boolean value) { interleavedFeatures = value; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index fa196cd3bbf..23eb814de81 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -788,8 +788,7 @@ public class RankProfile implements Cloneable { type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () -> new IllegalArgumentException(queryProfileType + " contains query feature " + feature.get() + " with type " + field.getType().asTensorType() + - ", but this is already defined " + - "in another query profile with type " + + ", but this is already defined in another query profile with type " + context.getType(feature.get()))); context.setType(feature.get(), type); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index f90a7e4f6cd..0ab8a2308a4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -288,7 +288,7 @@ public class Search implements ImmutableSearch { /** * Adds an extra field of this search definition not contained in a document * - * @param field to add to the searchdefinitions list of external fields. + * @param field to add to the schemas list of external fields */ public void addExtraField(SDField field) { if (fields.containsKey(field.getName())) { @@ -383,7 +383,7 @@ public class Search implements ImmutableSearch { * Consolidates a set of index settings for the same index into one * * @param indices The list of indexes to consolidate. - * @return The consolidated index + * @return the consolidated index */ private Index consolidateIndices(List<Index> indices) { Index first = indices.get(0); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 949539ff99f..eb68e6af203 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -230,20 +230,22 @@ public class SearchBuilder { sdocs.add(search.getDocument()); } } - SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); + var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); orderer.process(); for (SDDocumentType sdoc : orderer.getOrdered()) { new FieldOperationApplierForStructs().process(sdoc); new FieldOperationApplier().process(sdoc); } - DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList); + var resolver = new DocumentReferenceResolver(searchList); sdocs.forEach(resolver::resolveReferences); + var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList); + sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); if (validate) new DocumentGraphValidator().validateDocumentGraph(sdocs); - DocumentModelBuilder builder = new DocumentModelBuilder(model); + var builder = new DocumentModelBuilder(model); for (Search search : new SearchOrderer().order(searchList)) { new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list? process(search, deployLogger, new QueryProfiles(queryProfileRegistry, deployLogger), validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 76dff404568..8b5f7658475 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -240,6 +240,17 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce aaB.tensortype(attribute.tensorType().get().toString()); } aaB.imported(imported); + var dma = attribute.distanceMetric(); + if (attribute.hnswIndexParams().isPresent()) { + var ib = new AttributesConfig.Attribute.Index.Builder(); + var params = attribute.hnswIndexParams().get(); + ib.hnsw.enabled(true); + ib.hnsw.maxlinkspernode(params.maxLinksPerNode()); + ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert()); + var dm = AttributesConfig.Attribute.Index.Hnsw.Distancemetric.Enum.valueOf(dma.toString()); + ib.hnsw.distancemetric(dm); + aaB.index(ib); + } return aaB; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 245913d7822..fc8710fa1a1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -15,9 +15,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.derived.validation.Validation; +import com.yahoo.vespa.model.container.search.QueryProfiles; import java.io.IOException; import java.io.Writer; +import java.util.logging.Level; /** * A set of all derived configuration of a search definition. Use this as a facade to individual configurations when @@ -39,12 +41,13 @@ public class DerivedConfiguration { private VsmSummary streamingSummary; private IndexSchema indexSchema; private ImportedFields importedFields; + private QueryProfileRegistry queryProfiles; /** * Creates a complete derived configuration from a search definition. * Only used in tests. * - * @param search The search to derive a configuration from. Derived objects will be snapshots, but this argument is + * @param search the search to derive a configuration from. Derived objects will be snapshots, but this argument is * live. Which means that this object will be inconsistent when the given search definition is later * modified. * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} @@ -56,11 +59,11 @@ public class DerivedConfiguration { /** * Creates a complete derived configuration snapshot from a search definition. * - * @param search The search to derive a configuration from. Derived objects will be snapshots, but this + * @param search the search to derive a configuration from. Derived objects will be snapshots, but this * argument is live. Which means that this object will be inconsistent when the given * search definition is later modified. * @param deployLogger a {@link DeployLogger} for logging when doing operations on this - * @param deployProperties Properties set on deploy. + * @param deployProperties properties set on deploy * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} * @param queryProfiles the query profiles of this application */ @@ -72,6 +75,7 @@ public class DerivedConfiguration { ImportedMlModels importedModels) { Validator.ensureNotNull("Search definition", search); this.search = search; + this.queryProfiles = queryProfiles; if ( ! search.isDocumentsOnly()) { streamingFields = new VsmFields(search); streamingSummary = new VsmSummary(search); @@ -120,6 +124,10 @@ public class DerivedConfiguration { exportCfg(new DocumenttypesConfig(documentTypesCfg), toDirectory + "/" + "documenttypes.cfg"); } + public static void exportQueryProfiles(QueryProfileRegistry queryProfileRegistry, String toDirectory) throws IOException { + exportCfg(new QueryProfiles(queryProfileRegistry, (level, message) -> {}).getConfig(), toDirectory + "/" + "query-profiles.cfg"); + } + private static void exportCfg(ConfigInstance instance, String fileName) throws IOException { Writer writer = null; try { @@ -186,4 +194,7 @@ public class DerivedConfiguration { public ImportedFields getImportedFields() { return importedFields; } + + public QueryProfileRegistry getQueryProfiles() { return queryProfiles; } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java index 02b2a383318..da25680ca47 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java @@ -1,16 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.document.*; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.NumericDataType; +import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.document.*; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.searchdefinition.processing.ExactMatch; import com.yahoo.searchdefinition.processing.NGramMatch; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.search.config.IndexInfoConfig; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -34,8 +43,10 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { private static final String CMD_PLAIN_TOKENS = "plain-tokens"; private static final String CMD_MULTIVALUE = "multivalue"; private static final String CMD_FAST_SEARCH = "fast-search"; + private static final String CMD_PREDICATE = "predicate"; private static final String CMD_PREDICATE_BOUNDS = "predicate-bounds"; private static final String CMD_NUMERICAL = "numerical"; + private static final String CMD_PHRASE_SEGMENTING = "phrase-segmenting"; private Set<IndexCommand> commands = new java.util.LinkedHashSet<>(); private Map<String, String> aliases = new java.util.LinkedHashMap<>(); private Map<String, FieldSet> fieldSets; @@ -90,6 +101,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { protected void derive(ImmutableSDField field, Search search, boolean inPosition) { if (field.getDataType().equals(DataType.PREDICATE)) { + addIndexCommand(field, CMD_PREDICATE); Index index = field.getIndex(field.getName()); if (index != null) { BooleanIndexDefinition options = index.getBooleanIndexDefiniton(); @@ -286,21 +298,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { // TODO: Move this to the FieldSetSettings processor (and rename it) as that already has to look at this. private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) { - // Explicit query commands on the field set, overrides everything. - if (!fieldSet.queryCommands().isEmpty()) { - for (String qc : fieldSet.queryCommands()) { - iiB.command( - new IndexInfoConfig.Indexinfo.Command.Builder() - .indexname(fieldSet.getName()) - .command(qc)); - } - return; - } + for (String qc : fieldSet.queryCommands()) + iiB.command(new IndexInfoConfig.Indexinfo.Command.Builder().indexname(fieldSet.getName()).command(qc)); boolean anyIndexing = false; boolean anyAttributing = false; boolean anyLowerCasing = false; boolean anyStemming = false; boolean anyNormalizing = false; + String phraseSegmentingCommand = null; String stemmingCommand = null; Matching fieldSetMatching = fieldSet.getMatching(); // null if no explicit matching // First a pass over the fields to read some params to decide field settings implicitly: @@ -321,8 +326,13 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { if (field.getNormalizing().doRemoveAccents()) { anyNormalizing = true; } - if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType) + if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType) { fieldSetMatching = field.getMatching(); + } + Optional<String> explicitPhraseSegmentingCommand = field.getQueryCommands().stream().filter(c -> c.startsWith(CMD_PHRASE_SEGMENTING)).findFirst(); + if (explicitPhraseSegmentingCommand.isPresent()) { + phraseSegmentingCommand = explicitPhraseSegmentingCommand.get(); + } } if (anyIndexing && anyAttributing && fieldSet.getMatching() == null) { // We have both attributes and indexes and no explicit match setting -> @@ -365,6 +375,11 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { new IndexInfoConfig.Indexinfo.Command.Builder() .indexname(fieldSet.getName()) .command(CMD_NORMALIZE)); + if (phraseSegmentingCommand != null) + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(phraseSegmentingCommand)); } } else { // Assume only attribute fields @@ -400,9 +415,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } else if (fieldSetMatching.getType().equals(Matching.Type.TEXT)) { } - } - } private boolean hasMultiValueField(FieldSet fieldSet) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java index 60b8ee78c7b..39a67a69575 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java @@ -5,6 +5,7 @@ import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.document.StructuredDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; @@ -20,7 +21,7 @@ import java.util.List; import java.util.Map; /** - * Deriver of indexschema config containing information of all index fields with name and data type. + * Deriver of indexschema config containing information of all text index fields with name and data type. * * @author geirst */ @@ -44,9 +45,14 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { super.derive(search); } + private boolean isTensorField(ImmutableSDField field) { + return field.getDataType() instanceof TensorDataType; + } + private void deriveIndexFields(ImmutableSDField field, Search search) { - if (!field.doesIndexing() && - !field.isIndexStructureField()) + // Note: Indexes for tensor fields are NOT part of the index schema for text fields. + if ((!field.doesIndexing() && !field.isIndexStructureField()) || + isTensorField(field)) { return; } @@ -138,11 +144,10 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { return Collections.singletonList(field); } if (fieldType instanceof ArrayDataType) { - boolean header = field.isHeader(); List<Field> ret = new LinkedList<>(); - Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType(), header); + Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType()); for (Field flatField : flattenField(innerField)) { - ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()), header)); + ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()))); } return ret; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index fbcaf2a3a80..1661a80f238 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -24,6 +24,7 @@ import com.yahoo.document.datatypes.Float16FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.tensor.TensorType; +import static com.yahoo.searchdefinition.Index.DistanceMetric; import java.io.Serializable; import java.util.LinkedHashSet; @@ -66,6 +67,10 @@ public final class Attribute implements Cloneable, Serializable { /** This is set if the type of this is REFERENCE */ private final Optional<StructuredDataType> referenceDocumentType; + private Optional<DistanceMetric> distanceMetric = Optional.empty(); + + private Optional<HnswIndexParams> hnswIndexParams = Optional.empty(); + private boolean isPosition = false; private final Sorting sorting = new Sorting(); @@ -195,6 +200,12 @@ public final class Attribute implements Cloneable, Serializable { public Optional<TensorType> tensorType() { return tensorType; } public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; } + public static final DistanceMetric DEFAULT_DISTANCE_METRIC = DistanceMetric.EUCLIDEAN; + public DistanceMetric distanceMetric() { + return distanceMetric.orElse(DEFAULT_DISTANCE_METRIC); + } + public Optional<HnswIndexParams> hnswIndexParams() { return hnswIndexParams; } + public Sorting getSorting() { return sorting; } public void setRemoveIfZero(boolean remove) { this.removeIfZero = remove; } @@ -217,6 +228,8 @@ public final class Attribute implements Cloneable, Serializable { public void setUpperBound(long upperBound) { this.upperBound = upperBound; } public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; } public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } + public void setDistanceMetric(Optional<DistanceMetric> dm) { this.distanceMetric = dm; } + public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } public String getName() { return name; } public Type getType() { return type; } @@ -335,7 +348,7 @@ public final class Attribute implements Cloneable, Serializable { public int hashCode() { return Objects.hash( name, type, collectionType, sorting, isPrefetch(), fastAccess, removeIfZero, createIfNonExistent, - isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType); + isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType, hnswIndexParams); } @Override @@ -349,8 +362,8 @@ public final class Attribute implements Cloneable, Serializable { /** Returns whether these attributes describes the same entity, even if they have different names */ public boolean isCompatible(Attribute other) { - if ( ! this.type.equals(other.type)) return false; - if ( ! this.collectionType.equals(other.collectionType)) return false; + if (! this.type.equals(other.type)) return false; + if (! this.collectionType.equals(other.collectionType)) return false; if (this.isPrefetch() != other.isPrefetch()) return false; if (this.removeIfZero != other.removeIfZero) return false; if (this.createIfNonExistent != other.createIfNonExistent) return false; @@ -359,9 +372,11 @@ public final class Attribute implements Cloneable, Serializable { // if (this.noSearch != other.noSearch) return false; No backend consequences so compatible for now if (this.fastSearch != other.fastSearch) return false; if (this.huge != other.huge) return false; - if ( ! this.sorting.equals(other.sorting)) return false; - if (!this.tensorType.equals(other.tensorType)) return false; - if (!this.referenceDocumentType.equals(other.referenceDocumentType)) return false; + if (! this.sorting.equals(other.sorting)) return false; + if (! Objects.equals(tensorType, other.tensorType)) return false; + if (! Objects.equals(referenceDocumentType, other.referenceDocumentType)) return false; + if (! Objects.equals(distanceMetric, other.distanceMetric)) return false; + if (! Objects.equals(hnswIndexParams, other.hnswIndexParams)) return false; return true; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java index 55dedc4a1d7..22424286ef9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java @@ -25,10 +25,7 @@ public class FieldSet { public FieldSet addFieldName(String field) { fieldNames.add(field); return this; } public Set<String> getFieldNames() { return fieldNames; } public Set<ImmutableSDField> fields() { return fields; } - - public Set<String> queryCommands() { - return queryCommands; - } + public Set<String> queryCommands() { return queryCommands; } public void setMatching(Matching matching) { this.matching = matching; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java new file mode 100644 index 00000000000..2f084d3e513 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java @@ -0,0 +1,64 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import java.util.Locale; +import java.util.Optional; + +/** + * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. + * + * @author geirst + */ +public class HnswIndexParams { + + public static final int DEFAULT_MAX_LINKS_PER_NODE = 16; + public static final int DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT = 200; + + private final Optional<Integer> maxLinksPerNode; + private final Optional<Integer> neighborsToExploreAtInsert; + + public static class Builder { + private Optional<Integer> maxLinksPerNode = Optional.empty(); + private Optional<Integer> neighborsToExploreAtInsert = Optional.empty(); + + public void setMaxLinksPerNode(int value) { + maxLinksPerNode = Optional.of(value); + } + public void setNeighborsToExploreAtInsert(int value) { + neighborsToExploreAtInsert = Optional.of(value); + } + public HnswIndexParams build() { + return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert); + } + } + + public HnswIndexParams() { + this.maxLinksPerNode = Optional.empty(); + this.neighborsToExploreAtInsert = Optional.empty(); + } + + public HnswIndexParams(Optional<Integer> maxLinksPerNode, + Optional<Integer> neighborsToExploreAtInsert) { + this.maxLinksPerNode = maxLinksPerNode; + this.neighborsToExploreAtInsert = neighborsToExploreAtInsert; + } + + /** + * Creates a new instance where values from the given parameter instance are used where they are present, + * otherwise we use values from this. + */ + public HnswIndexParams overrideFrom(Optional<HnswIndexParams> other) { + if (! other.isPresent()) return this; + HnswIndexParams rhs = other.get(); + return new HnswIndexParams(rhs.maxLinksPerNode.or(() -> maxLinksPerNode), + rhs.neighborsToExploreAtInsert.or(() -> neighborsToExploreAtInsert)); + } + + public int maxLinksPerNode() { + return maxLinksPerNode.orElse(DEFAULT_MAX_LINKS_PER_NODE); + } + + public int neighborsToExploreAtInsert() { + return neighborsToExploreAtInsert.orElse(DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index 414a605c621..b3f98cc6f26 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -47,6 +47,7 @@ public class SDDocumentType implements Cloneable, Serializable { private FieldSets fieldSets; // Document references private Optional<DocumentReferences> documentReferences = Optional.empty(); + private TemporaryImportedFields temporaryImportedFields; static { VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName()); @@ -58,6 +59,7 @@ public class SDDocumentType implements Cloneable, Serializable { type.docType = docType.clone(); type.inheritedTypes.putAll(inheritedTypes); type.structType = structType; + // TODO this isn't complete; should it be..?! return type; } @@ -334,4 +336,11 @@ public class SDDocumentType implements Cloneable, Serializable { this.documentReferences = Optional.of(documentReferences); } + public TemporaryImportedFields getTemporaryImportedFields() { + return temporaryImportedFields; + } + + public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) { + this.temporaryImportedFields = temporaryImportedFields; + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index c657d29033a..30ce142d503 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -38,9 +38,8 @@ import java.util.TreeMap; /** * The field class represents a document field. It is used in - * the Document class to get and set fields. Each SDField has - * a name, a numeric ID, a data type, and a boolean that says whether it's - * a header field. The numeric ID is used when the fields are stored + * the Document class to get and set fields. Each SDField has a name, a numeric ID, + * a data type. The numeric ID is used when the fields are stored * in serialized form. * * @author bratseth @@ -92,7 +91,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, private NormalizeLevel normalizing = new NormalizeLevel(); /** Extra query commands of this field */ - private List<String> queryCommands=new java.util.ArrayList<>(0); + private List<String> queryCommands = new java.util.ArrayList<>(0); /** Summary fields defined in this field */ private Map<String, SummaryField> summaryFields = new java.util.LinkedHashMap<>(0); @@ -120,15 +119,14 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * Creates a new field. This method is only used to create reserved fields * @param name The name of the field * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field (true = "header"). */ - protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader, boolean populate) { - super(name, id, dataType, isHeader); - populate(populate, repo, name, dataType, isHeader); + protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) { + super(name, id, dataType); + populate(populate, repo, name, dataType); } - public SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader) { - this(repo, name, id, dataType, isHeader, true); + public SDField(SDDocumentType repo, String name, int id, DataType dataType) { + this(repo, name, id, dataType, true); } /** @@ -136,41 +134,35 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, @param name The name of the field @param dataType The datatype of the field - @param isHeader Whether this is a "header" field or a "content" field - (true = "header"). */ - public SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, boolean populate) { - super(name,dataType,isHeader); - populate(populate, repo, name, dataType, isHeader); + public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) { + super(name,dataType); + populate(populate, repo, name, dataType); } - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader) { - populate(populate,repo, name, dataType, isHeader, null, 0); + private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) { + populate(populate,repo, name, dataType, null, 0); } - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader, Matching fieldMatching, int recursion) { + private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) { if (populate || (dataType instanceof MapDataType)) { - populateWithStructFields(repo, name, dataType, isHeader, recursion); + populateWithStructFields(repo, name, dataType, recursion); populateWithStructMatching(repo, name, dataType, fieldMatching); } } - public SDField(String name, DataType dataType, boolean isHeader) { - this(null, name, dataType, isHeader, true); - } /** * Creates a new field. * * @param name The name of the field * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field (true = "header"). * @param owner the owning document (used to check for id collisions) */ - protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, boolean populate) { - super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType()); + protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) { + super(name, dataType, owner == null ? null : owner.getDocumentType()); this.ownerDocType=owner; - populate(populate, repo, name, dataType, isHeader); + populate(populate, repo, name, dataType); } /** @@ -178,27 +170,25 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * * @param name The name of the field * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field (true = "header"). * @param owner The owning document (used to check for id collisions) * @param fieldMatching The matching object to set for the field */ - protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, + protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, Matching fieldMatching, boolean populate, int recursion) { - super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType()); + super(name, dataType, owner == null ? null : owner.getDocumentType()); this.ownerDocType=owner; if (fieldMatching != null) this.setMatching(fieldMatching); - populate(populate, repo, name, dataType, isHeader, fieldMatching, recursion); + populate(populate, repo, name, dataType, fieldMatching, recursion); } /** - * Constructor for <b>header</b> fields * * @param name The name of the field * @param dataType The datatype of the field */ public SDField(SDDocumentType repo, String name, DataType dataType) { - this(repo, name,dataType,true, true); + this(repo, name,dataType, true); } public SDField(String name, DataType dataType) { this(null, name,dataType); @@ -277,7 +267,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } } - public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, boolean isHeader, int recursion) { + public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) { DataType dt = getFirstStructOrMapRecursive(); if (dt == null) { return; @@ -286,11 +276,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, MapDataType mdt = (MapDataType) dataType; SDField keyField = new SDField(sdoc, name.concat(".key"), mdt.getKeyType(), - isHeader, getOwnerDocType(), new Matching(), true, recursion + 1); + getOwnerDocType(), new Matching(), true, recursion + 1); structFields.put("key", keyField); SDField valueField = new SDField(sdoc, name.concat(".value"), mdt.getValueType(), - isHeader, getOwnerDocType(), new Matching(), true, recursion + 1); + getOwnerDocType(), new Matching(), true, recursion + 1); structFields.put("value", valueField); } else { if (recursion >= 10) { @@ -306,7 +296,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } for (Field field : subType.fieldSet()) { SDField subField = new SDField(sdoc, name.concat(".").concat(field.getName()), field.getDataType(), - isHeader, subType, new Matching(), true, recursion + 1); + subType, new Matching(), true, recursion + 1); structFields.put(field.getName(), subField); } } @@ -759,20 +749,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, return queryCommands.contains(name); } - /** - * A list of query commands - * - * @return a list of strings with query commands. - */ + /** Returns a list of query commands */ @Override - public List<String> getQueryCommands() { - return queryCommands; - } + public List<String> getQueryCommands() { return queryCommands; } - /** - * The document that this field was declared in, or null - * - */ + /** Returns the document that this field was declared in, or null */ private SDDocumentType getOwnerDocType() { return ownerDocType; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java index 886bf777d3a..04d11792379 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java @@ -8,12 +8,12 @@ import com.yahoo.document.DataType; */ public class TemporarySDField extends SDField { - public TemporarySDField(String name, DataType dataType, boolean isHeader, SDDocumentType owner) { - super(owner, name, dataType, isHeader, owner, false); + public TemporarySDField(String name, DataType dataType, SDDocumentType owner) { + super(owner, name, dataType, owner, false); } public TemporarySDField(String name, DataType dataType) { - super(null, name, dataType, true, false); + super(null, name, dataType, false); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java index 6fdf448a39b..a6707ec7ac0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java @@ -27,6 +27,7 @@ public class ExpressionTransforms { ImmutableList.of(new TensorFlowFeatureConverter(), new OnnxFeatureConverter(), new XgboostFeatureConverter(), + new LightGBMFeatureConverter(), new ConstantDereferencer(), new ConstantTensorTransformer(), new FunctionInliner(), diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java new file mode 100644 index 00000000000..5bde627dc0a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java @@ -0,0 +1,59 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.expressiontransforms; + +import com.yahoo.path.Path; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.vespa.model.ml.ConvertedModel; +import com.yahoo.vespa.model.ml.FeatureArguments; + +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Replaces instances of the lightgbm(model-path) pseudofeature with the + * native Vespa ranking expression implementing the same computation. + * + * @author lesters + */ +public class LightGBMFeatureConverter extends ExpressionTransformer<RankProfileTransformContext> { + + /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ + private final Map<Path, ConvertedModel> convertedLightGBMModels = new HashMap<>(); + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeature((ReferenceNode) node, context); + else if (node instanceof CompositeNode) + return super.transformChildren((CompositeNode) node, context); + else + return node; + } + + private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { + if ( ! feature.getName().equals("lightgbm")) return feature; + + try { + FeatureArguments arguments = asFeatureArguments(feature.getArguments()); + ConvertedModel convertedModel = + convertedLightGBMModels.computeIfAbsent(arguments.path(), + path -> ConvertedModel.fromSourceOrStore(path, true, context)); + return convertedModel.expression(arguments, context); + } catch (IllegalArgumentException | UncheckedIOException e) { + throw new IllegalArgumentException("Could not use LightGBM model from " + feature, e); + } + } + + private FeatureArguments asFeatureArguments(Arguments arguments) { + if (arguments.size() != 1) + throw new IllegalArgumentException("A lightgbm node must take a single argument pointing to " + + "the LightGBM model file under [application]/models"); + return new FeatureArguments(arguments); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java index 09e8b3ccdfa..ab9148992e0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java @@ -11,7 +11,7 @@ public interface FieldOperationContainer { /** Adds an operation */ void addOperation(FieldOperation op); - /** Apply all operations. Operations must be sorted in thjeir natural order before applying each operation. */ + /** Apply all operations. Operations must be sorted in their natural order before applying each operation. */ void applyOperations(SDField field); String getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java index 39f543c7db3..0c1f443dee3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.fieldoperation; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.Index.Type; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.HnswIndexParams; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; @@ -31,6 +32,9 @@ public class IndexOperation implements FieldOperation { private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); private Optional<Boolean> enableBm25 = Optional.empty(); + private Optional<String> distanceMetric = Optional.empty(); + private Optional<HnswIndexParams.Builder> hnswIndexParams = Optional.empty(); + public String getIndexName() { return indexName; } @@ -91,6 +95,12 @@ public class IndexOperation implements FieldOperation { if (enableBm25.isPresent()) { index.setInterleavedFeatures(enableBm25.get()); } + if (distanceMetric.isPresent()) { + index.setDistanceMetric(distanceMetric.get()); + } + if (hnswIndexParams.isPresent()) { + index.setHnswIndexParams(hnswIndexParams.get().build()); + } } public Type getType() { @@ -116,8 +126,17 @@ public class IndexOperation implements FieldOperation { public void setDensePostingListThreshold(double densePostingListThreshold) { this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold); } + public void setEnableBm25(boolean value) { enableBm25 = Optional.of(value); } + public void setDistanceMetric(String value) { + this.distanceMetric = Optional.of(value); + } + + public void setHnswIndexParams(HnswIndexParams.Builder params) { + this.hnswIndexParams = Optional.of(params); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java index a0d47d7fa81..0a29fae04bf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java @@ -4,9 +4,10 @@ package com.yahoo.searchdefinition.fieldoperation; import com.yahoo.searchdefinition.document.SDField; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class IndexingRewriteOperation implements FieldOperation { - public void apply(SDField field) { - } + + public void apply(SDField field) { } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index d1eb18c4916..0ffd13927b4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -71,7 +71,7 @@ public class AddExtraFieldsToDocument extends Processor { if (docField == null) { ImmutableSDField existingField = search.getField(field.getName()); if (existingField == null) { - SDField newField = new SDField(document, field.getName(), field.getDataType(), field.isHeader(), true); + SDField newField = new SDField(document, field.getName(), field.getDataType(), true); newField.setIsExtraField(true); document.addField(newField); } else if (!existingField.isImportedField()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index d6c334ee80b..6d3de23238d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -64,7 +64,7 @@ public class ImportedFieldsResolver extends Processor { } private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference, - ImmutableSDField targetField, boolean validate) { + ImmutableSDField targetField, boolean validate) { TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()), reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName())); ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference); @@ -175,7 +175,7 @@ public class ImportedFieldsResolver extends Processor { } private void validateTargetField(TemporaryImportedField importedField, - ImmutableSDField targetField, DocumentReference reference) { + ImmutableSDField targetField, DocumentReference reference) { if (!targetField.doesAttributing()) { fail(importedField, targetFieldAsString(targetField.getName(), reference) + ": Is not an attribute field. Only attribute fields supported"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java index 9a9e9bbba5f..89b8889b4ae 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java @@ -15,7 +15,10 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.logging.Level; /** @@ -45,9 +48,10 @@ public class RankingExpressionTypeResolver extends Processor { public void process(boolean validate, boolean documentsOnly) { if (documentsOnly) return; + Set<Reference> warnedAbout = new HashSet<>(); for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) { try { - resolveTypesIn(profile, validate); + resolveTypesIn(profile, validate, warnedAbout); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("In " + (search != null ? search + ", " : "") + profile, e); @@ -60,7 +64,7 @@ public class RankingExpressionTypeResolver extends Processor { * * @throws IllegalArgumentException if validate is true and the given rank profile does not produce valid types */ - private void resolveTypesIn(RankProfile profile, boolean validate) { + private void resolveTypesIn(RankProfile profile, boolean validate, Set<Reference> warnedAbout) { MapEvaluationTypeContext context = profile.typeContext(queryProfiles); for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : profile.getFunctions().entrySet()) { ExpressionFunction expressionFunction = function.getValue().function(); @@ -83,10 +87,14 @@ public class RankingExpressionTypeResolver extends Processor { profile.getSummaryFeatures().forEach(f -> resolveType(f, "summary feature " + f, context)); ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context); - if ( context.tensorsAreUsed() && ! context.queryFeaturesNotDeclared().isEmpty()) { - deployLogger.log(Level.WARNING, "The following query features are not declared in query profile " + + if ( context.tensorsAreUsed() && + ! context.queryFeaturesNotDeclared().isEmpty() && + ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) { + deployLogger.log(Level.WARNING, "The following query features used in '" + profile.getName() + + "' are not declared in query profile " + "types and will be interpreted as scalars, not tensors: " + context.queryFeaturesNotDeclared()); + warnedAbout.addAll(context.queryFeaturesNotDeclared()); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 8e54d7c00d6..c97ee2bd935 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -6,7 +6,8 @@ import com.yahoo.document.CollectionDataType; import com.yahoo.document.TensorDataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.HnswIndexParams; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -23,34 +24,71 @@ public class TensorFieldProcessor extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - for (SDField field : search.allConcreteFields()) { + for (var field : search.allConcreteFields()) { if ( field.getDataType() instanceof TensorDataType ) { - validateIndexingScripsForTensorField(field); - validateAttributeSettingForTensorField(field); + if (validate) { + validateIndexingScripsForTensorField(field); + validateAttributeSettingForTensorField(field); + } + processIndexSettingsForTensorField(field, validate); } else if (field.getDataType() instanceof CollectionDataType){ - validateDataTypeForCollectionField(field); + if (validate) { + validateDataTypeForCollectionField(field); + } } } } private void validateIndexingScripsForTensorField(SDField field) { - if (field.doesIndexing()) { - fail(search, field, "A field of type 'tensor' cannot be specified as an 'index' field."); + if (field.doesIndexing() && !isTensorTypeThatSupportsHnswIndex(field)) { + fail(search, field, "A tensor of type '" + tensorTypeToString(field) + "' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that."); + } + } + + private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) { + var type = ((TensorDataType)field.getDataType()).getTensorType(); + // Tensors with 1 indexed dimension supports a hnsw index (used for approximate nearest neighbor search). + if ((type.dimensions().size() == 1) && + type.dimensions().get(0).isIndexed()) { + return true; } + return false; + } + + private String tensorTypeToString(ImmutableSDField field) { + return ((TensorDataType)field.getDataType()).getTensorType().toString(); } private void validateAttributeSettingForTensorField(SDField field) { if (field.doesAttributing()) { - Attribute attribute = field.getAttributes().get(field.getName()); + var attribute = field.getAttributes().get(field.getName()); if (attribute != null && attribute.isFastSearch()) { fail(search, field, "An attribute of type 'tensor' cannot be 'fast-search'."); } } } + private void processIndexSettingsForTensorField(SDField field, boolean validate) { + if (!field.doesIndexing()) { + return; + } + if (isTensorTypeThatSupportsHnswIndex(field)) { + if (validate && !field.doesAttributing()) { + fail(search, field, "A tensor that has an index must also be an attribute."); + } + var index = field.getIndex(field.getName()); + // TODO: Calculate default params based on tensor dimension size + var params = new HnswIndexParams(); + if (index != null) { + params = params.overrideFrom(index.getHnswIndexParams()); + field.getAttribute().setDistanceMetric(index.getDistanceMetric()); + } + field.getAttribute().setHnswIndexParams(params); + } + } + private void validateDataTypeForCollectionField(SDField field) { if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType) fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index d0a0bbfb748..d6398bc348c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -61,7 +61,7 @@ public class UriHack extends Processor { String partName = uriName + "." + suffix; // I wonder if this is explicit in qrs or implicit in backend? // search.addFieldSetItem(uriName, partName); - SDField partField = new SDField(partName, generatedType, true); + SDField partField = new SDField(partName, generatedType); partField.setIndexStructureField(uriField.doesIndexing()); partField.setRankType(uriField.getRankType()); partField.setStemming(Stemming.NONE); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 02d500931d7..e0d015cc8b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -62,6 +62,7 @@ public class DocumentManager { } } } + private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) { for (AnnotationType type : types) { DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder(); @@ -110,6 +111,7 @@ public class DocumentManager { doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); } buildConfig(dt.getFieldSets(), doc); + buildImportedFieldsConfig(dt.getImportedFieldNames(), doc); } else if (type instanceof TemporaryStructuredDataType) { //Ignored } else if (type instanceof StructDataType) { @@ -164,4 +166,12 @@ public class DocumentManager { doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames())); } + private void buildImportedFieldsConfig(Collection<String> fieldNames, Datatype.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumentmanagerConfig.Datatype.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index d4228b52746..e6bf826dccc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -58,6 +58,7 @@ public class DocumentTypes { buildConfig(annotation, atb); } buildConfig(documentType.getFieldSets(), db); + buildImportedFieldsConfig(documentType.getImportedFieldNames(), db); builder.documenttype(db); } @@ -120,6 +121,14 @@ public class DocumentTypes { } } + private void buildImportedFieldsConfig(Collection<String> fieldNames, DocumenttypesConfig.Documenttype.Builder builder) { + for (String fieldName : fieldNames) { + var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder(); + ib.name(fieldName); + builder.importedfield(ib); + } + } + private void buildConfig(StructDataType type, DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, DocumenttypesConfig.Documenttype.Builder documentBuilder, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 72c475fb091..75b33e184d9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -461,6 +461,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon public FileReference sendFile(String relativePath) { return getRoot().getFileDistributor().sendFileToHost(relativePath, getHost()); } + public FileReference sendUri(String uri) { return getRoot().getFileDistributor().sendUriToHost(uri, getHost()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java index bf86bc4a453..48cd378a9a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java @@ -1,16 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.util.List; -import java.util.Map; - import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigInstance.Builder; import com.yahoo.config.model.producer.UserConfigRepo; +import java.io.PrintStream; +import java.util.List; +import java.util.Map; + /** * Interface that should be implemented by all config producing modules * in the vespa model. @@ -35,17 +33,6 @@ public interface ConfigProducer extends com.yahoo.config.ConfigInstance.Producer List<Service> getDescendantServices(); /** - * Writes files that need to be written. The files will usually - * only be written when the Vespa model is generated through the - * deploy-application script. - * This is primarily intended for debugging. - * - * @param directory directory to write files to - * @throws java.io.IOException if writing fails - */ - void writeFiles(File directory) throws IOException; - - /** * Dump the three of config producers to the specified stream. * * @param out The stream to print to, e.g. System.out diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index a952d63526b..db469c2091e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; -import java.io.File; -import java.util.Objects; - import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; +import java.util.Objects; + /** * A physical host, running a set of services. * The identity of a host is its hostname. Hosts are comparable on their host name. @@ -67,10 +66,6 @@ public final class Host extends AbstractConfigProducer<AbstractConfigProducer<?> } @Override - public void writeFiles(File directory) { - } - - @Override public void getConfig(SentinelConfig.Builder builder) { // TODO (MAJOR_RELEASE): This shouldn't really be here, but we need to make sure users can upgrade if we change sentinel to use hosts/<hostname>/sentinel instead of hosts/<hostname> // as config id. We should probably wait for a major release diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index eda562bea5a..557a61ec211 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; import java.net.UnknownHostException; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -34,13 +33,11 @@ public class HostSystem extends AbstractConfigProducer<Host> { private static Logger log = Logger.getLogger(HostSystem.class.getName()); - private Map<String,String> hostnames = new LinkedHashMap<>(); - private final Map<String, HostResource> hostname2host = new LinkedHashMap<>(); private final HostProvisioner provisioner; private final DeployLogger deployLogger; - public HostSystem(AbstractConfigProducer parent, String name, HostProvisioner provisioner, DeployLogger deployLogger) { + public HostSystem(AbstractConfigProducer<?> parent, String name, HostProvisioner provisioner, DeployLogger deployLogger) { super(parent, name); this.provisioner = provisioner; this.deployLogger = deployLogger; @@ -49,7 +46,8 @@ public class HostSystem extends AbstractConfigProducer<Host> { void checkName(String hostname) { // Give a warning if the host does not exist try { - Object address = java.net.InetAddress.getByName(hostname); + @SuppressWarnings("unused") + Object ignore = java.net.InetAddress.getByName(hostname); } catch (UnknownHostException e) { deployLogger.log(Level.WARNING, "Unable to lookup IP address of host: " + hostname); } @@ -78,36 +76,10 @@ public class HostSystem extends AbstractConfigProducer<Host> { return hostname2host.get(name); } - /** - * Returns the canonical name of a given host. This will cache names for faster lookup. - * - * @param hostname the hostname to retrieve the canonical hostname for. - * @return The canonical hostname, or null if unable to resolve. - * @throws UnknownHostException if the hostname cannot be resolved - */ - public String getCanonicalHostname(String hostname) throws UnknownHostException { - if ( ! hostnames.containsKey(hostname)) { - hostnames.put(hostname, lookupCanonicalHostname(hostname)); - } - return hostnames.get(hostname); - } - - /** - * Static helper method that looks up the canonical name of a given host. - * - * @param hostname the hostname to retrieve the canonical hostname for. - * @return The canonical hostname, or null if unable to resolve. - * @throws UnknownHostException if the hostname cannot be resolved - */ - // public - This is used by amenders outside this repo - public static String lookupCanonicalHostname(String hostname) throws UnknownHostException { - return java.net.InetAddress.getByName(hostname).getCanonicalHostName(); - } - @Override public String toString() { return "hosts [" + hostname2host.values().stream() - .map(host -> host.getHostname()) + .map(HostResource::getHostname) .collect(Collectors.joining(", ")) + "]"; } @@ -139,8 +111,8 @@ public class HostSystem extends AbstractConfigProducer<Host> { } } - public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, int groups, DeployLogger logger) { - List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, groups, new ProvisionDeployLogger(logger)); + public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, DeployLogger logger) { + List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, new ProvisionDeployLogger(logger)); // TODO: Even if HostResource owns a set of memberships, we need to return a map because the caller needs the current membership. Map<HostResource, ClusterMembership> retAllocatedHosts = new LinkedHashMap<>(); for (HostSpec spec : allocatedHosts) { @@ -169,7 +141,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { } Set<HostSpec> getHostSpecs() { - return getHosts().stream().map(host -> host.spec()).collect(Collectors.toCollection(LinkedHashSet::new)); + return getHosts().stream().map(HostResource::spec).collect(Collectors.toCollection(LinkedHashSet::new)); } /** A provision logger which forwards to a deploy logger */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 2cfd2e23d08..6f2123f248c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -23,11 +23,13 @@ import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.api.HostInfo; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.log.LogLevel; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; @@ -61,7 +63,6 @@ import com.yahoo.vespa.model.utils.internal.ReflectionUtil; import com.yahoo.yolean.Exceptions; import org.xml.sax.SAXException; -import java.io.File; import java.io.IOException; import java.io.Serializable; import java.lang.reflect.Constructor; @@ -123,6 +124,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private final FileDistributor fileDistributor; + private final Provisioned provisioned; + /** Creates a Vespa Model from internal model types only */ public VespaModel(ApplicationPackage app) throws IOException, SAXException { this(app, new NullConfigModelRegistry()); @@ -163,6 +166,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri configModelRegistry = new VespaConfigModelRegistry(configModelRegistry); VespaModelBuilder builder = new VespaDomBuilder(); this.applicationPackage = deployState.getApplicationPackage(); + this.provisioned = deployState.provisioned(); root = builder.getRoot(VespaModel.ROOT_CONFIGID, deployState, this); createGlobalRankProfiles(deployState.getDeployLogger(), deployState.getImportedModels(), @@ -205,7 +209,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** Creates a mutable model with no services instantiated */ public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException { - return new VespaModel(new NullConfigModelRegistry(), deployState, false, new FileDistributor(deployState.getFileRegistry(), null)); + return new VespaModel(new NullConfigModelRegistry(), deployState, false, + new FileDistributor(deployState.getFileRegistry(), List.of(), deployState.isHosted())); } private void validateWrapExceptions() { @@ -564,23 +569,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri id2producer.put(configId, descendant); } - /** - * Writes MODEL.cfg files for all config producers. - * - * @param baseDirectory dir to write files to - */ - public void writeFiles(File baseDirectory) throws IOException { - super.writeFiles(baseDirectory); - for (ConfigProducer cp : id2producer.values()) { - try { - File destination = new File(baseDirectory, cp.getConfigId().replace("/", File.separator)); - cp.writeFiles(destination); - } catch (IOException e) { - throw new IOException(cp.getConfigId() + ": " + e.getMessage()); - } - } - } - public Clients getClients() { return configModelRepo.getClients(); } @@ -629,11 +617,23 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri return Collections.unmodifiableMap(id2producer); } - /** - * Returns this root's model repository - */ + /** Returns this root's model repository */ public ConfigModelRepo configModelRepo() { return configModelRepo; } + /** If provisioning through the node repo, returns the provision requests issued during build of this */ + public Provisioned provisioned() { return provisioned; } + + /** Returns the id of all clusters in this */ + public Set<ClusterSpec.Id> allClusters() { + return hostSystem().getHosts().stream() + .map(HostResource::spec) + .filter(spec -> spec.membership().isPresent()) + .map(spec -> spec.membership().get().cluster().id()) + .collect(Collectors.toSet()); + } + + + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 54807697da4..631f4dab1a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -146,11 +146,13 @@ public class VespaModelFactory implements ModelFactory { .properties(modelContext.properties()) .vespaVersion(version()) .modelHostProvisioner(createHostProvisioner(modelContext)) + .provisioned(modelContext.provisioned()) .endpoints(modelContext.properties().endpoints()) .modelImporters(modelImporters) .zone(zone) .now(clock.instant()) - .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()); + .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()) + .wantedDockerImageRepo(modelContext.wantedDockerImageRepository()); modelContext.previousModel().ifPresent(builder::previousModel); return builder.build(validationParameters); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 5714d41ef67..24e88d7ef7d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -4,13 +4,14 @@ package com.yahoo.vespa.model.admin; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; /** * @author hmusum */ -public class LogserverContainerCluster extends ContainerCluster<LogserverContainer> implements ThreadpoolConfig.Producer { +public class LogserverContainerCluster extends ContainerCluster<LogserverContainer> { public LogserverContainerCluster(AbstractConfigProducer<?> parent, String name, DeployState deployState) { super(parent, name, name, deployState); @@ -27,6 +28,12 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain builder.maxthreads(10); } + @Override + public void getConfig(QrStartConfig.Builder builder) { + super.getConfig(builder); + builder.jvm.heapsize(384); + } + protected boolean messageBusEnabled() { return false; } private void addLogHandler() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index 2de4e5f5950..41d9df414ea 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -11,8 +11,7 @@ import com.yahoo.vespa.model.container.ContainerCluster; * * @author gjoranv */ -public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer> implements - ThreadpoolConfig.Producer +public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer> { public ClusterControllerContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { super(parent, subId, name, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index fd924eb2a0f..fccacc3210d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metricsproxy.http.metrics.MetricsV2Handler; +import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; @@ -20,6 +22,7 @@ import java.util.Map; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_NAME; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.createMetricsHandler; /** * Container running a metrics proxy. @@ -28,9 +31,11 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus */ public class MetricsProxyContainer extends Container implements NodeDimensionsConfig.Producer, + NodeInfoConfig.Producer, RpcConnectorConfig.Producer, VespaServicesConfig.Producer { + public static final int BASEPORT = 19092; final boolean isHostedVespa; @@ -46,6 +51,7 @@ public class MetricsProxyContainer extends Container implements addMetricsProxyComponent(NodeDimensions.class); addMetricsProxyComponent(RpcConnector.class); addMetricsProxyComponent(VespaServices.class); + addHandler(createMetricsHandler(MetricsV2Handler.class, MetricsV2Handler.V2_PATH)); } @Override @@ -53,8 +59,6 @@ public class MetricsProxyContainer extends Container implements return METRICS_PROXY_CONTAINER; } - static public int BASEPORT = 19092; - @Override public int getWantedPort() { return BASEPORT; @@ -121,7 +125,21 @@ public class MetricsProxyContainer extends Container implements } } - private void addMetricsProxyComponent(Class<?> componentClass) { + @Override + public void getConfig(NodeInfoConfig.Builder builder) { + builder.role(getNodeRole()) + .hostname(getHostName()); + } + + private String getNodeRole() { + String hostConfigId = getHost().getConfigId(); + if (! isHostedVespa) return hostConfigId; + return getHostResource().spec().membership() + .map(ClusterMembership::stringValue) + .orElse(hostConfigId); + } + + private void addMetricsProxyComponent(Class<?> componentClass) { addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index fcc6c3279de..20f2bfe6636 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -20,6 +20,9 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcServer; import ai.vespa.metricsproxy.service.ConfigSentinelClient; import ai.vespa.metricsproxy.service.SystemPollerProvider; +import ai.vespa.metricsproxy.telegraf.Telegraf; +import ai.vespa.metricsproxy.telegraf.TelegrafConfig; +import ai.vespa.metricsproxy.telegraf.TelegrafRegistry; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; @@ -67,7 +70,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC ApplicationDimensionsConfig.Producer, ConsumersConfig.Producer, MonitoringConfig.Producer, - ThreadpoolConfig.Producer, + TelegrafConfig.Producer, MetricsNodesConfig.Producer { public static final Logger log = Logger.getLogger(MetricsProxyContainerCluster.class.getName()); @@ -116,14 +119,30 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC addHttpHandler(ApplicationMetricsHandler.class, ApplicationMetricsHandler.V1_PATH); addMetricsProxyComponent(ApplicationMetricsRetriever.class); + + addTelegrafComponents(); } private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { + Handler<AbstractConfigProducer<?>> metricsHandler = createMetricsHandler(clazz, bindingPath); + addComponent(metricsHandler); + } + + static Handler<AbstractConfigProducer<?>> createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { Handler<AbstractConfigProducer<?>> metricsHandler = new Handler<>( new ComponentModel(clazz.getName(), null, METRICS_PROXY_BUNDLE_NAME, null)); metricsHandler.addServerBindings("http://*" + bindingPath, "http://*" + bindingPath + "/*"); - addComponent(metricsHandler); + return metricsHandler; + } + + private void addTelegrafComponents() { + getAdmin().ifPresent(admin -> { + if (admin.getUserMetrics().usesExternalMetricSystems()) { + addMetricsProxyComponent(Telegraf.class); + addMetricsProxyComponent(TelegrafRegistry.class); + } + }); } @Override @@ -156,6 +175,32 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC } @Override + public void getConfig(TelegrafConfig.Builder builder) { + builder.isHostedVespa(isHostedVespa()); + + var userConsumers = getUserMetricsConsumers(); + for (var consumer : userConsumers.values()) { + for (var cloudWatch : consumer.cloudWatches()) { + var cloudWatchBuilder = new TelegrafConfig.CloudWatch.Builder(); + cloudWatchBuilder + .region(cloudWatch.region()) + .namespace(cloudWatch.namespace()) + .consumer(cloudWatch.consumer()); + + cloudWatch.hostedAuth().ifPresent(hostedAuth -> cloudWatchBuilder + .accessKeyName(hostedAuth.accessKeyName) + .secretKeyName(hostedAuth.secretKeyName)); + + cloudWatch.sharedCredentials().ifPresent(sharedCredentials -> { + cloudWatchBuilder.file(sharedCredentials.file); + sharedCredentials.profile.ifPresent(cloudWatchBuilder::profile); + }); + builder.cloudWatch(cloudWatchBuilder); + } + } + } + + @Override public void getConfig(ThreadpoolConfig.Builder builder) { builder.maxthreads(10); } @@ -198,7 +243,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC Optional.of(monitoring.getInterval()) : Optional.empty(); } - private void addMetricsProxyComponent(Class<?> componentClass) { + private void addMetricsProxyComponent(Class<?> componentClass) { addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java new file mode 100644 index 00000000000..5351c3fb3a7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java @@ -0,0 +1,59 @@ +package com.yahoo.vespa.model.admin.monitoring; + +import java.util.Optional; + +/** + * Helper object for CloudWatch configuration. + * + * @author gjoranv + */ +public class CloudWatch { + private final String region; + private final String namespace; + private final MetricsConsumer consumer; + + private HostedAuth hostedAuth; + private SharedCredentials sharedCredentials; + + public CloudWatch(String region, String namespace, MetricsConsumer consumer) { + this.region = region; + this.namespace = namespace; + this.consumer = consumer; + } + + public String region() { return region; } + public String namespace() { return namespace; } + public String consumer() { return consumer.getId(); } + + public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); } + public Optional<SharedCredentials> sharedCredentials() {return Optional.ofNullable(sharedCredentials); } + + public void setHostedAuth(String accessKeyName, String secretKeyName) { + hostedAuth = new HostedAuth(accessKeyName, secretKeyName); + } + + public void setSharedCredentials(String file, Optional<String> profile) { + sharedCredentials = new SharedCredentials(file, profile); + } + + public static class HostedAuth { + public final String accessKeyName; + public final String secretKeyName; + + HostedAuth(String accessKeyName, String secretKeyName) { + this.accessKeyName = accessKeyName; + this.secretKeyName = secretKeyName; + } + } + + public static class SharedCredentials { + public final String file; + public final Optional<String> profile; + + SharedCredentials(String file, Optional<String> profile) { + this.file = file; + this.profile = profile; + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java index 19a62f47e39..c80cebe3d5b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java @@ -20,10 +20,12 @@ import static java.util.Collections.singleton; */ public class DefaultPublicMetrics { + public static final String DEFAULT_METRIC_SET_ID = "default"; + public static MetricSet defaultPublicMetricSet = createMetricSet(); private static MetricSet createMetricSet() { - return new MetricSet("public", + return new MetricSet(DEFAULT_METRIC_SET_ID, getAllMetrics(), singleton(defaultVespaMetricSet)); } @@ -84,6 +86,7 @@ public class DefaultPublicMetrics { metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); metrics.add(new Metric("content.proton.documentdb.matching.docs_reranked.rate")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.average")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index 529ed6ecf67..a8fbcf50b02 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -2,9 +2,13 @@ package com.yahoo.vespa.model.admin.monitoring; import javax.annotation.concurrent.Immutable; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; +import static java.util.Collections.unmodifiableList; + /** * Represents an arbitrary metric consumer * @@ -13,12 +17,15 @@ import java.util.Objects; */ @Immutable public class MetricsConsumer { + private final String id; private final MetricSet metricSet; + private final List<CloudWatch> cloudWatches = new ArrayList<>(); + /** - * @param id The consumer - * @param metricSet The metrics for this consumer + * @param id the consumer + * @param metricSet the metrics for this consumer */ public MetricsConsumer(String id, MetricSet metricSet) { this.id = Objects.requireNonNull(id, "A consumer must have a non-null id.");; @@ -38,4 +45,12 @@ public class MetricsConsumer { return metricSet.getMetrics(); } + public void addCloudWatch(CloudWatch cloudWatch) { + cloudWatches.add(cloudWatch); + } + + public List<CloudWatch> cloudWatches() { + return unmodifiableList(cloudWatches); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java index 58b77ee1297..c05cad89852 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java @@ -9,6 +9,7 @@ import java.util.Set; * @author gjoranv */ public class SystemMetrics { + public static final String CPU_UTIL = "cpu.util"; public static final String CPU_SYS_UTIL = "cpu.sys.util"; public static final String CPU_THROTTLED_TIME = "cpu.throttled_time.rate"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 57d938f8a71..141794256b8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -72,6 +72,9 @@ public class VespaMetricSet { metrics.add(new Metric("vds.server.network.tls-connections-broken")); metrics.add(new Metric("vds.server.network.failed-tls-config-reloads")); + // C++ Fnet metrics + metrics.add(new Metric("vds.server.fnet.num-connections")); + return metrics; } @@ -126,6 +129,8 @@ public class VespaMetricSet { metrics.add(new Metric("serverActiveThreads.count")); metrics.add(new Metric("serverActiveThreads.last")); + metrics.add(new Metric("jdisc.thread_pool.unhandled_exceptions.rate")); + metrics.add(new Metric("httpapi_latency.max")); metrics.add(new Metric("httpapi_latency.sum")); metrics.add(new Metric("httpapi_latency.count")); @@ -179,6 +184,15 @@ public class VespaMetricSet { metrics.add(new Metric("jdisc.http.request.content_size.sum")); metrics.add(new Metric("jdisc.http.request.content_size.count")); metrics.add(new Metric("jdisc.http.request.content_size.average")); // TODO: Remove in Vespa 8 + + metrics.add(new Metric("jdisc.http.ssl.handshake.failure.missing_client_cert.rate")); + metrics.add(new Metric("jdisc.http.ssl.handshake.failure.expired_client_cert.rate")); + metrics.add(new Metric("jdisc.http.ssl.handshake.failure.invalid_client_cert.rate")); + metrics.add(new Metric("jdisc.http.ssl.handshake.failure.incompatible_protocols.rate")); + metrics.add(new Metric("jdisc.http.ssl.handshake.failure.incompatible_ciphers.rate")); + metrics.add(new Metric("jdisc.http.ssl.handshake.failure.unknown.rate")); + + metrics.add(new Metric("jdisc.http.handler.unhandled_exceptions.rate")); return metrics; } @@ -288,6 +302,14 @@ public class VespaMetricSet { return metrics; } + private static void addSearchNodeExecutorMetrics(Set<Metric> metrics, String prefix) { + metrics.add(new Metric(prefix + ".queuesize.max")); + metrics.add(new Metric(prefix + ".queuesize.sum")); + metrics.add(new Metric(prefix + ".queuesize.count")); + metrics.add(new Metric(prefix + ".maxpending.last")); // TODO: Remove in Vespa 8 + metrics.add(new Metric(prefix + ".accepted.rate")); + } + private static Set<Metric> getSearchNodeMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); @@ -332,18 +354,12 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.search_protocol.docsum.requested_documents.count")); // Executors shared between all document dbs - metrics.add(new Metric("content.proton.executor.proton.maxpending.last")); - metrics.add(new Metric("content.proton.executor.proton.accepted.rate")); - metrics.add(new Metric("content.proton.executor.flush.maxpending.last")); - metrics.add(new Metric("content.proton.executor.flush.accepted.rate")); - metrics.add(new Metric("content.proton.executor.match.maxpending.last")); - metrics.add(new Metric("content.proton.executor.match.accepted.rate")); - metrics.add(new Metric("content.proton.executor.docsum.maxpending.last")); - metrics.add(new Metric("content.proton.executor.docsum.accepted.rate")); - metrics.add(new Metric("content.proton.executor.shared.maxpending.last")); - metrics.add(new Metric("content.proton.executor.shared.accepted.rate")); - metrics.add(new Metric("content.proton.executor.warmup.maxpending.last")); - metrics.add(new Metric("content.proton.executor.warmup.accepted.rate")); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.proton"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.flush"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.match"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.docsum"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.shared"); + addSearchNodeExecutorMetrics(metrics, "content.proton.executor.warmup"); // jobs metrics.add(new Metric("content.proton.documentdb.job.total.average")); @@ -357,18 +373,12 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.job.removed_documents_prune.average")); // Threading service (per document db) - metrics.add(new Metric("content.proton.documentdb.threading_service.master.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.master.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.summary.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.summary.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_inverter.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_inverter.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_writer.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_writer.accepted.rate")); - metrics.add(new Metric("content.proton.documentdb.threading_service.attribute_field_writer.maxpending.last")); - metrics.add(new Metric("content.proton.documentdb.threading_service.attribute_field_writer.accepted.rate")); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.master"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.summary"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index_field_inverter"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index_field_writer"); + addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.attribute_field_writer"); // lid space metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_bloat_factor.average")); @@ -453,10 +463,13 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.query_latency.sum")); metrics.add(new Metric("content.proton.documentdb.matching.query_latency.count")); metrics.add(new Metric("content.proton.documentdb.matching.query_latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.max")); - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.count")); + metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.max")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.sum")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.count")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.max")); + metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.count")); metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.queries.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doomed_queries.rate")); @@ -468,10 +481,13 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.max")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.sum")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.count")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.max")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.sum")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.count")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.max")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.max")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.count")); @@ -513,13 +529,24 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.sum")); metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.count")); metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.max")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.sum")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.count")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.max")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.sum")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.count")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.max")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.sum")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.count")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.max")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.sum")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.count")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); + metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.visitor.allthreads.completed.sum.rate")); metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate")); + metrics.add(new Metric("vds.visitor.allthreads.failed.sum.rate")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.max")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.sum")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.count")); @@ -528,19 +555,27 @@ public class VespaMetricSet { metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.sum")); metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.count")); metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.average")); // TODO: Remove in Vespa 8 - + + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.count")); @@ -560,13 +595,13 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.allthreads.splitbuckets.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.joinbuckets.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.failed.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.max")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.sum")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.count")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.filestor.alldisks.allthreads.setbucketstates.count.rate")); - //Distributor metrics.add(new Metric("vds.idealstate.buckets_rechecking.average")); metrics.add(new Metric("vds.idealstate.idealstate_diff.average")); @@ -589,6 +624,8 @@ public class VespaMetricSet { metrics.add(new Metric("vds.idealstate.garbage_collection.done_ok.rate")); metrics.add(new Metric("vds.idealstate.garbage_collection.done_failed.rate")); metrics.add(new Metric("vds.idealstate.garbage_collection.pending.average")); + metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.count")); + metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.rate")); metrics.add(new Metric("vds.distributor.puts.sum.latency.max")); metrics.add(new Metric("vds.distributor.puts.sum.latency.sum")); @@ -596,18 +633,24 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.puts.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.puts.sum.ok.rate")); metrics.add(new Metric("vds.distributor.puts.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.puts.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.puts.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.removes.sum.latency.max")); metrics.add(new Metric("vds.distributor.removes.sum.latency.sum")); metrics.add(new Metric("vds.distributor.removes.sum.latency.count")); metrics.add(new Metric("vds.distributor.removes.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.removes.sum.ok.rate")); metrics.add(new Metric("vds.distributor.removes.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.updates.sum.latency.max")); metrics.add(new Metric("vds.distributor.updates.sum.latency.sum")); metrics.add(new Metric("vds.distributor.updates.sum.latency.count")); metrics.add(new Metric("vds.distributor.updates.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.updates.sum.ok.rate")); metrics.add(new Metric("vds.distributor.updates.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.failures.test_and_set_failed.rate")); metrics.add(new Metric("vds.distributor.updates.sum.diverging_timestamp_updates.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.ok.rate")); metrics.add(new Metric("vds.distributor.removelocations.sum.failures.total.rate")); @@ -617,6 +660,7 @@ public class VespaMetricSet { metrics.add(new Metric("vds.distributor.gets.sum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("vds.distributor.gets.sum.ok.rate")); metrics.add(new Metric("vds.distributor.gets.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.gets.sum.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.max")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.sum")); metrics.add(new Metric("vds.distributor.visitor.sum.latency.count")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java index 9b0d9dbfadc..1f81f16a80b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java @@ -28,4 +28,13 @@ public class Metrics { return consumers.keySet().stream() .anyMatch(existing -> existing.equalsIgnoreCase(id)); } + + /** + * Returns true if any of the consumers have specified external metric systems. + */ + public boolean usesExternalMetricSystems() { + return consumers.values().stream() + .anyMatch(consumer -> ! consumer.cloudWatches().isEmpty()); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java new file mode 100644 index 00000000000..314ef9cc5a5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java @@ -0,0 +1,40 @@ +package com.yahoo.vespa.model.admin.monitoring.builder.xml; + +import com.yahoo.vespa.model.admin.monitoring.CloudWatch; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; +import org.w3c.dom.Element; + +import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalAttribute; +import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChild; + +/** + * @author gjoranv + */ +public class CloudWatchBuilder { + + private static final String REGION_ATTRIBUTE = "region"; + private static final String NAMESPACE_ATTRIBUTE = "namespace"; + private static final String CREDENTIALS_ELEMENT = "credentials"; + private static final String ACCESS_KEY_ATTRIBUTE = "access-key-name"; + private static final String SECRET_KEY_ATTRIBUTE = "secret-key-name"; + private static final String SHARED_CREDENTIALS_ELEMENT = "shared-credentials"; + private static final String PROFILE_ATTRIBUTE = "profile"; + private static final String FILE_ATTRIBUTE = "file"; + + public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) { + CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE), + cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE), + consumer); + + getOptionalChild(cloudwatchElement, CREDENTIALS_ELEMENT) + .ifPresent(elem -> cloudWatch.setHostedAuth(elem.getAttribute(ACCESS_KEY_ATTRIBUTE), + elem.getAttribute(SECRET_KEY_ATTRIBUTE))); + + getOptionalChild(cloudwatchElement, SHARED_CREDENTIALS_ELEMENT) + .ifPresent(elem -> cloudWatch.setSharedCredentials(elem.getAttribute(FILE_ATTRIBUTE), + getOptionalAttribute(elem, PROFILE_ATTRIBUTE))); + + return cloudWatch; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index b13fa4917e4..b686288868f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -42,7 +42,11 @@ public class MetricsBuilder { throwIfIllegalConsumerId(metrics, consumerId); MetricSet metricSet = buildMetricSet(consumerId, consumerElement); - metrics.addConsumer(new MetricsConsumer(consumerId, metricSet)); + var consumer = new MetricsConsumer(consumerId, metricSet); + for (Element cloudwatchElement : XML.getChildren(consumerElement, "cloudwatch")) { + consumer.addCloudWatch(CloudWatchBuilder.buildCloudWatch(cloudwatchElement, consumer)); + } + metrics.addConsumer(consumer); } return metrics; } @@ -58,11 +62,11 @@ public class MetricsBuilder { private MetricSet buildMetricSet(String consumerId, Element consumerElement) { List<Metric> metrics = XML.getChildren(consumerElement, "metric").stream() - .map(metricElement -> metricFromElement(metricElement)) + .map(MetricsBuilder::metricFromElement) .collect(Collectors.toCollection(LinkedList::new)); List<MetricSet> metricSets = XML.getChildren(consumerElement, "metric-set").stream() - .map(metricSetElement -> availableMetricSets.get(metricSetElement.getAttribute(ID_ATTRIBUTE))) + .map(metricSetElement -> getMetricSetOrThrow(metricSetElement.getAttribute(ID_ATTRIBUTE))) .collect(Collectors.toCollection(LinkedList::new)); metricSets.add(defaultVespaMetricSet); @@ -75,6 +79,11 @@ public class MetricsBuilder { return "user-metrics-" + consumerName; } + private MetricSet getMetricSetOrThrow(String id) { + if (! availableMetricSets.containsKey(id)) throw new IllegalArgumentException("No such metric-set: " + id); + return availableMetricSets.get(id); + } + private void throwIfIllegalConsumerId(Metrics metrics, String consumerId) { if (consumerId.equalsIgnoreCase(VESPA_CONSUMER_ID) && applicationType != ApplicationType.HOSTED_INFRASTRUCTURE) throw new IllegalArgumentException("'Vespa' is not allowed as metrics consumer id (case is ignored.)"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java new file mode 100644 index 00000000000..631ab0e2640 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java @@ -0,0 +1,48 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.CloudName; +import com.yahoo.vespa.model.VespaModel; + +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.collections.CollectionUtil.mkString; +import static com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator.needsAccessControlValidation; +import static com.yahoo.vespa.model.container.http.AccessControl.hasHandlerThatNeedsProtection; + +/** + * @author gjoranv + */ +public class AwsAccessControlValidator extends Validator { + + // NOTE: must be the same as the name in the declaration of the AWS cloud in hosted. + static final String AWS_CLOUD_NAME = "aws"; + + @Override + public void validate(VespaModel model, DeployState deployState) { + + if (! needsAccessControlValidation(model, deployState)) return; + if(! deployState.zone().cloud().equals(CloudName.from(AWS_CLOUD_NAME))) return; + + List<String> offendingClusters = new ArrayList<>(); + for (var cluster : model.getContainerClusters().values()) { + var http = cluster.getHttp(); + if (http == null + || ! http.getAccessControl().isPresent() + || ! http.getAccessControl().get().writeEnabled + || ! http.getAccessControl().get().readEnabled) + + if (hasHandlerThatNeedsProtection(cluster) || ! cluster.getAllServlets().isEmpty()) + offendingClusters.add(cluster.getName()); + } + if (! offendingClusters.isEmpty()) + deployState.validationOverrides() + .invalid(ValidationId.accessControl, + "Access-control must be enabled for read/write operations to container clusters in AWS production zones: " + + mkString(offendingClusters, "[", ", ", "]"), deployState.now()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java new file mode 100644 index 00000000000..462ac39fa84 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java @@ -0,0 +1,37 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; + +import java.util.List; + +import static java.util.stream.Collectors.toList; + +/** + * @author gjoranv + */ +public class CloudWatchValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + if (!deployState.isHosted()) return; + if (deployState.zone().system().isPublic()) return; + if (model.getAdmin().getApplicationType() != ConfigModelContext.ApplicationType.DEFAULT) return; + + var offendingConsumers = model.getAdmin().getUserMetrics().getConsumers().values().stream() + .filter(consumer -> !consumer.cloudWatches().isEmpty()) + .collect(toList()); + + if (! offendingConsumers.isEmpty()) { + throw new IllegalArgumentException("CloudWatch cannot be set up for non-public hosted Vespa and must " + + "be removed for consumers: " + consumerIds(offendingConsumers)); + } + } + + private List<String> consumerIds(List<MetricsConsumer> offendingConsumers) { + return offendingConsumers.stream().map(MetricsConsumer::getId).collect(toList()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index f9762ce58fa..43c1a88b0a1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -33,7 +33,7 @@ public class ComplexAttributeFieldsValidator extends Validator { continue; } SearchCluster searchCluster = (SearchCluster) cluster; - for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) { + for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) { validateComplexFields(searchCluster.getClusterName(), spec.getSearchDefinition().getSearch()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java index ac38336a405..d5bea2a2959 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.InstanceName; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ContainerModel; @@ -12,8 +13,7 @@ import java.util.List; import java.util.Optional; /** - * Validates that deployment spec (deployment.xml) has valid values (for now - * only global-service-id is validated) + * Validate deployment spec (deployment.xml). * * @author hmusum * @author bratseth @@ -30,11 +30,20 @@ public class DeploymentSpecValidator extends Validator { List<ContainerModel> containers = model.getRoot().configModelRepo().getModels(ContainerModel.class); for (DeploymentInstanceSpec instance : deploymentSpec.instances()) { instance.globalServiceId().ifPresent(globalServiceId -> { - if ( containers.stream().noneMatch(container -> container.getCluster().getName().equals(globalServiceId))) - throw new IllegalArgumentException("The global-service-id in " + instance + ", '" + globalServiceId + - "' specified in deployment.xml does not match any container cluster id"); + requireClusterId(containers, instance.name(), "Attribute 'globalServiceId'", globalServiceId); + }); + instance.endpoints().forEach(endpoint -> { + requireClusterId(containers, instance.name(), "Endpoint '" + endpoint.endpointId() + "'", + endpoint.containerId()); }); } } + private static void requireClusterId(List<ContainerModel> containers, InstanceName instanceName, String context, + String id) { + if (containers.stream().noneMatch(container -> container.getCluster().getName().equals(id))) + throw new IllegalArgumentException(context + " in instance " + instanceName + ": '" + id + + "' specified in deployment.xml does not match any container cluster ID"); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java index f00ad0f0dbb..4b8bbd4ff08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java @@ -12,7 +12,7 @@ public class EndpointCertificateSecretsValidator extends Validator { @Override public void validate(VespaModel model, DeployState deployState) { if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get() == EndpointCertificateSecrets.MISSING) { - throw new CertificateNotReadyException("TLS enabled, but could not retrieve certificate yet"); + throw new CertificateNotReadyException("TLS enabled, but could not yet retrieve certificate for application " + deployState.getProperties().applicationId().serializedForm()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 7f8ff6edd85..b6f7ab4ff62 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -33,24 +33,29 @@ import java.util.logging.Logger; /** * Validate rank setup for all search clusters (rank-profiles, index-schema, attributes configs), validating done - * by running through the binary 'vespa-verify-ranksetup' + * by running the binary 'vespa-verify-ranksetup-bin' * * @author vegardh */ public class RankSetupValidator extends Validator { private static final Logger log = Logger.getLogger(RankSetupValidator.class.getName()); - private final boolean force; + private static final String binaryName = "vespa-verify-ranksetup-bin "; - public RankSetupValidator(boolean force) { - this.force = force; + private final boolean ignoreValidationErrors; + + public RankSetupValidator(boolean ignoreValidationErrors) { + this.ignoreValidationErrors = ignoreValidationErrors; } @Override public void validate(VespaModel model, DeployState deployState) { File cfgDir = null; try { - cfgDir = Files.createTempDirectory("deploy_ranksetup").toFile(); + cfgDir = Files.createTempDirectory("verify-ranksetup." + + deployState.getProperties().applicationId().toFullString() + + ".") + .toFile(); for (AbstractSearchCluster cluster : model.getSearchClusters()) { // Skipping rank expression checking for streaming clusters, not implemented yet @@ -100,29 +105,29 @@ public class RankSetupValidator extends Validator { IOUtils.recursiveDeleteDir(dir); } - private void writeConfigs(String dir, AbstractConfigProducer producer) throws IOException { + private void writeConfigs(String dir, AbstractConfigProducer<?> producer) throws IOException { RankProfilesConfig.Builder rpcb = new RankProfilesConfig.Builder(); - RankProfilesConfig.Producer.class.cast(producer).getConfig(rpcb); + ((RankProfilesConfig.Producer) producer).getConfig(rpcb); RankProfilesConfig rpc = new RankProfilesConfig(rpcb); writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpc); IndexschemaConfig.Builder iscb = new IndexschemaConfig.Builder(); - IndexschemaConfig.Producer.class.cast(producer).getConfig(iscb); + ((IndexschemaConfig.Producer) producer).getConfig(iscb); IndexschemaConfig isc = new IndexschemaConfig(iscb); writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", isc); AttributesConfig.Builder acb = new AttributesConfig.Builder(); - AttributesConfig.Producer.class.cast(producer).getConfig(acb); + ((AttributesConfig.Producer) producer).getConfig(acb); AttributesConfig ac = new AttributesConfig(acb); writeConfig(dir, AttributesConfig.getDefName() + ".cfg", ac); RankingConstantsConfig.Builder rccb = new RankingConstantsConfig.Builder(); - RankingConstantsConfig.Producer.class.cast(producer).getConfig(rccb); + ((RankingConstantsConfig.Producer) producer).getConfig(rccb); RankingConstantsConfig rcc = new RankingConstantsConfig(rccb); writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rcc); ImportedFieldsConfig.Builder ifcb = new ImportedFieldsConfig.Builder(); - ImportedFieldsConfig.Producer.class.cast(producer).getConfig(ifcb); + ((ImportedFieldsConfig.Producer) producer).getConfig(ifcb); ImportedFieldsConfig ifc = new ImportedFieldsConfig(ifcb); writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifc); } @@ -132,8 +137,8 @@ public class RankSetupValidator extends Validator { } private boolean execValidate(String configId, SearchCluster sc, String sdName, DeployLogger deployLogger) { - String job = "vespa-verify-ranksetup-bin " + configId; - ProcessExecuter executer = new ProcessExecuter(); + String job = String.format("%s %s", binaryName, configId); + ProcessExecuter executer = new ProcessExecuter(true); try { Pair<Integer, String> ret = executer.exec(job); if (ret.getFirst() != 0) { @@ -147,27 +152,28 @@ public class RankSetupValidator extends Validator { } private void validateWarn(Exception e, DeployLogger deployLogger) { - String msg = "Unable to execute 'vespa-verify-ranksetup', validation of rank expressions will only take place when you start Vespa: " + + String msg = "Unable to execute '"+ binaryName + "', validation of rank expressions will only take place when you start Vespa: " + Exceptions.toMessageString(e); deployLogger.log(LogLevel.WARNING, msg); } private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger deployLogger) { - String errMsg = "For search cluster '" + sc.getClusterName() + "', search definition '" + sdName + "': error in rank setup. Details:\n"; + StringBuilder errMsg = new StringBuilder("For search cluster '").append(sc.getClusterName()).append("', ") + .append("search definition '").append(sdName).append("': error in rank setup. Details:\n"); for (String line : output.split("\n")) { // Remove debug lines from start script if (line.startsWith("debug\t")) continue; try { - LogMessage logmsg = LogMessage.parseNativeFormat(line); - errMsg = errMsg + logmsg.getLevel() + ": " + logmsg.getPayload() + "\n"; + LogMessage logMessage = LogMessage.parseNativeFormat(line); + errMsg.append(logMessage.getLevel()).append(": ").append(logMessage.getPayload()).append("\n"); } catch (InvalidLogFormatException e) { - errMsg = errMsg + line + "\n"; + errMsg.append(line).append("\n"); } } - if (force) { - deployLogger.log(LogLevel.WARNING, errMsg + "(Continuing because of force.)"); + if (ignoreValidationErrors) { + deployLogger.log(LogLevel.WARNING, errMsg.append("(Continuing since ignoreValidationErrors flag is set.)").toString()); } else { - throw new IllegalArgumentException(errMsg); + throw new IllegalArgumentException(errMsg.toString()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java index 907418ea9f0..9568ea5c27c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -9,7 +9,7 @@ import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; import java.io.FileNotFoundException; @@ -47,7 +47,7 @@ public class RankingConstantsValidator extends Validator { ApplicationPackage applicationPackage = deployState.getApplicationPackage(); ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Invalid constant tensor file(s):"); - for (SearchDefinition sd : deployState.getSearchDefinitions()) { + for (NamedSchema sd : deployState.getSchemas()) { for (RankingConstant rc : sd.getSearch().rankingConstants().asMap().values()) { try { validateRankingConstant(rc, applicationPackage); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java index 85ba75639eb..031ce0dbdd4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -15,7 +15,7 @@ import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.AbstractSearchCluster; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; import java.util.List; @@ -34,7 +34,7 @@ public class SearchDataTypeValidator extends Validator { if (cluster.isStreaming()) { continue; } - for (AbstractSearchCluster.SearchDefinitionSpec spec : cluster.getLocalSDS()) { + for (AbstractSearchCluster.SchemaSpec spec : cluster.getLocalSDS()) { SDDocumentType docType = spec.getSearchDefinition().getSearch().getDocument(); if (docType == null) { continue; @@ -44,7 +44,7 @@ public class SearchDataTypeValidator extends Validator { } } - private void validateDocument(AbstractSearchCluster cluster, SearchDefinition def, SDDocumentType doc) { + private void validateDocument(AbstractSearchCluster cluster, NamedSchema def, SDDocumentType doc) { for (SDDocumentType child : doc.getTypes()) { validateDocument(cluster, def, child); } @@ -84,7 +84,7 @@ public class SearchDataTypeValidator extends Validator { } } - private void disallowIndexingOfMaps(AbstractSearchCluster cluster, SearchDefinition def, Field field) { + private void disallowIndexingOfMaps(AbstractSearchCluster cluster, NamedSchema def, Field field) { DataType fieldType = field.getDataType(); if ((fieldType instanceof MapDataType) && (((SDField) field).doesIndexing())) { throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' cannot be indexed for search " + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 1e4a45428b8..22dd0289390 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -59,6 +59,8 @@ public class Validation { new SecretStoreValidator().validate(model, deployState); new EndpointCertificateSecretsValidator().validate(model, deployState); new AccessControlFilterValidator().validate(model, deployState); + new CloudWatchValidator().validate(model, deployState); + new AwsAccessControlValidator().validate(model, deployState); List<ConfigChangeAction> result = Collections.emptyList(); if (deployState.getProperties().isFirstTimeDeployment()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java index d14fe91a53b..162f6798462 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.VespaModel; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; @@ -21,35 +23,32 @@ public class ClusterSizeReductionValidator implements ChangeValidator { @Override public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) { - for (ContainerCluster currentCluster : current.getContainerClusters().values()) { - ContainerCluster nextCluster = next.getContainerClusters().get(currentCluster.getName()); - if (nextCluster == null) continue; - validate(currentCluster.getContainers().size(), - nextCluster.getContainers().size(), - currentCluster.getName(), + for (var clusterId : current.allClusters()) { + Capacity currentCapacity = current.provisioned().all().get(clusterId); + Capacity nextCapacity = next.provisioned().all().get(clusterId); + if (currentCapacity == null || nextCapacity == null) continue; + validate(currentCapacity, + nextCapacity, + clusterId, overrides, now); } - - for (ContentCluster currentCluster : current.getContentClusters().values()) { - ContentCluster nextCluster = next.getContentClusters().get(currentCluster.getName()); - if (nextCluster == null) continue; - validate(currentCluster.getSearch().getSearchNodes().size(), - nextCluster.getSearch().getSearchNodes().size(), - currentCluster.getName(), - overrides, - now); - } - return Collections.emptyList(); } - private void validate(int currentSize, int nextSize, String clusterName, ValidationOverrides overrides, Instant now) { + private void validate(Capacity current, + Capacity next, + ClusterSpec.Id clusterId, + ValidationOverrides overrides, + Instant now) { + int currentSize = current.minResources().nodes(); + int nextSize = next.minResources().nodes(); // don't allow more than 50% reduction, but always allow to reduce size with 1 if ( nextSize < ((double)currentSize) * 0.5 && nextSize != currentSize - 1) overrides.invalid(ValidationId.clusterSizeReduction, - "Size reduction in '" + clusterName + "' is too large. Current size: " + currentSize + - ", new size: " + nextSize + ". New size must be at least 50% of the current size", + "Size reduction in '" + clusterId.value() + "' is too large: " + + "New min size must be at least 50% of the current min size. " + + "Current size: " + currentSize + ", new size: " + nextSize, now); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index 8fdcf249bbc..5343a322382 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -5,6 +5,7 @@ import com.yahoo.collections.Pair; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.model.HostResource; @@ -15,6 +16,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -27,35 +29,43 @@ public class ResourcesReductionValidator implements ChangeValidator { @Override public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) { - var currentRequestedResourcesByClusterId = getRequestedResourcesByClusterId(current); - var nextRequestedResourcesByClusterId = getRequestedResourcesByClusterId(next); - - for (var clusterTypeAndId : currentRequestedResourcesByClusterId.keySet()) { - if (!nextRequestedResourcesByClusterId.containsKey(clusterTypeAndId)) continue; - validate(currentRequestedResourcesByClusterId.get(clusterTypeAndId), - nextRequestedResourcesByClusterId.get(clusterTypeAndId), - clusterTypeAndId.getSecond(), - overrides, - now); + for (var clusterId : current.allClusters()) { + Capacity currentCapacity = current.provisioned().all().get(clusterId); + Capacity nextCapacity = next.provisioned().all().get(clusterId); + if (currentCapacity == null || nextCapacity == null) continue; + validate(currentCapacity, nextCapacity, clusterId, overrides, now); } return List.of(); } - private void validate(NodeResources currentResources, NodeResources nextResources, ClusterSpec.Id clusterId, - ValidationOverrides overrides, Instant now) { + private void validate(Capacity current, + Capacity next, + ClusterSpec.Id clusterId, + ValidationOverrides overrides, + Instant now) { + if (current.minResources().nodeResources() == NodeResources.unspecified) return; + if (next.minResources().nodeResources() == NodeResources.unspecified) return; + List<String> illegalChanges = Stream.of( - validateResource("vCPU", currentResources.vcpu(), nextResources.vcpu()), - validateResource("memory GB", currentResources.memoryGb(), nextResources.memoryGb()), - validateResource("disk GB", currentResources.diskGb(), nextResources.diskGb())) + validateResource("vCPU", + current.minResources().nodeResources().vcpu(), + next.minResources().nodeResources().vcpu()), + validateResource("memory GB", + current.minResources().nodeResources().memoryGb(), + next.minResources().nodeResources().memoryGb()), + validateResource("disk GB", + current.minResources().nodeResources().diskGb(), + next.minResources().nodeResources().diskGb())) .flatMap(Optional::stream) .collect(Collectors.toList()); if (illegalChanges.isEmpty()) return; overrides.invalid(ValidationId.resourcesReduction, - "Resource reduction in '" + clusterId.value() + "' is too large. " + - String.join(" ", illegalChanges) + " New resources must be at least 50% of the current resources", - now); + "Resource reduction in '" + clusterId.value() + "' is too large. " + + String.join(" ", illegalChanges) + + " New min resources must be at least 50% of the current min resources", + now); } private static Optional<String> validateResource(String resourceName, double currentValue, double nextValue) { @@ -64,15 +74,4 @@ public class ResourcesReductionValidator implements ChangeValidator { return Optional.of(String.format(Locale.ENGLISH ,"Current %s: %.2f, new: %.2f.", resourceName, currentValue, nextValue)); } - private static Map<Pair<ClusterSpec.Type, ClusterSpec.Id>, NodeResources> getRequestedResourcesByClusterId(VespaModel vespaModel) { - return vespaModel.hostSystem().getHosts().stream() - .map(HostResource::spec) - .filter(spec -> spec.membership().isPresent() && spec.requestedResources().isPresent()) - .filter(spec -> !spec.membership().get().retired()) - .collect(Collectors.toMap( - spec -> new Pair<>(spec.membership().get().cluster().type(), spec.membership().get().cluster().id()), - spec -> spec.requestedResources().get(), - (e1, e2) -> e1)); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java index 97153e42ee5..df5fe15ca82 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java @@ -7,16 +7,16 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.InstanceName; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.Validator; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.component.Handler; import java.util.ArrayList; import java.util.List; import static com.yahoo.collections.CollectionUtil.mkString; -import static com.yahoo.vespa.model.container.http.AccessControl.isBuiltinGetOnly; +import static com.yahoo.config.provision.InstanceName.defaultName; +import static com.yahoo.vespa.model.container.http.AccessControl.hasHandlerThatNeedsProtection; /** * Validates that hosted applications in prod zones have write protection enabled. @@ -28,10 +28,7 @@ public class AccessControlOnFirstDeploymentValidator extends Validator { @Override public void validate(VespaModel model, DeployState deployState) { - if (! deployState.isHosted()) return; - if (! deployState.zone().environment().isProduction()) return; - if (deployState.zone().system().isPublic()) return; - if (model.getAdmin().getApplicationType() != ApplicationType.DEFAULT) return; + if (! needsAccessControlValidation(model, deployState)) return; List<String> offendingClusters = new ArrayList<>(); for (ContainerCluster<? extends Container> c : model.getContainerClusters().values()) { @@ -44,23 +41,19 @@ public class AccessControlOnFirstDeploymentValidator extends Validator { if (hasHandlerThatNeedsProtection(cluster) || ! cluster.getAllServlets().isEmpty()) offendingClusters.add(cluster.getName()); } - if (! offendingClusters.isEmpty() - && deployState.getApplicationPackage().getApplicationId().instance().equals(InstanceName.defaultName())) + if (! offendingClusters.isEmpty()) deployState.validationOverrides().invalid(ValidationId.accessControl, "Access-control must be enabled for write operations to container clusters in production zones: " + - mkString(offendingClusters, "[", ", ", "]."), deployState.now()); - } - - private boolean hasHandlerThatNeedsProtection(ApplicationContainerCluster cluster) { - return cluster.getHandlers().stream().anyMatch(this::handlerNeedsProtection); + mkString(offendingClusters, "[", ", ", "]"), deployState.now()); } - private boolean handlerNeedsProtection(Handler<?> handler) { - return ! isBuiltinGetOnly(handler) && hasNonMbusBinding(handler); - } + public static boolean needsAccessControlValidation(VespaModel model, DeployState deployState) { + if (! deployState.isHosted()) return false; + if (! deployState.zone().environment().isProduction()) return false; + if (deployState.zone().system().isPublic()) return false; + if (! deployState.getApplicationPackage().getApplicationId().instance().equals(defaultName())) return false; + if (model.getAdmin().getApplicationType() != ApplicationType.DEFAULT) return false; - private boolean hasNonMbusBinding(Handler<?> handler) { - return handler.getServerBindings().stream().anyMatch(binding -> ! binding.startsWith("mbus")); + return true; } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index a146e2cd7e7..d2f06da992c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -71,7 +71,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu Monitoring monitoring = getMonitoring(XML.getChild(adminElement,"monitoring"), deployState.isHosted()); Metrics metrics = new MetricsBuilder(applicationType, predefinedMetricSets) .buildMetrics(XML.getChild(adminElement, "metrics")); - FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent); + FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent, deployState.isHosted()); Admin admin = new Admin(parent, monitoring, metrics, multitenant, fileDistributionConfigProducer, deployState.isHosted()); admin.setApplicationType(applicationType); @@ -81,8 +81,8 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu return admin; } - private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer parent) { - return new FileDistributionConfigProducer(parent, fileRegistry, configServerSpecs); + private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer parent, boolean isHosted) { + return new FileDistributionConfigProducer(parent, fileRegistry, configServerSpecs, isHosted); } protected abstract void doBuildAdmin(DeployState deployState, Admin admin, Element adminE); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 61d0cd7cd1e..804a5442608 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -65,12 +65,12 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { createSlobroks(deployLogger, admin, allocateHosts(admin.hostSystem(), "slobroks", nodesSpecification)); } else { - createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.count(), 2)); + createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.minResources().nodes(), 2)); } } private void assignLogserver(DeployState deployState, NodesSpecification nodesSpecification, Admin admin) { - if (nodesSpecification.count() > 1) throw new IllegalArgumentException("You can only request a single log server"); + if (nodesSpecification.minResources().nodes() > 1) throw new IllegalArgumentException("You can only request a single log server"); if (deployState.getProperties().applicationId().instance().isTester()) return; // No logserver is needed on tester applications if (nodesSpecification.isDedicated()) { Collection<HostResource> hosts = allocateHosts(admin.hostSystem(), "logserver", nodesSpecification); @@ -79,7 +79,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { Logserver logserver = createLogserver(deployState.getDeployLogger(), admin, hosts); createContainerOnLogserverHost(deployState, admin, logserver.getHostResource()); } else if (containerModels.iterator().hasNext()) { - List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.count(), false); + List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.minResources().nodes(), false); if (hosts.isEmpty()) return; // No log server can be created (and none is needed) createLogserver(deployState.getDeployLogger(), admin, hosts); @@ -91,8 +91,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { private NodesSpecification createNodesSpecificationForLogserver() { DeployState deployState = context.getDeployState(); if (deployState.getProperties().useDedicatedNodeForLogserver() && - context.getApplicationType() == ConfigModelContext.ApplicationType.DEFAULT && - deployState.isHosted()) + context.getApplicationType() == ConfigModelContext.ApplicationType.DEFAULT && + deployState.isHosted()) return NodesSpecification.dedicated(1, context); else return NodesSpecification.nonDedicated(1, context); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java index f8f1f88e339..11fab0ada29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.text.XML; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import org.w3c.dom.Element; @@ -14,9 +15,13 @@ import org.w3c.dom.Element; */ public class DomClientProviderBuilder extends DomHandlerBuilder { + public DomClientProviderBuilder(ApplicationContainerCluster cluster) { + super(cluster); + } + @Override - protected Handler doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element clientElement) { - Handler<? super Component<?, ?>> client = getHandler(clientElement); + protected Handler doBuild(DeployState deployState, AbstractConfigProducer parent, Element clientElement) { + Handler<? super Component<?, ?>> client = createHandler(clientElement); for (Element binding : XML.getChildren(clientElement, "binding")) client.addClientBindings(XML.getValue(binding)); @@ -24,7 +29,7 @@ public class DomClientProviderBuilder extends DomHandlerBuilder { for (Element serverBinding : XML.getChildren(clientElement, "serverBinding")) client.addServerBindings(XML.getValue(serverBinding)); - DomComponentBuilder.addChildren(deployState, ancestor, clientElement, client); + DomComponentBuilder.addChildren(deployState, parent, clientElement, client); return client; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 558c4428d15..ac6d089cf24 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -1,38 +1,86 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; -import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; +import java.util.Set; + +import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_1; +import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_2; +import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1; +import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_2; +import static com.yahoo.vespa.model.container.ContainerCluster.VIP_HANDLER_BINDING; +import static java.util.logging.Level.INFO; + /** * @author gjoranv */ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> { + private static final Set<String> reservedBindings = Set.of(METRICS_V2_HANDLER_BINDING_1, + METRICS_V2_HANDLER_BINDING_2, + STATE_HANDLER_BINDING_1, + STATE_HANDLER_BINDING_2, + VIP_HANDLER_BINDING); + private final ApplicationContainerCluster cluster; + + public DomHandlerBuilder(ApplicationContainerCluster cluster) { + this.cluster = cluster; + } + @Override - protected Handler doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element handlerElement) { - Handler<? super Component<?, ?>> handler = getHandler(handlerElement); + protected Handler doBuild(DeployState deployState, AbstractConfigProducer parent, Element handlerElement) { + Handler<? super Component<?, ?>> handler = createHandler(handlerElement); for (Element binding : XML.getChildren(handlerElement, "binding")) - handler.addServerBindings(XML.getValue(binding)); + addServerBinding(handler, XML.getValue(binding), deployState.getDeployLogger()); for (Element clientBinding : XML.getChildren(handlerElement, "clientBinding")) handler.addClientBindings(XML.getValue(clientBinding)); - DomComponentBuilder.addChildren(deployState, ancestor, handlerElement, handler); + DomComponentBuilder.addChildren(deployState, parent, handlerElement, handler); return handler; } - protected Handler<? super Component<?, ?>> getHandler(Element handlerElement) { + Handler<? super Component<?, ?>> createHandler(Element handlerElement) { BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement); return new Handler<>(new ComponentModel(bundleSpec)); } + + private void addServerBinding(Handler<? super Component<?, ?>> handler, String binding, DeployLogger log) { + throwIfBindingIsReserved(binding, handler); + handler.addServerBindings(binding); + removeExistingServerBinding(binding, handler, log); + } + + private void throwIfBindingIsReserved(String binding, Handler<?> newHandler) { + for (var reserved : reservedBindings) { + if (binding.equals(reserved)) { + throw new IllegalArgumentException("Binding '" + binding + "' is a reserved Vespa binding and " + + "cannot be used by handler: " + newHandler.getComponentId()); + } + } + } + + private void removeExistingServerBinding(String binding, Handler<?> newHandler, DeployLogger log) { + for (var handler : cluster.getHandlers()) { + if (handler.getServerBindings().contains(binding)) { + handler.removeServerBinding(binding); + log.log(INFO, "Binding '" + binding + "' was already in use by handler '" + + handler.getComponentId() + "', but will now be taken over by handler: " + newHandler.getComponentId()); + } + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java index d34a11abdf4..80c95ad6b59 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java @@ -172,7 +172,12 @@ public class ModelElement { /** Returns the content of the attribute with the given name, or null if none */ public String stringAttribute(String name) { - if ( ! xml.hasAttribute(name)) return null; + return stringAttribute(name, null); + } + + /** Returns the content of the attribute with the given name, or the default value if none */ + public String stringAttribute(String name, String defaultValue) { + if ( ! xml.hasAttribute(name)) return defaultValue; return xml.getAttribute(name); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 2c88f965f1f..ad6eebe1ca5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -1,11 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.text.XML; @@ -15,9 +17,11 @@ import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; import org.w3c.dom.Element; import org.w3c.dom.Node; +import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; +import java.util.function.Function; +import java.util.regex.Pattern; /** * A common utility class to represent a requirement for nodes during model building. @@ -27,11 +31,9 @@ import java.util.Set; */ public class NodesSpecification { - private final boolean dedicated; - - private final int count; + private final ClusterResources min, max; - private final int groups; + private final boolean dedicated; /** The Vespa version we want the nodes to run */ private Version version; @@ -43,50 +45,76 @@ public class NodesSpecification { private final boolean required; private final boolean canFail; - - private final boolean exclusive; - - /** Whether this requires running container and content processes co-located on the same node. */ - private final boolean combined; - /** The resources each node should have, or empty to use the default */ - private final Optional<NodeResources> resources; - - /** The identifier of the custom docker image layer to use (not supported yet) */ - private final Optional<String> dockerImage; + private final boolean exclusive; - private NodesSpecification(boolean dedicated, int count, int groups, Version version, - boolean required, boolean canFail, boolean exclusive, boolean combined, - Optional<NodeResources> resources, Optional<String> dockerImage) { + /** The repo part of a docker image (without tag), optional */ + private final Optional<String> dockerImageRepo; + + /** The ID of the cluster referencing this node specification, if any */ + private final Optional<String> combinedId; + + private NodesSpecification(ClusterResources min, + ClusterResources max, + boolean dedicated, Version version, + boolean required, boolean canFail, boolean exclusive, + Optional<String> dockerImageRepo, + Optional<String> combinedId) { + if (max.smallerThan(min)) + throw new IllegalArgumentException("Min resources must be larger or equal to max resources, but " + + max + " is smaller than " + min); + + // Non-scaled resources must be equal + if ( ! min.nodeResources().justNonNumbers().equals(max.nodeResources().justNonNumbers())) + throw new IllegalArgumentException("Min and max resources must have the same non-numeric settings, but " + + "min is " + min + " and max " + max); + if (min.nodeResources().bandwidthGbps() != max.nodeResources().bandwidthGbps()) + throw new IllegalArgumentException("Min and max resources must have the same bandwith, but " + + "min is " + min + " and max " + max); + + this.min = min; + this.max = max; this.dedicated = dedicated; - this.count = count; - this.groups = groups; this.version = version; this.required = required; this.canFail = canFail; this.exclusive = exclusive; - this.resources = resources; - this.dockerImage = dockerImage; - this.combined = combined; + this.dockerImageRepo = dockerImageRepo; + this.combinedId = combinedId; } - private NodesSpecification(boolean dedicated, boolean canFail, boolean combined, Version version, ModelElement nodesElement) { - this(dedicated, - nodesElement.integerAttribute("count", 1), - nodesElement.integerAttribute("groups", 1), - version, - nodesElement.booleanAttribute("required", false), - canFail, - nodesElement.booleanAttribute("exclusive", false), - combined, - getResources(nodesElement), - Optional.ofNullable(nodesElement.stringAttribute("docker-image"))); + private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, + ModelElement nodesElement, Optional<String> dockerImageRepo) { + var resolvedElement = resolveElement(nodesElement); + var combinedId = findCombinedId(nodesElement, resolvedElement); + var resources = toResources(resolvedElement); + return new NodesSpecification(resources.getFirst(), + resources.getSecond(), + dedicated, + version, + resolvedElement.booleanAttribute("required", false), + canFail, + resolvedElement.booleanAttribute("exclusive", false), + dockerImageToUse(resolvedElement, dockerImageRepo), + combinedId); } - private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement) { - var resolvedElement = resolveElement(nodesElement); - boolean combined = resolvedElement != nodesElement || isReferencedByOtherElement(nodesElement); - return new NodesSpecification(dedicated, canFail, combined, version, resolvedElement); + private static Pair<ClusterResources, ClusterResources> toResources(ModelElement nodesElement) { + Pair<Integer, Integer> nodes = toRange(nodesElement.stringAttribute("count"), 1, Integer::parseInt); + Pair<Integer, Integer> groups = toRange(nodesElement.stringAttribute("groups"), 1, Integer::parseInt); + var min = new ClusterResources(nodes.getFirst(), groups.getFirst(), nodeResources(nodesElement).getFirst()); + var max = new ClusterResources(nodes.getSecond(), groups.getSecond(), nodeResources(nodesElement).getSecond()); + return new Pair<>(min, max); + } + + /** Returns the ID of the cluster referencing this node specification, if any */ + private static Optional<String> findCombinedId(ModelElement nodesElement, ModelElement resolvedElement) { + if (resolvedElement != nodesElement) { + // Specification for a container cluster referencing nodes in a content cluster + return containerIdOf(nodesElement); + } + // Specification for a content cluster that is referenced by a container cluster + return containerIdReferencing(nodesElement); } /** Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element */ @@ -94,7 +122,8 @@ public class NodesSpecification { return create(true, ! context.getDeployState().getProperties().isBootstrap(), context.getDeployState().getWantedNodeVespaVersion(), - nodesElement); + nodesElement, + context.getDeployState().getWantedDockerImageRepo()); } /** @@ -110,7 +139,7 @@ public class NodesSpecification { } /** - * Returns a requirement for undedicated or dedicated nodes taken from the <code>nodes</code> element + * Returns a requirement for non-dedicated or dedicated nodes taken from the <code>nodes</code> element * contained in the given parent element, or empty if the parent element is null, or the nodes elements * is not present. */ @@ -121,37 +150,42 @@ public class NodesSpecification { if (nodesElement == null) return Optional.empty(); return Optional.of(create(nodesElement.booleanAttribute("dedicated", false), ! context.getDeployState().getProperties().isBootstrap(), - context.getDeployState().getWantedNodeVespaVersion(), nodesElement)); + context.getDeployState().getWantedNodeVespaVersion(), + nodesElement, + context.getDeployState().getWantedDockerImageRepo())); } - /** Returns a requirement from <code>count</code> nondedicated nodes in one group */ + /** + * Returns a requirement from <code>count</code> non-dedicated nodes in one group + */ public static NodesSpecification nonDedicated(int count, ConfigModelContext context) { - return new NodesSpecification(false, - count, - 1, + return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified), + new ClusterResources(count, 1, NodeResources.unspecified), + false, context.getDeployState().getWantedNodeVespaVersion(), false, ! context.getDeployState().getProperties().isBootstrap(), false, - false, - Optional.empty(), + context.getDeployState().getWantedDockerImageRepo(), Optional.empty()); } /** Returns a requirement from <code>count</code> dedicated nodes in one group */ public static NodesSpecification dedicated(int count, ConfigModelContext context) { - return new NodesSpecification(true, - count, - 1, + return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified), + new ClusterResources(count, 1, NodeResources.unspecified), + true, context.getDeployState().getWantedNodeVespaVersion(), false, ! context.getDeployState().getProperties().isBootstrap(), false, - false, - Optional.empty(), + context.getDeployState().getWantedDockerImageRepo(), Optional.empty()); } + public ClusterResources minResources() { return min; } + public ClusterResources maxResources() { return max; } + /** * Returns whether this requires dedicated nodes. * Otherwise the model encountering this request should reuse nodes requested for other purposes whenever possible. @@ -165,42 +199,48 @@ public class NodesSpecification { */ public boolean isExclusive() { return exclusive; } - /** Returns the number of nodes required */ - public int count() { return count; } - - /** Returns the number of host groups this specifies. Default is 1 */ - public int groups() { return groups; } - public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, DeployLogger logger) { - if (combined) + if (combinedId.isPresent()) clusterType = ClusterSpec.Type.combined; - ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId, version, exclusive); - return hostSystem.allocateHosts(cluster, Capacity.fromCount(count, resources, required, canFail), groups, logger); + ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId) + .vespaVersion(version) + .exclusive(exclusive) + .combinedId(combinedId.map(ClusterSpec.Id::from)) + .dockerImageRepo(dockerImageRepo) + .build(); + return hostSystem.allocateHosts(cluster, Capacity.from(min, max, required, canFail), logger); } - private static Optional<NodeResources> getResources(ModelElement nodesElement) { + private static Pair<NodeResources, NodeResources> nodeResources(ModelElement nodesElement) { ModelElement resources = nodesElement.child("resources"); if (resources != null) { - return Optional.of(new NodeResources(resources.requiredDoubleAttribute("vcpu"), - parseGbAmount(resources.requiredStringAttribute("memory"), "B"), - parseGbAmount(resources.requiredStringAttribute("disk"), "B"), - Optional.ofNullable(resources.stringAttribute("bandwidth")) - .map(b -> parseGbAmount(b, "BPS")) - .orElse(0.3), - parseOptionalDiskSpeed(resources.stringAttribute("disk-speed")), - parseOptionalStorageType(resources.stringAttribute("storage-type")))); + return nodeResourcesFromResorcesElement(resources); } else if (nodesElement.stringAttribute("flavor") != null) { // legacy fallback - return Optional.of(NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor"))); + var flavorResources = NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor")); + return new Pair<>(flavorResources, flavorResources); } - else { // Get the default - return Optional.empty(); + else { + return new Pair<>(NodeResources.unspecified, NodeResources.unspecified); } } + private static Pair<NodeResources, NodeResources> nodeResourcesFromResorcesElement(ModelElement element) { + Pair<Double, Double> vcpu = toRange(element.requiredStringAttribute("vcpu"), .0, Double::parseDouble); + Pair<Double, Double> memory = toRange(element.requiredStringAttribute("memory"), .0, s -> parseGbAmount(s, "B")); + Pair<Double, Double> disk = toRange(element.requiredStringAttribute("disk"), .0, s -> parseGbAmount(s, "B")); + Pair<Double, Double> bandwith = toRange(element.stringAttribute("bandwith"), .3, s -> parseGbAmount(s, "BPS")); + NodeResources.DiskSpeed diskSpeed = parseOptionalDiskSpeed(element.stringAttribute("disk-speed")); + NodeResources.StorageType storageType = parseOptionalStorageType(element.stringAttribute("storage-type")); + + var min = new NodeResources(vcpu.getFirst(), memory.getFirst(), disk.getFirst(), bandwith.getFirst(), diskSpeed, storageType); + var max = new NodeResources(vcpu.getSecond(), memory.getSecond(), disk.getSecond(), bandwith.getSecond(), diskSpeed, storageType); + return new Pair<>(min, max); + } + private static double parseGbAmount(String byteAmount, String unit) { byteAmount = byteAmount.strip(); byteAmount = byteAmount.toUpperCase(); @@ -280,24 +320,35 @@ public class NodesSpecification { return new ModelElement(referencedNodesElement); } - /** Returns whether the given nodesElement is referenced by any other nodes element */ - private static boolean isReferencedByOtherElement(ModelElement nodesElement) { + /** Returns the ID of the parent container element of nodesElement, if any */ + private static Optional<String> containerIdOf(ModelElement nodesElement) { + var element = nodesElement.getXml(); + for (var containerTag : List.of("container", "jdisc")) { + var container = findParentByTag(containerTag, element); + if (container.isEmpty()) continue; + return container.map(el -> el.getAttribute("id")); + } + return Optional.empty(); + } + + /** Returns the ID of the container element referencing nodesElement, if any */ + private static Optional<String> containerIdReferencing(ModelElement nodesElement) { var element = nodesElement.getXml(); var services = findParentByTag("services", element); - if (services.isEmpty()) return false; + if (services.isEmpty()) return Optional.empty(); var content = findParentByTag("content", element); - if (content.isEmpty()) return false; + if (content.isEmpty()) return Optional.empty(); var contentClusterId = content.get().getAttribute("id"); - if (contentClusterId.isEmpty()) return false; + if (contentClusterId.isEmpty()) return Optional.empty(); for (var rootChild : XML.getChildren(services.get())) { if ( ! ContainerModelBuilder.isContainerTag(rootChild)) continue; var nodes = XML.getChild(rootChild, "nodes"); if (nodes == null) continue; if (!contentClusterId.equals(nodes.getAttribute("of"))) continue; - return true; + return Optional.of(rootChild.getAttribute("id")); } - return false; + return Optional.empty(); } private static Optional<Element> findChildById(Element parent, String id) { @@ -319,11 +370,33 @@ public class NodesSpecification { return new IllegalArgumentException("referenced service '" + referenceId + "' is not defined"); } + private static Optional<String> dockerImageToUse(ModelElement nodesElement, Optional<String> dockerImage) { + String dockerImageFromElement = nodesElement.stringAttribute("docker-image"); + return dockerImageFromElement == null ? dockerImage : Optional.of(dockerImageFromElement); + } + + /** Parses a value ("value") or value range ("[min-value, max-value]") */ + private static <T> Pair<T, T> toRange(String s, T defaultValue, Function<String, T> valueParser) { + try { + if (s == null) return new Pair<>(defaultValue, defaultValue); + s = s.trim(); + if (s.startsWith("[") && s.endsWith("]")) { + String[] numbers = s.substring(1, s.length() - 1).split(","); + if (numbers.length != 2) throw new IllegalArgumentException(); + return new Pair<>(valueParser.apply(numbers[0].trim()), valueParser.apply(numbers[1].trim())); + } else { + return new Pair<>(valueParser.apply(s), valueParser.apply(s)); + } + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Expected a number or range on the form [min, max], but got '" + s + "'", e); + } + } + @Override public String toString() { - return "specification of " + count + (dedicated ? " dedicated " : " ") + "nodes" + - (resources.isPresent() ? " with resources " + resources.get() : "") + - (groups > 1 ? " in " + groups + " groups" : ""); + return "specification of " + (dedicated ? "dedicated " : "") + + (min.equals(max) ? min : "min " + min + " max " + max); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 97b78e1b9b1..c9caca1831f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -141,8 +141,7 @@ public class VespaDomBuilder extends VespaModelBuilder { } private void initializeService(AbstractService t, DeployState deployState, - HostSystem hostSystem, Element producerSpec) - { + HostSystem hostSystem, Element producerSpec) { initializeProducer(t, deployState, producerSpec); if (producerSpec != null) { if (producerSpec.hasAttribute(JVM_OPTIONS)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index efd00528d54..63e6af03c44 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; +import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; @@ -9,6 +10,9 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.BundlesConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.container.handler.metrics.MetricsProxyApiConfig; +import com.yahoo.container.handler.metrics.MetricsV2Handler; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.messagebus.MbusServerProvider; import com.yahoo.jdisc.http.ServletPathsConfig; @@ -17,8 +21,10 @@ import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ConfigProducerGroup; +import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.Servlet; import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; import com.yahoo.vespa.model.container.jersey.RestApi; @@ -45,7 +51,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat RankProfilesConfig.Producer, RankingConstantsConfig.Producer, ServletPathsConfig.Producer, - ContainerMbusConfig.Producer { + ContainerMbusConfig.Producer, + MetricsProxyApiConfig.Producer { + + public static final String METRICS_V2_HANDLER_CLASS = MetricsV2Handler.class.getName(); + public static final String METRICS_V2_HANDLER_BINDING_1 = "http://*" + MetricsV2Handler.V2_PATH; + public static final String METRICS_V2_HANDLER_BINDING_2 = METRICS_V2_HANDLER_BINDING_1 + "/*"; private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); @@ -58,6 +69,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private MbusParams mbusParams; private boolean messageBusEnabled = true; + private final double softStartSeconds; private Integer memoryPercentage = null; @@ -72,7 +84,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.DeprecatedSecretStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); + addMetricsV2Handler(); addTestrunnerComponentsIfTester(deployState); + softStartSeconds = deployState.getProperties().defaultSoftStartSeconds(); } @Override @@ -99,6 +113,13 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } } + public void addMetricsV2Handler() { + Handler<AbstractConfigProducer<?>> handler = new Handler<>( + new ComponentModel(METRICS_V2_HANDLER_CLASS, null, null, null)); + handler.addServerBindings(METRICS_V2_HANDLER_BINDING_1, METRICS_V2_HANDLER_BINDING_2); + addComponent(handler); + } + private void addTestrunnerComponentsIfTester(DeployState deployState) { if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-testrunner-components-jar-with-dependencies.jar"))); @@ -188,10 +209,17 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } @Override + public void getConfig(MetricsProxyApiConfig.Builder builder) { + builder.metricsPort(MetricsProxyContainer.BASEPORT) + .metricsApiPath(ApplicationMetricsHandler.VALUES_PATH); + } + + @Override public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); builder.jvm.verbosegc(true) .availableProcessors(0) + .compressedClassSpaceSize(0) //TODO Reduce, next step is 512m .minHeapsize(1536) .heapsize(1536); if (getMemoryPercentage().isPresent()) { @@ -223,6 +251,11 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat null)))); } + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + builder.softStartSeconds(softStartSeconds); + } + public static class MbusParams { // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) final Double maxConcurrentFactor; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 7e2d6680827..31c8724d634 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -140,7 +140,7 @@ public abstract class Container extends AbstractService implements if (http == null) { return defaultHttpServer; } else { - return http.getHttpServer(); + return http.getHttpServer().orElse(null); } } @@ -228,10 +228,10 @@ public abstract class Container extends AbstractService implements // XXX unused - remove: from.allocatePort("http/1"); portsMeta.on(offset++).tag("http").tag("external"); - } else if (getHttp().getHttpServer() == null) { + } else if (getHttp().getHttpServer().isEmpty()) { // no http server ports } else { - for (ConnectorFactory connectorFactory : getHttp().getHttpServer().getConnectorFactories()) { + for (ConnectorFactory connectorFactory : getHttp().getHttpServer().get().getConnectorFactories()) { int port = getPort(connectorFactory); String name = "http/" + connectorFactory.getName(); from.requirePort(port, name); @@ -280,7 +280,7 @@ public abstract class Container extends AbstractService implements final Http http = getHttp(); if (http != null) { // TODO: allow the user to specify health port manually - if (http.getHttpServer() == null) { + if (http.getHttpServer().isEmpty()) { return -1; } else { return getRelativePort(0); @@ -303,7 +303,7 @@ public abstract class Container extends AbstractService implements .slobrokId(serviceSlobrokId())). filedistributor(filedistributorConfig()); if (clusterName != null) { - builder.discriminator(clusterName+"."+name); + builder.discriminator(clusterName + "." + name); } else { builder.discriminator(name); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6fa446bf365..e2b1f97a6eb 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -19,6 +19,7 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.core.document.ContainerDocumentConfig; import com.yahoo.container.handler.ThreadPoolProvider; +import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.container.jdisc.state.StateHandler; @@ -99,7 +100,9 @@ public abstract class ContainerCluster<CONTAINER extends Container> DocprocConfig.Producer, ClusterInfoConfig.Producer, RoutingProviderConfig.Producer, - ConfigserverConfig.Producer { + ConfigserverConfig.Producer, + ThreadpoolConfig.Producer +{ /** * URI prefix used for internal, usually programmatic, APIs. URIs using this @@ -111,15 +114,20 @@ public abstract class ContainerCluster<CONTAINER extends Container> public static final String APPLICATION_STATUS_HANDLER_CLASS = "com.yahoo.container.handler.observability.ApplicationStatusHandler"; public static final String BINDINGS_OVERVIEW_HANDLER_CLASS = BindingsOverviewHandler.class.getName(); - public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler"; public static final String LOG_HANDLER_CLASS = com.yahoo.container.handler.LogHandler.class.getName(); public static final String DEFAULT_LINGUISTICS_PROVIDER = "com.yahoo.language.provider.DefaultLinguisticsProvider"; public static final String CMS = "-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1"; public static final String G1GC = "-XX:+UseG1GC -XX:MaxTenuringThreshold=15"; + public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler"; + public static final String STATE_HANDLER_BINDING_1 = "http://*" + StateHandler.STATE_API_ROOT; + public static final String STATE_HANDLER_BINDING_2 = STATE_HANDLER_BINDING_1 + "/*"; + public static final String ROOT_HANDLER_PATH = "/"; public static final String ROOT_HANDLER_BINDING = "http://*" + ROOT_HANDLER_PATH; + public static final String VIP_HANDLER_BINDING = "http://*/status.html"; + private final String name; protected List<CONTAINER> containers = new ArrayList<>(); @@ -200,15 +208,11 @@ public abstract class ContainerCluster<CONTAINER extends Container> public void addMetricStateHandler() { Handler<AbstractConfigProducer<?>> stateHandler = new Handler<>( new ComponentModel(STATE_HANDLER_CLASS, null, null, null)); - stateHandler.addServerBindings("http://*" + StateHandler.STATE_API_ROOT, - "http://*" + StateHandler.STATE_API_ROOT + "/*"); + stateHandler.addServerBindings(STATE_HANDLER_BINDING_1, STATE_HANDLER_BINDING_2); addComponent(stateHandler); } public void addDefaultRootHandler() { - if (hasHandlerWithBinding(ROOT_HANDLER_BINDING)) - return; - Handler<AbstractConfigProducer<?>> handler = new Handler<>( new ComponentModel(BundleInstantiationSpecification.getFromStrings( BINDINGS_OVERVIEW_HANDLER_CLASS, null, null), null)); // null bundle, as the handler is in container-disc @@ -216,15 +220,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> addComponent(handler); } - private boolean hasHandlerWithBinding(String binding) { - Collection<Handler<?>> handlers = getHandlers(); - for (Handler handler : handlers) { - if (handler.getServerBindings().contains(binding)) - return true; - } - return false; - } - public void addApplicationStatusHandler() { Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>( new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings( @@ -235,7 +230,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> public void addVipHandler() { Handler<?> vipHandler = Handler.fromClassName(FileStatusHandlerComponent.CLASS); - vipHandler.addServerBindings("http://*/status.html"); + vipHandler.addServerBindings(VIP_HANDLER_BINDING); addComponent(vipHandler); } @@ -484,6 +479,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> builder.jvm .verbosegc(false) .availableProcessors(2) + .compressedClassSpaceSize(32) .minHeapsize(32) .heapsize(512) .heapSizeAsPercentageOfPhysicalMemory(0) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java index c671749cff0..a466dabe984 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java @@ -4,7 +4,12 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.component.ComponentId; import com.yahoo.config.model.producer.AbstractConfigProducer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + /** * A group of config producers that have a component id. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java index ee61b34987a..3d9a1b2e665 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java @@ -10,6 +10,7 @@ import com.yahoo.osgi.provider.model.ComponentModel; * @author Tony Vaagenes */ public class FileStatusHandlerComponent extends Handler implements VipStatusConfig.Producer { + public static final String CLASS = "com.yahoo.container.handler.VipStatusHandler"; private final String fileName; @@ -26,4 +27,5 @@ public class FileStatusHandlerComponent extends Handler implements VipStatusConf builder.accessdisk(true). statusfile(fileName); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index e07c3216850..82484e07773 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -8,7 +8,9 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; /** * Models a jdisc RequestHandler (including ClientProvider). @@ -21,7 +23,7 @@ import java.util.List; */ public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> { - private List<String> serverBindings = new ArrayList<>(); + private Set<String> serverBindings = new LinkedHashSet<>(); private List<String> clientBindings = new ArrayList<>(); public Handler(ComponentModel model) { @@ -40,12 +42,16 @@ public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component< serverBindings.addAll(Arrays.asList(bindings)); } + public void removeServerBinding(String binding) { + serverBindings.remove(binding); + } + public void addClientBindings(String... bindings) { clientBindings.addAll(Arrays.asList(bindings)); } - public final List<String> getServerBindings() { - return Collections.unmodifiableList(serverBindings); + public final Set<String> getServerBindings() { + return Collections.unmodifiableSet(serverBindings); } public final List<String> getClientBindings() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index f3758def2b1..470b82496a3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.container.StatisticsConfig; +import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.net.HostName; import com.yahoo.vespa.defaults.Defaults; @@ -29,7 +30,8 @@ public class ConfigserverCluster extends AbstractConfigProducer ZookeeperServerConfig.Producer, ConfigserverConfig.Producer, StatisticsConfig.Producer, - HealthMonitorConfig.Producer { + HealthMonitorConfig.Producer, + VipStatusConfig.Producer { private final CloudConfigOptions options; private ContainerCluster containerCluster; @@ -116,8 +118,8 @@ public class ConfigserverCluster extends AbstractConfigProducer } builder.serverId(HostName.getLocalhost()); - if (!containerCluster.getHttp().getHttpServer().getConnectorFactories().isEmpty()) { - builder.httpport(containerCluster.getHttp().getHttpServer().getConnectorFactories().get(0).getListenPort()); + if (!containerCluster.getHttp().getHttpServer().get().getConnectorFactories().isEmpty()) { + builder.httpport(containerCluster.getHttp().getHttpServer().get().getConnectorFactories().get(0).getListenPort()); } if (options.useVespaVersionInRequest().isPresent()) { builder.useVespaVersionInRequest(options.useVespaVersionInRequest().get()); @@ -178,4 +180,8 @@ public class ConfigserverCluster extends AbstractConfigProducer builder.snapshot_interval(60.0); } + @Override + public void getConfig(VipStatusConfig.Builder builder) { + builder.initiallyInRotation(false); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index d3ba2718d71..dd48e65c340 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http; -import com.google.common.collect.ImmutableList; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; @@ -15,7 +15,6 @@ import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -24,23 +23,23 @@ import java.util.stream.Stream; * Helper class for http access control. * * @author gjoranv + * @author bjorncs */ public final class AccessControl { public static final ComponentId ACCESS_CONTROL_CHAIN_ID = ComponentId.fromString("access-control-chain"); - private static final List<String> UNPROTECTED_HANDLERS = ImmutableList.of( + public static final List<String> UNPROTECTED_HANDLERS = List.of( FileStatusHandlerComponent.CLASS, ContainerCluster.APPLICATION_STATUS_HANDLER_CLASS, ContainerCluster.BINDINGS_OVERVIEW_HANDLER_CLASS, ContainerCluster.STATE_HANDLER_CLASS, - ContainerCluster.LOG_HANDLER_CLASS + ContainerCluster.LOG_HANDLER_CLASS, + ApplicationContainerCluster.METRICS_V2_HANDLER_CLASS ); public static final class Builder { private String domain; - private String applicationId; - private Optional<String> vespaDomain = Optional.empty(); private boolean readEnabled = false; private boolean writeEnabled = true; private final Set<String> excludeBindings = new LinkedHashSet<>(); @@ -48,9 +47,8 @@ public final class AccessControl { private Collection<Servlet> servlets = Collections.emptyList(); private final DeployLogger logger; - public Builder(String domain, String applicationId, DeployLogger logger) { + public Builder(String domain, DeployLogger logger) { this.domain = domain; - this.applicationId = applicationId; this.logger = logger; } @@ -69,52 +67,37 @@ public final class AccessControl { return this; } - public Builder vespaDomain(String vespaDomain) { - this.vespaDomain = Optional.ofNullable(vespaDomain); - return this; - } - - public Builder setHandlers(Collection<Handler<?>> handlers) { - this.handlers = handlers; - return this; - } - - public Builder setServlets(Collection<Servlet> servlets) { - this.servlets = servlets; + public Builder setHandlers(ApplicationContainerCluster cluster) { + this.handlers = cluster.getHandlers(); + this.servlets = cluster.getAllServlets(); return this; } public AccessControl build() { - return new AccessControl(domain, applicationId, writeEnabled, readEnabled, - excludeBindings, vespaDomain, servlets, handlers, logger); + return new AccessControl(domain, writeEnabled, readEnabled, + excludeBindings, servlets, handlers, logger); } } public final String domain; - public final String applicationId; public final boolean readEnabled; public final boolean writeEnabled; - public final Optional<String> vespaDomain; private final Set<String> excludedBindings; private final Collection<Handler<?>> handlers; private final Collection<Servlet> servlets; private final DeployLogger logger; private AccessControl(String domain, - String applicationId, boolean writeEnabled, boolean readEnabled, Set<String> excludedBindings, - Optional<String> vespaDomain, Collection<Servlet> servlets, Collection<Handler<?>> handlers, DeployLogger logger) { this.domain = domain; - this.applicationId = applicationId; this.readEnabled = readEnabled; this.writeEnabled = writeEnabled; this.excludedBindings = Collections.unmodifiableSet(excludedBindings); - this.vespaDomain = vespaDomain; this.handlers = handlers; this.servlets = servlets; this.logger = logger; @@ -125,6 +108,10 @@ public final class AccessControl { .collect(Collectors.toCollection(ArrayList::new)); } + public static boolean hasHandlerThatNeedsProtection(ApplicationContainerCluster cluster) { + return cluster.getHandlers().stream().anyMatch(AccessControl::handlerNeedsProtection); + } + private Stream<Binding> getHandlerBindings() { return handlers.stream() .filter(this::shouldHandlerBeProtected) @@ -144,7 +131,7 @@ public final class AccessControl { && handler.getServerBindings().stream().noneMatch(excludedBindings::contains); } - public static boolean isBuiltinGetOnly(Handler<?> handler) { + private static boolean isBuiltinGetOnly(Handler<?> handler) { return UNPROTECTED_HANDLERS.contains(handler.getClassId().getName()); } @@ -159,4 +146,13 @@ public final class AccessControl { private static Stream<String> servletBindings(Servlet servlet) { return Stream.of("http://*/").map(protocol -> protocol + servlet.bindingPath); } + + private static boolean handlerNeedsProtection(Handler<?> handler) { + return ! isBuiltinGetOnly(handler) && hasNonMbusBinding(handler); + } + + private static boolean hasNonMbusBinding(Handler<?> handler) { + return handler.getServerBindings().stream().anyMatch(binding -> ! binding.startsWith("mbus")); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java index 400ddf80cf9..0fcf7b2d06c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -8,46 +8,39 @@ import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.component.chain.ChainedComponent; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; /** * Represents the http servers and filters of a container cluster. * * @author Tony Vaagenes + * @author bjorncs */ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> implements ServerConfig.Producer { - private FilterChains filterChains; - private JettyHttpServer httpServer; - private List<Binding> bindings; - private final Optional<AccessControl> accessControl; + private final FilterChains filterChains; + private final List<Binding> bindings = new CopyOnWriteArrayList<>(); + private volatile JettyHttpServer httpServer; + private volatile AccessControl accessControl; - public Http(List<Binding> bindings) { - this(bindings, null); + public Http(FilterChains chains) { + super("http"); + this.filterChains = chains; } - public Http(List<Binding> bindings, AccessControl accessControl) { - super( "http"); - this.bindings = Collections.unmodifiableList(bindings); - this.accessControl = Optional.ofNullable(accessControl); - } - - public void setFilterChains(FilterChains filterChains) { - this.filterChains = filterChains; - } - - public void setBindings(List<Binding> bindings) { - this.bindings = Collections.unmodifiableList(bindings); + public void setAccessControl(AccessControl accessControl) { + if (this.accessControl != null) throw new IllegalStateException("Access control already assigned"); + this.accessControl = accessControl; } public FilterChains getFilterChains() { return filterChains; } - public JettyHttpServer getHttpServer() { - return httpServer; + public Optional<JettyHttpServer> getHttpServer() { + return Optional.ofNullable(httpServer); } public void setHttpServer(JettyHttpServer newServer) { @@ -76,25 +69,21 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl } public Optional<AccessControl> getAccessControl() { - return accessControl; + return Optional.ofNullable(accessControl); } @Override public void getConfig(ServerConfig.Builder builder) { for (Binding binding : bindings) { builder.filter(new ServerConfig.Filter.Builder() - .id(binding.filterId().stringValue()) - .binding(binding.binding())); + .id(binding.filterId().stringValue()) + .binding(binding.binding())); } } @Override public void validate() { - validate(bindings); - } - - void validate(Collection<Binding> bindings) { - if (bindings.isEmpty()) return; + if (((Collection<Binding>) bindings).isEmpty()) return; if (filterChains == null) throw new IllegalArgumentException("Null FilterChains are not allowed when there are filter bindings"); @@ -107,5 +96,4 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl throw new RuntimeException("Can't find filter " + binding.filterId() + " for binding " + binding.binding()); } } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java index 4f84a01ff94..4a331718985 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java @@ -8,6 +8,7 @@ import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.SimpleComponent; +import java.util.List; import java.util.Optional; import static com.yahoo.component.ComponentSpecification.fromString; @@ -16,6 +17,7 @@ import static com.yahoo.component.ComponentSpecification.fromString; * Configure SSL using file references * * @author mortent + * @author bjorncs */ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements ConnectorConfig.Producer { public static final String COMPONENT_ID_PREFIX = "configured-ssl-provider@"; @@ -26,8 +28,16 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C private final String certificatePath; private final String caCertificatePath; private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication; + private final List<String> cipherSuites; + private final List<String> protocolVersions; - public ConfiguredFilebasedSslProvider(String servername, String privateKeyPath, String certificatePath, String caCertificatePath, String clientAuthentication) { + public ConfiguredFilebasedSslProvider(String servername, + String privateKeyPath, + String certificatePath, + String caCertificatePath, + String clientAuthentication, + List<String> cipherSuites, + List<String> protocolVersions) { super(new ComponentModel( new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername), fromString(COMPONENT_CLASS), @@ -36,15 +46,21 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C this.certificatePath = certificatePath; this.caCertificatePath = caCertificatePath; this.clientAuthentication = mapToConfigEnum(clientAuthentication); + this.cipherSuites = cipherSuites; + this.protocolVersions = protocolVersions; } @Override public void getConfig(ConnectorConfig.Builder builder) { - builder.ssl.enabled(true); - builder.ssl.privateKeyFile(privateKeyPath); - builder.ssl.certificateFile(certificatePath); - builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")); - builder.ssl.clientAuth(clientAuthentication); + builder.ssl( + new ConnectorConfig.Ssl.Builder() + .enabled(true) + .privateKeyFile(privateKeyPath) + .certificateFile(certificatePath) + .caCertificateFile(Optional.ofNullable(caCertificatePath).orElse("")) + .clientAuth(clientAuthentication) + .enabledCipherSuites(cipherSuites) + .enabledProtocols(protocolVersions)); } public SimpleComponent getComponent() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 12db3b87243..fb8e9dffbbb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -7,6 +7,7 @@ import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; +import java.time.Duration; import java.util.List; /** @@ -17,42 +18,49 @@ import java.util.List; public class HostedSslConnectorFactory extends ConnectorFactory { private static final List<String> INSECURE_WHITELISTED_PATHS = List.of("/status.html"); + private static final String DEFAULT_HOSTED_TRUSTSTORE = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"; private final boolean enforceClientAuth; + private final String proxyProtocol; /** - * Create connector factory that uses a certificate provided by the config-model / configserver. + * Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore. */ - public static HostedSslConnectorFactory withProvidedCertificate(String serverName, EndpointCertificateSecrets endpointCertificateSecrets) { - return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificates*/null), false); + // TODO Enforce client authentication + public static HostedSslConnectorFactory withProvidedCertificate(String proxyProtocol, String serverName, EndpointCertificateSecrets endpointCertificateSecrets) { + return new HostedSslConnectorFactory(proxyProtocol, + createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null), false); } /** * Create connector factory that uses a certificate provided by the config-model / configserver and a truststore configured by the application. */ - public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { - return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, tlsCaCertificates), true); + public static HostedSslConnectorFactory withProvidedCertificateAndTruststore( + String proxyProtocol, String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { + return new HostedSslConnectorFactory(proxyProtocol, + createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates), true); } /** * Create connector factory that uses the default certificate and truststore provided by Vespa (through Vespa-global TLS configuration). */ - public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName) { - return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true); + public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String proxyProtocol, String serverName) { + return new HostedSslConnectorFactory(proxyProtocol, new DefaultSslProvider(serverName), true); } - private HostedSslConnectorFactory(SimpleComponent sslProviderComponent, boolean enforceClientAuth) { + private HostedSslConnectorFactory(String proxyProtocol, SimpleComponent sslProviderComponent, boolean enforceClientAuth) { super("tls4443", 4443, sslProviderComponent); + this.proxyProtocol = proxyProtocol; this.enforceClientAuth = enforceClientAuth; } private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider( - String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { + String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificatesPath, String tlsCaCertificates) { return new ConfiguredDirectSslProvider( serverName, endpointCertificateSecrets.key(), endpointCertificateSecrets.certificate(), - /*caCertificatePath*/null, + tlsCaCertificatesPath, tlsCaCertificates, ClientAuth.Enum.WANT_AUTH); } @@ -60,9 +68,27 @@ public class HostedSslConnectorFactory extends ConnectorFactory { @Override public void getConfig(ConnectorConfig.Builder connectorBuilder) { super.getConfig(connectorBuilder); - connectorBuilder.tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder() - .pathWhitelist(INSECURE_WHITELISTED_PATHS) - .enable(enforceClientAuth)); + connectorBuilder + .tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder() + .pathWhitelist(INSECURE_WHITELISTED_PATHS) + .enable(enforceClientAuth)) + .proxyProtocol(configureProxyProtocol()) + .idleTimeout(Duration.ofMinutes(3).toSeconds()) + .maxConnectionLife(Duration.ofMinutes(10).toSeconds()); + } + + private ConnectorConfig.ProxyProtocol.Builder configureProxyProtocol() { + ConnectorConfig.ProxyProtocol.Builder proxyProtocolBuilder = new ConnectorConfig.ProxyProtocol.Builder(); + switch (proxyProtocol) { + case "https-only": + return proxyProtocolBuilder.enabled(false).mixedMode(false); + case "https+proxy-protocol": + return proxyProtocolBuilder.enabled(true).mixedMode(true); + case "proxy-protocol-only": + return proxyProtocolBuilder.enabled(true).mixedMode(false); + default: + throw new IllegalArgumentException("Unknown proxy-protocol settings: " + proxyProtocol); + } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index b37caf22216..bfde9b9add1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -6,19 +6,18 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; -import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.http.AccessControl; +import com.yahoo.vespa.model.container.http.Binding; import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; -import com.yahoo.vespa.model.container.http.Binding; import org.w3c.dom.Element; import java.util.ArrayList; @@ -55,24 +54,18 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> filterChains = new FilterChainsBuilder().newChainsInstance(ancestor); } - Http http = new Http(bindings, accessControl); - http.setFilterChains(filterChains); - - buildHttpServers(deployState, ancestor, http, spec); - + Http http = new Http(filterChains); + http.getBindings().addAll(bindings); + http.setAccessControl(accessControl); + http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec)); return http; } private AccessControl buildAccessControl(DeployState deployState, AbstractConfigProducer ancestor, Element accessControlElem) { - String application = XmlHelper.getOptionalChildValue(accessControlElem, "application") - .orElse(getDeployedApplicationId(deployState, ancestor).value()); - - AccessControl.Builder builder = new AccessControl.Builder(accessControlElem.getAttribute("domain"), application, deployState.getDeployLogger()); + AthenzDomain domain = getAccessControlDomain(deployState, accessControlElem); + AccessControl.Builder builder = new AccessControl.Builder(domain.value(), deployState.getDeployLogger()); - getContainerCluster(ancestor).ifPresent(cluster -> { - builder.setHandlers(cluster.getHandlers()); - builder.setServlets(cluster.getAllServlets()); - }); + getContainerCluster(ancestor).ifPresent(builder::setHandlers); XmlHelper.getOptionalAttribute(accessControlElem, "read").ifPresent( readAttr -> builder.readEnabled(Boolean.valueOf(readAttr))); @@ -85,17 +78,29 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> .map(XML::getValue) .forEach(builder::excludeBinding); } - XmlHelper.getOptionalChildValue(accessControlElem, "vespa-domain").ifPresent(builder::vespaDomain); return builder.build(); } - /** - * Returns the id of the deployed application, or the default value if not explicitly set (self-hosted). - */ - private static ApplicationName getDeployedApplicationId(DeployState deployState, AbstractConfigProducer ancestor) { - return getContainerCluster(ancestor) - .map(cluster -> deployState.getProperties().applicationId().application()) - .orElse(ApplicationId.defaultId().application()); + // TODO Fail if domain is not provided through deploy properties + private static AthenzDomain getAccessControlDomain(DeployState deployState, Element accessControlElem) { + AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null); + AthenzDomain explicitDomain = XmlHelper.getOptionalAttribute(accessControlElem, "domain") + .map(AthenzDomain::from) + .orElse(null); + if (tenantDomain == null) { + if (explicitDomain == null) { + throw new IllegalStateException("No Athenz domain provided for 'access-control'"); + } + deployState.getDeployLogger().log(Level.WARNING, "Athenz tenant is not provided by deploy call. This will soon be handled as failure."); + } + if (explicitDomain != null) { + if (tenantDomain != null && !explicitDomain.equals(tenantDomain)) { + throw new IllegalArgumentException( + String.format("Domain in access-control ('%s') does not match tenant domain ('%s')", explicitDomain.value(), tenantDomain.value())); + } + deployState.getDeployLogger().log(Level.WARNING, "Domain in 'access-control' is deprecated and will be removed soon"); + } + return tenantDomain != null ? tenantDomain : explicitDomain; } private static Optional<ApplicationContainerCluster> getContainerCluster(AbstractConfigProducer configProducer) { @@ -125,10 +130,6 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> return result; } - private void buildHttpServers(DeployState deployState, AbstractConfigProducer ancestor, Http http, Element spec) { - http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec)); - } - static int readPort(ModelElement spec, boolean isHosted, DeployLogger logger) { Integer port = spec.integerAttribute("port"); if (port == null) @@ -139,13 +140,9 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> int legalPortInHostedVespa = Container.BASEPORT; if (isHosted && port != legalPortInHostedVespa && ! spec.booleanAttribute("required", false)) { - // TODO: After January 2020: - // - Set required='true' for the http server on port 4443 in the tester services.xml in InternalStepRunner - // - Enable 2 currently ignored tests in this module - // - throw IllegalArgumentException here instead of warning - logger.log(Level.WARNING, "Illegal port " + port + " in http server '" + - spec.stringAttribute("id") + "'" + - ": Port must be set to " + legalPortInHostedVespa); + throw new IllegalArgumentException("Illegal port " + port + " in http server '" + + spec.stringAttribute("id") + "'" + + ": Port must be set to " + legalPortInHostedVespa); } return port; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java index db831a1ec2f..562026ab4dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -9,13 +9,17 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; -import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider; import com.yahoo.vespa.model.container.http.ssl.ConfiguredFilebasedSslProvider; +import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider; import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; import org.w3c.dom.Element; +import java.util.Arrays; +import java.util.List; import java.util.Optional; +import static java.util.stream.Collectors.toList; + /** * @author Einar M R Rosenvinge * @author mortent @@ -40,12 +44,16 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil String certificateFile = XML.getValue(XML.getChild(sslConfigurator, "certificate-file")); Optional<String> caCertificateFile = XmlHelper.getOptionalChildValue(sslConfigurator, "ca-certificates-file"); Optional<String> clientAuthentication = XmlHelper.getOptionalChildValue(sslConfigurator, "client-authentication"); + List<String> cipherSuites = extractOptionalCommaSeparatedList(sslConfigurator, "cipher-suites"); + List<String> protocols = extractOptionalCommaSeparatedList(sslConfigurator, "protocols"); return new ConfiguredFilebasedSslProvider( serverName, privateKeyFile, certificateFile, caCertificateFile.orElse(null), - clientAuthentication.orElse(null)); + clientAuthentication.orElse(null), + cipherSuites, + protocols); } else if (sslProviderConfigurator != null) { String className = sslProviderConfigurator.getAttribute("class"); String bundle = sslProviderConfigurator.getAttribute("bundle"); @@ -55,4 +63,13 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil } } + private static List<String> extractOptionalCommaSeparatedList(Element sslElement, String listElementName) { + return XmlHelper.getOptionalChildValue(sslElement, listElementName) + .map(element -> + Arrays.stream(element.split(",")) + .filter(listEntry -> !listEntry.isBlank()) + .map(String::trim) + .collect(toList())) + .orElse(List.of()); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index e19d81e7fb2..4c4b1ca7f82 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -58,7 +58,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> private void initializeDispatchers(Collection<AbstractSearchCluster> searchClusters) { for (AbstractSearchCluster searchCluster : searchClusters) { if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue; - owningCluster.addComponent(new DispatcherComponent((IndexedSearchCluster)searchCluster)); + var dispatcher = new DispatcherComponent((IndexedSearchCluster)searchCluster); + owningCluster.addComponent(dispatcher); } } @@ -130,7 +131,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> AbstractSearchCluster sys = findClusterWithId(searchClusters, i); QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder(). name(sys.getClusterName()); - for (AbstractSearchCluster.SearchDefinitionSpec spec : sys.getLocalSDS()) { + for (AbstractSearchCluster.SchemaSpec spec : sys.getLocalSDS()) { scB.searchdef(spec.getSearchDefinition().getName()); } scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java index 704188e80e8..284aa3b46c0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; +import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.model.container.component.Component; @@ -13,7 +14,7 @@ import com.yahoo.vespa.model.search.IndexedSearchCluster; * * @author bratseth */ -public class DispatcherComponent extends Component<DispatcherComponent, ComponentModel> +public class DispatcherComponent extends Component<AbstractConfigProducer<?>, ComponentModel> implements DispatchConfig.Producer { private final IndexedSearchCluster indexedSearchCluster; @@ -21,14 +22,17 @@ public class DispatcherComponent extends Component<DispatcherComponent, Componen public DispatcherComponent(IndexedSearchCluster indexedSearchCluster) { super(toComponentModel(indexedSearchCluster)); this.indexedSearchCluster = indexedSearchCluster; + String clusterName = indexedSearchCluster.getClusterName(); + var rpcResoucePool = new RpcResourcePoolComponent(clusterName); + inject(rpcResoucePool); + addComponent(rpcResoucePool); } private static ComponentModel toComponentModel(IndexedSearchCluster indexedSearchCluster) { String dispatcherComponentId = "dispatcher." + indexedSearchCluster.getClusterName(); // used by ClusterSearcher return new ComponentModel(dispatcherComponentId, - "com.yahoo.search.dispatch.Dispatcher", - BundleMapper.searchAndDocprocBundle, - null); + com.yahoo.search.dispatch.Dispatcher.class.getName(), + BundleMapper.searchAndDocprocBundle); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java index 0abb0803405..0a9618e7b08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java @@ -232,8 +232,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer return propB; } - private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig( - String fullName, Object value) { + private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig(String fullName, Object value) { QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder(); if (value instanceof SubstituteString) value=value.toString(); // Send only types understood by configBuilder downwards @@ -251,7 +250,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer qtB.matchaspath(true); for (QueryProfileType inherited : profileType.inherited()) qtB.inherit(inherited.getId().stringValue()); - List<FieldDescription> fields=new ArrayList<>(profileType.declaredFields().values()); + List<FieldDescription> fields = new ArrayList<>(profileType.declaredFields().values()); Collections.sort(fields); for (FieldDescription field : fields) qtB.field(createConfig(field)); @@ -260,22 +259,20 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer private QueryProfilesConfig.Queryprofiletype.Field.Builder createConfig(FieldDescription field) { QueryProfilesConfig.Queryprofiletype.Field.Builder fB = new QueryProfilesConfig.Queryprofiletype.Field.Builder(); - fB. - name(field.getName()). - type(field.getType().stringValue()); + fB.name(field.getName()).type(field.getType().stringValue()); if ( ! field.isOverridable()) fB.overridable(false); if (field.isMandatory()) fB.mandatory(true); - String aliases=toSpaceSeparatedString(field.getAliases()); - if (!aliases.isEmpty()) + String aliases = toSpaceSeparatedString(field.getAliases()); + if ( ! aliases.isEmpty()) fB.alias(aliases); return fB; } public String toSpaceSeparatedString(List<String> list) { - StringBuilder b=new StringBuilder(); - for (Iterator<String> i=list.iterator(); i.hasNext(); ) { + StringBuilder b = new StringBuilder(); + for (Iterator<String> i = list.iterator(); i.hasNext(); ) { b.append(i.next()); if (i.hasNext()) b.append(" "); @@ -290,10 +287,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer } } - /** - * The config produced by this - * @return query profiles config - */ + /** Returns the config produced by this */ public QueryProfilesConfig getConfig() { QueryProfilesConfig.Builder qB = new QueryProfilesConfig.Builder(); getConfig(qB); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java new file mode 100644 index 00000000000..2689c2ce71b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java @@ -0,0 +1,18 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.xml.BundleMapper; + +public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent, ComponentModel> { + + public RpcResourcePoolComponent(String clusterName) { + super(toComponentModel(clusterName)); + } + + private static ComponentModel toComponentModel(String clusterName) { + String componentId = "rpcresourcepool." + clusterName; + return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), BundleMapper.searchAndDocprocBundle); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java index e05b2d27e09..4ecc666a9f2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java @@ -118,7 +118,7 @@ public class LocalProvider extends Provider implements public List<String> getDocumentTypes() { List<String> documentTypes = new ArrayList<>(); - for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) { + for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) { documentTypes.add(spec.getSearchDefinition().getSearch().getDocument().getName()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index 2828fcf09d0..19d1b6546a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -9,6 +9,8 @@ import org.w3c.dom.Element; import java.util.Arrays; import java.util.List; +import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS; + /** * This object builds a bundle instantiation spec from an XML element. * @@ -36,7 +38,7 @@ public class BundleInstantiationSpecificationBuilder { private static void validate(BundleInstantiationSpecification instSpec) { List<String> forbiddenClasses = Arrays.asList( - "com.yahoo.search.handler.SearchHandler", + SEARCH_HANDLER_CLASS, "com.yahoo.processing.handler.ProcessingHandler"); for (String forbiddenClass: forbiddenClasses) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index aef2697a5dd..4bd9f5fa8b0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -19,12 +19,15 @@ import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; @@ -57,9 +60,11 @@ import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChains; +import com.yahoo.vespa.model.container.http.AccessControl; import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; @@ -88,6 +93,7 @@ import java.util.function.Consumer; import java.util.regex.Pattern; import java.util.stream.Collectors; +import static com.yahoo.vespa.model.container.http.AccessControl.ACCESS_CONTROL_CHAIN_ID; import static java.util.logging.Level.WARNING; /** @@ -106,6 +112,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private static final String DEPRECATED_CONTAINER_TAG = "jdisc"; private static final String ENVIRONMENT_VARIABLES_ELEMENT = "environment-variables"; + static final String SEARCH_HANDLER_CLASS = com.yahoo.search.handler.SearchHandler.class.getName(); + static final String SEARCH_HANDLER_BINDING = "http://*/search/*"; + public enum Networking { disable, enable } private ApplicationPackage app; @@ -175,7 +184,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { DocumentFactoryBuilder.buildDocumentFactories(cluster, spec); addConfiguredComponents(deployState, cluster, spec); addSecretStore(cluster, spec); - addHandlers(deployState, cluster, spec); addRestApis(deployState, spec, cluster); addServlets(deployState, spec, cluster); @@ -188,8 +196,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addDefaultHandlersExceptStatus(); addStatusHandlers(cluster, context.getDeployState().isHosted()); + addUserHandlers(deployState, cluster, spec); - addHttp(deployState, spec, cluster, context.getApplicationType(), deployState.getProperties().applicationId().instance().isTester()); + addHttp(deployState, spec, cluster, context); addAccessLogs(deployState, cluster, spec); addRoutingAliases(cluster, spec, deployState.zone().environment()); @@ -287,7 +296,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addClientProviders(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { for (Element clientSpec: XML.getChildren(spec, "client")) { - cluster.addComponent(new DomClientProviderBuilder().build(deployState, cluster, clientSpec)); + cluster.addComponent(new DomClientProviderBuilder(cluster).build(deployState, cluster, clientSpec)); } } @@ -302,7 +311,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(cluster::addComponent); } - if (accessLogElements.isEmpty() && cluster.getSearch() != null) + if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault()) cluster.addDefaultSearchAccessLog(); } @@ -311,21 +320,23 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } - private void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ApplicationType applicationType, boolean isTesterApplication) { + private void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { Element httpElement = XML.getChild(spec, "http"); if (httpElement != null) { cluster.setHttp(buildHttp(deployState, cluster, httpElement)); } - if (deployState.isHosted() && applicationType == ApplicationType.DEFAULT && !isTesterApplication) { + if (isHostedTenantApplication(context)) { + addHostedImplicitHttpIfNotPresent(cluster); + addHostedImplicitAccessControlIfNotPresent(deployState, cluster); addAdditionalHostedConnector(deployState, cluster); } } private void addAdditionalHostedConnector(DeployState deployState, ApplicationContainerCluster cluster) { - addImplicitHttpIfNotPresent(cluster); - JettyHttpServer server = cluster.getHttp().getHttpServer(); + JettyHttpServer server = cluster.getHttp().getHttpServer().get(); String serverName = server.getComponentId().getName(); + String proxyProtocol = deployState.getProperties().proxyProtocol(); // If the deployment contains certificate/private key reference, setup TLS port if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); @@ -334,27 +345,47 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); HostedSslConnectorFactory connectorFactory = authorizeClient - ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) - : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets); + ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(proxyProtocol, serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) + : HostedSslConnectorFactory.withProvidedCertificate(proxyProtocol, serverName, endpointCertificateSecrets); server.addConnector(connectorFactory); } else { - server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName)); + server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(proxyProtocol, serverName)); } } - private static void addImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) { + private static boolean isHostedTenantApplication(ConfigModelContext context) { + var deployState = context.getDeployState(); + boolean isTesterApplication = deployState.getProperties().applicationId().instance().isTester(); + return deployState.isHosted() && context.getApplicationType() == ApplicationType.DEFAULT && !isTesterApplication; + } + + private static void addHostedImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) { if(cluster.getHttp() == null) { - Http http = new Http(Collections.emptyList()); - http.setFilterChains(new FilterChains(cluster)); - cluster.setHttp(http); + cluster.setHttp(new Http(new FilterChains(cluster))); } - if(cluster.getHttp().getHttpServer() == null) { + if(cluster.getHttp().getHttpServer().isEmpty()) { JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer")); cluster.getHttp().setHttpServer(defaultHttpServer); defaultHttpServer.addConnector(new ConnectorFactory("SearchServer", Defaults.getDefaults().vespaWebServicePort())); } } + private void addHostedImplicitAccessControlIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) { + Http http = cluster.getHttp(); + if (http.getAccessControl().isPresent()) return; // access control added explicitly + AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null); + if (tenantDomain == null) return; // tenant domain not present, cannot add access control. this should eventually be a failure. + AccessControl accessControl = + new AccessControl.Builder(tenantDomain.value(), deployState.getDeployLogger()) + .setHandlers(cluster) + .readEnabled(false) + .writeEnabled(false) + .build(); + http.getFilterChains().add(new Chain<>(FilterChains.emptyChainSpec(ACCESS_CONTROL_CHAIN_ID))); + http.setAccessControl(accessControl); + http.getBindings().addAll(accessControl.getBindings()); + } + private Http buildHttp(DeployState deployState, ApplicationContainerCluster cluster, Element httpElement) { Http http = new HttpBuilder().build(deployState, cluster, httpElement); @@ -441,10 +472,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { containerSearch.setPageTemplates(PageTemplates.create(applicationPackage)); } - private void addHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { + private void addUserHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { for (Element component: XML.getChildren(spec, "handler")) { cluster.addComponent( - new DomHandlerBuilder().build(deployState, cluster, component)); + new DomHandlerBuilder(cluster).build(deployState, cluster, component)); } } @@ -532,14 +563,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState().zone(), jvmGCOptions, context.getDeployState().isHosted())); } + /** + * Add nodes to cluster according to the given containerElement. + * + * Note: DO NOT change allocation behaviour to allow version X and Y of the config-model to allocate a different set + * of nodes. Such changes must be guarded by a common condition (e.g. feature flag) so the behaviour can be changed + * simultaneously for all active config models. + */ private void addNodesFromXml(ApplicationContainerCluster cluster, Element containerElement, ConfigModelContext context) { Element nodesElement = XML.getChild(containerElement, "nodes"); - if (nodesElement == null) { // default single node on localhost - ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa()); - HostResource host = allocateSingleNodeHost(cluster, log, containerElement, context); - node.setHostResource(host); - node.initService(context.getDeployLogger()); - cluster.addContainers(Collections.singleton(node)); + if (nodesElement == null) { + cluster.addContainers(allocateWithoutNodesTag(cluster, context)); } else { List<ApplicationContainer> nodes = createNodes(cluster, nodesElement, context); @@ -615,30 +649,33 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { " must be an integer percentage ending by the '%' sign"); } } - - /** Creates a single host when there is no nodes tag */ - private HostResource allocateSingleNodeHost(ApplicationContainerCluster cluster, DeployLogger logger, Element containerElement, ConfigModelContext context) { + + /** Allocate a container cluster without a nodes tag */ + private List<ApplicationContainer> allocateWithoutNodesTag(ApplicationContainerCluster cluster, ConfigModelContext context) { DeployState deployState = context.getDeployState(); HostSystem hostSystem = cluster.hostSystem(); if (deployState.isHosted()) { - Optional<HostResource> singleContentHost = getHostResourceFromContentClusters(cluster, containerElement, context); - if (singleContentHost.isPresent()) { // there is a content cluster; put the container on its first node - return singleContentHost.get(); - } - else { // request 1 node - ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from(cluster.getName()), - deployState.getWantedNodeVespaVersion(), - false); - Capacity capacity = Capacity.fromCount(1, - Optional.empty(), - false, - ! deployState.getProperties().isBootstrap()); - return hostSystem.allocateHosts(clusterSpec, capacity, 1, logger).keySet().iterator().next(); - } - } else { - return hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); + // request just enough nodes to satisfy environment capacity requirement + ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, + ClusterSpec.Id.from(cluster.getName())) + .vespaVersion(deployState.getWantedNodeVespaVersion()) + .dockerImageRepo(deployState.getWantedDockerImageRepo()) + .build(); + int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; + Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified), + false, + !deployState.getProperties().isBootstrap()); + var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); + return createNodesFromHosts(log, hosts, cluster); } + return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); + } + + private List<ApplicationContainer> singleHostContainerCluster(ApplicationContainerCluster cluster, HostResource host, ConfigModelContext context) { + ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa()); + node.setHostResource(host); + node.initService(context.getDeployLogger()); + return List.of(node); } private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { @@ -652,13 +689,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private List<ApplicationContainer> createNodesFromNodeType(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { NodeType type = NodeType.valueOf(nodesElement.getAttribute("type")); - ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from(cluster.getName()), - context.getDeployState().getWantedNodeVespaVersion(), - false); + ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName())) + .vespaVersion(context.getDeployState().getWantedNodeVespaVersion()) + .dockerImageRepo(context.getDeployState().getWantedDockerImageRepo()) + .build(); Map<HostResource, ClusterMembership> hosts = cluster.getRoot().hostSystem().allocateHosts(clusterSpec, - Capacity.fromRequiredNodeType(type), 1, log); + Capacity.fromRequiredNodeType(type), log); return createNodesFromHosts(context.getDeployLogger(), hosts, cluster); } @@ -766,7 +803,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler"); - String[] defaultBindings = {"http://*/search/*"}; + String[] defaultBindings = {SEARCH_HANDLER_BINDING}; for (String binding: serverBindings(searchElement, defaultBindings)) { searchHandler.addServerBindings(binding); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 75161859068..fcaba66ef69 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -16,8 +16,8 @@ import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SearchCluster; -import com.yahoo.vespa.model.search.SearchDefinition; -import com.yahoo.vespa.model.search.SearchDefinitionXMLHandler; +import com.yahoo.vespa.model.search.NamedSchema; +import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchNode; import com.yahoo.vespa.model.search.StreamingSearchCluster; import com.yahoo.vespa.model.search.TransactionLogServer; @@ -52,6 +52,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot private final String clusterName; private final Map<String, NewDocumentType> documentDefinitions; private final Set<NewDocumentType> globallyDistributedDocuments; + private Double visibilityDelay = 0.0; /** The search nodes of this if it does not have an indexed cluster */ private List<SearchNode> nonIndexed = new ArrayList<>(); @@ -135,21 +136,21 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) { - List<ModelElement> indexedDefs = getIndexedSearchDefinitions(clusterElem); + List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); if (!indexedDefs.isEmpty()) { IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, deployState); isc.setRoutingSelector(clusterElem.childAsString("documents.selection")); Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); if (visibilityDelay != null) { - isc.setVisibilityDelay(visibilityDelay); + search.setVisibilityDelay(visibilityDelay); } search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs); } } - private List<ModelElement> getIndexedSearchDefinitions(ModelElement clusterElem) { + private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) { List<ModelElement> indexedDefs = new ArrayList<>(); ModelElement docElem = clusterElem.child("documents"); if (docElem == null) { @@ -179,29 +180,36 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot this.flushOnShutdown = flushOnShutdown; } + public void setVisibilityDelay(double delay) { + this.visibilityDelay=delay; + if (hasIndexedCluster()) { + indexedCluster.setVisibilityDelay(delay); + } + } + private void addSearchCluster(DeployState deployState, SearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) { - addSearchDefinitions(deployState, documentDefs, cluster); + addSchemas(deployState, documentDefs, cluster); if (queryTimeout != null) { cluster.setQueryTimeout(queryTimeout); } cluster.defaultDocumentsConfig(); - cluster.deriveSearchDefinitions(deployState); + cluster.deriveSchemas(deployState); addCluster(cluster); } - private void addSearchDefinitions(DeployState deployState, List<ModelElement> searchDefs, AbstractSearchCluster sc) { + private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, AbstractSearchCluster sc) { for (ModelElement e : searchDefs) { - SearchDefinitionXMLHandler searchDefinitionXMLHandler = new SearchDefinitionXMLHandler(e); - SearchDefinition searchDefinition = - searchDefinitionXMLHandler.getResponsibleSearchDefinition(deployState.getSearchDefinitions()); + SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e); + NamedSchema searchDefinition = + schemaDefinitionXMLHandler.getResponsibleSearchDefinition(deployState.getSchemas()); if (searchDefinition == null) throw new RuntimeException("Search definition parsing error or file does not exist: '" + - searchDefinitionXMLHandler.getName() + "'"); + schemaDefinitionXMLHandler.getName() + "'"); // TODO: remove explicit building of user configs when the complete content model is built using builders. - sc.getLocalSDS().add(new AbstractSearchCluster.SearchDefinitionSpec(searchDefinition, - UserConfigBuilder.build(e.getXml(), deployState, deployState.getDeployLogger()))); + sc.getLocalSDS().add(new AbstractSearchCluster.SchemaSpec(searchDefinition, + UserConfigBuilder.build(e.getXml(), deployState, deployState.getDeployLogger()))); //need to get the document names from this sdfile sc.addDocumentNames(searchDefinition); } @@ -307,7 +315,6 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot @Override public void getConfig(ProtonConfig.Builder builder) { - double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0; builder.feeding.concurrency(0.40); // As if specified 0.8 in services.xml boolean hasAnyNonIndexedCluster = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java index 0d15207b6ce..0f9eb5341ab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java @@ -11,18 +11,25 @@ public class DispatchTuning { public static final DispatchTuning empty = new DispatchTuning.Builder().build(); - public enum DispatchPolicy { ROUNDROBIN, ADAPTIVE}; + public enum DispatchPolicy { ROUNDROBIN, ADAPTIVE} private final Integer maxHitsPerPartition; private DispatchPolicy dispatchPolicy; private final Double minGroupCoverage; private final Double minActiveDocsCoverage; + public Double getTopkProbability() { + return topkProbability; + } + + private final Double topkProbability; + private DispatchTuning(Builder builder) { maxHitsPerPartition = builder.maxHitsPerPartition; dispatchPolicy = builder.dispatchPolicy; minGroupCoverage = builder.minGroupCoverage; minActiveDocsCoverage = builder.minActiveDocsCoverage; + topkProbability = builder.topKProbability; } /** Returns the max number of hits to fetch from each partition, or null to fetch all */ @@ -46,6 +53,7 @@ public class DispatchTuning { private DispatchPolicy dispatchPolicy; private Double minGroupCoverage; private Double minActiveDocsCoverage; + private Double topKProbability; public DispatchTuning build() { return new DispatchTuning(this); @@ -55,6 +63,10 @@ public class DispatchTuning { this.maxHitsPerPartition = maxHitsPerPartition; return this; } + public Builder setTopKProbability(Double topKProbability) { + this.topKProbability = topKProbability; + return this; + } public Builder setDispatchPolicy(String policy) { if (policy != null) dispatchPolicy = toDispatchPolicy(policy); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index adfc703f747..f0c374b398f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -376,6 +376,10 @@ public class StorageGroup { * specified using a group count attribute. * <li>Neither element is present: Create a single node. * </ul> + * + * Note: DO NOT change allocation behaviour to allow version X and Y of the config-model to allocate a different + * set of nodes. Such changes must be guarded by a common condition (e.g. feature flag) so the behaviour can be + * changed simultaneously for all active config models. */ private GroupBuilder collectGroup(boolean isHosted, Optional<ModelElement> groupElement, Optional<ModelElement> nodesElement, String name, String index) { StorageGroup group = new StorageGroup( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 9b17412b83a..6dd3e619ec2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -136,10 +136,7 @@ public class ContentCluster extends AbstractConfigProducer implements c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, redundancyBuilder, c); validateThatGroupSiblingsAreUnique(c.clusterName, c.rootGroup); c.search.handleRedundancy(c.redundancy); - - IndexedSearchCluster index = c.search.getIndexed(); - if (index != null) - setupIndexedCluster(index, contentElement, deployState.getDeployLogger()); + setupSearchCluster(c.search, contentElement, deployState.getDeployLogger()); if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) ) throw new RuntimeException("Indexed search requires proton as engine"); @@ -166,17 +163,25 @@ public class ContentCluster extends AbstractConfigProducer implements return c; } - private void setupIndexedCluster(IndexedSearchCluster index, ModelElement element, DeployLogger logger) { + private void setupSearchCluster(ContentSearchCluster csc, ModelElement element, DeployLogger logger) { ContentSearch search = DomContentSearchBuilder.build(element); + Double visibilityDelay = search.getVisibilityDelay(); + if (visibilityDelay != null) { + csc.setVisibilityDelay(visibilityDelay); + } + if (csc.hasIndexedCluster()) { + setupIndexedCluster(csc.getIndexed(), search, element, logger); + } + + + } + private void setupIndexedCluster(IndexedSearchCluster index, ContentSearch search, ModelElement element, DeployLogger logger) { Double queryTimeout = search.getQueryTimeout(); if (queryTimeout != null) { Preconditions.checkState(index.getQueryTimeout() == null, - "In " + index + ": You may not specify query-timeout in both proton and content."); + "In " + index + ": You may not specify query-timeout in both proton and content."); index.setQueryTimeout(queryTimeout); } - Double visibilityDelay = search.getVisibilityDelay(); - if (visibilityDelay != null) - index.setVisibilityDelay(visibilityDelay); index.setSearchCoverage(DomSearchCoverageBuilder.build(element)); index.setDispatchSpec(DomDispatchBuilder.build(element)); @@ -281,7 +286,7 @@ public class ContentCluster extends AbstractConfigProducer implements .orElse(NodesSpecification.nonDedicated(3, context)); Collection<HostResource> hosts = nodesSpecification.isDedicated() ? getControllerHosts(nodesSpecification, admin, clusterName, context) : - drawControllerHosts(nodesSpecification.count(), rootGroup, containers); + drawControllerHosts(nodesSpecification.minResources().nodes(), rootGroup, containers); clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone"), hosts, clusterName, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java index b53d66632a8..d599a1a1aca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java @@ -23,6 +23,7 @@ public class DomTuningDispatchBuilder { return builder.build(); } builder.setMaxHitsPerPartition(dispatchElement.childAsInteger("max-hits-per-partition")); + builder.setTopKProbability(dispatchElement.childAsDouble("top-k-probability")); builder.setDispatchPolicy(dispatchElement.childAsString("dispatch-policy")); builder.setMinGroupCoverage(dispatchElement.childAsDouble("min-group-coverage")); builder.setMinActiveDocsCoverage(dispatchElement.childAsDouble("min-active-docs-coverage")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java index 9662540e8df..2c6808b5773 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java @@ -20,8 +20,9 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer { private final Map<Host, FileDistributionConfigProvider> fileDistributionConfigProviders = new IdentityHashMap<>(); private final FileDistributor fileDistributor; - public FileDistributionConfigProducer(AbstractConfigProducer ancestor, FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpec) { - this(ancestor, new FileDistributor(fileRegistry, configServerSpec)); + public FileDistributionConfigProducer(AbstractConfigProducer ancestor, FileRegistry fileRegistry, + List<ConfigServerSpec> configServerSpec, boolean isHosted) { + this(ancestor, new FileDistributor(fileRegistry, configServerSpec, isHosted)); } private FileDistributionConfigProducer(AbstractConfigProducer parent, FileDistributor fileDistributor) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index 576b009c846..5ccf86f9ba8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -11,7 +11,10 @@ import com.yahoo.vespa.model.Host; import java.util.*; /** - * Responsible for directing distribution of files to hosts. + * Sends RPC requests to hosts (tenant hosts and config servers) to start download of files. This is used during prepare + * of an application. Services themselves will also request files, the work done in this class is done so that hosts can + * start downloading files before services gets new config that needs these files. This also tries to make sure that + * all config servers (not just the one where the application was deployed) have the files available. * * @author Tony Vaagenes */ @@ -19,61 +22,42 @@ public class FileDistributor { private final FileRegistry fileRegistry; private final List<ConfigServerSpec> configServerSpecs; + private final boolean isHosted; - /** A map from files to the hosts to which that file should be distributed */ + /** A map from file reference to the hosts to which that file reference should be distributed */ private final Map<FileReference, Set<Host>> filesToHosts = new LinkedHashMap<>(); + public FileDistributor(FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpecs, boolean isHosted) { + this.fileRegistry = fileRegistry; + this.configServerSpecs = configServerSpecs; + this.isHosted = isHosted; + } + /** * Adds the given file to the associated application packages' registry of file and marks the file - * for distribution to the given hosts. + * for distribution to the given host. * <b>Note: This class receives ownership of the given collection.</b> * * @return the reference to the file, created by the application package */ - public FileReference sendFileToHosts(String relativePath, Collection<Host> hosts) { - FileReference reference = fileRegistry.addFile(relativePath); - addToFilesToDistribute(reference, hosts); - - return reference; + public FileReference sendFileToHost(String relativePath, Host host) { + return addFileReference(fileRegistry.addFile(relativePath), host); } /** * Adds the given file to the associated application packages' registry of file and marks the file - * for distribution to the given hosts. + * for distribution to the given host. * <b>Note: This class receives ownership of the given collection.</b> * * @return the reference to the file, created by the application package */ - public FileReference sendUriToHosts(String uri, Collection<Host> hosts) { - FileReference reference = fileRegistry.addUri(uri); - if (reference != null) { - addToFilesToDistribute(reference, hosts); - } - - return reference; - } - - /** Same as sendFileToHost(relativePath,Collections.singletonList(host) */ - public FileReference sendFileToHost(String relativePath, Host host) { - return sendFileToHosts(relativePath, Arrays.asList(host)); - } - public FileReference sendUriToHost(String uri, Host host) { - return sendUriToHosts(uri, Arrays.asList(host)); - } - - private void addToFilesToDistribute(FileReference reference, Collection<Host> hosts) { - Set<Host> oldHosts = getHosts(reference); - oldHosts.addAll(hosts); - } - - private Set<Host> getHosts(FileReference reference) { - return filesToHosts.computeIfAbsent(reference, k -> new HashSet<>()); + return addFileReference(fileRegistry.addUri(uri), host); } - public FileDistributor(FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpecs) { - this.fileRegistry = fileRegistry; - this.configServerSpecs = configServerSpecs; + private FileReference addFileReference(FileReference reference, Host host) { + filesToHosts.computeIfAbsent(reference, k -> new HashSet<>()).add(host); + return reference; } /** Returns the files which has been marked for distribution to the given host */ @@ -107,16 +91,20 @@ public class FileDistributor { // should only be called during deploy public void sendDeployedFiles(FileDistribution dbHandler) { String fileSourceHost = fileSourceHost(); - for (Host host : getTargetHosts()) { - if ( ! host.getHostname().equals(fileSourceHost)) { - dbHandler.startDownload(host.getHostname(), ConfigProxy.BASEPORT, filesToSendToHost(host)); - } - } + // Ask other config servers to download, for redundancy - if (configServerSpecs != null) - configServerSpecs.stream() - .filter(configServerSpec -> !configServerSpec.getHostName().equals(fileSourceHost)) - .forEach(spec -> dbHandler.startDownload(spec.getHostName(), spec.getConfigServerPort(), allFilesToSend())); + configServerSpecs.stream() + .filter(spec -> !spec.getHostName().equals(fileSourceHost)) + .forEach(spec -> dbHandler.startDownload(spec.getHostName(), spec.getConfigServerPort(), allFilesToSend())); + + // Skip starting download for application hosts when on hosted, since this is just a hint and requests for files + // will fail until the application is activated (this call is done when preparing an application deployment) + // due to authorization of RPC requests on config servers only considering files belonging to active applications + if (isHosted) return; + + getTargetHosts().stream() + .filter(host -> ! host.getHostname().equals(fileSourceHost)) + .forEach(host -> dbHandler.startDownload(host.getHostname(), ConfigProxy.BASEPORT, filesToSendToHost(host))); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index 8a88e720bed..fe6c6c52e2d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -29,7 +29,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer protected int index; private Double visibilityDelay = 0.0; private List<String> documentNames = new ArrayList<>(); - private List<SearchDefinitionSpec> localSDS = new LinkedList<>(); + private List<SchemaSpec> localSDS = new LinkedList<>(); public AbstractSearchCluster(AbstractConfigProducer parent, String clusterName, int index) { super(parent, "cluster." + clusterName); @@ -38,11 +38,11 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer } public void prepareToDistributeFiles(List<SearchNode> backends) { - for (SearchDefinitionSpec sds : localSDS) + for (SchemaSpec sds : localSDS) sds.getSearchDefinition().getSearch().rankingConstants().sendTo(backends); } - public void addDocumentNames(SearchDefinition searchDefinition) { + public void addDocumentNames(NamedSchema searchDefinition) { String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName(); documentNames.add(dName); } @@ -50,7 +50,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer /** Returns a List with document names used in this search cluster */ public List<String> getDocumentNames() { return documentNames; } - public List<SearchDefinitionSpec> getLocalSDS() { + public List<SchemaSpec> getLocalSDS() { return localSDS; } @@ -107,18 +107,17 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer } } - public static final class SearchDefinitionSpec { + public static final class SchemaSpec { - private final SearchDefinition searchDefinition; + private final NamedSchema searchDefinition; private final UserConfigRepo userConfigRepo; - public SearchDefinitionSpec(SearchDefinition searchDefinition, - UserConfigRepo userConfigRepo) { + public SchemaSpec(NamedSchema searchDefinition, UserConfigRepo userConfigRepo) { this.searchDefinition = searchDefinition; this.userConfigRepo = userConfigRepo; } - public SearchDefinition getSearchDefinition() { + public NamedSchema getSearchDefinition() { return searchDefinition; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 1b4c03a2182..56adc227df4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -53,6 +53,7 @@ public class IndexedSearchCluster extends SearchCluster private final DispatchGroup rootDispatch; private DispatchSpec dispatchSpec; private final boolean useAdaptiveDispatch; + private final double defaultTopKProbability; private List<SearchNode> searchNodes = new ArrayList<>(); /** @@ -70,6 +71,7 @@ public class IndexedSearchCluster extends SearchCluster unionCfg = new UnionConfiguration(this, documentDbs); rootDispatch = new DispatchGroup(this); useAdaptiveDispatch = deployState.getProperties().useAdaptiveDispatch(); + defaultTopKProbability = deployState.getProperties().defaultTopKProbability(); } @Override @@ -154,8 +156,7 @@ public class IndexedSearchCluster extends SearchCluster private void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) { ddbB.inputdoctypename(sdoc.getInputDocType()) - .configid(sdoc.getConfigId()) - .visibilitydelay(getVisibilityDelay()); + .configid(sdoc.getConfigId()); } @Override @@ -196,13 +197,15 @@ public class IndexedSearchCluster extends SearchCluster routingSelector = sb.toString(); } } + @Override - protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, DeployState deployState) { - for (SearchDefinitionSpec spec : localSearches) { + protected void deriveAllSchemas(List<SchemaSpec> localSearches, DeployState deployState) { + for (SchemaSpec spec : localSearches) { com.yahoo.searchdefinition.Search search = spec.getSearchDefinition().getSearch(); if ( ! (search instanceof DocumentOnlySearch)) { DocumentDatabase db = new DocumentDatabase(this, search.getName(), - new DerivedConfiguration(search, deployState.getDeployLogger(), + new DerivedConfiguration(search, + deployState.getDeployLogger(), deployState.getProperties(), deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), @@ -306,7 +309,11 @@ public class IndexedSearchCluster extends SearchCluster } if (useAdaptiveDispatch) builder.distributionPolicy(DistributionPolicy.ADAPTIVE); - + if (tuning.dispatch.getTopkProbability() != null) { + builder.topKProbability(tuning.dispatch.getTopkProbability()); + } else { + builder.topKProbability(defaultTopKProbability); + } if (tuning.dispatch.getMinActiveDocsCoverage() != null) builder.minActivedocsPercentage(tuning.dispatch.getMinActiveDocsCoverage()); if (tuning.dispatch.getMinGroupCoverage() != null) @@ -334,6 +341,7 @@ public class IndexedSearchCluster extends SearchCluster if (searchCoverage.getMaxWaitAfterCoverageFactor() != null) builder.maxWaitAfterCoverageFactor(searchCoverage.getMaxWaitAfterCoverageFactor()); } + builder.warmuptime(5.0); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java index 860f89792e2..ba81073709e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java @@ -8,7 +8,8 @@ import java.util.Collection; /** * @author Tony Vaagenes */ -public class SearchDefinition { +// TODO: This class is quite pointless +public class NamedSchema { private final Search search; private final String name; @@ -23,15 +24,15 @@ public class SearchDefinition { return name; } - public SearchDefinition(String name, Search search) { + public NamedSchema(String name, Search search) { this.name = name; this.search = search; } //Find search definition from a collection with the name specified - public static SearchDefinition findByName(final String searchDefinitionName, Collection<SearchDefinition> searchDefinitions) { - for (SearchDefinition candidate : searchDefinitions) { - if (candidate.getName().equals(searchDefinitionName) ) + public static NamedSchema findByName(String schemaName, Collection<NamedSchema> schemas) { + for (NamedSchema candidate : schemas) { + if (candidate.getName().equals(schemaName) ) return candidate; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java index c337c77d2a2..5d8b0b688d8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java @@ -30,6 +30,7 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { public void getConfig(ProtonConfig.Builder builder) { setHwInfo(builder); tuneDiskWriteSpeed(builder); + tuneRequestThreads(builder); tuneDocumentStoreMaxFileSize(builder.summary.log); tuneFlushStrategyMemoryLimits(builder.flush.memory); tuneFlushStrategyTlsSize(builder.flush.memory); @@ -105,6 +106,12 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { } } + private void tuneRequestThreads(ProtonConfig.Builder builder) { + int numCores = (int)Math.ceil(nodeFlavor.getMinCpuCores()); + builder.numsearcherthreads(numCores); + builder.numsummarythreads(numCores); + } + private void tuneWriteFilter(ProtonConfig.Writefilter.Builder builder) { // "Reserve" 1GB of memory for other processes running on the content node (config-proxy, cluster-controller, metrics-proxy) double reservedMemoryGb = 1; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java index 1054253e3f0..b505b5e681c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java @@ -7,15 +7,15 @@ import java.io.Serializable; import java.util.List; /** - * Represents a single searchdefinition file. + * Represents a single schema file. * * @author arnej27959 */ -public class SearchDefinitionXMLHandler implements Serializable { +public class SchemaDefinitionXMLHandler implements Serializable { private String sdName; - public SearchDefinitionXMLHandler(ModelElement elem) { + public SchemaDefinitionXMLHandler(ModelElement elem) { sdName = elem.stringAttribute("name"); if (sdName == null) { sdName = elem.stringAttribute("type"); @@ -24,8 +24,8 @@ public class SearchDefinitionXMLHandler implements Serializable { public String getName() { return sdName; } - public SearchDefinition getResponsibleSearchDefinition(List<SearchDefinition> searchDefinitions) { - return SearchDefinition.findByName( getName(), searchDefinitions ); + public NamedSchema getResponsibleSearchDefinition(List<NamedSchema> schemas) { + return NamedSchema.findByName(getName(), schemas ); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 60b3cb6987c..0139e949c7a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -35,22 +35,14 @@ public abstract class SearchCluster extends AbstractSearchCluster super(parent, clusterName, index); } - public void writeFiles(File directory) throws java.io.IOException { - if (!directory.isDirectory() && !directory.mkdirs()) { - throw new java.io.IOException("Cannot create directory: "+ directory); - } - writeSdFiles(directory); - super.writeFiles(directory); - } - /** * Must be called after cluster is built, to derive SD configs * Derives the search definitions from the application package.. * Also stores the document names contained in the search * definitions. */ - public void deriveSearchDefinitions(DeployState deployState) { - deriveAllSearchDefinitions(getLocalSDS(), deployState); + public void deriveSchemas(DeployState deployState) { + deriveAllSchemas(getLocalSDS(), deployState); } @Override @@ -140,7 +132,7 @@ public abstract class SearchCluster extends AbstractSearchCluster return false; } - protected abstract void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, DeployState deployState); + protected abstract void deriveAllSchemas(List<SchemaSpec> localSearches, DeployState deployState); public abstract void defaultDocumentsConfig(); public abstract DerivedConfiguration getSdConfig(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index d668adea116..28e7b3eb37a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -91,7 +91,7 @@ public class StreamingSearchCluster extends SearchCluster implements } @Override - protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> local, DeployState deployState) { + protected void deriveAllSchemas(List<SchemaSpec> local, DeployState deployState) { if (local.size() == 1) { deriveSingleSearchDefinition(local.get(0).getSearchDefinition().getSearch(), deployState); } else if (local.size() > 1){ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index f5a91297e9e..f93bf6fc872 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -241,7 +241,12 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ public Integer level = null; public void getConfig(ProtonConfig.Summary.Cache.Compression.Builder compression) { - if (type != null) compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name)); + if (type != null) compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name)); + if (level != null) compression.level(level); + } + + public void getConfig(ProtonConfig.Summary.Log.Compact.Compression.Builder compression) { + if (type != null) compression.type(ProtonConfig.Summary.Log.Compact.Compression.Type.Enum.valueOf(type.name)); if (level != null) compression.level(level); } @@ -281,6 +286,12 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ } } + public void getConfig(ProtonConfig.Summary.Log.Compact.Builder compact) { + if (compression != null) { + compression.getConfig(compact.compression); + } + } + public void getConfig(ProtonConfig.Summary.Log.Chunk.Builder chunk) { if (outputInt) { if (maxSize!=null) chunk.maxbytes(maxSize.intValue()); @@ -288,7 +299,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ throw new IllegalStateException("Fix this, chunk does not have long types"); } if (compression != null) { - compression.getConfig(chunk.compression); + compression.getConfig(chunk.compression); } } } @@ -303,6 +314,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ if (minFileSizeFactor!=null) log.minfilesizefactor(minFileSizeFactor); if (chunk != null) { chunk.getConfig(log.chunk); + chunk.getConfig(log.compact); } } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 52665ff56a9..3560cf2cd84 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -197,6 +197,7 @@ TOKEN : < NL: "\n" > | < ANNOTATION: "annotation" > | < ANNOTATIONREFERENCE: "annotationreference" > +| < SCHEMA: "schema" > | < SEARCH: "search" > | < DIVERSITY: "diversity" > | < MIN_GROUPS: "min-groups" > @@ -332,6 +333,10 @@ TOKEN : | < UPPERBOUND: "upper-bound" > | < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" > | < ENABLE_BM25: "enable-bm25" > +| < HNSW: "hnsw" > +| < MAXLINKSPERNODE: "max-links-per-node" > +| < DISTANCEMETRIC: "distance-metric" > +| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" > | < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? > | < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > | < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? > @@ -400,38 +405,38 @@ Search search(DocumentTypeManager docMan, String dir) : Search search; } { - (<NL>)* (search = rootSearch(dir) | search = rootDocument(dir)) + (<NL>)* (search = rootSchema(dir) | search = rootDocument(dir)) { return search; } } /** - * This rule consumes a proper search block. This and rootDocument() are the only rules that should ever consume + * This rule consumes a proper schema block. This and rootDocument() are the only rules that should ever consume * trailing newline tokens. * - * @param dir The directory containing the file being parsed. - * @return The search definition object. + * @param dir the directory containing the file being parsed. + * @return the schema definition object. */ -Search rootSearch(String dir) : +Search rootSchema(String dir) : { String name; Search search; } { - ( <SEARCH> name = identifier() { search = new Search(name, app); + ( ( <SCHEMA> | <SEARCH> ) name = identifier() { search = new Search(name, app); rankProfileRegistry.add(new DefaultRankProfile(search, rankProfileRegistry)); rankProfileRegistry.add(new UnrankedRankProfile(search, rankProfileRegistry));} - lbrace() (rootSearchItem(search) (<NL>)*)* <RBRACE> (<NL>)* <EOF>) + lbrace() (rootSchemaItem(search) (<NL>)*)* <RBRACE> (<NL>)* <EOF>) { return search; } } /** - * Consumes an element of a search block. This and rootSearch() are the only rules that should ever consume + * Consumes an element of a schema block. This and rootSearch() are the only rules that should ever consume * trailing newline tokens. * * @param search The search object to modify. * @return Null. */ -Object rootSearchItem(Search search) : { } +Object rootSchemaItem(Search search) : { } { ( document(search) | documentSummary(search) @@ -449,10 +454,10 @@ Object rootSearchItem(Search search) : { } } /** - * Consumes a search definition that contains only documents to be used for inheritance, etc. + * Consumes a schema definition that contains only documents to be used for inheritance, etc. * - * @param dir The directory containing the file being parsed. - * @return The search definition object. + * @param dir the directory containing the file being parsed. + * @return the schema definition object. */ Search rootDocument(String dir) : { @@ -478,7 +483,7 @@ Object rootDocumentItem(Search search) : { } /** * Consumes a use-document statement. This currently does nothing. * - * @param search The search object to modify. + * @param search the search object to modify. */ void useDocument(Search search) : { } { @@ -488,7 +493,7 @@ void useDocument(Search search) : { } /** * Consumes a document element. The name defaults to the search's name, but may be set. * - * @param search The search object to add content to. + * @param search the search object to add content to. */ void document(Search search) : { @@ -507,7 +512,7 @@ void document(Search search) : /** * Consumes a document element, explicitly named * - * @param search The search object to add content to. + * @param search the search object to add content to. */ void namedDocument(Search search) : { @@ -637,7 +642,7 @@ void field(SDDocumentType document, Search search) : if (name != null && com.yahoo.searchdefinition.Search.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, true, document); + field = new TemporarySDField(name, type, document); } lbrace() (fieldBody(field, search, document) (<NL>)*)* <RBRACE> { @@ -651,27 +656,35 @@ void field(SDDocumentType document, Search search) : void fieldSet(Search search) : { - String name; -} -{ - <FIELDSET> name = identifier() lbrace() - (fieldSetItem(name, search)(<NL>)*)+ - <RBRACE> -} - -void fieldSetItem(String setName, Search search) : -{ + String setName; String field; String queryCommand; - SDField matchSettings = new SDField(setName, DataType.STRING); // match etc for fieldset represented as SDField or ease of parsing + List queryCommands = new ArrayList(); + FieldOperationContainer matchSetting; + List matchSettings = new ArrayList(); } { - ( <FIELDS><COLON> field=identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } - ( <COMMA> field=identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } )* ) - | - ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { search.fieldSets().userFieldSets().get(setName).queryCommands().add(queryCommand);} - | - ( match(matchSettings) ) { matchSettings.applyOperations(); search.fieldSets().userFieldSets().get(setName).setMatching(matchSettings.getMatching());} + <FIELDSET> setName = identifier() lbrace() + (( + ( <FIELDS><COLON> field = identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } + ( <COMMA> field = identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } )* ) + | + ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); } + | + ( matchSetting = match(new SDField(setName, DataType.STRING)) ) { matchSettings.add(matchSetting); } + )(<NL>)*)+ + <RBRACE> + { + // Apply settings after parsing since all user field items must be set first + + for (Object command : queryCommands) + search.fieldSets().userFieldSets().get(setName).queryCommands().add((String)command); + + for (Object setting : matchSettings) { + ((SDField)setting).applyOperations(); + search.fieldSets().userFieldSets().get(setName).setMatching(((SDField)setting).getMatching()); + } + } } /** @@ -923,7 +936,7 @@ void structFieldDefinition(SDDocumentType struct) : if (name != null && com.yahoo.searchdefinition.Search.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } - field = new TemporarySDField(name, type, true, struct); + field = new TemporarySDField(name, type, struct); struct.addField(field); } lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> { @@ -1531,7 +1544,7 @@ void queryCommand(FieldOperationContainer container) : QueryCommandOperation field = new QueryCommandOperation(); } { - <QUERYCOMMAND> <COLON> command = identifierWithDash() + <QUERYCOMMAND> <COLON> ( command = identifierWithDash() | command = quotedString() ) { field.addQueryCommand(command); container.addOperation(field); @@ -1551,11 +1564,11 @@ void alias(FieldOperationContainer container) : } } -Object match(FieldOperationContainer field) : { } +FieldOperationContainer match(FieldOperationContainer field) : { } { <MATCH> ( (<COLON> matchType(field)) | (lbrace() (matchItem(field) (<NL>)*)* <RBRACE>) ) - { return null; } + { return field; } } /** @@ -1803,10 +1816,35 @@ Object indexBody(IndexOperation index) : | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); } | <ENABLE_BM25> { index.setEnableBm25(true); } + | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { index.setDistanceMetric(str); } + | hnswIndex(index) { } ) { return null; } } +void hnswIndex(IndexOperation index) : +{ + HnswIndexParams.Builder params = new HnswIndexParams.Builder(); +} +{ + ( LOOKAHEAD(<HNSW> lbrace()) + <HNSW> ( (lbrace() (hnswIndexBody(params) (<NL>)*)* <RBRACE>) ) | + <HNSW> ) + { + index.setHnswIndexParams(params); + } +} + +void hnswIndexBody(HnswIndexParams.Builder params) : +{ + int num; + String str; +} +{ + ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); } + | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); } ) +} + /** * Consumes a constant block of a search element. * @@ -2583,6 +2621,7 @@ String identifier() : { } | <REFERENCE> | <REMOVEIFZERO> | <RERANKCOUNT> + | <SCHEMA> | <SEARCH> | <SECONDARY> | <SECONDPHASE> diff --git a/config-model/src/main/perl/vespa-deploy b/config-model/src/main/perl/vespa-deploy index 59a84f5b0c0..a128e4a8d4c 100755 --- a/config-model/src/main/perl/vespa-deploy +++ b/config-model/src/main/perl/vespa-deploy @@ -154,7 +154,7 @@ my $command = shift; $command ||= "help"; # The '--insecure' parameter is sadly required as it is not possible to disable or alter hostname verification with curl -my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper --insecure -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200'; +my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200'; my $CURL_PUT = $curl_command . ' --write-out \%{http_code} --request PUT'; my $CURL_GET = $curl_command . ' --request GET'; diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 7a3e2916f94..1eac11a879c 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -82,10 +82,27 @@ Metrics = element metrics { element metric { attribute id { xsd:Name } & attribute display-name { xsd:Name }? - }* + }* & + Cloudwatch? }+ } +Cloudwatch = element cloudwatch { + attribute region { xsd:Name } & + attribute namespace { xsd:string { pattern = "[\w_\-/#:\.]+" } } & + ( + element credentials { + attribute access-key-name { xsd:Name } & + attribute secret-key-name { xsd:Name } + } + | + element shared-credentials { + attribute file { string } & + attribute profile { xsd:Name }? + } + )? +} + ClusterControllers = element cluster-controllers { attribute standalone-zookeeper { xsd:string }? & element cluster-controller { diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index e3ad942e7b3..878faabfec1 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -17,14 +17,14 @@ anyElement = element * { JavaId = xsd:string { pattern = "([a-zA-Z_$][a-zA-Z\d_$]*\.)*[a-zA-Z_$][a-zA-Z\d_$]*" } Nodes = element nodes { - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute docker-image { xsd:string }? & Resources? } Resources = element resources { - attribute vcpu { xsd:double { minExclusive = "0.0" } } & + attribute vcpu { xsd:double { minExclusive = "0.0" } | xsd:string } & attribute memory { xsd:string } & attribute disk { xsd:string } & attribute disk-speed { xsd:string }? & @@ -32,12 +32,13 @@ Resources = element resources { } OptionalDedicatedNodes = element nodes { - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute docker-image { xsd:string }? & attribute dedicated { xsd:boolean }? & - attribute exclusive { xsd:boolean }? + attribute exclusive { xsd:boolean }? & + Resources? } GenericConfig = element config { diff --git a/config-model/src/main/resources/schema/container.rnc b/config-model/src/main/resources/schema/container.rnc index 3f0e1f626ac..034de1113c2 100644 --- a/config-model/src/main/resources/schema/container.rnc +++ b/config-model/src/main/resources/schema/container.rnc @@ -23,11 +23,10 @@ Server = element server { } AccessControl = element access-control { - attribute domain { xsd:NCName } & - attribute read { string "true" | string "false" }? & - attribute write { string "true" | string "false" }? & - element vespa-domain { xsd:NCName }? & - element application { xsd:NCName }? & + attribute domain { xsd:NCName }? & # TODO Vespa 8 Remove + attribute read { string "true" | string "false" }? & # TODO Vespa 8 Remove + attribute write { string "true" | string "false" }? & # TODO Vespa 8 Remove + element vespa-domain { xsd:NCName }? & # TODO Remove after end of March 2020 element exclude { Binding+ }? diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 6e4346d96ee..3c8b60fb84b 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -96,7 +96,9 @@ Ssl = element ssl { element private-key-file { string } & element certificate-file { string } & element ca-certificates-file { string }? & - element client-authentication { string "disabled" | string "want" | string "need" }? + element client-authentication { string "disabled" | string "want" | string "need" }? & + element cipher-suites { string }? & + element protocols { string }? } SslProvider = element ssl-provider { @@ -237,7 +239,7 @@ NodesOfContainerCluster = element nodes { attribute type { xsd:string } | ( - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute exclusive { xsd:boolean }? & diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 8b3868c132e..481d82ebb4b 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -57,11 +57,14 @@ PersistenceThread = element thread { ## Declare which storage threads each disk should have. PersistenceThreads = element persistence-threads { + ## The number of threads to create + attribute count { xsd:integer }? & + ## All of the below settings are deprecated. ## Operations with priority worse than this can be blocked attribute highest-priority-to-block { xsd:string } ? & ## Operations with priority better than this can block others attribute lowest-priority-to-block-others { xsd:string } ? & - Thread+ + Thread* } MinNodeRatioPerGroup = element min-node-ratio-per-group { @@ -82,6 +85,7 @@ DispatchTuning = element dispatch { element dispatch-policy { string "round-robin" | string "adaptive" | string "random" }? & element min-group-coverage { xsd:double }? & element min-active-docs-coverage { xsd:double }? & + element top-k-probability { xsd:double }? & element use-local-node { string "true" | string "false" }? } @@ -218,11 +222,11 @@ ContentNodes = element nodes { attribute vespamalloc-debug-stacktrace { xsd:string }? & ( ( - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute docker-image { xsd:string }? & - attribute groups { xsd:positiveInteger }? + attribute groups { xsd:positiveInteger | xsd:string }? ) | ContentNode + @@ -263,12 +267,12 @@ Group = element group { | ( element nodes { - attribute count { xsd:positiveInteger } & + attribute count { xsd:positiveInteger | xsd:string } & attribute flavor { xsd:string }? & attribute required { xsd:boolean }? & attribute exclusive { xsd:boolean }? & attribute docker-image { xsd:string }? & - attribute groups { xsd:positiveInteger }? + attribute groups { xsd:positiveInteger | xsd:string }? } ) | diff --git a/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg b/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg deleted file mode 100644 index 6e18bdf64e7..00000000000 --- a/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg +++ /dev/null @@ -1,3 +0,0 @@ -namespace=config -foo bar -baz []678 diff --git a/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd index ab5e42f983d..247df8a0241 100644 --- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd +++ b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd @@ -33,8 +33,12 @@ search test { expression: xgboost("xgboost_2_2") } + function my_lightgbm() { + expression: lightgbm("lightgbm_regression") + } + first-phase { - expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost + expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost + my_lightgbm } } diff --git a/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index c5b1dfef610..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19125 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg deleted file mode 100644 index 7aed9bdc244..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19131 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg deleted file mode 100644 index 49e5f59b9be..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19137 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg deleted file mode 100644 index 8d5d4fdde7f..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19143 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index be7d8e44c16..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19110 -slobrok.name "search/cluster.music/rtx/0/clustercontroller" -slobrok.config search/cluster.music/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg deleted file mode 100644 index f39dc6adc03..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19112 -slobrok.name "search/cluster.music/rtx/1/clustercontroller" -slobrok.config search/cluster.music/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index 5c9f46bf8ce..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19156 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg deleted file mode 100644 index 69f91eab48c..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19105 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg deleted file mode 100644 index eebb1cb6b40..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19111 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg deleted file mode 100644 index 82e9aafc5a8..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19162 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index aa557a9ae04..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19144 -slobrok.name "search/cluster.rt/rtx/0/clustercontroller" -slobrok.config search/cluster.rt/rtx -servicemonitor.autodisable true -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg deleted file mode 100644 index 1407c3ff209..00000000000 --- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19146 -slobrok.name "search/cluster.rt/rtx/1/clustercontroller" -slobrok.config search/cluster.rt/rtx -servicemonitor.autodisable true -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index c5b1dfef610..00000000000 --- a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19125 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index d0888e9be96..00000000000 --- a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19115 -slobrok.name "search/cluster.music/rtx/0/clustercontroller" -slobrok.config search/cluster.music/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index 53dcc3f9686..00000000000 --- a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19118 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index 474b8c68cdb..00000000000 --- a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19108 -slobrok.name "search/cluster.music/rtx/0/clustercontroller" -slobrok.config search/cluster.music/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index 6dabdfc6af7..00000000000 --- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19115 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index 3c1b537236a..00000000000 --- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19105 -slobrok.name "search/cluster.music1/rtx/0/clustercontroller" -slobrok.config search/cluster.music1/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg deleted file mode 100644 index b6d62fc678b..00000000000 --- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19126 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg deleted file mode 100644 index 887eaa6b634..00000000000 --- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg +++ /dev/null @@ -1,7 +0,0 @@ -listenport 19132 -filesizemax 50000000 -servername "tls" -basedir "tls" -usefsync false -maxthreads 4 -crcmethod xxh64 diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg deleted file mode 100644 index c00fcc456d6..00000000000 --- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg +++ /dev/null @@ -1,6 +0,0 @@ -port 19116 -slobrok.name "search/cluster.music2/rtx/0/clustercontroller" -slobrok.config search/cluster.music2/rtx -servicemonitor.autodisable false -servicemonitor.autoenable false -servicemonitor.timeout 120 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg new file mode 100644 index 00000000000..7b50176625d --- /dev/null +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -0,0 +1,104 @@ +enablecompression false +datatype[0].id 1381038251 +datatype[0].structtype[0].name "position" +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].compresstype NONE +datatype[0].structtype[0].compresslevel 0 +datatype[0].structtype[0].compressthreshold 95 +datatype[0].structtype[0].compressminsize 800 +datatype[0].structtype[0].field[0].name "x" +datatype[0].structtype[0].field[0].datatype 0 +datatype[0].structtype[0].field[0].detailedtype "" +datatype[0].structtype[0].field[1].name "y" +datatype[0].structtype[0].field[1].datatype 0 +datatype[0].structtype[0].field[1].detailedtype "" +datatype[1].id 595216861 +datatype[1].referencetype[0].target_type_id -1318255918 +datatype[2].id 542332920 +datatype[2].referencetype[0].target_type_id 443162583 +datatype[3].id 959075962 +datatype[3].structtype[0].name "ad.header" +datatype[3].structtype[0].version 0 +datatype[3].structtype[0].compresstype NONE +datatype[3].structtype[0].compresslevel 0 +datatype[3].structtype[0].compressthreshold 95 +datatype[3].structtype[0].compressminsize 800 +datatype[3].structtype[0].field[0].name "campaign_ref" +datatype[3].structtype[0].field[0].datatype 595216861 +datatype[3].structtype[0].field[0].detailedtype "" +datatype[3].structtype[0].field[1].name "person_ref" +datatype[3].structtype[0].field[1].datatype 542332920 +datatype[3].structtype[0].field[1].detailedtype "" +datatype[4].id -255288561 +datatype[4].structtype[0].name "ad.body" +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].compresstype NONE +datatype[4].structtype[0].compresslevel 0 +datatype[4].structtype[0].compressthreshold 95 +datatype[4].structtype[0].compressminsize 800 +datatype[5].id 2987301 +datatype[5].documenttype[0].name "ad" +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0].name "document" +datatype[5].documenttype[0].inherits[0].version 0 +datatype[5].documenttype[0].headerstruct 959075962 +datatype[5].documenttype[0].bodystruct -255288561 +datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +datatype[5].documenttype[0].importedfield[0].name "my_cool_field" +datatype[5].documenttype[0].importedfield[1].name "my_swag_field" +datatype[5].documenttype[0].importedfield[2].name "my_name" +datatype[6].id -2041471955 +datatype[6].structtype[0].name "campaign.header" +datatype[6].structtype[0].version 0 +datatype[6].structtype[0].compresstype NONE +datatype[6].structtype[0].compresslevel 0 +datatype[6].structtype[0].compressthreshold 95 +datatype[6].structtype[0].compressminsize 800 +datatype[6].structtype[0].field[0].name "cool_field" +datatype[6].structtype[0].field[0].datatype 2 +datatype[6].structtype[0].field[0].detailedtype "" +datatype[6].structtype[0].field[1].name "swag_field" +datatype[6].structtype[0].field[1].datatype 4 +datatype[6].structtype[0].field[1].detailedtype "" +datatype[7].id 1448849794 +datatype[7].structtype[0].name "campaign.body" +datatype[7].structtype[0].version 0 +datatype[7].structtype[0].compresstype NONE +datatype[7].structtype[0].compresslevel 0 +datatype[7].structtype[0].compressthreshold 95 +datatype[7].structtype[0].compressminsize 800 +datatype[8].id -1318255918 +datatype[8].documenttype[0].name "campaign" +datatype[8].documenttype[0].version 0 +datatype[8].documenttype[0].inherits[0].name "document" +datatype[8].documenttype[0].inherits[0].version 0 +datatype[8].documenttype[0].headerstruct -2041471955 +datatype[8].documenttype[0].bodystruct 1448849794 +datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "cool_field" +datatype[8].documenttype[0].fieldsets{[document]}.fields[1] "swag_field" +datatype[9].id 3129224 +datatype[9].structtype[0].name "person.header" +datatype[9].structtype[0].version 0 +datatype[9].structtype[0].compresstype NONE +datatype[9].structtype[0].compresslevel 0 +datatype[9].structtype[0].compressthreshold 95 +datatype[9].structtype[0].compressminsize 800 +datatype[9].structtype[0].field[0].name "name" +datatype[9].structtype[0].field[0].datatype 2 +datatype[9].structtype[0].field[0].detailedtype "" +datatype[10].id -2003767395 +datatype[10].structtype[0].name "person.body" +datatype[10].structtype[0].version 0 +datatype[10].structtype[0].compresstype NONE +datatype[10].structtype[0].compresslevel 0 +datatype[10].structtype[0].compressthreshold 95 +datatype[10].structtype[0].compressminsize 800 +datatype[11].id 443162583 +datatype[11].documenttype[0].name "person" +datatype[11].documenttype[0].version 0 +datatype[11].documenttype[0].inherits[0].name "document" +datatype[11].documenttype[0].inherits[0].version 0 +datatype[11].documenttype[0].headerstruct 3129224 +datatype[11].documenttype[0].bodystruct -2003767395 +datatype[11].documenttype[0].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg new file mode 100644 index 00000000000..7859703ffe0 --- /dev/null +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -0,0 +1,141 @@ +enablecompression false +documenttype[0].id 2987301 +documenttype[0].name "ad" +documenttype[0].version 0 +documenttype[0].headerstruct 959075962 +documenttype[0].bodystruct -255288561 +documenttype[0].inherits[0].id 8 +documenttype[0].datatype[0].id 959075962 +documenttype[0].datatype[0].type STRUCT +documenttype[0].datatype[0].array.element.id 0 +documenttype[0].datatype[0].map.key.id 0 +documenttype[0].datatype[0].map.value.id 0 +documenttype[0].datatype[0].wset.key.id 0 +documenttype[0].datatype[0].wset.createifnonexistent false +documenttype[0].datatype[0].wset.removeifzero false +documenttype[0].datatype[0].annotationref.annotation.id 0 +documenttype[0].datatype[0].sstruct.name "ad.header" +documenttype[0].datatype[0].sstruct.version 0 +documenttype[0].datatype[0].sstruct.compression.type NONE +documenttype[0].datatype[0].sstruct.compression.level 0 +documenttype[0].datatype[0].sstruct.compression.threshold 95 +documenttype[0].datatype[0].sstruct.compression.minsize 200 +documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref" +documenttype[0].datatype[0].sstruct.field[0].id 23963250 +documenttype[0].datatype[0].sstruct.field[0].datatype 595216861 +documenttype[0].datatype[0].sstruct.field[0].detailedtype "" +documenttype[0].datatype[0].sstruct.field[1].name "person_ref" +documenttype[0].datatype[0].sstruct.field[1].id 100779805 +documenttype[0].datatype[0].sstruct.field[1].datatype 542332920 +documenttype[0].datatype[0].sstruct.field[1].detailedtype "" +documenttype[0].datatype[1].id -255288561 +documenttype[0].datatype[1].type STRUCT +documenttype[0].datatype[1].array.element.id 0 +documenttype[0].datatype[1].map.key.id 0 +documenttype[0].datatype[1].map.value.id 0 +documenttype[0].datatype[1].wset.key.id 0 +documenttype[0].datatype[1].wset.createifnonexistent false +documenttype[0].datatype[1].wset.removeifzero false +documenttype[0].datatype[1].annotationref.annotation.id 0 +documenttype[0].datatype[1].sstruct.name "ad.body" +documenttype[0].datatype[1].sstruct.version 0 +documenttype[0].datatype[1].sstruct.compression.type NONE +documenttype[0].datatype[1].sstruct.compression.level 0 +documenttype[0].datatype[1].sstruct.compression.threshold 95 +documenttype[0].datatype[1].sstruct.compression.minsize 200 +documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +documenttype[0].referencetype[0].id 595216861 +documenttype[0].referencetype[0].target_type_id -1318255918 +documenttype[0].referencetype[1].id 542332920 +documenttype[0].referencetype[1].target_type_id 443162583 +documenttype[0].importedfield[0].name "my_cool_field" +documenttype[0].importedfield[1].name "my_swag_field" +documenttype[0].importedfield[2].name "my_name" +documenttype[1].id -1318255918 +documenttype[1].name "campaign" +documenttype[1].version 0 +documenttype[1].headerstruct -2041471955 +documenttype[1].bodystruct 1448849794 +documenttype[1].inherits[0].id 8 +documenttype[1].datatype[0].id -2041471955 +documenttype[1].datatype[0].type STRUCT +documenttype[1].datatype[0].array.element.id 0 +documenttype[1].datatype[0].map.key.id 0 +documenttype[1].datatype[0].map.value.id 0 +documenttype[1].datatype[0].wset.key.id 0 +documenttype[1].datatype[0].wset.createifnonexistent false +documenttype[1].datatype[0].wset.removeifzero false +documenttype[1].datatype[0].annotationref.annotation.id 0 +documenttype[1].datatype[0].sstruct.name "campaign.header" +documenttype[1].datatype[0].sstruct.version 0 +documenttype[1].datatype[0].sstruct.compression.type NONE +documenttype[1].datatype[0].sstruct.compression.level 0 +documenttype[1].datatype[0].sstruct.compression.threshold 95 +documenttype[1].datatype[0].sstruct.compression.minsize 200 +documenttype[1].datatype[0].sstruct.field[0].name "cool_field" +documenttype[1].datatype[0].sstruct.field[0].id 1588702436 +documenttype[1].datatype[0].sstruct.field[0].datatype 2 +documenttype[1].datatype[0].sstruct.field[0].detailedtype "" +documenttype[1].datatype[0].sstruct.field[1].name "swag_field" +documenttype[1].datatype[0].sstruct.field[1].id 1691224741 +documenttype[1].datatype[0].sstruct.field[1].datatype 4 +documenttype[1].datatype[0].sstruct.field[1].detailedtype "" +documenttype[1].datatype[1].id 1448849794 +documenttype[1].datatype[1].type STRUCT +documenttype[1].datatype[1].array.element.id 0 +documenttype[1].datatype[1].map.key.id 0 +documenttype[1].datatype[1].map.value.id 0 +documenttype[1].datatype[1].wset.key.id 0 +documenttype[1].datatype[1].wset.createifnonexistent false +documenttype[1].datatype[1].wset.removeifzero false +documenttype[1].datatype[1].annotationref.annotation.id 0 +documenttype[1].datatype[1].sstruct.name "campaign.body" +documenttype[1].datatype[1].sstruct.version 0 +documenttype[1].datatype[1].sstruct.compression.type NONE +documenttype[1].datatype[1].sstruct.compression.level 0 +documenttype[1].datatype[1].sstruct.compression.threshold 95 +documenttype[1].datatype[1].sstruct.compression.minsize 200 +documenttype[1].fieldsets{[document]}.fields[0] "cool_field" +documenttype[1].fieldsets{[document]}.fields[1] "swag_field" +documenttype[2].id 443162583 +documenttype[2].name "person" +documenttype[2].version 0 +documenttype[2].headerstruct 3129224 +documenttype[2].bodystruct -2003767395 +documenttype[2].inherits[0].id 8 +documenttype[2].datatype[0].id 3129224 +documenttype[2].datatype[0].type STRUCT +documenttype[2].datatype[0].array.element.id 0 +documenttype[2].datatype[0].map.key.id 0 +documenttype[2].datatype[0].map.value.id 0 +documenttype[2].datatype[0].wset.key.id 0 +documenttype[2].datatype[0].wset.createifnonexistent false +documenttype[2].datatype[0].wset.removeifzero false +documenttype[2].datatype[0].annotationref.annotation.id 0 +documenttype[2].datatype[0].sstruct.name "person.header" +documenttype[2].datatype[0].sstruct.version 0 +documenttype[2].datatype[0].sstruct.compression.type NONE +documenttype[2].datatype[0].sstruct.compression.level 0 +documenttype[2].datatype[0].sstruct.compression.threshold 95 +documenttype[2].datatype[0].sstruct.compression.minsize 200 +documenttype[2].datatype[0].sstruct.field[0].name "name" +documenttype[2].datatype[0].sstruct.field[0].id 1160796772 +documenttype[2].datatype[0].sstruct.field[0].datatype 2 +documenttype[2].datatype[0].sstruct.field[0].detailedtype "" +documenttype[2].datatype[1].id -2003767395 +documenttype[2].datatype[1].type STRUCT +documenttype[2].datatype[1].array.element.id 0 +documenttype[2].datatype[1].map.key.id 0 +documenttype[2].datatype[1].map.value.id 0 +documenttype[2].datatype[1].wset.key.id 0 +documenttype[2].datatype[1].wset.createifnonexistent false +documenttype[2].datatype[1].wset.removeifzero false +documenttype[2].datatype[1].annotationref.annotation.id 0 +documenttype[2].datatype[1].sstruct.name "person.body" +documenttype[2].datatype[1].sstruct.version 0 +documenttype[2].datatype[1].sstruct.compression.type NONE +documenttype[2].datatype[1].sstruct.compression.level 0 +documenttype[2].datatype[1].sstruct.compression.threshold 95 +documenttype[2].datatype[1].sstruct.compression.minsize 200 +documenttype[2].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg index 97f480745cf..0a76e44c4ac 100644 --- a/config-model/src/test/derived/advanced/attributes.cfg +++ b/config-model/src/test/derived/advanced/attributes.cfg @@ -19,3 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg index 56fd15f9f5d..006400c09d4 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "elem_array.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -40,3 +44,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg index 022bdbd31a4..cd91c15700b 100644 --- a/config-model/src/test/derived/attributeprefetch/attributes.cfg +++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multibyte" attribute[].datatype INT8 attribute[].collectiontype ARRAY @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsbyte" attribute[].datatype INT8 attribute[].collectiontype WEIGHTEDSET @@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singleint" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multiint" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsint" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET @@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlelong" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multilong" attribute[].datatype INT64 attribute[].collectiontype ARRAY @@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wslong" attribute[].datatype INT64 attribute[].collectiontype WEIGHTEDSET @@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlefloat" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multifloat" attribute[].datatype FLOAT attribute[].collectiontype ARRAY @@ -229,6 +269,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsfloat" attribute[].datatype FLOAT attribute[].collectiontype WEIGHTEDSET @@ -250,6 +294,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singledouble" attribute[].datatype DOUBLE attribute[].collectiontype SINGLE @@ -271,6 +319,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multidouble" attribute[].datatype DOUBLE attribute[].collectiontype ARRAY @@ -292,6 +344,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsdouble" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET @@ -313,6 +369,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "singlestring" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -334,6 +394,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "multistring" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -355,6 +419,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "wsstring" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -376,3 +444,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg index 7a21001a9ed..29e4c209ef2 100644 --- a/config-model/src/test/derived/attributes/attributes.cfg +++ b/config-model/src/test/derived/attributes/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a3" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a5" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a6" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b1" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b3" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b4" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b5" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b6" attribute[].datatype INT64 attribute[].collectiontype ARRAY @@ -229,6 +269,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b7" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET @@ -250,6 +294,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a9" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -271,6 +319,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a10" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -292,6 +344,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a11" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -313,6 +369,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a12" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -334,6 +394,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a7_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -355,6 +419,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "a8_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -376,3 +444,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg index dda746da5b4..5606e5ea730 100644 --- a/config-model/src/test/derived/complex/attributes.cfg +++ b/config-model/src/test/derived/complex/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "fleeting" attribute[].datatype FLOAT attribute[].collectiontype ARRAY @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "fleeting2" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "foundat" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "collapseby" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "ts" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "combineda" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year_arr" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year_sub" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,3 +219,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg new file mode 100644 index 00000000000..b61fd7e2ee5 --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/attributes.cfg @@ -0,0 +1,25 @@ +attribute[].name "t1" +attribute[].datatype TENSOR +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "tensor(x[128])" +attribute[].imported false +attribute[].index.hnsw.enabled true +attribute[].index.hnsw.maxlinkspernode 32 +attribute[].index.hnsw.distancemetric ANGULAR +attribute[].index.hnsw.neighborstoexploreatinsert 300 diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg new file mode 100644 index 00000000000..e9fc265ca67 --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg @@ -0,0 +1,5 @@ +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "test" +ilscript[].docfield[] "t1" +ilscript[].content[] "clear_state | guard { input t1 | attribute t1 | index t1; }" diff --git a/config-model/src/test/derived/hnsw_index/test.sd b/config-model/src/test/derived/hnsw_index/test.sd new file mode 100644 index 00000000000..207ed764a87 --- /dev/null +++ b/config-model/src/test/derived/hnsw_index/test.sd @@ -0,0 +1,14 @@ +search test { + document test { + field t1 type tensor(x[128]) { + indexing: attribute | index + index { + distance-metric: angular + hnsw { + max-links-per-node: 32 + neighbors-to-explore-at-insert: 300 + } + } + } + } +} diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg index db2280a7846..8f68068c8e7 100644 --- a/config-model/src/test/derived/imported_position_field/attributes.cfg +++ b/config-model/src/test/derived/imported_position_field/attributes.cfg @@ -1,42 +1,50 @@ -attribute[0].name "parent_ref" -attribute[0].datatype REFERENCE -attribute[0].collectiontype SINGLE -attribute[0].removeifzero false -attribute[0].createifnonexistent false -attribute[0].fastsearch false -attribute[0].huge false -attribute[0].ismutable false -attribute[0].sortascending true -attribute[0].sortfunction UCA -attribute[0].sortstrength PRIMARY -attribute[0].sortlocale "" -attribute[0].enablebitvectors false -attribute[0].enableonlybitvector false -attribute[0].fastaccess false -attribute[0].arity 8 -attribute[0].lowerbound -9223372036854775808 -attribute[0].upperbound 9223372036854775807 -attribute[0].densepostinglistthreshold 0.4 -attribute[0].tensortype "" -attribute[0].imported false -attribute[1].name "my_pos_zcurve" -attribute[1].datatype INT64 -attribute[1].collectiontype SINGLE -attribute[1].removeifzero false -attribute[1].createifnonexistent false -attribute[1].fastsearch true -attribute[1].huge false -attribute[1].ismutable false -attribute[1].sortascending true -attribute[1].sortfunction UCA -attribute[1].sortstrength PRIMARY -attribute[1].sortlocale "" -attribute[1].enablebitvectors false -attribute[1].enableonlybitvector false -attribute[1].fastaccess false -attribute[1].arity 8 -attribute[1].lowerbound -9223372036854775808 -attribute[1].upperbound 9223372036854775807 -attribute[1].densepostinglistthreshold 0.4 -attribute[1].tensortype "" -attribute[1].imported true
\ No newline at end of file +attribute[].name "parent_ref" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_pos_zcurve" +attribute[].datatype INT64 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg index ce6ff5e54ae..eb87c6cf53e 100644 --- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg +++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg @@ -1,168 +1,200 @@ -attribute[0].name "parent_ref" -attribute[0].datatype REFERENCE -attribute[0].collectiontype SINGLE -attribute[0].removeifzero false -attribute[0].createifnonexistent false -attribute[0].fastsearch false -attribute[0].huge false -attribute[0].ismutable false -attribute[0].sortascending true -attribute[0].sortfunction UCA -attribute[0].sortstrength PRIMARY -attribute[0].sortlocale "" -attribute[0].enablebitvectors false -attribute[0].enableonlybitvector false -attribute[0].fastaccess false -attribute[0].arity 8 -attribute[0].lowerbound -9223372036854775808 -attribute[0].upperbound 9223372036854775807 -attribute[0].densepostinglistthreshold 0.4 -attribute[0].tensortype "" -attribute[0].imported false -attribute[1].name "my_elem_array.name" -attribute[1].datatype STRING -attribute[1].collectiontype SINGLE -attribute[1].removeifzero false -attribute[1].createifnonexistent false -attribute[1].fastsearch true -attribute[1].huge false -attribute[1].ismutable false -attribute[1].sortascending true -attribute[1].sortfunction UCA -attribute[1].sortstrength PRIMARY -attribute[1].sortlocale "" -attribute[1].enablebitvectors false -attribute[1].enableonlybitvector false -attribute[1].fastaccess false -attribute[1].arity 8 -attribute[1].lowerbound -9223372036854775808 -attribute[1].upperbound 9223372036854775807 -attribute[1].densepostinglistthreshold 0.4 -attribute[1].tensortype "" -attribute[1].imported true -attribute[2].name "my_elem_array.weight" -attribute[2].datatype INT32 -attribute[2].collectiontype SINGLE -attribute[2].removeifzero false -attribute[2].createifnonexistent false -attribute[2].fastsearch false -attribute[2].huge false -attribute[2].ismutable false -attribute[2].sortascending true -attribute[2].sortfunction UCA -attribute[2].sortstrength PRIMARY -attribute[2].sortlocale "" -attribute[2].enablebitvectors false -attribute[2].enableonlybitvector false -attribute[2].fastaccess false -attribute[2].arity 8 -attribute[2].lowerbound -9223372036854775808 -attribute[2].upperbound 9223372036854775807 -attribute[2].densepostinglistthreshold 0.4 -attribute[2].tensortype "" -attribute[2].imported true -attribute[3].name "my_elem_map.key" -attribute[3].datatype STRING -attribute[3].collectiontype SINGLE -attribute[3].removeifzero false -attribute[3].createifnonexistent false -attribute[3].fastsearch true -attribute[3].huge false -attribute[3].ismutable false -attribute[3].sortascending true -attribute[3].sortfunction UCA -attribute[3].sortstrength PRIMARY -attribute[3].sortlocale "" -attribute[3].enablebitvectors false -attribute[3].enableonlybitvector false -attribute[3].fastaccess false -attribute[3].arity 8 -attribute[3].lowerbound -9223372036854775808 -attribute[3].upperbound 9223372036854775807 -attribute[3].densepostinglistthreshold 0.4 -attribute[3].tensortype "" -attribute[3].imported true -attribute[4].name "my_elem_map.value.name" -attribute[4].datatype STRING -attribute[4].collectiontype SINGLE -attribute[4].removeifzero false -attribute[4].createifnonexistent false -attribute[4].fastsearch true -attribute[4].huge false -attribute[4].ismutable false -attribute[4].sortascending true -attribute[4].sortfunction UCA -attribute[4].sortstrength PRIMARY -attribute[4].sortlocale "" -attribute[4].enablebitvectors false -attribute[4].enableonlybitvector false -attribute[4].fastaccess false -attribute[4].arity 8 -attribute[4].lowerbound -9223372036854775808 -attribute[4].upperbound 9223372036854775807 -attribute[4].densepostinglistthreshold 0.4 -attribute[4].tensortype "" -attribute[4].imported true -attribute[5].name "my_elem_map.value.weight" -attribute[5].datatype INT32 -attribute[5].collectiontype SINGLE -attribute[5].removeifzero false -attribute[5].createifnonexistent false -attribute[5].fastsearch false -attribute[5].huge false -attribute[5].ismutable false -attribute[5].sortascending true -attribute[5].sortfunction UCA -attribute[5].sortstrength PRIMARY -attribute[5].sortlocale "" -attribute[5].enablebitvectors false -attribute[5].enableonlybitvector false -attribute[5].fastaccess false -attribute[5].arity 8 -attribute[5].lowerbound -9223372036854775808 -attribute[5].upperbound 9223372036854775807 -attribute[5].densepostinglistthreshold 0.4 -attribute[5].tensortype "" -attribute[5].imported true -attribute[6].name "my_str_int_map.key" -attribute[6].datatype STRING -attribute[6].collectiontype SINGLE -attribute[6].removeifzero false -attribute[6].createifnonexistent false -attribute[6].fastsearch true -attribute[6].huge false -attribute[6].ismutable false -attribute[6].sortascending true -attribute[6].sortfunction UCA -attribute[6].sortstrength PRIMARY -attribute[6].sortlocale "" -attribute[6].enablebitvectors false -attribute[6].enableonlybitvector false -attribute[6].fastaccess false -attribute[6].arity 8 -attribute[6].lowerbound -9223372036854775808 -attribute[6].upperbound 9223372036854775807 -attribute[6].densepostinglistthreshold 0.4 -attribute[6].tensortype "" -attribute[6].imported true -attribute[7].name "my_str_int_map.value" -attribute[7].datatype INT32 -attribute[7].collectiontype SINGLE -attribute[7].removeifzero false -attribute[7].createifnonexistent false -attribute[7].fastsearch false -attribute[7].huge false -attribute[7].ismutable false -attribute[7].sortascending true -attribute[7].sortfunction UCA -attribute[7].sortstrength PRIMARY -attribute[7].sortlocale "" -attribute[7].enablebitvectors false -attribute[7].enableonlybitvector false -attribute[7].fastaccess false -attribute[7].arity 8 -attribute[7].lowerbound -9223372036854775808 -attribute[7].upperbound 9223372036854775807 -attribute[7].densepostinglistthreshold 0.4 -attribute[7].tensortype "" -attribute[7].imported true
\ No newline at end of file +attribute[].name "parent_ref" +attribute[].datatype REFERENCE +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_array.name" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_array.weight" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.key" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.value.name" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_elem_map.value.weight" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_str_int_map.key" +attribute[].datatype STRING +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch true +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 +attribute[].name "my_str_int_map.value" +attribute[].datatype INT32 +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "" +attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg index 15b1771a2e8..d3a51523e05 100644 --- a/config-model/src/test/derived/importedfields/attributes.cfg +++ b/config-model/src/test/derived/importedfields/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "b_ref_with_summary" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_string_field" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_array_field" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_int_wset_field" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET @@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "my_ancient_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,3 +194,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg index a83ec45c5e9..388b212689a 100644 --- a/config-model/src/test/derived/indexschema/index-info.cfg +++ b/config-model/src/test/derived/indexschema/index-info.cfg @@ -44,6 +44,8 @@ indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "sd" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "sd" +indexinfo[].command[].command "phrase-segmenting false" +indexinfo[].command[].indexname "sd" indexinfo[].command[].command "literal-boost" indexinfo[].command[].indexname "pos.x" indexinfo[].command[].command "index" @@ -307,6 +309,16 @@ indexinfo[].command[].indexname "exactexplicit" indexinfo[].command[].command "exact ARNOLD" indexinfo[].command[].indexname "exactexplicit" indexinfo[].command[].command "dynteaser" +indexinfo[].command[].indexname "exactexplicit" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "exactexplicit" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "exactexplicit" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "exactexplicit" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "exactexplicit" +indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "exactexplicit2" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "exactexplicit2" @@ -322,6 +334,8 @@ indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "gram" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "gram" +indexinfo[].command[].command "phrase-segmenting false" +indexinfo[].command[].indexname "gram" indexinfo[].command[].command "ngram 2" indexinfo[].command[].indexname "nostem1" indexinfo[].command[].command "lowercase" diff --git a/config-model/src/test/derived/indexschema/indexschema.sd b/config-model/src/test/derived/indexschema/indexschema.sd index 49f0f7dfca6..01f552a8c32 100644 --- a/config-model/src/test/derived/indexschema/indexschema.sd +++ b/config-model/src/test/derived/indexschema/indexschema.sd @@ -24,6 +24,7 @@ search indexschema { field sd type string { indexing: index rank:literal + query-command: "phrase-segmenting false" } field pos type position { indexing: attribute @@ -130,17 +131,17 @@ search indexschema { } fieldset exactexplicit { - fields:sa, sb query-command: "exact ARNOLD" + fields:sa, sb query-command: dynteaser } fieldset exactexplicit2 { - fields:sc, sd match { exact exact-terminator: "Arnold" } + fields:sc, sd } fieldset gram { diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg index 4a081edbf54..397d8878792 100644 --- a/config-model/src/test/derived/inheritance/attributes.cfg +++ b/config-model/src/test/derived/inheritance/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "overridden" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "onlymother" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg index 13f59d4925f..3b30af10a8f 100644 --- a/config-model/src/test/derived/inheritfromparent/attributes.cfg +++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg @@ -19,3 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg index 8901acf63d1..4de78c52efd 100644 --- a/config-model/src/test/derived/map_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_map.value" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg index 665edcdf45d..72e60b857a2 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "str_elem_map.value.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_elem_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "int_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -103,3 +119,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg index 7f9592dafc8..4d19dc69b33 100644 --- a/config-model/src/test/derived/music/attributes.cfg +++ b/config-model/src/test/derived/music/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "hiphopvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "metalvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY @@ -229,3 +269,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml b/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml new file mode 100644 index 00000000000..b8140f34617 --- /dev/null +++ b/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml @@ -0,0 +1 @@ +<query-profile id="default" type="root" /> diff --git a/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml b/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml new file mode 100644 index 00000000000..895e0663181 --- /dev/null +++ b/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml @@ -0,0 +1,3 @@ +<query-profile-type id="root" inherits="native"> + <field name="ranking.features.query(q_vec)" type="tensor<float>(x[5])" /> +</query-profile-type> diff --git a/config-model/src/test/derived/nearestneighbor/test.sd b/config-model/src/test/derived/nearestneighbor/test.sd new file mode 100644 index 00000000000..ab5f6d85448 --- /dev/null +++ b/config-model/src/test/derived/nearestneighbor/test.sd @@ -0,0 +1,27 @@ +search test { + document test { + field id type int { + indexing: attribute | summary + } + field vec type tensor<float>(x[5]) { + indexing: attribute | summary + } + field vec_hnsw type tensor<float>(x[5]) { + indexing: attribute | index | summary + index { + hnsw { + max-links-per-node: 16 + neighbors-to-explore-at-insert: 200 + } + } + } + } + rank-profile default { + first-phase { + expression: 10000 - itemRawScore(nns) + } + } + document-summary minimal { + summary id type int {} + } +} diff --git a/config-model/src/test/derived/neuralnet/query-profiles.cfg b/config-model/src/test/derived/neuralnet/query-profiles.cfg new file mode 100644 index 00000000000..817640a89fd --- /dev/null +++ b/config-model/src/test/derived/neuralnet/query-profiles.cfg @@ -0,0 +1,54 @@ +queryprofile[].id "default" +queryprofile[].type "DefaultQueryProfileType" +queryprofiletype[].id "DefaultQueryProfileType" +queryprofiletype[].strict false +queryprofiletype[].matchaspath false +queryprofiletype[].inherit[] "native" +queryprofiletype[].field[].name "ranking" +queryprofiletype[].field[].type "query-profile:ranking_0_0" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].id "ranking_0_0" +queryprofiletype[].strict false +queryprofiletype[].matchaspath false +queryprofiletype[].inherit[] "ranking" +queryprofiletype[].field[].name "features" +queryprofiletype[].field[].type "query-profile:features_0_1" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "rankfeature" +queryprofiletype[].id "features_0_1" +queryprofiletype[].strict false +queryprofiletype[].matchaspath false +queryprofiletype[].field[].name "query(W_0)" +queryprofiletype[].field[].type "tensor(hidden[9],x[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(W_1)" +queryprofiletype[].field[].type "tensor(hidden[9],out[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(W_out)" +queryprofiletype[].field[].type "tensor(out[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(b_0)" +queryprofiletype[].field[].type "tensor(hidden[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(b_1)" +queryprofiletype[].field[].type "tensor(out[9])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +queryprofiletype[].field[].name "query(b_out)" +queryprofiletype[].field[].type "tensor(out[1])" +queryprofiletype[].field[].overridable true +queryprofiletype[].field[].mandatory false +queryprofiletype[].field[].alias "" +enableGroupingSessionCache true diff --git a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml index e74152638fb..42336098a9a 100644 --- a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml +++ b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml @@ -1,5 +1,5 @@ <!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<query-profile-type id="DefaultQueryProfileType"> +<query-profile-type id="DefaultQueryProfileType" inherits="native"> <field name="ranking.features.query(W_0)" type="tensor(x[9],hidden[9])" /> <field name="ranking.features.query(b_0)" type="tensor(hidden[9])" /> <field name="ranking.features.query(W_1)" type="tensor(hidden[9],out[9])" /> diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg index b33c2fbdf9b..2aed2288773 100644 --- a/config-model/src/test/derived/newrank/attributes.cfg +++ b/config-model/src/test/derived/newrank/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "scorekey" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE @@ -208,3 +244,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg index 47e07e2a524..3a9daf7af94 100644 --- a/config-model/src/test/derived/predicate_attribute/attributes.cfg +++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg @@ -19,3 +19,7 @@ attribute[].upperbound 200 attribute[].densepostinglistthreshold 0.2 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/predicate_attribute/index-info.cfg b/config-model/src/test/derived/predicate_attribute/index-info.cfg index 3d9f57dd84b..4ebac65e1f5 100644 --- a/config-model/src/test/derived/predicate_attribute/index-info.cfg +++ b/config-model/src/test/derived/predicate_attribute/index-info.cfg @@ -4,6 +4,8 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "sddocname" indexinfo[].command[].command "word" indexinfo[].command[].indexname "some_predicate_field" +indexinfo[].command[].command "predicate" +indexinfo[].command[].indexname "some_predicate_field" indexinfo[].command[].command "predicate-bounds [3..200]" indexinfo[].command[].indexname "some_predicate_field" indexinfo[].command[].command "index" diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg index d7922a0de69..0a8cbd82186 100644 --- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg +++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "attributefield2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,3 +44,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg index 12dbf896edc..e83b187a0cc 100644 --- a/config-model/src/test/derived/reference_fields/attributes.cfg +++ b/config-model/src/test/derived/reference_fields/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "other_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "yet_another_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE @@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg index e88dfde03bb..83c310ca5ca 100644 --- a/config-model/src/test/derived/sorting/attributes.cfg +++ b/config-model/src/test/derived/sorting/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "syntaxcheck2" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "infieldonly" attribute[].datatype STRING attribute[].collectiontype SINGLE @@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index 4634e120a3a..780f47ee3d5 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor<float>(x[2],y[1])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f3" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor(x{})" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f4" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor(x[10],y[10])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f5" attribute[].datatype TENSOR attribute[].collectiontype SINGLE @@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor<float>(x[10])" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "f6" attribute[].datatype FLOAT attribute[].collectiontype SINGLE @@ -103,3 +119,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index e6ffc37e871..82535324864 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "along" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "abool" attribute[].datatype BOOL attribute[].collectiontype SINGLE @@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "ashortfloat" attribute[].datatype FLOAT16 attribute[].collectiontype SINGLE @@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "arrayfield" attribute[].datatype INT32 attribute[].collectiontype ARRAY @@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield2" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield3" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "setfield4" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "tagfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "juletre" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -229,6 +269,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "album1" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET @@ -250,6 +294,10 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 attribute[].name "other" attribute[].datatype INT64 attribute[].collectiontype SINGLE @@ -271,3 +319,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd index e61b64dc0ed..a8c801b1421 100644 --- a/config-model/src/test/examples/simple.sd +++ b/config-model/src/test/examples/simple.sd @@ -3,7 +3,7 @@ # You can get a reasonable configuration by only configuring # a document # ...this has become less and less simple over time actually -search simple { +schema simple { document simple { diff --git a/config-model/src/test/integration/lightgbm/models/regression.json b/config-model/src/test/integration/lightgbm/models/regression.json new file mode 100644 index 00000000000..cf0488ecd8b --- /dev/null +++ b/config-model/src/test/integration/lightgbm/models/regression.json @@ -0,0 +1,275 @@ +{ + "name": "tree", + "version": "v3", + "num_class": 1, + "num_tree_per_iteration": 1, + "label_index": 0, + "max_feature_idx": 3, + "average_output": false, + "objective": "regression", + "feature_names": [ + "numerical_1", + "numerical_2", + "categorical_1", + "categorical_2" + ], + "monotone_constraints": [], + "tree_info": [ + { + "tree_index": 0, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 68.5353012084961, + "threshold": 0.46643291586559305, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 2.1594397038037663, + "leaf_weight": 469, + "leaf_count": 469 + }, + "right_child": { + "split_index": 1, + "split_feature": 3, + "split_gain": 41.27640151977539, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.246035, + "internal_weight": 531, + "internal_count": 531, + "left_child": { + "leaf_index": 1, + "leaf_value": 2.235297305276056, + "leaf_weight": 302, + "leaf_count": 302 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 2.1792953471546546, + "leaf_weight": 229, + "leaf_count": 229 + } + } + } + }, + { + "tree_index": 1, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 2, + "split_gain": 64.22250366210938, + "threshold": "3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": 0.03070842919354316, + "leaf_weight": 399, + "leaf_count": 399 + }, + "right_child": { + "split_index": 1, + "split_feature": 0, + "split_gain": 36.74250030517578, + "threshold": 0.5102250691730842, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.204906, + "internal_weight": 601, + "internal_count": 601, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.04439151147520909, + "leaf_weight": 315, + "leaf_count": 315 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.005117411709368601, + "leaf_weight": 286, + "leaf_count": 286 + } + } + } + }, + { + "tree_index": 2, + "num_leaves": 3, + "num_cat": 0, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 1, + "split_gain": 57.1327018737793, + "threshold": 0.668665477622446, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 40.859100341796875, + "threshold": 0.008118820676863816, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": -0.162926, + "internal_weight": 681, + "internal_count": 681, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.15361238490967524, + "leaf_weight": 21, + "leaf_count": 21 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": -0.01192330846157292, + "leaf_weight": 660, + "leaf_count": 660 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": 0.03499044894987518, + "leaf_weight": 319, + "leaf_count": 319 + } + } + }, + { + "tree_index": 3, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 0, + "split_gain": 54.77090072631836, + "threshold": 0.5201391072644542, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.02141000620783247, + "leaf_weight": 543, + "leaf_count": 543 + }, + "right_child": { + "split_index": 1, + "split_feature": 2, + "split_gain": 27.200700759887695, + "threshold": "0||1", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0.255704, + "internal_weight": 457, + "internal_count": 457, + "left_child": { + "leaf_index": 1, + "leaf_value": -0.004121485787596721, + "leaf_weight": 191, + "leaf_count": 191 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.04534090904886873, + "leaf_weight": 266, + "leaf_count": 266 + } + } + } + }, + { + "tree_index": 4, + "num_leaves": 3, + "num_cat": 1, + "shrinkage": 0.1, + "tree_structure": { + "split_index": 0, + "split_feature": 3, + "split_gain": 51.84349822998047, + "threshold": "2||3||4", + "decision_type": "==", + "default_left": false, + "missing_type": "NaN", + "internal_value": 0, + "internal_weight": 0, + "internal_count": 1000, + "left_child": { + "split_index": 1, + "split_feature": 1, + "split_gain": 39.352699279785156, + "threshold": 0.27283279016959255, + "decision_type": "<=", + "default_left": true, + "missing_type": "NaN", + "internal_value": 0.188414, + "internal_weight": 593, + "internal_count": 593, + "left_child": { + "leaf_index": 0, + "leaf_value": -0.01924803254356527, + "leaf_weight": 184, + "leaf_count": 184 + }, + "right_child": { + "leaf_index": 2, + "leaf_value": 0.03643772842347651, + "leaf_weight": 409, + "leaf_count": 409 + } + }, + "right_child": { + "leaf_index": 1, + "leaf_value": -0.02701711918923075, + "leaf_weight": 407, + "leaf_count": 407 + } + } + } + ], + "pandas_categorical": [ + [ + "a", + "b", + "c", + "d", + "e" + ], + [ + "i", + "j", + "k", + "l", + "m" + ] + ] +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index cb1577417b4..fe82f2406f2 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -20,7 +20,7 @@ import com.yahoo.searchdefinition.DocumentOnlySearch; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; import org.junit.After; import org.junit.Rule; import org.junit.Test; @@ -31,7 +31,6 @@ import java.io.File; import java.io.IOException; import java.io.Reader; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -60,9 +59,9 @@ public class ApplicationDeployTest { public void testVespaModel() throws SAXException, IOException { ApplicationPackageTester tester = ApplicationPackageTester.create(TESTDIR + "app1"); VespaModel model = new VespaModel(tester.app()); - List<SearchDefinition> searchDefinitions = tester.getSearchDefinitions(); - assertEquals(searchDefinitions.size(), 5); - for (SearchDefinition searchDefinition : searchDefinitions) { + List<NamedSchema> schemas = tester.getSchemas(); + assertEquals(schemas.size(), 5); + for (NamedSchema searchDefinition : schemas) { Search s = searchDefinition.getSearch(); switch (s.getName()) { case "music": @@ -72,21 +71,18 @@ public class ApplicationDeployTest { break; case "product": assertTrue(s instanceof DocumentOnlySearch); - assertEquals(s.getDocument().getField("title").getDataType(), DataType.STRING); + assertEquals(DataType.STRING, s.getDocument().getField("title").getDataType()); break; default: fail(); } } - File[] truth = new File[]{new File(TESTSDDIR + "laptop.sd"), - new File(TESTSDDIR + "music.sd"), - new File(TESTSDDIR + "pc.sd"), - new File(TESTSDDIR + "product.sd"), - new File(TESTSDDIR + "sock.sd")}; - Arrays.sort(truth); - List<File> appSdFiles = tester.app().getSearchDefinitionFiles(); - Collections.sort(appSdFiles); - assertEquals(appSdFiles, Arrays.asList(truth)); + assertEquals(Set.of(new File(TESTSDDIR + "laptop.sd"), + new File(TESTSDDIR + "music.sd"), + new File(TESTSDDIR + "pc.sd"), + new File(TESTSDDIR + "product.sd"), + new File(TESTSDDIR + "sock.sd")), + new HashSet<>(tester.app().getSearchDefinitionFiles())); List<FilesApplicationPackage.Component> components = tester.app().getComponents(); assertEquals(1, components.size()); @@ -103,7 +99,7 @@ public class ApplicationDeployTest { // Check that getFilename works ArrayList<String> sdFileNames = new ArrayList<>(); - for (SearchDefinition sd : searchDefinitions) + for (NamedSchema sd : schemas) sdFileNames.add(sd.getFilename()); Collections.sort(sdFileNames); assertEquals("laptop.sd", sdFileNames.get(0)); @@ -190,11 +186,11 @@ public class ApplicationDeployTest { File tmpDir = tmpFolder.getRoot(); IOUtils.copyDirectory(new File(TESTDIR, "app1"), tmpDir); ApplicationPackageTester tester = ApplicationPackageTester.create(tmpDir.getAbsolutePath()); - assertEquals(5, tester.getSearchDefinitions().size()); - File sdDir = new File(tmpDir, "searchdefinitions"); + assertEquals(5, tester.getSchemas().size()); + File sdDir = new File(tmpDir, "schemas"); File sd = new File(sdDir, "testfoo.sd"); IOUtils.writeFile(sd, "search testfoo { document testfoo { field bar type string { } } }", false); - assertEquals(6, tester.getSearchDefinitions().size()); + assertEquals(6, tester.getSchemas().size()); } @Test @@ -300,6 +296,16 @@ public class ApplicationDeployTest { @Test public void testGetJarEntryName() { + JarEntry e = new JarEntry("/schemas/foo.sd"); + assertEquals(ApplicationPackage.getFileName(e), "foo.sd"); + e = new JarEntry("bar"); + assertEquals(ApplicationPackage.getFileName(e), "bar"); + e = new JarEntry(""); + assertEquals(ApplicationPackage.getFileName(e), ""); + } + + @Test + public void testGetJarEntryNameForLegacyPath() { JarEntry e = new JarEntry("/searchdefinitions/foo.sd"); assertEquals(ApplicationPackage.getFileName(e), "foo.sd"); e = new JarEntry("bar"); diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java index 87b6efa83d6..8e7d5aadb36 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; import java.io.File; import java.io.IOException; @@ -39,8 +39,8 @@ public class ApplicationPackageTester { public FilesApplicationPackage app() { return applicationPackage; } - public List<SearchDefinition> getSearchDefinitions() { - return new DeployState.Builder().applicationPackage(app()).build().getSearchDefinitions(); + public List<NamedSchema> getSchemas() { + return new DeployState.Builder().applicationPackage(app()).build().getSchemas(); } public static ApplicationPackageTester create(String applicationPackageDir) { diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java index 38e438e4d3a..f8ab3cc54c8 100644 --- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java +++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java @@ -9,6 +9,7 @@ import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo; @@ -53,6 +54,9 @@ public class MockModelContext implements ModelContext { } @Override + public Provisioned provisioned() { return new Provisioned(); } + + @Override public DeployLogger deployLogger() { return new BaseDeployLogger(); } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 7b4b650295c..7208d8c5fc1 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -264,7 +264,7 @@ public class ModelProvisioningTest { assertEquals("Heap size is lowered with combined clusters", 17, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); - assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Type.combined, model); + assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); } } @@ -1205,6 +1205,62 @@ public class ModelProvisioningTest { } @Test + public void testRequestingRangesMin() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='container'>" + + " <nodes count='[4, 6]'>" + + " <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" + + " </nodes>" + + " </container>" + + " <content version='1.0' id='foo'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='[6, 20]' groups='[3,4]'>" + + " <resources vcpu='8' memory='200Gb' disk='1Pb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + + int totalHosts = 10; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(new NodeResources(11.5, 10, 30, 0.3), 6); + tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20); + VespaModel model = tester.createModel(services, true); + assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); + } + + @Test + public void testRequestingRangesMax() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='container'>" + + " <nodes count='[4, 6]'>" + + " <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" + + " </nodes>" + + " </container>" + + " <content version='1.0' id='foo'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='[6, 20]' groups='[3,4]'>" + + " <resources vcpu='8' memory='200Gb' disk='1Pb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + + int totalHosts = 26; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 6); + tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20); + VespaModel model = tester.createModel(services, true, true); + assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); + } + + @Test public void testContainerOnly() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + @@ -1308,14 +1364,14 @@ public class ModelProvisioningTest { " </http>" + "</container>"; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(1); + tester.addHosts(2); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getHosts().size()); + assertEquals(2, model.getHosts().size()); assertEquals(1, model.getContainerClusters().size()); + assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); } @Test - @Ignore // TODO: Enable when turning the port check on public void testThatStandaloneSyntaxOnHostedVespaRequiresDefaultPort() { try { String services = @@ -1375,7 +1431,7 @@ public class ModelProvisioningTest { } @Test - public void testNoNodeTagMeans1Node() { + public void testNoNodeTagMeansTwoNodes() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + @@ -1390,16 +1446,16 @@ public class ModelProvisioningTest { " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(1); + tester.addHosts(3); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().hostSystem().getHosts().size()); - assertEquals(1, model.getAdmin().getSlobroks().size()); - assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); + assertEquals(3, model.getRoot().hostSystem().getHosts().size()); + assertEquals(2, model.getAdmin().getSlobroks().size()); + assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); } @Test - public void testNoNodeTagMeans1NodeNoContent() { + public void testNoNodeTagMeansTwoNodesNoContent() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + @@ -1409,11 +1465,11 @@ public class ModelProvisioningTest { " </container>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(1); + tester.addHosts(2); VespaModel model = tester.createModel(services, true); - assertEquals(1, model.getRoot().hostSystem().getHosts().size()); - assertEquals(1, model.getAdmin().getSlobroks().size()); - assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); + assertEquals(2, model.getRoot().hostSystem().getHosts().size()); + assertEquals(2, model.getAdmin().getSlobroks().size()); + assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); } @Test @@ -1806,12 +1862,17 @@ public class ModelProvisioningTest { assertTrue(logdConfig.logserver().use()); } - private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Type type, VespaModel model) { - assertEquals("Nodes in cluster " + id + " with type " + type, nodeCount, + private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Id combinedId, + ClusterSpec.Type type, VespaModel model) { + assertEquals("Nodes in cluster " + id + " with type " + type + (combinedId != null ? ", combinedId " + combinedId : ""), nodeCount, model.hostSystem().getHosts().stream() .map(h -> h.spec().membership().get().cluster()) - .filter(spec -> spec.id().equals(id) && spec.type().equals(type)) + .filter(spec -> spec.id().equals(id) && spec.type().equals(type) && spec.combinedId().equals(Optional.ofNullable(combinedId))) .count()); } + private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Type type, VespaModel model) { + assertProvisioned(nodeCount, id, null, type, model); + } + } diff --git a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java index 94602d5201a..127c121197b 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.document.test; import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import org.junit.Test; @@ -17,7 +17,7 @@ import static org.junit.Assert.*; * @author Thomas Gundersen * @author bratseth */ -public class SDDocumentTypeTestCase extends SearchDefinitionTestCase { +public class SDDocumentTypeTestCase extends SchemaTestCase { // Verify that we can register and retrieve fields. @Test diff --git a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java index 7dcbd92655b..b3109c3c2e4 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.document.test; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import org.junit.Test; @@ -11,7 +11,7 @@ import static org.junit.Assert.fail; /** * @author Thomas Gundersen */ -public class SDFieldTestCase extends SearchDefinitionTestCase { +public class SDFieldTestCase extends SchemaTestCase { @Test public void testIdSettingConflict() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java index 846166ae93c..6a40778c9c4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java @@ -18,7 +18,7 @@ import static org.junit.Assert.assertTrue; * * @author bratseth */ -public class ArraysTestCase extends SearchDefinitionTestCase { +public class ArraysTestCase extends SchemaTestCase { @Test public void testArrayImporting() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java index cfd02c22b89..af1e061b7fa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java @@ -16,7 +16,7 @@ import static org.junit.Assert.assertTrue; * * @author Einar M R Rosenvinge */ -public class ArraysWeightedSetsTestCase extends SearchDefinitionTestCase { +public class ArraysWeightedSetsTestCase extends SchemaTestCase { @Test public void testArrayWeightedSetsImporting() throws java.io.IOException, com.yahoo.searchdefinition.parser.ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index 084cbcfdfc0..83cad4cf266 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -25,7 +25,7 @@ import static org.junit.Assert.*; * * @author bratseth */ -public class AttributeSettingsTestCase extends SearchDefinitionTestCase { +public class AttributeSettingsTestCase extends SchemaTestCase { @Rule public final ExpectedException exceptionRule = ExpectedException.none(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java new file mode 100644 index 00000000000..2c13427760f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java @@ -0,0 +1,15 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * Convenience class for tests that need to set attribute properties on fields. + */ +public class AttributeUtils { + + public static void addAttributeAspect(SDField field) { + field.parseIndexingScript("{ attribute }"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java index 8ccb1ed969a..3bb464c5fa5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java @@ -14,7 +14,7 @@ import static org.junit.Assert.assertEquals; * * @author bratseth */ -public class CommentTestCase extends SearchDefinitionTestCase { +public class CommentTestCase extends SchemaTestCase { @Test public void testComments() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java index 8378ec811a5..e46208c770d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java @@ -36,7 +36,7 @@ public class DocumentReferenceResolverTest { // Create foo document with document reference to bar and add another field SDField fooRefToBarField = new SDField ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType())); - addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooRefToBarField); SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT); Search fooSearch = new Search(); SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); @@ -59,7 +59,7 @@ public class DocumentReferenceResolverTest { // Create foo document with document reference to non-existing document bar SDField fooRefToBarField = new SDField( "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar"))); - addAttributeAspect(fooRefToBarField); + AttributeUtils.addAttributeAspect(fooRefToBarField); Search fooSearch = new Search(); SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); fooDocument.addField(fooRefToBarField); @@ -95,8 +95,4 @@ public class DocumentReferenceResolverTest { resolver.resolveReferences(fooDocument); } - private static void addAttributeAspect(SDField fooRefToBarField) { - fooRefToBarField.parseIndexingScript("{ attribute }"); - } - } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java index be3bae05c5b..1c7b3e19663 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertSame; /** * @author Einar M R Rosenvinge */ -public class FieldOfTypeDocumentTestCase extends SearchDefinitionTestCase { +public class FieldOfTypeDocumentTestCase extends SchemaTestCase { @Test public void testDocument() throws IOException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java new file mode 100644 index 00000000000..fcbb89b5c42 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java @@ -0,0 +1,66 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.DataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedField; +import org.junit.Test; + +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ImportedFieldsEnumeratorTest { + + @Test + public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() { + Search parentSearch = new Search(); + SDDocumentType parentDocument = new SDDocumentType("parent", parentSearch); + var parentField = new SDField("their_field", DataType.INT); + AttributeUtils.addAttributeAspect(parentField); + parentDocument.addField(parentField); + parentSearch.addDocument(parentDocument); + + Search fooSearch = new Search(); + SDField fooRefToParent = new SDField( + "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(fooRefToParent); + var fooImports = fooSearch.temporaryImportedFields().get(); + fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field")); + fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field")); + SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch); + fooSearch.addDocument(fooDocument); + + Search barSearch = new Search(); + SDField barRefToParent = new SDField( + "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType())); + AttributeUtils.addAttributeAspect(barRefToParent); + var barImports = barSearch.temporaryImportedFields().get(); + barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field")); + SDDocumentType barDocument = new SDDocumentType("bar", barSearch); + barSearch.addDocument(barDocument); + + var enumerator = new ImportedFieldsEnumerator(List.of(parentSearch, fooSearch, barSearch)); + + enumerator.enumerateImportedFields(parentDocument); + assertImportedFieldsAre(parentDocument, List.of()); // No imported fields in parent + + enumerator.enumerateImportedFields(fooDocument); + assertImportedFieldsAre(fooDocument, List.of("my_first_import", "my_second_import")); + + enumerator.enumerateImportedFields(barDocument); + assertImportedFieldsAre(barDocument, List.of("my_cool_import")); + } + + private void assertImportedFieldsAre(SDDocumentType documentType, List<String> expectedNames) { + assertNotNull(documentType.getTemporaryImportedFields()); + var actualNames = documentType.getTemporaryImportedFields().fields().keySet(); + var expectedNameSet = new HashSet<>(expectedNames); + assertEquals(expectedNameSet, actualNames); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 519828497fe..4453f327bb4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -18,7 +18,7 @@ import static org.junit.Assert.fail; /** * @author bratseth */ -public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionTestCase { +public class IncorrectRankingExpressionFileRefTestCase extends SchemaTestCase { @Test public void testIncorrectRef() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java index c145c0e5634..91ab5e2b5df 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java @@ -11,7 +11,7 @@ import static org.junit.Assert.fail; * * @author bratseth */ -public class IncorrectSummaryTypesTestCase extends SearchDefinitionTestCase { +public class IncorrectSummaryTypesTestCase extends SchemaTestCase { @Test public void testImportingIncorrect() throws ParseException { try { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java index 2cfb542d06b..f992d5ee0ba 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java @@ -15,7 +15,7 @@ import static org.junit.Assert.assertEquals; * * @author bratseth */ -public class IndexSettingsTestCase extends SearchDefinitionTestCase { +public class IndexSettingsTestCase extends SchemaTestCase { @Test public void testStemmingSettings() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java index 21ba3a5e80a..70119ad42f9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java @@ -11,7 +11,7 @@ import static org.junit.Assert.assertNotNull; * * @author frodelu */ -public class IndexingParsingTestCase extends SearchDefinitionTestCase { +public class IndexingParsingTestCase extends SchemaTestCase { @Test public void requireThatIndexingExpressionsCanBeParsed() throws Exception { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java index df9f5778614..5721dbf06e8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java @@ -11,7 +11,7 @@ import java.io.IOException; * * @author bratseth */ -public class MultipleSummariesTestCase extends SearchDefinitionTestCase { +public class MultipleSummariesTestCase extends SchemaTestCase { @Test public void testArrayImporting() throws IOException, ParseException { SearchBuilder.buildFromFile("src/test/examples/multiplesummaries.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java index d2360453976..47b6905c677 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java @@ -14,7 +14,7 @@ import static org.junit.Assert.fail; * * @author Lars Christian Jensen */ -public class NameFieldCheckTestCase extends SearchDefinitionTestCase { +public class NameFieldCheckTestCase extends SchemaTestCase { @Test public void testNameField() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java index 5ac37bf0a3a..64527e7f7a5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java @@ -13,7 +13,7 @@ import static org.junit.Assert.assertTrue; * * @author bratseth */ -public class OutsideTestCase extends SearchDefinitionTestCase { +public class OutsideTestCase extends SchemaTestCase { @Test public void testOutsideIndex() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index 5ba508e3ef3..e2f2c1fd407 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -34,7 +34,7 @@ import static org.junit.Assert.assertTrue; * * @author bratseth */ -public class RankProfileTestCase extends SearchDefinitionTestCase { +public class RankProfileTestCase extends SchemaTestCase { @Test public void testRankProfileInheritance() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java index f4666f7fb3b..3fe2861de0c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java @@ -13,7 +13,7 @@ import static org.junit.Assert.assertEquals; /** * @author bratseth */ -public class RankPropertiesTestCase extends SearchDefinitionTestCase { +public class RankPropertiesTestCase extends SchemaTestCase { @Test public void testRankPropertyInheritance() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index 51508414205..d84d967a184 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -17,7 +17,7 @@ import static org.junit.Assert.*; /** * @author bratseth */ -public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase { +public class RankingExpressionConstantsTestCase extends SchemaTestCase { @Test public void testConstants() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index 58e62353e5c..e0679eb5175 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -17,7 +17,7 @@ import static org.junit.Assert.assertTrue; /** * @author bratseth */ -public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase { +public class RankingExpressionInliningTestCase extends SchemaTestCase { @Test public void testFunctionInliningPreserveArithmeticOrdering() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 3d842be129f..5c1134f928c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; /** * @author lesters */ -public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase { +public class RankingExpressionShadowingTestCase extends SchemaTestCase { @Test public void testBasicFunctionShadowing() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index c5027af2a0c..c1fe5e42dfa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -15,7 +15,7 @@ import static org.junit.Assert.fail; /** * @author bratseth */ -public class RankingExpressionValidationTestCase extends SearchDefinitionTestCase { +public class RankingExpressionValidationTestCase extends SchemaTestCase { @Test public void testInvalidExpressionProducesException() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java index 5a5fc1cc312..1a939d71937 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java @@ -11,7 +11,7 @@ import static org.junit.Assert.assertNotNull; /** * @author bratseth */ -public class ReservedWordsAsFieldNamesTestCase extends SearchDefinitionTestCase { +public class ReservedWordsAsFieldNamesTestCase extends SchemaTestCase { @Test public void testIt() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java index fd4bb393c49..0ae39b7f8b6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java @@ -15,7 +15,7 @@ import static org.junit.Assert.*; * * @author hmusum */ -public class SearchDefinitionsParsingTestCase extends SearchDefinitionTestCase { +public class SchemaParsingTestCase extends SchemaTestCase { @Test public void requireThatIndexingExpressionsCanBeParsed() throws Exception { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java index ba6da8792fa..7f3ea7d14bc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java @@ -10,7 +10,7 @@ import java.io.IOException; import static helpers.CompareConfigTestHelper.assertSerializedConfigEquals; import static helpers.CompareConfigTestHelper.assertSerializedConfigFileEquals; -public abstract class SearchDefinitionTestCase { +public abstract class SchemaTestCase { protected static void assertConfigFile(String filename, String cfg) throws IOException { assertSerializedConfigFileEquals(filename, cfg); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java index 66ff1877994..018703153ac 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; -import com.yahoo.document.Document; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.RankType; @@ -31,7 +30,7 @@ import static org.junit.Assert.fail; * * @author bratseth */ -public class SearchImporterTestCase extends SearchDefinitionTestCase { +public class SearchImporterTestCase extends SchemaTestCase { @Test @SuppressWarnings("deprecation") diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java index 9b27d338ced..e5b8ec85d75 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java @@ -16,7 +16,7 @@ import static org.junit.Assert.assertNull; * * @author bratseth */ -public class StemmingSettingTestCase extends SearchDefinitionTestCase { +public class StemmingSettingTestCase extends SchemaTestCase { @Test public void testStemmingSettings() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java index 001ad64e2da..77df5b391dc 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java @@ -14,7 +14,7 @@ import static org.junit.Assert.fail; * * @author bratseth */ -public class StructTestCase extends SearchDefinitionTestCase { +public class StructTestCase extends SchemaTestCase { @Test public void testStruct() throws IOException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index d67df3a5239..a345cabe909 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -1,11 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.vespa.configmodel.producers.DocumentManager; @@ -19,22 +21,27 @@ import java.io.IOException; * * @author bratseth */ -public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase { +public abstract class AbstractExportingTestCase extends SchemaTestCase { private static final String tempDir = "temp/"; private static final String searchDefRoot = "src/test/derived/"; - private DerivedConfiguration derive(String dirName, String searchDefinitionName) throws IOException, ParseException { + private DerivedConfiguration derive(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException { File toDir = new File(tempDir + dirName); toDir.mkdirs(); deleteContent(toDir); SearchBuilder builder = SearchBuilder.createFromDirectory(searchDefRoot + dirName + "/"); - return derive(dirName, searchDefinitionName, builder); + return derive(dirName, searchDefinitionName, properties, builder); } - private DerivedConfiguration derive(String dirName, String searchDefinitionName, SearchBuilder builder) throws IOException { + private DerivedConfiguration derive(String dirName, + String searchDefinitionName, + TestProperties properties, + SearchBuilder builder) throws IOException { DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName), + new BaseDeployLogger(), + properties, builder.getRankProfileRegistry(), builder.getQueryProfileRegistry(), new ImportedMlModels()); @@ -53,6 +60,7 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase String path = exportConfig(name, config); DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), new DocumentmanagerConfig.Builder()), path); DerivedConfiguration.exportDocuments(new DocumentTypes().produce(builder.getModel(), new DocumenttypesConfig.Builder()), path); + DerivedConfiguration.exportQueryProfiles(builder.getQueryProfileRegistry(), path); return config; } @@ -76,7 +84,11 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase } protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName) throws IOException, ParseException { - DerivedConfiguration derived = derive(dirName, searchDefinitionName); + return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties()); + } + + protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException { + DerivedConfiguration derived = derive(dirName, searchDefinitionName, properties); assertCorrectConfigFiles(dirName); return derived; } @@ -87,7 +99,7 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase */ protected DerivedConfiguration assertCorrectDeriving(SearchBuilder builder, String dirName) throws IOException { builder.build(); - DerivedConfiguration derived = derive(dirName, null, builder); + DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder); assertCorrectConfigFiles(dirName); return derived; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java index 79ec3027c20..80a92a5b5ec 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -20,7 +20,7 @@ import static org.junit.Assert.assertFalse; * * @author bratseth */ -public class AttributeListTestCase extends SearchDefinitionTestCase { +public class AttributeListTestCase extends SchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java index b47a268d95a..07762fc6937 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -16,7 +16,7 @@ import static org.junit.Assert.assertEquals; * * @author vegardh */ -public class CasingTestCase extends SearchDefinitionTestCase { +public class CasingTestCase extends SchemaTestCase { @Test public void testCasing() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java index 1209da6d64b..8b09a4efd57 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.document.DataType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import org.junit.Test; import java.util.List; @@ -16,7 +16,7 @@ import static org.junit.Assert.assertEquals; * * @author bratseth */ -public class DeriverTestCase extends SearchDefinitionTestCase { +public class DeriverTestCase extends SchemaTestCase { @Test public void testDeriveDocManager() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index aaac1631722..47862a2611b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; @@ -17,7 +17,7 @@ import org.junit.Test; * * @author bratseth */ -public class EmptyRankProfileTestCase extends SearchDefinitionTestCase { +public class EmptyRankProfileTestCase extends SchemaTestCase { @Test public void testDeriving() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 61065cd4bcc..3c55aa808b5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -150,4 +151,9 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectConfigFiles("tensor2"); } + @Test + public void testHnswIndex() throws IOException, ParseException { + assertCorrectDeriving("hnsw_index"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java index c8ba5168e1c..69c247b94d4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.RankType; import org.junit.Test; @@ -14,7 +14,7 @@ import static org.junit.Assert.*; * * @author geirst */ -public class NativeRankTypeDefinitionsTestCase extends SearchDefinitionTestCase { +public class NativeRankTypeDefinitionsTestCase extends SchemaTestCase { @Test public void testTables() { assertEquals(NativeTable.Type.FIRST_OCCURRENCE.getName(), "firstOccurrenceTable"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java new file mode 100644 index 00000000000..9f57b22fd58 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java @@ -0,0 +1,40 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.component.ComponentId; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class NearestNeighborTestCase extends AbstractExportingTestCase { + + @Test + public void testNearestNeighbor() throws IOException, ParseException { + try { + ComponentId.resetGlobalCountersForTests(); + DerivedConfiguration c = assertCorrectDeriving("nearestneighbor"); + + CompiledQueryProfileRegistry queryProfiles = + QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile(); + Query q = new Query("?ranking.features.query(q_vec)=[1,2,3,4,5,6]", // length is 6, not 5 + queryProfiles.getComponent("default")); + fail("This should fail when q_vec is parsed as a tensor"); + } catch (QueryException e) { + // success + assertEquals("Invalid request parameter", e.getMessage()); + } catch (RuntimeException e) { + e.printStackTrace(); + throw e; + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java index b299c7fa299..a6171901d2d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java @@ -1,16 +1,34 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; import java.io.IOException; +import com.yahoo.component.ComponentId; + +import static org.junit.Assert.assertEquals; + public class NeuralNetTestCase extends AbstractExportingTestCase { @Test public void testNeuralNet() throws IOException, ParseException { - assertCorrectDeriving("neuralnet"); + ComponentId.resetGlobalCountersForTests(); + DerivedConfiguration c = assertCorrectDeriving("neuralnet"); + + // Verify that query profiles end up correct when passed through the same intermediate forms as a full system + CompiledQueryProfileRegistry queryProfiles = + QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile(); + Query q = new Query("?test=foo&ranking.features.query(b_1)=[1,2,3,4,5,6,7,8,9]", + queryProfiles.getComponent("default")); + assertEquals("tensor(out[9]):[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]", + q.properties().get("ranking.features.query(b_1)").toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java index b770024ebf1..0c677456a87 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.searchdefinition.DocumentReference; import com.yahoo.searchdefinition.DocumentReferences; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.TemporarySDField; @@ -26,47 +26,47 @@ import static org.junit.Assert.assertEquals; * @author bratseth * @author bjorncs */ -public class SearchOrdererTestCase extends SearchDefinitionTestCase { +public class SearchOrdererTestCase extends SchemaTestCase { - private static Map<String, Search> createSearchDefinitions() { - Map<String, Search> searchDefinitions = new HashMap<>(); + private static Map<String, Search> createSchemas() { + Map<String, Search> schemas = new HashMap<>(); - Search grandParent = createSearchDefinition("grandParent", searchDefinitions); + Search grandParent = createSchema("grandParent", schemas); - Search mother = createSearchDefinition("mother", searchDefinitions); + Search mother = createSchema("mother", schemas); inherit(mother, grandParent); - Search father = createSearchDefinition("father", searchDefinitions); + Search father = createSchema("father", schemas); inherit(father, grandParent); createDocumentReference(father, mother, "wife_ref"); - Search daugther = createSearchDefinition("daughter", searchDefinitions); + Search daugther = createSchema("daughter", schemas); inherit(daugther, father); inherit(daugther, mother); - Search son = createSearchDefinition("son", searchDefinitions); + Search son = createSchema("son", schemas); inherit(son, father); inherit(son, mother); - Search product = createSearchDefinition("product", searchDefinitions); + Search product = createSchema("product", schemas); - Search pc = createSearchDefinition("pc", searchDefinitions); + Search pc = createSchema("pc", schemas); inherit(pc, product); - Search pcAccessory = createSearchDefinition("accessory-pc", searchDefinitions); + Search pcAccessory = createSchema("accessory-pc", schemas); inherit(pcAccessory, product); createDocumentReference(pcAccessory, pc, "pc_ref"); - createSearchDefinition("alone", searchDefinitions); + createSchema("alone", schemas); - return searchDefinitions; + return schemas; } - private static Search createSearchDefinition(String name, Map<String, Search> searchDefinitions) { + private static Search createSchema(String name, Map<String, Search> schemas) { Search search = new Search(name, null); SDDocumentType document = new SDDocumentType(name); document.setDocumentReferences(new DocumentReferences(emptyMap())); search.addDocument(document); - searchDefinitions.put(search.getName(), search); + schemas.put(search.getName(), search); return search; } @@ -75,13 +75,13 @@ public class SearchOrdererTestCase extends SearchDefinitionTestCase { } private static void assertOrder(List<String> expectedSearchOrder, List<String> inputNames) { - Map<String, Search> searchDefinitions = createSearchDefinitions(); - List<Search> inputSearchDefinitions = inputNames.stream() - .map(searchDefinitions::get) + Map<String, Search> schemas = createSchemas(); + List<Search> inputSchemas = inputNames.stream() + .map(schemas::get) .map(Objects::requireNonNull) .collect(toList()); List<String> actualSearchOrder = new SearchOrderer() - .order(inputSearchDefinitions) + .order(inputSchemas) .stream() .map(Search::getName) .collect(toList()); @@ -104,31 +104,37 @@ public class SearchOrdererTestCase extends SearchDefinitionTestCase { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("grandParent", "mother", "father", "daughter", "son", "product", "pc", "alone")); } + @Test public void testOneLevelReordering() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("grandParent", "daughter", "son", "mother", "father", "pc", "product", "alone")); } + @Test public void testMultiLevelReordering() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("daughter", "son", "mother", "father", "grandParent", "pc", "product", "alone")); } + @Test public void testAloneIsKeptInPlaceWithMultiLevelReordering() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("alone", "daughter", "son", "mother", "father", "grandParent", "pc", "product")); } + @Test public void testPartialMultiLevelReordering() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("daughter", "grandParent", "mother", "son", "father", "product", "pc", "alone")); } + @Test public void testMultilevelReorderingAccrossHierarchies() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone")); } + @Test public void referees_are_ordered_before_referrer() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "accessory-pc", "son"), diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index f85f9994e04..07d7405b1db 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -25,7 +25,7 @@ import static org.junit.Assert.assertTrue; * * @author bratseth */ -public class SummaryMapTestCase extends SearchDefinitionTestCase { +public class SummaryMapTestCase extends SchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/simple.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java index b82620b1cf5..afbc9f52f6b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -20,7 +20,7 @@ import static org.junit.Assert.assertNull; * * @author bratseth */ -public class SummaryTestCase extends SearchDefinitionTestCase { +public class SummaryTestCase extends SchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index 24575df8c91..c03e915aa8b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.document.DataType; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.processing.Processing; @@ -20,7 +20,7 @@ import static org.junit.Assert.assertFalse; * * @author bratseth */ -public class TypeConversionTestCase extends SearchDefinitionTestCase { +public class TypeConversionTestCase extends SchemaTestCase { /** Tests that exact-string stuff is not spilled over to the default index */ @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java new file mode 100644 index 00000000000..e3dcc925e5e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java @@ -0,0 +1,45 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.searchdefinition.document; + +import java.util.Optional; +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class HnswIndexParamsTestCase { + + @Test + public void override_from() throws Exception { + var empty = new HnswIndexParams(); + var builder = new HnswIndexParams.Builder(); + builder.setMaxLinksPerNode(7); + var one = builder.build(); + builder.setNeighborsToExploreAtInsert(42); + var three = builder.build(); + builder.setMaxLinksPerNode(17); + builder.setNeighborsToExploreAtInsert(500); + var four = builder.build(); + + assertThat(empty.maxLinksPerNode(), is(16)); + assertThat(empty.neighborsToExploreAtInsert(), is(200)); + + assertThat(one.maxLinksPerNode(), is(7)); + assertThat(three.neighborsToExploreAtInsert(), is(42)); + + assertThat(four.maxLinksPerNode(), is(17)); + assertThat(four.neighborsToExploreAtInsert(), is(500)); + + var five = four.overrideFrom(Optional.of(empty)); + assertThat(five.maxLinksPerNode(), is(17)); + assertThat(five.neighborsToExploreAtInsert(), is(500)); + + var six = four.overrideFrom(Optional.of(one)); + assertThat(six.maxLinksPerNode(), is(7)); + assertThat(six.neighborsToExploreAtInsert(), is(500)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java index 7236ccbc117..35ce4dff730 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -17,7 +17,7 @@ import static org.junit.Assert.assertFalse; * @author vegardh * */ -public class AttributesExactMatchTestCase extends SearchDefinitionTestCase { +public class AttributesExactMatchTestCase extends SchemaTestCase { @Test public void testAttributesExactMatch() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java index ac3ba1d98d9..9a4357c5d65 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; @@ -18,7 +18,7 @@ import static org.junit.Assert.fail; /** * @author Mathias Mølster Lidal */ -public class BoldingTestCase extends SearchDefinitionTestCase { +public class BoldingTestCase extends SchemaTestCase { @Test public void testBoldingNonString() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java index 620cee49ac4..809ccdb3a3a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; @@ -18,7 +18,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -public class ImplicitSearchFieldsTestCase extends SearchDefinitionTestCase { +public class ImplicitSearchFieldsTestCase extends SchemaTestCase { @Test public void testRequireThatExtraFieldsAreIncluded() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java index f2d81414b5a..c9ea57c5b9b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.document.*; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; @@ -13,7 +13,7 @@ import org.junit.Test; import java.io.IOException; import static org.junit.Assert.*; -public class ImplicitStructTypesTestCase extends SearchDefinitionTestCase { +public class ImplicitStructTypesTestCase extends SchemaTestCase { @Test public void testRequireThatImplicitStructsAreCreated() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java index 7acbf67772a..ae00e4f3079 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.documentmodel.DocumentSummary; import org.junit.Test; @@ -13,7 +13,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -public class ImplicitSummaryFieldsTestCase extends SearchDefinitionTestCase { +public class ImplicitSummaryFieldsTestCase extends SchemaTestCase { @Test public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java index d313c2391fd..7863c544b60 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java @@ -7,7 +7,7 @@ import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; @@ -28,7 +28,7 @@ import static org.junit.Assert.assertEquals; /** * @author Simon Thoresen Hult */ -public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { +public class IndexingScriptRewriterTestCase extends SchemaTestCase { @Test public void testSetLanguageRewriting() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java index cac50354dc2..fcf1c39f5b4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -20,7 +20,7 @@ import static org.junit.Assert.assertTrue; /** * @author baldersheim */ -public class IntegerIndex2AttributeTestCase extends SearchDefinitionTestCase { +public class IntegerIndex2AttributeTestCase extends SchemaTestCase { @Test public void testIntegerIndex2Attribute() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java index 385d1df90ad..c792d3bf40b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; @@ -21,7 +21,7 @@ import static org.junit.Assert.fail; /** * @author bratseth */ -public class NGramTestCase extends SearchDefinitionTestCase { +public class NGramTestCase extends SchemaTestCase { @Test public void testNGram() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java index 0d6334a5223..4ab56f809c9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -14,7 +14,7 @@ import java.io.IOException; * @author vegardh * */ -public class RankModifierTestCase extends SearchDefinitionTestCase { +public class RankModifierTestCase extends SchemaTestCase { @Test public void testLiteral() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 08dd5148b29..0cd6674751e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -15,6 +15,7 @@ import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import java.util.HashMap; @@ -33,6 +34,7 @@ class RankProfileSearchFixture { private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(), new OnnxImporter(), + new LightGBMImporter(), new XGBoostImporter()); private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); private final QueryProfileRegistry queryProfileRegistry; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java index d740884d3e5..502fc4472bc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java @@ -7,7 +7,7 @@ import com.yahoo.searchdefinition.RankProfile.RankProperty; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -16,7 +16,7 @@ import java.util.List; import static org.junit.Assert.fail; -public class RankPropertyVariablesTestCase extends SearchDefinitionTestCase { +public class RankPropertyVariablesTestCase extends SchemaTestCase { @Test public void testRankPropVariables() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java index 9e9ef2589be..a306e0f2c90 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java @@ -275,7 +275,7 @@ public class RankingExpressionTypeResolverTestCase { builder.build(true, logger); String message = logger.findMessage("The following query features"); assertNotNull(message); - assertEquals("WARNING: The following query features are not declared in query profile types and " + + assertEquals("WARNING: The following query features used in 'my_rank_profile' are not declared in query profile types and " + "will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]", message); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java new file mode 100644 index 00000000000..79d19371f1c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java @@ -0,0 +1,88 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.After; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author lesters + */ +public class RankingExpressionWithLightGBMTestCase { + + private final Path applicationDir = Path.fromString("src/test/integration/lightgbm/"); + + private final static String lightGBMExpression = + "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)"; + + @After + public void removeGeneratedModelFiles() { + IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + } + + @Test + public void testLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + + @Test + public void testNestedLightGBMReference() { + RankProfileSearchFixture search = fixtureWith("5 + sum(lightgbm('regression.json'))"); + search.assertFirstPhaseExpression("5 + reduce(" + lightGBMExpression + ", sum)", "my_profile"); + } + + @Test + public void testImportingFromStoredExpressions() throws IOException { + RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')"); + search.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + + // At this point the expression is stored - copy application to another location which do not have a models dir + Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); + try { + storedApplicationDirectory.toFile().mkdirs(); + IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(), + storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); + RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(storedApplicationDirectory); + RankProfileSearchFixture searchFromStored = fixtureWith("lightgbm('regression.json')"); + searchFromStored.assertFirstPhaseExpression(lightGBMExpression, "my_profile"); + } + finally { + IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); + } + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) { + return fixtureWith(firstPhaseExpression, null, null, + new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(applicationDir)); + } + + private RankProfileSearchFixture fixtureWith(String firstPhaseExpression, + String constant, + String field, + RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage application) { + try { + RankProfileSearchFixture fixture = new RankProfileSearchFixture( + application, + application.getQueryProfiles(), + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: " + firstPhaseExpression + + " }\n" + + " }", + constant, + field); + fixture.compileRankProfile("my_profile", applicationDir.append("models")); + return fixture; + } catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + +} + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 96fa59a77cc..b3eda9b7e13 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -19,7 +19,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; -public class RankingExpressionsTestCase extends SearchDefinitionTestCase { +public class RankingExpressionsTestCase extends SchemaTestCase { @Test public void testFunctions() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java index dbcfc8c202d..ca8744a07bb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; @@ -16,7 +16,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTestCase { +public class SummaryFieldsMustHaveValidSourceTestCase extends SchemaTestCase { @Test public void requireThatInvalidSourceIsCaught() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java index b6569357495..b9702c6c4f7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; -import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.config.model.test.TestUtil; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; + +import static com.yahoo.searchdefinition.SearchBuilder.createFromString; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -16,7 +21,7 @@ public class TensorFieldTestCase { @Test public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type array<tensor(x{})> {}")); + createFromString(getSd("field f1 type array<tensor(x{})> {}")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -28,11 +33,12 @@ public class TensorFieldTestCase { @Test public void requireThatTensorFieldCannotBeIndexField() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); + createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("For search 'test', field 'f1': A field of type 'tensor' cannot be specified as an 'index' field.", + assertEquals("For search 'test', field 'f1': A tensor of type 'tensor(x{})' does not support having an 'index'. " + + "Currently, only tensors with 1 indexed dimension supports that.", e.getMessage()); } } @@ -40,7 +46,7 @@ public class TensorFieldTestCase { @Test public void requireThatTensorAttributeCannotBeFastSearch() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }")); + createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -51,7 +57,7 @@ public class TensorFieldTestCase { @Test public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException { try { - SearchBuilder.createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); + createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -59,8 +65,67 @@ public class TensorFieldTestCase { } } + @Test + public void hnsw_index_is_default_turned_off() throws ParseException { + var attr = createFromString(getSd("field t1 type tensor(x[64]) { indexing: attribute }")) + .getSearch().getAttribute("t1"); + assertFalse(attr.hnswIndexParams().isPresent()); + } + + @Test + public void hnsw_index_gets_default_parameters_if_not_specified() throws ParseException { + assertHnswIndexParams("", 16, 200); + assertHnswIndexParams("index: hnsw", 16, 200); + } + + @Test + public void hnsw_index_parameters_can_be_specified() throws ParseException { + assertHnswIndexParams("index { hnsw { max-links-per-node: 32 } }", 32, 200); + assertHnswIndexParams("index { hnsw { neighbors-to-explore-at-insert: 300 } }", 16, 300); + assertHnswIndexParams(joinLines("index {", + " hnsw {", + " max-links-per-node: 32", + " neighbors-to-explore-at-insert: 300", + " }", + "}"), + 32, 300); + } + + @Test + public void tensor_with_hnsw_index_must_be_an_attribute() throws ParseException { + try { + createFromString(getSd("field t1 type tensor(x[64]) { indexing: index }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 't1': A tensor that has an index must also be an attribute.", e.getMessage()); + } + } + private static String getSd(String field) { - return "search test {\n document test {\n" + field + "}\n}\n"; + return joinLines("search test {", + " document test {", + " " + field, + " }", + "}"); + } + + private void assertHnswIndexParams(String indexSpec, int maxLinksPerNode, int neighborsToExploreAtInsert) throws ParseException { + var sd = getSdWithIndexSpec(indexSpec); + System.out.println(sd); + var search = createFromString(sd).getSearch(); + var attr = search.getAttribute("t1"); + var params = attr.hnswIndexParams(); + assertTrue(params.isPresent()); + assertEquals(maxLinksPerNode, params.get().maxLinksPerNode()); + assertEquals(neighborsToExploreAtInsert, params.get().neighborsToExploreAtInsert()); + } + + private String getSdWithIndexSpec(String indexSpec) { + return getSd(joinLines("field t1 type tensor(x[64]) {", + " indexing: attribute | index", + " " + indexSpec, + "}")); } private void assertStartsWith(String prefix, String string) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index f90320ad686..8308b638497 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -13,7 +13,7 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; @@ -25,7 +25,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -public class TensorTransformTestCase extends SearchDefinitionTestCase { +public class TensorTransformTestCase extends SchemaTestCase { @Test public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java index ef6bc57223d..957b5c55889 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -12,7 +12,7 @@ import java.io.IOException; import static org.junit.Assert.assertNotNull; /** @author bratseth */ -public class WeightedSetSummaryToTestCase extends SearchDefinitionTestCase { +public class WeightedSetSummaryToTestCase extends SchemaTestCase { @Test public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java new file mode 100644 index 00000000000..9144ad411b2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.searchdefinition.SchemaTestCase; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; + +import java.io.IOException; + +/** + * Utility functions for testing generated configs for reference/imported fields. + */ +public abstract class AbstractReferenceFieldTestCase extends SchemaTestCase { + + private static String TEST_FOLDER = "src/test/configmodel/types/references/"; + + protected void assertDocumentConfigs(DocumentModel model, + String cfgFileSpec) throws IOException { + assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg"); + assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg"); + } + + protected void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException { + DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder()); + assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString()); + } + + protected void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException { + DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); + assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java new file mode 100644 index 00000000000..599ae77a456 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.config.model.test.TestUtil.joinLines; + +public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenceFieldTestCase { + + @Test + public void imported_fields_are_included_in_generated_document_configs() throws ParseException, IOException { + assertDocumentConfigs(new TestDocumentModelBuilder().addCampaign().addPerson().build(joinLines( + "search ad {", + " document ad {", + " field campaign_ref type reference<campaign> { indexing: attribute }", + " field person_ref type reference<person> { indexing: attribute }", + " }", + " import field campaign_ref.cool_field as my_cool_field {}", + " import field campaign_ref.swag_field as my_swag_field {}", + " import field person_ref.name as my_name {}", + "}")), + "multiple_imported_fields"); + } + + private static class TestDocumentModelBuilder { + private final SearchBuilder builder = new SearchBuilder(); + public TestDocumentModelBuilder addCampaign() throws ParseException { + builder.importString(joinLines("search campaign {", + " document campaign {", + " field cool_field type string { indexing: attribute }", + " field swag_field type long { indexing: attribute }", + " }", + "}")); + return this; + } + public TestDocumentModelBuilder addPerson() throws ParseException { + builder.importString(joinLines("search person {", + " document person {", + " field name type string { indexing: attribute }", + " }", + "}")); + return this; + } + public DocumentModel build(String adSdContent) throws ParseException { + builder.importString(adSdContent); + builder.build(); + return builder.getModel(); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java index e9a7a6ed33e..55980ee5fea 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java @@ -1,15 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.ReferenceDataType; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.configmodel.producers.DocumentManager; -import com.yahoo.vespa.configmodel.producers.DocumentTypes; import org.junit.Test; import java.io.IOException; @@ -20,7 +15,7 @@ import static org.junit.Assert.assertEquals; /** * @author geirst */ -public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionTestCase { +public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReferenceFieldTestCase { @Test public void reference_fields_can_reference_other_document_types() throws ParseException, IOException { @@ -60,24 +55,6 @@ public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionT assertEquals(campaignRefType.getTargetType(), campaignType); } - private static String TEST_FOLDER = "src/test/configmodel/types/references/"; - - private void assertDocumentConfigs(DocumentModel model, - String cfgFileSpec) throws IOException { - assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg"); - assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg"); - } - - private void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException { - DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder()); - assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString()); - } - - private void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException { - DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder()); - assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString()); - } - private static class TestDocumentModelBuilder { private final SearchBuilder builder = new SearchBuilder(); public TestDocumentModelBuilder addCampaign() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java index f5034d0530b..91152648b10 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; @@ -13,7 +13,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -public class DocumentModelBuilderTestCase extends SearchDefinitionTestCase { +public class DocumentModelBuilderTestCase extends SchemaTestCase { @Test public void testDocumentManagerSimple() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index 234841f2b6c..6aea0593f8a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; -import com.yahoo.component.Version; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.provision.ClusterMembership; @@ -9,17 +8,12 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import org.junit.Test; -import java.util.Arrays; import java.util.Optional; -import static com.yahoo.config.provision.ClusterSpec.Type.admin; import static com.yahoo.config.provision.ClusterSpec.Type.container; -import static com.yahoo.config.provision.ClusterSpec.Type.content; import static org.hamcrest.Matchers.endsWith; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; /** * @author gjoranv @@ -54,7 +48,7 @@ public class HostResourceTest { } private static ClusterSpec clusterSpec(ClusterSpec.Type type, String id) { - return ClusterSpec.from(type, ClusterSpec.Id.from(id), ClusterSpec.Group.from(0), Version.fromString("6.42"), false); + return ClusterSpec.specification(type, ClusterSpec.Id.from(id)).group(ClusterSpec.Group.from(0)).vespaVersion("6.42").build(); } private static HostResource hostResourceWithMemberships(ClusterMembership membership) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java index 90ec1779f39..cf1ae637cf9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; -import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.MockModelContext; import com.yahoo.config.model.NullConfigModelRegistry; @@ -23,7 +22,6 @@ import org.junit.Before; import org.junit.Test; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -105,23 +103,23 @@ public class VespaModelFactoryTest { @Override public HostSpec allocateHost(String alias) { return new HostSpec(hostName, - Collections.emptyList(), - ClusterMembership.from(ClusterSpec.from(ClusterSpec.Type.admin, - new ClusterSpec.Id(routingClusterName), - ClusterSpec.Group.from(0), - Version.fromString("6.42"), false), + List.of(), + ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), 0)); } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - return Collections.singletonList(new HostSpec(hostName, - Collections.emptyList(), - ClusterMembership.from(ClusterSpec.from(ClusterSpec.Type.container, - new ClusterSpec.Id(routingClusterName), - ClusterSpec.Group.from(0), - Version.fromString("6.42"), false), - 0))); + return prepare(cluster, capacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + return List.of(new HostSpec(hostName, + List.of(), + ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), + 0))); } }; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index cd67e432b8f..4b23a7e2e71 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -52,7 +52,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { @Before public void setup() { - sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); } @Test @@ -457,7 +457,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { private VespaModel createVespaModel(String servicesXml, boolean isHosted) throws IOException, SAXException { ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() .withServices(servicesXml) - .withSearchDefinitions(sds) + .withSchemas(sds) .build(); // Need to create VespaModel to make deploy properties have effect DeployLogger logger = new DeployLoggerStub(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java new file mode 100644 index 00000000000..12a10a7e354 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java @@ -0,0 +1,244 @@ +package com.yahoo.vespa.model.admin.metricsproxy; + +import ai.vespa.metricsproxy.core.ConsumersConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.admin.monitoring.Metric; +import com.yahoo.vespa.model.admin.monitoring.MetricSet; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly; +import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID; +import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; +import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; +import static java.util.Collections.singleton; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests for {@link MetricsProxyContainerCluster} related to metrics consumers. + * + * @author gjoranv + */ +public class MetricsConsumersTest { + + private static int numPublicDefaultMetrics = defaultPublicMetricSet.getMetrics().size(); + private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); + private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); + private static int numSystemMetrics = systemMetricSet.getMetrics().size(); + private static int numNetworkMetrics = networkMetricSet.getMetrics().size(); + private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void default_public_consumer_is_set_up_for_self_hosted() { + ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); + assertEquals(2, config.consumer().size()); + assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); + + int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics; + assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size()); + } + + @Test + public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() { + ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted); + assertEquals(2, config.consumer().size()); + assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); + assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); + } + + @Test + public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() { + ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); + assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); + assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size()); + } + + @Test + public void vespa_consumer_can_be_amended_via_admin_object() { + VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); + var additionalMetric = new Metric("additional-metric"); + model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric))); + + ConsumersConfig config = consumersConfigFromModel(model); + assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size()); + + ConsumersConfig.Consumer vespaConsumer = config.consumer(0); + assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric)); + } + + @Test + public void vespa_is_a_reserved_consumer_id() { + assertReservedConsumerId("Vespa"); + } + + @Test + public void default_is_a_reserved_consumer_id() { + assertReservedConsumerId("default"); + } + + private void assertReservedConsumerId(String consumerId) { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='" + consumerId + "'/>", + " </metrics>", + " </admin>", + "</services>" + ); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id"); + consumersConfigFromXml(services, self_hosted); + } + + @Test + public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() { + String services = String.join("\n", + "<services application-type='hosted-infrastructure'>", + " <admin version='4.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='Vespa'>", + " <metric id='custom.metric1'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + VespaModel hostedModel = getModel(services, hosted); + ConsumersConfig config = consumersConfigFromModel(hostedModel); + assertEquals(2, config.consumer().size()); + + // All default metrics are retained + ConsumersConfig.Consumer vespaConsumer = config.consumer(0); + assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size()); + + Metric customMetric1 = new Metric("custom.metric1"); + assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1)); + } + + @Test + public void consumer_id_is_case_insensitive() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='A'/>", + " <consumer id='a'/>", + " </metrics>", + " </admin>", + "</services>" + ); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("'a' is used as id for two metrics consumers"); + consumersConfigFromXml(services, self_hosted); + } + + @Test + public void non_existent_metric_set_causes_exception() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-non-existent-default-set'>", + " <metric-set id='non-existent'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("No such metric-set: non-existent"); + consumersConfigFromXml(services, self_hosted); + } + + @Test + public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-metrics-only'>", + " <metric id='custom.metric1'/>", + " <metric id='custom.metric2'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + ConsumersConfig.Consumer consumer = getCustomConsumer(services); + + assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size()); + + Metric customMetric1 = new Metric("custom.metric1"); + Metric customMetric2 = new Metric("custom.metric2"); + assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1)); + assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2)); + } + + @Test + public void consumer_with_default_metric_set_has_all_its_metrics_plus_all_system_metrics_plus_its_own() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-public-default-set'>", + " <metric-set id='default'/>", + " <metric id='custom.metric'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + ConsumersConfig.Consumer consumer = getCustomConsumer(services); + + assertEquals(numPublicDefaultMetrics + numSystemMetrics + 1, consumer.metric().size()); + + Metric customMetric = new Metric("custom.metric"); + assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); + } + + @Test + public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='consumer-with-vespa-set'>", + " <metric-set id='vespa'/>", + " <metric id='my.extra.metric'/>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + ConsumersConfig.Consumer consumer = getCustomConsumer(services); + assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size()); + + Metric customMetric = new Metric("my.extra.metric"); + assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index de40e557265..bed77bd5c77 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -5,10 +5,9 @@ package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler; import ai.vespa.metricsproxy.http.application.MetricsNodesConfig; +import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.http.yamas.YamasHandler; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; @@ -21,13 +20,9 @@ import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.util.Collection; @@ -39,26 +34,17 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.M import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_TENANT; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getApplicationDimensionsConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getMetricsNodesConfig; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getQrStartConfig; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID; -import static java.util.Collections.singleton; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly; import static java.util.stream.Collectors.toList; import static org.hamcrest.CoreMatchers.endsWith; import static org.hamcrest.CoreMatchers.hasItem; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -67,15 +53,6 @@ import static org.junit.Assert.assertTrue; */ public class MetricsProxyContainerClusterTest { - private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); - private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); - private static int numSystemMetrics = systemMetricSet.getMetrics().size(); - private static int numNetworkMetrics = networkMetricSet.getMetrics().size(); - private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics; - - @Rule - public ExpectedException thrown = ExpectedException.none(); - @Test public void metrics_proxy_bundle_is_included_in_bundles_config() { VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); @@ -104,10 +81,11 @@ public class MetricsProxyContainerClusterTest { assertEquals(512, qrStartConfig.jvm().heapsize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); assertEquals(2, qrStartConfig.jvm().availableProcessors()); - assertEquals(false, qrStartConfig.jvm().verbosegc()); + assertFalse(qrStartConfig.jvm().verbosegc()); assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts()); assertEquals(512, qrStartConfig.jvm().stacksize()); assertEquals(0, qrStartConfig.jvm().directMemorySizeCache()); + assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize()); assertEquals(75, qrStartConfig.jvm().baseMaxDirectMemorySize()); } @@ -130,161 +108,6 @@ public class MetricsProxyContainerClusterTest { } @Test - public void default_public_consumer_is_set_up_for_self_hosted() { - ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); - assertEquals(2, config.consumer().size()); - assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); - - int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics; - assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size()); - } - - @Test - public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() { - ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted); - assertEquals(2, config.consumer().size()); - assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); - assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID); - } - - @Test - public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() { - ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted); - assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID); - assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size()); - } - - @Test - public void vespa_consumer_can_be_amended_via_admin_object() { - VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); - var additionalMetric = new Metric("additional-metric"); - model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric))); - - ConsumersConfig config = consumersConfigFromModel(model); - assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size()); - - ConsumersConfig.Consumer vespaConsumer = config.consumer(0); - assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric)); - } - - @Test - public void vespa_is_a_reserved_consumer_id() { - assertReservedConsumerId("Vespa"); - } - - @Test - public void default_is_a_reserved_consumer_id() { - assertReservedConsumerId("default"); - } - - private void assertReservedConsumerId(String consumerId) { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='" + consumerId + "'/>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id"); - consumersConfigFromXml(services, self_hosted); - } - - @Test - public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() { - String services = String.join("\n", - "<services application-type='hosted-infrastructure'>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='Vespa'>", - " <metric id='custom.metric1'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - VespaModel hostedModel = getModel(services, hosted); - ConsumersConfig config = consumersConfigFromModel(hostedModel); - assertEquals(2, config.consumer().size()); - - // All default metrics are retained - ConsumersConfig.Consumer vespaConsumer = config.consumer(0); - assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1)); - } - - @Test - public void consumer_id_is_case_insensitive() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='A'/>", - " <consumer id='a'/>", - " </metrics>", - " </admin>", - "</services>" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'a' is used as id for two metrics consumers"); - consumersConfigFromXml(services, self_hosted); - } - - @Test - public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-metrics-only'>", - " <metric id='custom.metric1'/>", - " <metric id='custom.metric2'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - - assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - Metric customMetric2 = new Metric("custom.metric2"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1)); - assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2)); - } - - @Test - public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() { - String services = String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='consumer-with-vespa-set'>", - " <metric-set id='vespa'/>", - " <metric id='my.extra.metric'/>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size()); - - Metric customMetric = new Metric("my.extra.metric"); - assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); - } - - @Test public void hosted_application_propagates_application_dimensions() { VespaModel hostedModel = getModel(servicesWithAdminOnly(), hosted); ApplicationDimensionsConfig config = getApplicationDimensionsConfig(hostedModel); @@ -314,16 +137,6 @@ public class MetricsProxyContainerClusterTest { assertEquals(MetricsV1Handler.VALUES_PATH, node.metricsPath()); } - private static String servicesWithAdminOnly() { - return String.join("\n", - "<services>", - " <admin version='4.0'>", - " <adminserver hostalias='node1'/>", - " </admin>", - "</services>" - ); - } - private static String servicesWithTwoNodes() { return String.join("\n", "<services>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index 621cebd6246..eddad6fce89 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -1,6 +1,7 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; @@ -10,10 +11,10 @@ import com.yahoo.vespa.model.test.VespaModelTester; import org.junit.Test; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CONTAINER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.containerConfigId; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig; @@ -90,13 +91,28 @@ public class MetricsProxyContainerTest { String services = servicesWithContent(); VespaModel hostedModel = getModel(services, hosted); assertEquals(1, hostedModel.getHosts().size()); - String configId = CLUSTER_CONFIG_ID + "/" + hostedModel.getHosts().iterator().next().getHostname(); + String configId = containerConfigId(hostedModel, hosted); NodeDimensionsConfig config = getNodeDimensionsConfig(hostedModel, configId); assertEquals("content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_TYPE)); assertEquals("my-content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_ID)); } + @Test + public void metrics_v2_handler_is_set_up_with_node_info_config() { + String services = servicesWithContent(); + VespaModel hostedModel = getModel(services, hosted); + + var container = (MetricsProxyContainer)hostedModel.id2producer().get(containerConfigId(hostedModel, hosted)); + var handlers = container.getHandlers().getComponents(); + + assertEquals(1, handlers.size()); + var metricsV2Handler = handlers.iterator().next(); + + NodeInfoConfig config = hostedModel.getConfig(NodeInfoConfig.class, metricsV2Handler.getConfigId()); + assertTrue(config.role().startsWith("content/my-content/0/")); + assertTrue(config.hostname().startsWith("node-1-3-9-")); + } @Test public void vespa_services_config_has_all_services() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index f3140aafdaf..8ecb13d7ae5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -49,12 +49,22 @@ class MetricsProxyModelTester { return tester.createModel(servicesXml, true); } - static String configId(VespaModel model, MetricsProxyModelTester.TestMode mode) { + static String containerConfigId(VespaModel model, MetricsProxyModelTester.TestMode mode) { return (mode == hosted) ? CLUSTER_CONFIG_ID + "/" + model.getHosts().iterator().next().getHostname() : CONTAINER_CONFIG_ID; } + static String servicesWithAdminOnly() { + return String.join("\n", + "<services>", + " <admin version='4.0'>", + " <adminserver hostalias='node1'/>", + " </admin>", + "</services>" + ); + } + static boolean checkMetric(ConsumersConfig.Consumer consumer, Metric metric) { for (ConsumersConfig.Consumer.Metric m : consumer.metric()) { if (metric.name.equals(m.name()) && metric.outputName.equals(m.outputname())) @@ -77,32 +87,32 @@ class MetricsProxyModelTester { } static ConsumersConfig consumersConfigFromModel(VespaModel model) { - return new ConsumersConfig((ConsumersConfig.Builder) model.getConfig(new ConsumersConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(ConsumersConfig.class, CLUSTER_CONFIG_ID); } static MetricsNodesConfig getMetricsNodesConfig(VespaModel model) { - return new MetricsNodesConfig((MetricsNodesConfig.Builder) model.getConfig(new MetricsNodesConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(MetricsNodesConfig.class, CLUSTER_CONFIG_ID); } static ApplicationDimensionsConfig getApplicationDimensionsConfig(VespaModel model) { - return new ApplicationDimensionsConfig((ApplicationDimensionsConfig.Builder) model.getConfig(new ApplicationDimensionsConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(ApplicationDimensionsConfig.class, CLUSTER_CONFIG_ID); } static QrStartConfig getQrStartConfig(VespaModel model) { - return new QrStartConfig((QrStartConfig.Builder) model.getConfig(new QrStartConfig.Builder(), CLUSTER_CONFIG_ID)); + return model.getConfig(QrStartConfig.class, CLUSTER_CONFIG_ID); } static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model, String configId) { - return new NodeDimensionsConfig((NodeDimensionsConfig.Builder) model.getConfig(new NodeDimensionsConfig.Builder(), configId)); + return model.getConfig(NodeDimensionsConfig.class, configId); } static VespaServicesConfig getVespaServicesConfig(String servicesXml) { VespaModel model = getModel(servicesXml, self_hosted); - return new VespaServicesConfig((VespaServicesConfig.Builder) model.getConfig(new VespaServicesConfig.Builder(), CONTAINER_CONFIG_ID)); + return model.getConfig(VespaServicesConfig.class, CONTAINER_CONFIG_ID); } static RpcConnectorConfig getRpcConnectorConfig(VespaModel model) { - return new RpcConnectorConfig((RpcConnectorConfig.Builder) model.getConfig(new RpcConnectorConfig.Builder(), CONTAINER_CONFIG_ID)); + return model.getConfig(RpcConnectorConfig.class, CONTAINER_CONFIG_ID); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java new file mode 100644 index 00000000000..9a904592744 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java @@ -0,0 +1,151 @@ +package com.yahoo.vespa.model.admin.metricsproxy; + +import ai.vespa.metricsproxy.telegraf.Telegraf; +import ai.vespa.metricsproxy.telegraf.TelegrafConfig; +import ai.vespa.metricsproxy.telegraf.TelegrafRegistry; +import com.yahoo.component.ComponentId; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; + +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author gjoranv + */ +public class TelegrafTest { + + @Test + public void telegraf_components_are_set_up_when_cloudwatch_is_configured() { + String services = servicesWithCloudwatch(); + VespaModel hostedModel = getModel(services, hosted); + + var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap(); + assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(Telegraf.class.getName()))); + assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(TelegrafRegistry.class.getName()))); + } + + @Test + public void telegraf_components_are_not_set_up_when_no_external_systems_are_added_in_services() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='foo' />", + " </metrics>", + " </admin>", + "</services>"); + VespaModel hostedModel = getModel(services, hosted); + + var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap(); + assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(Telegraf.class.getName())))); + assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(TelegrafRegistry.class.getName())))); + } + + @Test + public void telegraf_config_is_generated_for_cloudwatch_in_services() { + String services = servicesWithCloudwatch(); + VespaModel hostedModel = getModel(services, hosted); + TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + assertTrue(config.isHostedVespa()); + + var cloudWatch0 = config.cloudWatch(0); + assertEquals("cloudwatch-consumer", cloudWatch0.consumer()); + assertEquals("us-east-1", cloudWatch0.region()); + assertEquals("my-namespace", cloudWatch0.namespace()); + assertEquals("my-access-key", cloudWatch0.accessKeyName()); + assertEquals("my-secret-key", cloudWatch0.secretKeyName()); + assertEquals("default", cloudWatch0.profile()); + } + + private String servicesWithCloudwatch() { + return String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='cloudwatch-consumer'>", + " <metric id='my-metric'/>", + " <cloudwatch region='us-east-1' namespace='my-namespace' >", + " <credentials access-key-name='my-access-key' ", + " secret-key-name='my-secret-key' />", + " </cloudwatch>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + } + + @Test + public void multiple_cloudwatches_are_allowed_for_the_same_consumer() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='cloudwatch-consumer'>", + " <metric id='my-metric'/>", + " <cloudwatch region='us-east-1' namespace='namespace-1' >", + " <credentials access-key-name='access-key-1' ", + " secret-key-name='secret-key-1' />", + " </cloudwatch>", + " <cloudwatch region='us-east-1' namespace='namespace-2' >", + " <shared-credentials profile='profile-2' />", + " </cloudwatch>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + VespaModel hostedModel = getModel(services, hosted); + TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + + var cloudWatch0 = config.cloudWatch(0); + assertEquals("cloudwatch-consumer", cloudWatch0.consumer()); + assertEquals("us-east-1", cloudWatch0.region()); + assertEquals("namespace-1", cloudWatch0.namespace()); + assertEquals("access-key-1", cloudWatch0.accessKeyName()); + assertEquals("secret-key-1", cloudWatch0.secretKeyName()); + assertEquals("default", cloudWatch0.profile()); + + var cloudWatch1 = config.cloudWatch(1); + assertEquals("cloudwatch-consumer", cloudWatch1.consumer()); + assertEquals("us-east-1", cloudWatch1.region()); + assertEquals("namespace-2", cloudWatch1.namespace()); + assertEquals("", cloudWatch1.accessKeyName()); + assertEquals("", cloudWatch1.secretKeyName()); + assertEquals("profile-2", cloudWatch1.profile()); + } + + @Test + public void profile_named_default_is_used_when_no_profile_is_given_in_shared_credentials() { + String services = String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='cloudwatch-consumer'>", + " <metric id='my-metric'/>", + " <cloudwatch region='us-east-1' namespace='foo' >", + " <shared-credentials file='/path/to/file' />", + " </cloudwatch>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + VespaModel model = getModel(services, self_hosted); + TelegrafConfig config = model.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + assertEquals("default", config.cloudWatch(0).profile()); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlValidatorTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlValidatorTestBase.java new file mode 100644 index 00000000000..58b65199729 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlValidatorTestBase.java @@ -0,0 +1,174 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static com.yahoo.config.provision.Environment.prod; +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author gjoranv + */ +public abstract class AccessControlValidatorTestBase { + + protected Validator validator; + protected Zone zone; + + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + private static String servicesXml(boolean addHandler, boolean protection) { + return joinLines("<services version='1.0'>", + " <container id='default' version='1.0'>", + addHandler ? httpHandlerXml : "", + " <http>", + " <filtering>", + " <access-control domain='foo' read='" + protection + "' write='" + protection + "' />", + " </filtering>", + " </http>", + " </container>", + "</services>"); + } + + private static final String httpHandlerXml = + joinLines(" <handler id='foo'>", + " <binding>http://foo/bar</binding>", + " </handler>"); + + @Test + public void cluster_with_protection_passes_validation() throws IOException, SAXException { + DeployState deployState = deployState(servicesXml(true, true)); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + validator.validate(model, deployState); + } + + @Test + public void cluster_with_no_handlers_passes_validation_without_protection() throws IOException, SAXException{ + DeployState deployState = deployState(servicesXml(false, false)); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + validator.validate(model, deployState); + } + + @Test + public void cluster_without_custom_components_passes_validation_without_protection() throws IOException, SAXException{ + String servicesXml = joinLines("<services version='1.0'>", + " <container id='default' version='1.0' />", + "</services>"); + DeployState deployState = deployState(servicesXml); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + validator.validate(model, deployState); + } + + @Test + public void cluster_with_handler_fails_validation_without_protection() throws IOException, SAXException{ + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(containsString("Access-control must be enabled")); + exceptionRule.expectMessage(containsString("production zones: [default]")); + + DeployState deployState = deployState(servicesXml(true, false)); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + validator.validate(model, deployState); + } + + @Test + public void no_http_element_has_same_effect_as_no_write_protection() throws IOException, SAXException{ + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage(containsString("Access-control must be enabled")); + exceptionRule.expectMessage(containsString("production zones: [default]")); + + String servicesXml = joinLines("<services version='1.0'>", + " <container id='default' version='1.0'>", + httpHandlerXml, + " </container>", + "</services>"); + DeployState deployState = deployState(servicesXml); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + validator.validate(model, deployState); + } + + @Test + public void cluster_with_mbus_handler_passes_validation_without_write_protection() throws IOException, SAXException{ + String servicesXml = joinLines("<services version='1.0'>", + " <container id='default' version='1.0'>", + " <handler id='foo'>", + " <binding>mbus://*/foo</binding>", + " </handler>", + " </container>", + "</services>"); + DeployState deployState = deployState(servicesXml); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + validator.validate(model, deployState); + } + + @Test + public void write_protection_is_not_required_for_non_default_application_type() throws IOException, SAXException{ + String servicesXml = joinLines("<services version='1.0' application-type='hosted-infrastructure'>", + " <container id='default' version='1.0'>", + httpHandlerXml, + " </container>", + "</services>"); + DeployState deployState = deployState(servicesXml); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + validator.validate(model, deployState); + } + + @Test + public void write_protection_is_not_required_with_validation_override() throws IOException, SAXException{ + DeployState deployState = deployState(servicesXml(true, false), + "<validation-overrides><allow until='2000-01-30'>access-control</allow></validation-overrides>", + LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant()); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + validator.validate(model, deployState); + } + + private DeployState deployState(String servicesXml) { + return deployState(servicesXml, "<validation-overrides></validation-overrides>", Instant.now()); + } + + private DeployState deployState(String servicesXml, String validationOverrides, Instant now) { + ApplicationPackage app = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .withValidationOverrides(validationOverrides) + .build(); + + DeployState.Builder builder = new DeployState.Builder() + .applicationPackage(app) + .zone(zone) + .properties(new TestProperties().setHostedVespa(true)) + .now(now); + final DeployState deployState = builder.build(); + + assertTrue("Test must emulate a hosted deployment.", deployState.isHosted()); + assertEquals("Test must emulate a prod environment.", prod, deployState.zone().environment()); + + return deployState; + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java new file mode 100644 index 00000000000..4eb69ec405a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import org.junit.Before; + +import static com.yahoo.vespa.model.application.validation.AwsAccessControlValidator.AWS_CLOUD_NAME; + +/** + * @author gjoranv + */ +public class AwsAccessControlValidatorTest extends AccessControlValidatorTestBase { + + @Before + public void setup() { + validator = new AwsAccessControlValidator(); + zone = new Zone(CloudName.from(AWS_CLOUD_NAME), SystemName.main, Environment.prod, RegionName.from("foo")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java new file mode 100644 index 00000000000..40b8223479d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java @@ -0,0 +1,101 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static com.yahoo.config.provision.Environment.prod; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + + +/** + * @author gjoranv + */ +public class CloudWatchValidatorTest { + + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void cloudwatch_in_public_zones_passes_validation() throws IOException, SAXException { + DeployState deployState = deployState(servicesWithCloudwatch(), true, true); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + new CloudWatchValidator().validate(model, deployState); + } + + @Test + public void cloudwatch_passes_validation_for_self_hosted_vespa() throws IOException, SAXException { + DeployState deployState = deployState(servicesWithCloudwatch(), false, false); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + new CloudWatchValidator().validate(model, deployState); + } + + @Test + public void cloudwatch_in_non_public_zones_fails_validation() throws IOException, SAXException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage( + "CloudWatch cannot be set up for non-public hosted Vespa and must be removed for consumers: [cloudwatch-consumer]"); + + DeployState deployState = deployState(servicesWithCloudwatch(), true, false); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + + new CloudWatchValidator().validate(model, deployState); + } + + private static DeployState deployState(String servicesXml, boolean isHosted, boolean isPublic) { + ApplicationPackage app = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build(); + + DeployState.Builder builder = new DeployState.Builder() + .applicationPackage(app) + .properties(new TestProperties().setHostedVespa(isHosted)); + if (isHosted) { + var system = isPublic ? SystemName.Public : SystemName.main; + builder.zone(new Zone(system, Environment.prod, RegionName.from("foo"))); + } + final DeployState deployState = builder.build(); + + if (isHosted) { + assertTrue("Test must emulate a hosted deployment.", deployState.isHosted()); + assertEquals("Test must emulate a prod environment.", prod, deployState.zone().environment()); + } + return deployState; + } + + private String servicesWithCloudwatch() { + return String.join("\n", + "<services>", + " <admin version='2.0'>", + " <adminserver hostalias='node1'/>", + " <metrics>", + " <consumer id='cloudwatch-consumer'>", + " <metric id='my-metric'/>", + " <cloudwatch region='us-east-1' namespace='my-namespace' >", + " <credentials access-key-name='my-access-key' ", + " secret-key-name='my-secret-key' />", + " </cloudwatch>", + " </consumer>", + " </metrics>", + " </admin>", + "</services>" + ); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java index c6d56455d44..e0366b62933 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.test.MockApplicationPackage; @@ -11,8 +10,7 @@ import org.xml.sax.SAXException; import java.io.IOException; -import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** @@ -21,48 +19,71 @@ import static org.junit.Assert.fail; public class DeploymentSpecValidatorTest { @Test - public void testDeploymentWithNonExistentGlobalId() throws IOException, SAXException { - final String simpleHosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + - "<hosts> " + - "<host name=\"localhost\">" + - "<alias>node0</alias>" + - "</host>" + - "</hosts>"; - - final String services = "<services version='1.0'>" + - " <admin version='2.0'>" + - " <adminserver hostalias='node0' />" + - " </admin>" + - " <container id='default' version='1.0'>" + - " <search/>" + - " <nodes>" + - " <node hostalias='node0'/>" + - " </nodes>" + - " </container>" + - "</services>"; - - final String deploymentSpec = "<?xml version='1.0' encoding='UTF-8'?>" + + public void testDeploymentWithNonExistentGlobalId() { + var deploymentXml = "<?xml version='1.0' encoding='UTF-8'?>" + "<deployment version='1.0'>" + " <test />" + " <prod global-service-id='non-existing'>" + " <region active='true'>us-east</region>" + " </prod>" + "</deployment>"; + assertValidationError("Attribute 'globalServiceId' in instance default: 'non-existing' specified in " + + "deployment.xml does not match any container cluster ID", deploymentXml); + } - ApplicationPackage app = new MockApplicationPackage.Builder() + @Test + public void testEndpointNonExistentContainerId() { + var deploymentXml = "<?xml version='1.0' encoding='UTF-8'?>" + + "<deployment version='1.0'>" + + " <test />" + + " <prod>" + + " <region active='true'>us-east</region>" + + " </prod>" + + " <endpoints>" + + " <endpoint container-id='non-existing'/>" + + " </endpoints>" + + "</deployment>"; + assertValidationError("Endpoint 'default' in instance default: 'non-existing' specified in " + + "deployment.xml does not match any container cluster ID", deploymentXml); + } + + private static void assertValidationError(String message, String deploymentXml) { + var simpleHosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + + "<hosts> " + + "<host name=\"localhost\">" + + "<alias>node0</alias>" + + "</host>" + + "</hosts>"; + + var services = "<services version='1.0'>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0' />" + + " </admin>" + + " <container id='default' version='1.0'>" + + " <search/>" + + " <nodes>" + + " <node hostalias='node0'/>" + + " </nodes>" + + " </container>" + + "</services>"; + + var app = new MockApplicationPackage.Builder() .withHosts(simpleHosts) .withServices(services) - .withDeploymentSpec(deploymentSpec) + .withDeploymentSpec(deploymentXml) .build(); - DeployState.Builder builder = new DeployState.Builder().applicationPackage(app); + var builder = new DeployState.Builder().applicationPackage(app); try { - final DeployState deployState = builder.build(); + var deployState = builder.build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); new DeploymentSpecValidator().validate(model, deployState); fail("Did not get expected exception"); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("specified in deployment.xml does not match any container cluster id")); + assertEquals(message, e.getMessage()); + } catch (SAXException|IOException e) { + throw new RuntimeException(e); } } } + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java index 21df39ebde8..318a0630c4d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java @@ -47,7 +47,7 @@ public class EndpointCertificateSecretsValidatorTest { VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); exceptionRule.expect(CertificateNotReadyException.class); - exceptionRule.expectMessage("TLS enabled, but could not retrieve certificate yet"); + exceptionRule.expectMessage("TLS enabled, but could not yet retrieve certificate for application default:default:default"); new EndpointCertificateSecretsValidator().validate(model, deployState); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java index f7b6d7b25e1..915b3c01e1b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java @@ -6,6 +6,7 @@ import com.yahoo.collections.Pair; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.provision.InMemoryProvisioner; @@ -32,7 +33,7 @@ import static com.yahoo.config.model.test.MockApplicationPackage.MUSIC_SEARCHDEF */ public class ValidationTester { - private final HostProvisioner hostProvisioner; + private final InMemoryProvisioner hostProvisioner; /** Creates a validation tester with 1 node available */ public ValidationTester() { @@ -45,7 +46,7 @@ public class ValidationTester { } /** Creates a validation tester with a given host provisioner */ - public ValidationTester(HostProvisioner hostProvisioner) { + public ValidationTester(InMemoryProvisioner hostProvisioner) { this.hostProvisioner = hostProvisioner; } @@ -63,9 +64,10 @@ public class ValidationTester { Environment environment, String validationOverrides) { Instant now = LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant(); + Provisioned provisioned = hostProvisioner.startProvisionedRecording(); ApplicationPackage newApp = new MockApplicationPackage.Builder() .withServices(services) - .withSearchDefinitions(ImmutableList.of(MUSIC_SEARCHDEFINITION, BOOK_SEARCHDEFINITION)) + .withSchemas(ImmutableList.of(MUSIC_SEARCHDEFINITION, BOOK_SEARCHDEFINITION)) .withValidationOverrides(validationOverrides) .build(); VespaModelCreatorWithMockPkg newModelCreator = new VespaModelCreatorWithMockPkg(newApp); @@ -77,6 +79,7 @@ public class ValidationTester { .applicationPackage(newApp) .properties(new TestProperties().setHostedVespa(true)) .modelHostProvisioner(hostProvisioner) + .provisioned(provisioned) .now(now); if (previousModel != null) deployStateBuilder.previousModel(previousModel); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java index 895aa4f6a36..87684aca174 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java @@ -27,7 +27,9 @@ public class ClusterSizeReductionValidatorTest { fail("Expected exception due to cluster size reduction"); } catch (IllegalArgumentException expected) { - assertEquals("cluster-size-reduction: Size reduction in 'default' is too large. Current size: 30, new size: 14. New size must be at least 50% of the current size. " + + assertEquals("cluster-size-reduction: Size reduction in 'default' is too large: " + + "New min size must be at least 50% of the current min size. " + + "Current size: 30, new size: 14. " + ValidationOverrides.toAllowMessage(ValidationId.clusterSizeReduction), Exceptions.toMessageString(expected)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index bb209e58e24..81b7f870a10 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -201,7 +201,7 @@ public class ConfigValueChangeValidatorTest { " </engine>\n" + " </content>\n" + "</services>", - createSearchDefinitions(docTypes) + createSchemas(docTypes) ).create(); } @@ -213,7 +213,7 @@ public class ConfigValueChangeValidatorTest { "</documents>"; } - private static List<String> createSearchDefinitions(List<String> docTypes) { + private static List<String> createSchemas(List<String> docTypes) { return docTypes.stream() .map(type -> "search " + type + " { document " + type + " { } }") .collect(Collectors.toList()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java index 43ad1bc0e8a..80127ac6854 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; -import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; import org.junit.Test; import java.time.Instant; @@ -41,7 +41,7 @@ public class IndexedSearchClusterChangeValidatorTest { public static VespaModel newOneDocModel(String sdContent) { return new ApplicationPackageBuilder(). addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")). - addSearchDefinition(new SearchDefinitionBuilder(). + addSchemas(new SchemaBuilder(). name("d1").content(sdContent).build()).buildCreator().create(); } @@ -52,9 +52,9 @@ public class IndexedSearchClusterChangeValidatorTest { public static VespaModel newTwoDocModel(String d1Content, String d2Content) { return new ApplicationPackageBuilder(). addCluster(new ContentClusterBuilder().name("foo").docTypes("d1", "d2")). - addSearchDefinition(new SearchDefinitionBuilder(). + addSchemas(new SchemaBuilder(). name("d1").content(d1Content).build()). - addSearchDefinition(new SearchDefinitionBuilder(). + addSchemas(new SchemaBuilder(). name("d2").content(d2Content).build()). buildCreator().create(); } @@ -67,9 +67,9 @@ public class IndexedSearchClusterChangeValidatorTest { return new ApplicationPackageBuilder(). addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")). addCluster(new ContentClusterBuilder().name("bar").docTypes("d2")). - addSearchDefinition(new SearchDefinitionBuilder(). + addSchemas(new SchemaBuilder(). name("d1").content(d1Content).build()). - addSearchDefinition(new SearchDefinitionBuilder(). + addSchemas(new SchemaBuilder(). name("d2").content(d2Content).build()). buildCreator().create(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java index 1322a9061ed..9a363789798 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java @@ -30,7 +30,7 @@ public class ResourcesReductionValidatorTest { fail("Expected exception due to resources reduction"); } catch (IllegalArgumentException expected) { assertEquals("resources-reduction: Resource reduction in 'default' is too large. " + - "Current memory GB: 64.00, new: 16.00. New resources must be at least 50% of the current resources. " + + "Current memory GB: 64.00, new: 16.00. New min resources must be at least 50% of the current min resources. " + ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction), Exceptions.toMessageString(expected)); } @@ -45,7 +45,7 @@ public class ResourcesReductionValidatorTest { } catch (IllegalArgumentException expected) { assertEquals("resources-reduction: Resource reduction in 'default' is too large. " + "Current vCPU: 8.00, new: 3.00. Current memory GB: 64.00, new: 16.00. Current disk GB: 800.00, new: 200.00. " + - "New resources must be at least 50% of the current resources. " + + "New min resources must be at least 50% of the current min resources. " + ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction), Exceptions.toMessageString(expected)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java index c5fee3efa99..8edbc964bfb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java @@ -8,7 +8,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.DocType; -import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; import org.junit.Test; import java.time.Instant; @@ -40,7 +40,7 @@ public class StreamingSearchClusterChangeValidatorTest { public static VespaModel createOneDocModel(String sdContent) { return new ApplicationPackageBuilder() .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1")))) - .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(sdContent).build()) + .addSchemas(new SchemaBuilder().name("d1").content(sdContent).build()) .buildCreator().create(); } @@ -51,8 +51,8 @@ public class StreamingSearchClusterChangeValidatorTest { public static VespaModel createTwoDocModel(String d1Content, String d2Content) { return new ApplicationPackageBuilder() .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1"), DocType.streaming("d2")))) - .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(d1Content).build()) - .addSearchDefinition(new SearchDefinitionBuilder().name("d2").content(d2Content).build()) + .addSchemas(new SchemaBuilder().name("d1").content(d1Content).build()) + .addSchemas(new SchemaBuilder().name("d2").content(d2Content).build()) .buildCreator().create(); } @@ -64,8 +64,8 @@ public class StreamingSearchClusterChangeValidatorTest { return new ApplicationPackageBuilder() .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1")))) .addCluster(new ContentClusterBuilder().name("bar").docTypes(Arrays.asList(DocType.streaming("d2")))) - .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(d1Content).build()) - .addSearchDefinition(new SearchDefinitionBuilder().name("d2").content(d2Content).build()) + .addSchemas(new SchemaBuilder().name("d1").content(d1Content).build()) + .addSchemas(new SchemaBuilder().name("d2").content(d2Content).build()) .buildCreator().create(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java index 3827da08679..20ff9afd530 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java @@ -6,7 +6,7 @@ import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeActi import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; -import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; import com.yahoo.vespa.model.search.DocumentDatabase; import java.util.Arrays; @@ -34,7 +34,7 @@ public abstract class ContentClusterFixture { private static ContentCluster createCluster(String sdContent) throws Exception { return new ContentClusterBuilder().build( ContentClusterUtils.createMockRoot( - Arrays.asList(new SearchDefinitionBuilder().content(sdContent).build()))); + Arrays.asList(new SchemaBuilder().content(sdContent).build()))); } protected DocumentDatabase currentDb() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 4064e53dfb7..3dfcef70aba 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -203,6 +203,7 @@ public class DocumentTypeChangeValidatorTest { headerfields, new StructDataType("bodyfields"), new FieldSets(), + Collections.emptySet(), Collections.emptySet()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java index c8f1224693b..d1a78a548d9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java @@ -1,172 +1,21 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.first; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.model.VespaModel; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.xml.sax.SAXException; - -import java.io.IOException; -import java.time.Instant; -import java.time.LocalDate; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static com.yahoo.config.provision.Environment.prod; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import com.yahoo.vespa.model.application.validation.AccessControlValidatorTestBase; +import org.junit.Before; /** * @author gjoranv */ -public class AccessControlOnFirstDeploymentValidatorTest { - - @Rule - public final ExpectedException exceptionRule = ExpectedException.none(); - - private static String servicesXml(boolean addHandler, boolean writeProtection) { - return joinLines("<services version='1.0'>", - " <container id='default' version='1.0'>", - addHandler ? httpHandlerXml : "", - " <http>", - " <filtering>", - " <access-control domain='foo' write='" + writeProtection + "' />", - " </filtering>", - " </http>", - " </container>", - "</services>"); - } - - private static final String httpHandlerXml = - joinLines(" <handler id='foo'>", - " <binding>http://foo/bar</binding>", - " </handler>"); - - @Test - public void cluster_with_write_protection_passes_validation() throws IOException, SAXException{ - DeployState deployState = deployState(servicesXml(true, true)); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new AccessControlOnFirstDeploymentValidator().validate(model, deployState); - } - - @Test - public void cluster_with_no_handlers_passes_validation_without_write_protection() throws IOException, SAXException{ - DeployState deployState = deployState(servicesXml(false, false)); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new AccessControlOnFirstDeploymentValidator().validate(model, deployState); - } - - @Test - public void cluster_without_custom_components_passes_validation_without_write_protection() throws IOException, SAXException{ - String servicesXml = joinLines("<services version='1.0'>", - " <container id='default' version='1.0' />", - "</services>"); - DeployState deployState = deployState(servicesXml); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new AccessControlOnFirstDeploymentValidator().validate(model, deployState); - } - - @Test - public void cluster_with_handler_fails_validation_without_write_protection() throws IOException, SAXException{ - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "Access-control must be enabled for write operations to container clusters in production zones: [default]"); - - DeployState deployState = deployState(servicesXml(true, false)); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new AccessControlOnFirstDeploymentValidator().validate(model, deployState); - } - - @Test - public void no_http_element_has_same_effect_as_no_write_protection() throws IOException, SAXException{ - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage( - "Access-control must be enabled for write operations to container clusters in production zones: [default]"); - - String servicesXml = joinLines("<services version='1.0'>", - " <container id='default' version='1.0'>", - httpHandlerXml, - " </container>", - "</services>"); - DeployState deployState = deployState(servicesXml); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new AccessControlOnFirstDeploymentValidator().validate(model, deployState); - } - - @Test - public void cluster_with_mbus_handler_passes_validation_without_write_protection() throws IOException, SAXException{ - String servicesXml = joinLines("<services version='1.0'>", - " <container id='default' version='1.0'>", - " <handler id='foo'>", - " <binding>mbus://*/foo</binding>", - " </handler>", - " </container>", - "</services>"); - DeployState deployState = deployState(servicesXml); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new AccessControlOnFirstDeploymentValidator().validate(model, deployState); - } - - @Test - public void write_protection_is_not_required_for_non_default_application_type() throws IOException, SAXException{ - String servicesXml = joinLines("<services version='1.0' application-type='hosted-infrastructure'>", - " <container id='default' version='1.0'>", - httpHandlerXml, - " </container>", - "</services>"); - DeployState deployState = deployState(servicesXml); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new AccessControlOnFirstDeploymentValidator().validate(model, deployState); - } - - @Test - public void write_protection_is_not_required_with_validation_override() throws IOException, SAXException{ - DeployState deployState = deployState(servicesXml(true, false), - "<validation-overrides><allow until='2000-01-30'>access-control</allow></validation-overrides>", - LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant()); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new AccessControlOnFirstDeploymentValidator().validate(model, deployState); - } - - private static DeployState deployState(String servicesXml) { - return deployState(servicesXml, "<validation-overrides></validation-overrides>", Instant.now()); - } - - private static DeployState deployState(String servicesXml, String validationOverrides, Instant now) { - ApplicationPackage app = new MockApplicationPackage.Builder() - .withServices(servicesXml) - .withValidationOverrides(validationOverrides) - .build(); - - DeployState.Builder builder = new DeployState.Builder() - .applicationPackage(app) - .zone(new Zone(Environment.prod, RegionName.from("foo")) ) - .properties(new TestProperties().setHostedVespa(true)) - .now(now); - final DeployState deployState = builder.build(); - - assertTrue("Test must emulate a hosted deployment.", deployState.isHosted()); - assertEquals("Test must emulate a prod environment.", prod, deployState.zone().environment()); +public class AccessControlOnFirstDeploymentValidatorTest extends AccessControlValidatorTestBase { - return deployState; + @Before + public void setup() { + validator = new AccessControlOnFirstDeploymentValidator(); + zone = new Zone(Environment.prod, RegionName.from("foo")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 31a2a6496d7..b4fe4175707 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -236,8 +236,8 @@ public class ContentBuilderTest extends DomBuilderTest { } @Test - public void requireThatContentStreamingHandlesMultipleSearchDefinitions() { - final String musicClusterId = "music-cluster-id"; + public void requireThatContentStreamingHandlesMultipleSchemas() { + String musicClusterId = "music-cluster-id"; ContentCluster cluster = createContentWithBooksToo( "<content version='1.0' id='" + musicClusterId + "'>" + @@ -825,8 +825,8 @@ public class ContentBuilderTest extends DomBuilderTest { VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withHosts(getHosts()) .withServices(combined) - .withSearchDefinitions(Arrays.asList(MockApplicationPackage.MUSIC_SEARCHDEFINITION, - MockApplicationPackage.BOOK_SEARCHDEFINITION)) + .withSchemas(Arrays.asList(MockApplicationPackage.MUSIC_SEARCHDEFINITION, + MockApplicationPackage.BOOK_SEARCHDEFINITION)) .build()) .create(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java index 060fff5bf66..73dd1ca3f3b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.collections.CollectionUtil; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; -import com.yahoo.vespa.model.content.DispatchTuning; import com.yahoo.vespa.model.search.Tuning; import org.junit.Test; import org.w3c.dom.Element; @@ -228,6 +227,8 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest { assertEquals(cfg.summary().log().chunk().maxbytes(), 256); assertEquals(cfg.summary().log().chunk().compression().type(), ProtonConfig.Summary.Log.Chunk.Compression.Type.LZ4); assertEquals(cfg.summary().log().chunk().compression().level(), 5); + assertEquals(cfg.summary().log().compact().compression().type(), ProtonConfig.Summary.Log.Compact.Compression.Type.LZ4); + assertEquals(cfg.summary().log().compact().compression().level(), 5); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index d1c5e344c24..33cf0635349 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -97,6 +97,7 @@ public class ContainerClusterTest { cluster.getConfig(qsB); QrStartConfig qsC= new QrStartConfig(qsB); assertEquals(expectedMemoryPercentage, qsC.jvm().heapSizeAsPercentageOfPhysicalMemory()); + assertEquals(0, qsC.jvm().compressedClassSpaceSize()); } @Test @@ -156,6 +157,7 @@ public class ContainerClusterTest { QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder); assertEquals(32, qrStartConfig.jvm().minHeapsize()); assertEquals(512, qrStartConfig.jvm().heapsize()); + assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); @@ -200,6 +202,34 @@ public class ContainerClusterTest { } @Test + public void requireThatSoftStartSecondsCanBeControlledByProperties() { + DeployState state = new DeployState.Builder().properties(new TestProperties().setSoftStartSeconds(300.0)) + .build(); + MockRoot root = new MockRoot("foo", state); + ApplicationContainerCluster cluster = createContainerCluster(root, false); + addContainer(root.deployLogger(), cluster, "c1", "host-c1"); + + ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); + cluster.getConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + assertEquals(500, threadpoolConfig.maxthreads()); + assertEquals(300.0, threadpoolConfig.softStartSeconds(), 0.0); + } + + @Test + public void requireThatDefaultThreadPoolConfigIsSane() { + MockRoot root = new MockRoot("foo"); + ApplicationContainerCluster cluster = createContainerCluster(root, false); + addContainer(root.deployLogger(), cluster, "c1", "host-c1"); + + ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); + cluster.getConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + assertEquals(500, threadpoolConfig.maxthreads()); + assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); + } + + @Test public void requireThatRoutingProviderIsDisabledForNonHosted() { DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(false)).build(); MockRoot root = new MockRoot("foo", state); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index 7cd26b2f1e6..28e23ce3222 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -3,8 +3,13 @@ package com.yahoo.vespa.model.container.xml; import com.google.common.collect.ImmutableSet; import com.yahoo.collections.CollectionUtil; +import com.yahoo.component.ComponentId; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.container.jdisc.state.StateHandler; +import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.http.AccessControl; import com.yahoo.vespa.model.container.http.Http; @@ -16,14 +21,19 @@ import org.w3c.dom.Element; import java.util.Collection; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -64,10 +74,7 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( " <http>", " <filtering>", - " <access-control domain='my-domain'>", - " <application>my-app</application>", - " <vespa-domain>custom-vespa-domain</vespa-domain>", - " </access-control>", + " <access-control domain='my-domain'/>", " </filtering>", " </http>"); @@ -76,8 +83,6 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { AccessControl accessControl = http.getAccessControl().get(); assertEquals("Wrong domain.", "my-domain", accessControl.domain); - assertEquals("Wrong application.", "my-app", accessControl.applicationId); - assertEquals("Wrong vespa-domain.", "custom-vespa-domain", accessControl.vespaDomain.get()); } @Test @@ -234,6 +239,58 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { } + + @Test + public void access_control_is_implicitly_added_for_hosted_apps() { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + nodesXml, + "</container>" ); + AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); + DeployState state = new DeployState.Builder().properties( + new TestProperties() + .setAthenzDomain(tenantDomain) + .setHostedVespa(true)) + .build(); + createModel(root, state, null, clusterElem); + Optional<AccessControl> maybeAccessControl = + ((ApplicationContainer) root.getProducer("container/container.0")).getHttp().getAccessControl(); + assertThat(maybeAccessControl.isPresent(), is(true)); + AccessControl accessControl = maybeAccessControl.get(); + assertThat(accessControl.writeEnabled, is(false)); + assertThat(accessControl.readEnabled, is(false)); + assertThat(accessControl.domain, equalTo(tenantDomain.value())); + } + + @Test + public void access_control_is_implicitly_added_for_hosted_apps_with_existing_http_element() { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + " <http>", + " <server port='" + getDefaults().vespaWebServicePort() + "' id='main' />", + " <filtering>", + " <filter id='outer' />", + " <request-chain id='myChain'>", + " <filter id='inner' />", + " </request-chain>", + " </filtering>", + " </http>", + nodesXml, + "</container>" ); + AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); + DeployState state = new DeployState.Builder().properties( + new TestProperties() + .setAthenzDomain(tenantDomain) + .setHostedVespa(true)) + .build(); + createModel(root, state, null, clusterElem); + Http http = ((ApplicationContainer) root.getProducer("container/container.0")).getHttp(); + assertThat(http.getAccessControl().isPresent(), is(true)); + assertThat(http.getFilterChains().hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID), is(true)); + assertThat(http.getFilterChains().hasChain(ComponentId.fromString("myChain")), is(true)); + } + + private String httpWithExcludedBinding(String excludedBinding) { return joinLines( " <http>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index 4ea10c9a1c1..00ab175f496 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -18,30 +18,22 @@ import static org.junit.Assert.assertEquals; /** * @author gjoranv - * @since 5.5 */ public class AccessLogTest extends ContainerModelBuilderTestBase { @Test - public void default_access_log_is_only_added_when_search_is_present() { + public void default_access_log_is_added_by_default() { Element cluster1Elem = DomBuilderTest.parse( "<container id='cluster1' version='1.0'>", - "<search />", - nodesXml, - "</container>"); - Element cluster2Elem = DomBuilderTest.parse( - "<container id='cluster2' version='1.0'>", " <nodes>", " <node hostalias='mockhost' baseport='1234' />", " </nodes>", "</container>" ); - createModel(root, cluster1Elem, cluster2Elem); + createModel(root, cluster1Elem); assertNotNull(getJsonAccessLog("cluster1")); - assertNull( getJsonAccessLog("cluster2")); assertNull(getVespaAccessLog("cluster1")); - assertNull(getVespaAccessLog("cluster2")); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 1bbc4ea2684..dcd1c46e52f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -25,6 +25,7 @@ import com.yahoo.container.QrConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.handler.VipStatusHandler; +import com.yahoo.container.handler.metrics.MetricsV2Handler; import com.yahoo.container.handler.observability.ApplicationStatusHandler; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.servlet.ServletConfigConfig; @@ -39,7 +40,6 @@ import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ApplicationContainer; -import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; @@ -47,7 +47,6 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.hamcrest.Matchers; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -66,12 +65,17 @@ import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern; import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BINDING; +import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.isEmptyString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -149,7 +153,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - @Ignore // TODO: Enable when turning the port check on public void fail_if_http_port_is_not_default_in_hosted_vespa() throws Exception { try { String servicesXml = @@ -224,26 +227,49 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertThat(defaultRootHandler.serverBindings(), contains("http://*/")); JdiscBindingsConfig.Handlers applicationStatusHandler = config.handlers(ApplicationStatusHandler.class.getName()); - assertThat(applicationStatusHandler.serverBindings(), - contains("http://*/ApplicationStatus")); + assertThat(applicationStatusHandler.serverBindings(), contains("http://*/ApplicationStatus")); JdiscBindingsConfig.Handlers fileRequestHandler = config.handlers(VipStatusHandler.class.getName()); - assertThat(fileRequestHandler.serverBindings(), - contains("http://*/status.html")); + assertThat(fileRequestHandler.serverBindings(), contains("http://*/status.html")); + + JdiscBindingsConfig.Handlers metricsV2Handler = config.handlers(MetricsV2Handler.class.getName()); + assertThat(metricsV2Handler.serverBindings(), contains("http://*/metrics/v2", "http://*/metrics/v2/*")); } @Test - public void default_root_handler_is_disabled_when_user_adds_a_handler_with_same_binding() { + public void default_root_handler_binding_can_be_stolen_by_user_configured_handler() { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>" + " <handler id='userRootHandler'>" + - " <binding>" + ContainerCluster.ROOT_HANDLER_BINDING + "</binding>" + + " <binding>" + ROOT_HANDLER_BINDING + "</binding>" + " </handler>" + "</container>"); createModel(root, clusterElem); + // The handler is still set up. ComponentsConfig.Components userRootHandler = getComponent(componentsConfig(), BindingsOverviewHandler.class.getName()); - assertThat(userRootHandler, nullValue()); + assertThat(userRootHandler, notNullValue()); + + // .. but it has no bindings + var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); + assertThat(discBindingsConfig.handlers(BindingsOverviewHandler.class.getName()), is(nullValue())); + } + + @Test + public void reserved_binding_cannot_be_stolen_by_user_configured_handler() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>" + + " <handler id='userHandler'>" + + " <binding>" + STATE_HANDLER_BINDING_1 + "</binding>" + + " </handler>" + + "</container>"); + try { + createModel(root, clusterElem); + fail("Expected exception when stealing a reserved binding."); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Binding 'http://*/state/v1' is a reserved Vespa binding " + + "and cannot be used by handler: userHandler")); + } } @Test @@ -598,7 +624,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .setMultitenant(true) .setHostedVespa(true)) .build()); - assertEquals(1, model.hostSystem().getHosts().size()); + assertEquals(2, model.hostSystem().getHosts().size()); } @Test(expected = IllegalArgumentException.class) @@ -783,7 +809,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0"); // Verify that there are two connectors - List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().getConnectorFactories(); + List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories(); assertEquals(2, connectorFactories.size()); List<Integer> ports = connectorFactories.stream() .map(ConnectorFactory::getListenPort) @@ -801,6 +827,10 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertEquals("CERT", connectorConfig.ssl().certificate()); assertEquals("KEY", connectorConfig.ssl().privateKey()); assertEquals(4443, connectorConfig.listenPort()); + + assertThat("Connector must use Athenz truststore in a non-public system.", + connectorConfig.ssl().caCertificateFile(), equalTo("/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem")); + assertThat(connectorConfig.ssl().caCertificate(), isEmptyString()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index 9ecd33f4273..eda90b03147 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -28,7 +28,11 @@ import java.util.Map; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.core.IsNull.notNullValue; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThat; + /** * @author einarmr @@ -204,24 +208,25 @@ public class DocprocBuilderTest extends DomBuilderTest { @Test public void testBundlesConfig() { - assertThat(bundlesConfig.bundle().size(), is(0)); + assertTrue(bundlesConfig.bundle().isEmpty()); } @Test public void testSchemaMappingConfig() { - assertThat(schemamappingConfig.fieldmapping().size(), is(0)); + assertTrue(schemamappingConfig.fieldmapping().isEmpty()); } @Test public void testQrStartConfig() { QrStartConfig.Jvm jvm = qrStartConfig.jvm(); - assertThat(jvm.server(), is(true)); - assertThat(jvm.verbosegc(), is(true)); - assertThat(jvm.gcopts(), is("-XX:+UseG1GC -XX:MaxTenuringThreshold=15")); - assertThat(jvm.minHeapsize(), is(1536)); - assertThat(jvm.heapsize(), is(1536)); - assertThat(jvm.stacksize(), is(512)); - assertThat(qrStartConfig.ulimitv(), is("")); + assertTrue(jvm.server()); + assertTrue(jvm.verbosegc()); + assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", jvm.gcopts()); + assertEquals(1536, jvm.minHeapsize()); + assertEquals(1536, jvm.heapsize()); + assertEquals(512, jvm.stacksize()); + assertTrue(qrStartConfig.ulimitv().isEmpty()); + assertEquals(0, jvm.compressedClassSpaceSize()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 68f507c810d..0f9bd506310 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -152,6 +152,14 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas " <client-authentication>need</client-authentication>", " </ssl>", " </server>", + " <server port='9003' id='with-ciphers-and-protocols'>", + " <ssl>", + " <private-key-file>/foo/key</private-key-file>", + " <certificate-file>/foo/cert</certificate-file>", + " <cipher-suites>TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384</cipher-suites>", + " <protocols>TLSv1.3</protocols>", + " </ssl>", + " </server>", " </http>", nodesXml, "", @@ -179,6 +187,13 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas assertThat(needClientAuth.ssl().caCertificateFile(), is(equalTo(""))); assertThat(needClientAuth.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH))); + ConnectorConfig withCiphersAndProtocols = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/with-ciphers-and-protocols/configured-ssl-provider@with-ciphers-and-protocols"); + assertTrue(withCiphersAndProtocols.ssl().enabled()); + assertThat(withCiphersAndProtocols.ssl().privateKeyFile(), is(equalTo("/foo/key"))); + assertThat(withCiphersAndProtocols.ssl().certificateFile(), is(equalTo("/foo/cert"))); + assertThat(withCiphersAndProtocols.ssl().enabledCipherSuites(), is(equalTo(List.of("TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384")))); + assertThat(withCiphersAndProtocols.ssl().enabledProtocols(), is(equalTo(List.of("TLSv1.3")))); + ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class); connectorFactories.forEach(connectorFactory -> assertChildComponentExists(connectorFactory, ConfiguredFilebasedSslProvider.COMPONENT_CLASS)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index 1f0b0188681..47f37a75fd0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -16,8 +16,11 @@ import org.w3c.dom.Element; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; import static com.yahoo.test.Matchers.hasItemWithMethod; +import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_BINDING; +import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.*; @@ -54,8 +57,6 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { if (guiHandler == null) fail(); } - - @Test public void search_handler_bindings_can_be_overridden() { Element clusterElem = DomBuilderTest.parse( @@ -91,6 +92,25 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { assertThat(discBindingsConfig, not(containsString("/search/*"))); } + @Test + public void search_handler_binding_can_be_stolen_by_user_configured_handler() { + var myHandler = "replaces_search_handler"; + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <search />", + " <handler id='" + myHandler + "'>", + " <binding>" + SEARCH_HANDLER_BINDING + "</binding>", + " </handler>", + nodesXml, + "</container>"); + + createModel(root, clusterElem); + + var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); + assertThat(discBindingsConfig.handlers(myHandler).serverBindings(0), is(SEARCH_HANDLER_BINDING)); + assertThat(discBindingsConfig.handlers(SEARCH_HANDLER_CLASS), is(nullValue())); + } + // TODO: remove test when all containers are named 'container' @Test public void cluster_with_only_search_gets_qrserver_as_service_name() { @@ -190,7 +210,7 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { private VespaModel getVespaModelWithMusic(String hosts, String services) { - return new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create(); + return new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSchemas("music")).create(); } private String hostsXml() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 305bcc1d7d5..4d5df7c1965 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -26,7 +26,7 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; -import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; import com.yahoo.vespa.model.routing.DocumentProtocol; import com.yahoo.vespa.model.routing.Routing; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; @@ -218,7 +218,7 @@ public class ContentClusterTest extends ContentBaseTest { "\n" + "</services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); VespaModel model = (new VespaModelCreatorWithMockPkg(null, xml, sds)).create(); assertEquals(2, model.getContentClusters().get("bar").getDocumentDefinitions().size()); ContainerCluster cluster = model.getAdmin().getClusterControllers(); @@ -259,7 +259,7 @@ public class ContentClusterTest extends ContentBaseTest { " </services>"; DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(properties); - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1"); return (new VespaModelCreatorWithMockPkg(null, services, sds)).create(deployStateBuilder); } @Test @@ -301,7 +301,7 @@ public class ContentClusterTest extends ContentBaseTest { "\n" + "</services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); assertTrue(model.getContentClusters().get("bar").getPersistence() instanceof ProtonEngine.Factory); @@ -340,7 +340,7 @@ public class ContentClusterTest extends ContentBaseTest { " </content>\n" + "</services>\n"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); try{ new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); assertTrue("Deploying without redundancy should fail", false); @@ -697,7 +697,7 @@ public class ContentClusterTest extends ContentBaseTest { "</services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); { @@ -818,7 +818,7 @@ public class ContentClusterTest extends ContentBaseTest { " </group>" + "</content>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("true"); + List<String> sds = ApplicationPackageUtils.generateSchemas("true"); new VespaModelCreatorWithMockPkg(null, xml, sds).create(); } @@ -865,7 +865,7 @@ public class ContentClusterTest extends ContentBaseTest { " </group>" + "</content>" + "</services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("bunnies", "hares", "rabbits"); + List<String> sds = ApplicationPackageUtils.generateSchemas("bunnies", "hares", "rabbits"); return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); } @@ -912,8 +912,8 @@ public class ContentClusterTest extends ContentBaseTest { DeployState.Builder deployStateBuilder = new DeployState.Builder() .zone(zone) .properties(new TestProperties().setHostedVespa(true)); - List<String> searchDefinitions = SearchDefinitionBuilder.createSearchDefinitions("test"); - MockRoot root = ContentClusterUtils.createMockRoot(searchDefinitions, deployStateBuilder); + List<String> schemas = SchemaBuilder.createSchemas("test"); + MockRoot root = ContentClusterUtils.createMockRoot(schemas, deployStateBuilder); ContentCluster cluster = ContentClusterUtils.createCluster(clusterXml, root); root.freezeModelTopology(); cluster.validate(); @@ -933,6 +933,17 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(distributionBits, storDistributormanagerConfig.minsplitcount()); } + private void verifyTopKProbabilityPropertiesControl(double topKProbability) { + VespaModel model = createEnd2EndOneNode(new TestProperties().setTopKProbability(topKProbability)); + + ContentCluster cc = model.getContentClusters().get("storage"); + DispatchConfig.Builder builder = new DispatchConfig.Builder(); + cc.getSearch().getConfig(builder); + + DispatchConfig cfg = new DispatchConfig(builder); + assertEquals(topKProbability, cfg.topKProbability(), 0.0); + } + private void verifyRoundRobinPropertiesControl(boolean useAdaptiveDispatch) { VespaModel model = createEnd2EndOneNode(new TestProperties().setUseAdaptiveDispatch(useAdaptiveDispatch)); @@ -946,7 +957,6 @@ public class ContentClusterTest extends ContentBaseTest { } else { assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, cfg.distributionPolicy()); } - } @Test @@ -955,5 +965,12 @@ public class ContentClusterTest extends ContentBaseTest { verifyRoundRobinPropertiesControl(true); } + @Test + public void default_topKprobability_controlled_by_properties() { + verifyTopKProbabilityPropertiesControl(1.0); + verifyTopKProbabilityPropertiesControl(0.999); + verifyTopKProbabilityPropertiesControl(0.77); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 98fa179b219..3415044b088 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.DocType; -import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; import org.junit.Test; import java.util.ArrayList; @@ -17,7 +17,7 @@ import java.util.List; import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; -import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions; +import static com.yahoo.vespa.model.content.utils.SchemaBuilder.createSchemas; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -38,7 +38,7 @@ public class ContentSearchClusterTest { private static ContentCluster createClusterWithTwoDocumentType() throws Exception { return createCluster(new ContentClusterBuilder().docTypes("foo", "bar").getXml(), - createSearchDefinitions("foo", "bar")); + createSchemas("foo", "bar")); } private static ContentCluster createClusterWithGlobalType() throws Exception { @@ -55,7 +55,7 @@ public class ContentSearchClusterTest { "<node distribution-key='1' hostalias='mockhost'/>", "</group>")); String clusterXml = builder.getXml(); - return createCluster(clusterXml, createSearchDefinitions(docTypes)); + return createCluster(clusterXml, createSchemas(docTypes)); } private static ContentClusterBuilder createClusterBuilderWithGlobalType() { @@ -127,18 +127,19 @@ public class ContentSearchClusterTest { } private static ContentCluster createClusterWithThreeDocumentTypes() throws Exception { - List<String> searchDefinitions = new ArrayList<>(); - searchDefinitions.add(new SearchDefinitionBuilder().name("a") - .content(joinLines("field ref_to_b type reference<b> { indexing: attribute }", - "field ref_to_c type reference<c> { indexing: attribute }")).build()); - searchDefinitions.add(new SearchDefinitionBuilder().name("b") - .content("field ref_to_c type reference<c> { indexing: attribute }").build()); - searchDefinitions.add(new SearchDefinitionBuilder().name("c").build()); - return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList( - DocType.index("a"), - DocType.indexGlobal("b"), - DocType.indexGlobal("c"))).getXml(), - searchDefinitions); + List<String> schemas = new ArrayList<>(); + schemas.add(new SchemaBuilder().name("a") + .content(joinLines("field ref_to_b type reference<b> { indexing: attribute }", + "field ref_to_c type reference<c> { indexing: attribute }")) + .build()); + schemas.add(new SchemaBuilder().name("b") + .content("field ref_to_c type reference<c> { indexing: attribute }") + .build()); + schemas.add(new SchemaBuilder().name("c").build()); + return createCluster(new ContentClusterBuilder().docTypes(List.of(DocType.index("a"), + DocType.indexGlobal("b"), + DocType.indexGlobal("c"))).getXml(), + schemas); } private static BucketspacesConfig getBucketspacesConfig(ContentCluster cluster) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java index f708d7673e2..8a46aaaa230 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java @@ -19,11 +19,13 @@ public class DispatchTuningTest { .setDispatchPolicy("round-robin") .setMinGroupCoverage(7.5) .setMinActiveDocsCoverage(12.5) + .setTopKProbability(18.3) .build(); assertEquals(69, dispatch.getMaxHitsPerPartition().intValue()); assertEquals(7.5, dispatch.getMinGroupCoverage().doubleValue(), 0.0); assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0); assertTrue(DispatchTuning.DispatchPolicy.ROUNDROBIN == dispatch.getDispatchPolicy()); + assertEquals(18.3, dispatch.getTopkProbability(), 0.0); } @Test public void requireThatRandomDispatchWork() { @@ -52,6 +54,7 @@ public class DispatchTuningTest { assertNull(dispatch.getDispatchPolicy()); assertNull(dispatch.getMinActiveDocsCoverage()); assertNull(dispatch.getMinGroupCoverage()); + assertNull(dispatch.getTopkProbability()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index f36ef6c3ba3..365dc74274d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -24,7 +24,7 @@ public class DistributorTest { ContentCluster parseCluster(String xml) { try { - List<String> searchDefs = ApplicationPackageUtils.generateSearchDefinitions("music", "movies", "bunnies"); + List<String> searchDefs = ApplicationPackageUtils.generateSchemas("music", "movies", "bunnies"); MockRoot root = ContentClusterUtils.createMockRoot(searchDefs); return ContentClusterUtils.createCluster(xml, root); } catch (Exception e) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java index 992edf6b1bb..51d0afc1f93 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java @@ -50,7 +50,7 @@ public class GenericConfigTest { @Before public void getVespaModel() { - model = (new VespaModelCreatorWithMockPkg(ContentBaseTest.getHosts(), servicesXml(), ApplicationPackageUtils.generateSearchDefinitions("type1"))).create(); + model = (new VespaModelCreatorWithMockPkg(ContentBaseTest.getHosts(), servicesXml(), ApplicationPackageUtils.generateSchemas("type1"))).create(); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index ecf8f100288..504c3d9ba9c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -112,12 +112,12 @@ public class IndexedTest extends ContentBaseTest { } private VespaModelCreatorWithMockPkg getIndexedVespaModelCreator() { - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2", "type3"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2", "type3"); return new VespaModelCreatorWithMockPkg(getHosts(), createProtonIndexedVespaServices(Arrays.asList("type1", "type2", "type3")), sds); } private VespaModel getStreamingVespaModel() { - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1"); return new VespaModelCreatorWithMockPkg(getHosts(), createProtonStreamingVespaServices(Arrays.asList("type1")), sds).create(); } @@ -229,7 +229,7 @@ public class IndexedTest extends ContentBaseTest { " </content>\n" + " </services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("docstorebench"); + List<String> sds = ApplicationPackageUtils.generateSchemas("docstorebench"); VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create(); ProtonConfig.Builder pb = new ProtonConfig.Builder(); model.getConfig(pb, "docstore/search/cluster.docstore/0"); @@ -252,7 +252,7 @@ public class IndexedTest extends ContentBaseTest { " </content>" + "</services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("index_me", "store_me"); + List<String> sds = ApplicationPackageUtils.generateSchemas("index_me", "store_me"); VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create(); ProtonConfig.Builder pb = new ProtonConfig.Builder(); model.getConfig(pb, "docstore/search/cluster.docstore/0"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index 55d070d7247..177b86c953e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -164,7 +164,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { " </container>\n" + "</services>\n"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("music", "title", "artist"); + List<String> sds = ApplicationPackageUtils.generateSchemas("music", "title", "artist"); VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create(); assertIndexing(model, new DocprocClusterSpec("dokprok")); @@ -448,7 +448,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { } private VespaModel getIndexedSearchVespaModel(String xml) { - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("music", "album", "artist"); + List<String> sds = ApplicationPackageUtils.generateSchemas("music", "album", "artist"); return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 21c384dfc69..883d8b89765 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -112,11 +112,7 @@ public class StorageClusterTest { "<cluster id=\"bees\">\n" + " <documents/>" + " <tuning>\n" + - " <persistence-threads>\n" + - " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" + - " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" + - " <thread count=\"1\"/>\n" + - " </persistence-threads>\n" + + " <persistence-threads count=\"7\"/>\n" + " </tuning>\n" + " <group>" + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + @@ -130,6 +126,44 @@ public class StorageClusterTest { stc.getConfig(builder); StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(7, config.num_threads()); + assertEquals(false, config.enable_multibit_split_optimalization()); + } + { + assertEquals(1, stc.getChildren().size()); + StorageNode sn = stc.getChildren().values().iterator().next(); + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + sn.getConfig(builder); + StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(7, config.num_threads()); + } + } + + @Test + public void testPersistenceThreadsOld() throws Exception { + + StorageCluster stc = parse( + "<cluster id=\"bees\">\n" + + " <documents/>" + + " <tuning>\n" + + " <persistence-threads>\n" + + " <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" + + " <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" + + " <thread count=\"1\"/>\n" + + " </persistence-threads>\n" + + " </tuning>\n" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</cluster>", + new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) + ); + + { + StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); + stc.getConfig(builder); + StorFilestorConfig config = new StorFilestorConfig(builder); + assertEquals(4, config.num_threads()); assertEquals(false, config.enable_multibit_split_optimalization()); } @@ -161,7 +195,7 @@ public class StorageClusterTest { StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); stc.getConfig(builder); StorFilestorConfig config = new StorFilestorConfig(builder); - assertEquals(6, config.num_threads()); + assertEquals(8, config.num_threads()); } { assertEquals(1, stc.getChildren().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java index c0ddd49069d..e099476ebb6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java @@ -49,7 +49,7 @@ public class StorageContentTest extends ContentBaseTest { } private VespaModel getStorageVespaModel(String cluster1docs, String cluster2docs) { - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2", "type3"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2", "type3"); return new VespaModelCreatorWithMockPkg(getHosts(), createStorageVespaServices(cluster1docs, cluster2docs), sds).create(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 62221c206fd..852844fe451 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -176,7 +176,7 @@ public class ClusterTest { " </tuning>", " </content>", "</services>")) - .withSearchDefinitions(ApplicationPackageUtils.generateSearchDefinition("my_document")) + .withSchemas(ApplicationPackageUtils.generateSearchDefinition("my_document")) .build(); List<Content> contents = new TestDriver().buildModel(app).getConfigModels(Content.class); assertEquals(1, contents.size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java index 7fa27f74d74..abfb03e41dd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java @@ -47,6 +47,7 @@ public class DomDispatchTuningBuilderTest { assertNull(dispatch.getMinGroupCoverage()); assertNull(dispatch.getMinActiveDocsCoverage()); assertNull(dispatch.getDispatchPolicy()); + assertNull(dispatch.getTopkProbability()); } @Test @@ -58,12 +59,14 @@ public class DomDispatchTuningBuilderTest { " <max-hits-per-partition>69</max-hits-per-partition>" + " <min-group-coverage>7.5</min-group-coverage>" + " <min-active-docs-coverage>12.5</min-active-docs-coverage>" + + " <top-k-probability>0.999</top-k-probability>" + " </dispatch>" + " </tuning>" + "</content>"); assertEquals(69, dispatch.getMaxHitsPerPartition().intValue()); assertEquals(7.5, dispatch.getMinGroupCoverage().doubleValue(), 0.0); assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0); + assertEquals(0.999, dispatch.getTopkProbability().doubleValue(), 0.0); } @Test public void requireThatTuningDispatchPolicyRoundRobin() throws Exception { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java index 1b7f3e9b14d..5fc213fad1d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java @@ -14,7 +14,7 @@ import java.util.List; public class ApplicationPackageBuilder { private List<ContentClusterBuilder> contentClusters = new ArrayList<>(); - private List<String> searchDefinitions = new ArrayList<>(); + private List<String> schemas = new ArrayList<>(); public ApplicationPackageBuilder() { } @@ -24,13 +24,13 @@ public class ApplicationPackageBuilder { return this; } - public ApplicationPackageBuilder addSearchDefinition(String searchDefinition) { - searchDefinitions.add(searchDefinition); + public ApplicationPackageBuilder addSchemas(String schemas) { + this.schemas.add(schemas); return this; } public VespaModelCreatorWithMockPkg buildCreator() { - return new VespaModelCreatorWithMockPkg(null, getServices(), searchDefinitions); + return new VespaModelCreatorWithMockPkg(null, getServices(), schemas); } private String getServices() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index 62d2bc51830..b1550d90f35 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -30,46 +30,46 @@ import java.util.Optional; public class ContentClusterUtils { public static MockRoot createMockRoot(String[] hosts) { - return createMockRoot(hosts, SearchDefinitionBuilder.createSearchDefinitions("test")); + return createMockRoot(hosts, SchemaBuilder.createSchemas("test")); } - private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions) { - return createMockRoot(provisioner, searchDefinitions, new DeployState.Builder()); + private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> schemas) { + return createMockRoot(provisioner, schemas, new DeployState.Builder()); } - private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions, DeployState.Builder deployStateBuilder) { - ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSearchDefinitions(searchDefinitions).build(); + private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> schemas, DeployState.Builder deployStateBuilder) { + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSchemas(schemas).build(); DeployState deployState = deployStateBuilder.applicationPackage(applicationPackage) .modelHostProvisioner(provisioner) .build(); return new MockRoot("", deployState); } - public static MockRoot createMockRoot(String[] hosts, List<String> searchDefinitions) { - return createMockRoot(new InMemoryProvisioner(true, hosts), searchDefinitions); + public static MockRoot createMockRoot(String[] hosts, List<String> schemas) { + return createMockRoot(new InMemoryProvisioner(true, hosts), schemas); } - public static MockRoot createMockRoot(List<String> searchDefinitions) { - return createMockRoot(new SingleNodeProvisioner(), searchDefinitions); + public static MockRoot createMockRoot(List<String> schemas) { + return createMockRoot(new SingleNodeProvisioner(), schemas); } - public static MockRoot createMockRoot(List<String> searchDefinitions, DeployState.Builder deployStateBuilder) { - return createMockRoot(new SingleNodeProvisioner(), searchDefinitions, deployStateBuilder); + public static MockRoot createMockRoot(List<String> schemas, DeployState.Builder deployStateBuilder) { + return createMockRoot(new SingleNodeProvisioner(), schemas, deployStateBuilder); } public static ContentCluster createCluster(String clusterXml, MockRoot root) { Document doc = XML.getDocument(clusterXml); Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), false, - new FileDistributionConfigProducer(root, new MockFileRegistry(), null), + new FileDistributionConfigProducer(root, new MockFileRegistry(), List.of(), false), root.getDeployState().isHosted()); ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(), - null,null, root, null); + null,null, root, null); return new ContentCluster.Builder(admin).build(Collections.emptyList(), context, doc.getDocumentElement()); } - public static ContentCluster createCluster(String clusterXml, List<String> searchDefinitions) throws Exception { - MockRoot root = createMockRoot(searchDefinitions); + public static ContentCluster createCluster(String clusterXml, List<String> schemas) throws Exception { + MockRoot root = createMockRoot(schemas); ContentCluster cluster = createCluster(clusterXml, root); root.freezeModelTopology(); cluster.validate(); @@ -77,7 +77,7 @@ public class ContentClusterUtils { } public static ContentCluster createCluster(String clusterXml) throws Exception { - return createCluster(clusterXml, SearchDefinitionBuilder.createSearchDefinitions("test")); + return createCluster(clusterXml, SchemaBuilder.createSchemas("test")); } public static String createClusterXml(String groupXml, int redundancy, int searchableCopies) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java index c622bcfaec5..c18dac17064 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java @@ -12,20 +12,20 @@ import static com.yahoo.config.model.test.TestUtil.joinLines; * * @author geirst */ -public class SearchDefinitionBuilder { +public class SchemaBuilder { private String name = "test"; private String content = ""; - public SearchDefinitionBuilder() { + public SchemaBuilder() { } - public SearchDefinitionBuilder name(String name) { + public SchemaBuilder name(String name) { this.name = name; return this; } - public SearchDefinitionBuilder content(String content) { + public SchemaBuilder content(String content) { this.content = content; return this; } @@ -38,10 +38,10 @@ public class SearchDefinitionBuilder { "}"); } - public static List<String> createSearchDefinitions(String ... docTypes) { + public static List<String> createSchemas(String ... docTypes) { return Arrays.asList(docTypes) .stream() - .map(type -> new SearchDefinitionBuilder().name(type).build()) + .map(type -> new SchemaBuilder().name(type).build()) .collect(Collectors.toList()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java index 131a5344116..55672c5aa16 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java @@ -10,31 +10,34 @@ import org.junit.Test; import java.io.File; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; /** * @author bratseth */ public class FileDistributorTestCase { + @Test public void fileDistributor() { MockHosts hosts = new MockHosts(); - FileDistributor fileDistributor = new FileDistributor(new MockFileRegistry(), null); + FileDistributor fileDistributor = new FileDistributor(new MockFileRegistry(), List.of(), false); String file1 = "component/path1"; String file2 = "component/path2"; - FileReference ref1 = fileDistributor.sendFileToHosts(file1, Arrays.asList(hosts.host1, hosts.host2)); - FileReference ref2 = fileDistributor.sendFileToHosts(file2, Arrays.asList(hosts.host3)); + FileReference ref1 = fileDistributor.sendFileToHost(file1, hosts.host1); + fileDistributor.sendFileToHost(file1, hosts.host2); // same file reference as above + FileReference ref2 = fileDistributor.sendFileToHost(file2, hosts.host3); assertEquals(new HashSet<>(Arrays.asList(hosts.host1, hosts.host2, hosts.host3)), fileDistributor.getTargetHosts()); - assertTrue( ref1 != null ); - assertTrue( ref2 != null ); + assertNotNull(ref1); + assertNotNull(ref2); MockFileDistribution dbHandler = new MockFileDistribution(); fileDistributor.sendDeployedFiles(dbHandler); @@ -54,4 +57,5 @@ public class FileDistributorTestCase { return null; } } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java index ce36ecc4a1c..7a3b76db7f8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; -import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import com.google.common.collect.ImmutableList; import com.yahoo.config.model.ApplicationPackageTester; import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; @@ -10,8 +9,10 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankingConstant; +import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; +import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; @@ -35,6 +36,7 @@ public class ImportedModelTester { private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(), new OnnxImporter(), + new LightGBMImporter(), new XGBoostImporter(), new VespaImporter()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java index c5c475360a3..ced7243adf5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java @@ -45,7 +45,7 @@ public class MlModelsTest { private void verify(VespaModel model) { assertEquals("Global models are created (although not used directly here", - 4, model.rankProfileList().getRankProfiles().size()); + 5, model.rankProfileList().getRankProfiles().size()); RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder(); model.getSearchClusters().get(0).getConfig(builder); @@ -71,8 +71,9 @@ public class MlModelsTest { "rankingExpression(mnist_softmax_tensorflow).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))\n" + "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" + "rankingExpression(my_xgboost).rankingScript: if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)\n" + + "rankingExpression(my_lightgbm).rankingScript: if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)\n" + "vespa.rank.firstphase: rankingExpression(firstphase)\n" + - "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost)\n" + + "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost) + rankingExpression(my_lightgbm)\n" + "vespa.type.attribute.argument: tensor<float>(d0[],d1[784])\n"; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index 3d4ac7f2eeb..2d3ddc33afb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -96,9 +96,10 @@ public class ModelEvaluationTest { cluster.getConfig(cb); RankingConstantsConfig constantsConfig = new RankingConstantsConfig(cb); - assertEquals(4, config.rankprofile().size()); + assertEquals(5, config.rankprofile().size()); Set<String> modelNames = config.rankprofile().stream().map(v -> v.name()).collect(Collectors.toSet()); assertTrue(modelNames.contains("xgboost_2_2")); + assertTrue(modelNames.contains("lightgbm_regression")); assertTrue(modelNames.contains("mnist_saved")); assertTrue(modelNames.contains("mnist_softmax")); assertTrue(modelNames.contains("mnist_softmax_saved")); @@ -112,13 +113,18 @@ public class ModelEvaluationTest { ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter(MockFileAcquirer.returnFile(null)) .importFrom(config, constantsConfig)); - assertEquals(4, evaluator.models().size()); + assertEquals(5, evaluator.models().size()); Model xgboost = evaluator.models().get("xgboost_2_2"); assertNotNull(xgboost); assertNotNull(xgboost.evaluatorOf()); assertNotNull(xgboost.evaluatorOf("xgboost_2_2")); + Model lightgbm = evaluator.models().get("lightgbm_regression"); + assertNotNull(lightgbm); + assertNotNull(lightgbm.evaluatorOf()); + assertNotNull(lightgbm.evaluatorOf("lightgbm_regression")); + Model tensorflow_mnist = evaluator.models().get("mnist_saved"); assertNotNull(tensorflow_mnist); assertEquals(1, tensorflow_mnist.functions().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java index a4b414ba0da..78c6d2eacc9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java @@ -69,6 +69,13 @@ public class NodeFlavorTuningTest { } @Test + public void require_that_num_search_threads_and_summary_threads_follow_cores() { + ProtonConfig cfg = configFromNumCoresSetting(4.5); + assertEquals(5, cfg.numsearcherthreads()); + assertEquals(5, cfg.numsummarythreads()); + } + + @Test public void require_that_fast_disk_is_reflected_in_proton_config() { ProtonConfig cfg = configFromDiskSetting(true); assertEquals(200, cfg.hwinfo().disk().writespeed(), delta); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 177e741937d..97417f5a522 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -90,7 +90,7 @@ public class DocumentDatabaseTestCase { private void assertSingleSD(String mode) { final List<String> sds = Arrays.asList("type1"); VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); assertEquals(1, indexedSearchCluster.getDocumentDbs().size()); @@ -111,7 +111,7 @@ public class DocumentDatabaseTestCase { sds.add(nameAndMode.getType()); } return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); } @Test @@ -211,10 +211,10 @@ public class DocumentDatabaseTestCase { } @Test - public void requireThatWeCanHaveMultipleSearchDefinitions() { - final List<String> sds = Arrays.asList("type1", "type2", "type3"); + public void testMultipleSchemas() { + List<String> sds = List.of("type1", "type2", "type3"); VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); String type1Id = "test/search/cluster.test/type1"; @@ -264,7 +264,7 @@ public class DocumentDatabaseTestCase { public void requireThatRelevantConfigIsAvailableForClusterSearcher() { final List<String> sds = Arrays.asList("type1", "type2"); VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); String searcherId = "container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher"; { // documentdb-info config @@ -325,7 +325,7 @@ public class DocumentDatabaseTestCase { private void assertDocumentDBConfigAvailableForStreaming(String mode) { final List<String> sds = Arrays.asList("type"); VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type"); assertEquals(1, dcfg.documentdb().size()); @@ -343,7 +343,7 @@ public class DocumentDatabaseTestCase { List<String> documentDBConfigIds, Map<String, List<String>> expectedAttributesMap) { VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); ProtonConfig proton = getProtonCfg(contentSearchCluster); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java index 1c4e005cb67..ea4b3db5ebb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java @@ -18,12 +18,15 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.search.AbstractSearchCluster; -import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + /** * Unit tests for SearchCluster. Please use this instead of SearchModelTestCase if possible and @@ -124,7 +127,7 @@ public class SearchClusterTest { " </content>" + "</services>"; - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create(); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSchemas("music")).create(); ContainerCluster containerCluster1 = (ContainerCluster)model.getConfigProducer("j1").get(); assertFalse(containerCluster1.getSearch().getChains().localProviders().isEmpty()); @@ -160,23 +163,30 @@ public class SearchClusterTest { AbstractSearchCluster searchCluster2 = model.getSearchClusters().get(xbulkIndex); assertEquals("xbulk", searchCluster2.getClusterName()); - Component<?,?> normalDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.normal")); - assertNotNull(normalDispatcher); - assertEquals("dispatcher.normal", normalDispatcher.getComponentId().stringValue()); - assertEquals("com.yahoo.search.dispatch.Dispatcher", normalDispatcher.getClassId().stringValue()); - assertEquals("j1/component/dispatcher.normal", normalDispatcher.getConfigId()); - DispatchConfig.Builder normalDispatchConfigBuilder = new DispatchConfig.Builder(); - model.getConfig(normalDispatchConfigBuilder, "j1/component/dispatcher.normal"); - assertEquals("node2host", normalDispatchConfigBuilder.build().node(0).host()); - - Component<?,?> xbulkDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.xbulk")); - assertNotNull(xbulkDispatcher); - assertEquals("dispatcher.xbulk", xbulkDispatcher.getComponentId().stringValue()); - assertEquals("com.yahoo.search.dispatch.Dispatcher", xbulkDispatcher.getClassId().stringValue()); - assertEquals("j1/component/dispatcher.xbulk", xbulkDispatcher.getConfigId()); - DispatchConfig.Builder xbulkDispatchConfigBuilder = new DispatchConfig.Builder(); - model.getConfig(xbulkDispatchConfigBuilder, "j1/component/dispatcher.xbulk"); - assertEquals("node0host", xbulkDispatchConfigBuilder.build().node(0).host()); + verifyDispatch(model, containerCluster1, "normal", "node2host"); + verifyDispatch(model, containerCluster1, "xbulk", "node0host"); + } + + private void verifyDispatch(VespaModel model, ContainerCluster containerCluster, String cluster, String host) { + Component<?,?> dispatcher = (Component<?, ?>)containerCluster.getComponentsMap().get(new ComponentId("dispatcher." + cluster)); + assertNotNull(dispatcher); + assertEquals("dispatcher." + cluster, dispatcher.getComponentId().stringValue()); + assertEquals("com.yahoo.search.dispatch.Dispatcher", dispatcher.getClassId().stringValue()); + assertEquals("j1/component/dispatcher." + cluster, dispatcher.getConfigId()); + DispatchConfig.Builder dispatchConfigBuilder = new DispatchConfig.Builder(); + model.getConfig(dispatchConfigBuilder, dispatcher.getConfigId()); + assertEquals(host, dispatchConfigBuilder.build().node(0).host()); + + assertTrue(dispatcher.getInjectedComponentIds().contains("rpcresourcepool." + cluster)); + + Component<?,?> rpcResourcePool = (Component<?, ?>)dispatcher.getChildren().get("rpcresourcepool." + cluster); + assertNotNull(rpcResourcePool); + assertEquals("rpcresourcepool." + cluster, rpcResourcePool.getComponentId().stringValue()); + assertEquals("com.yahoo.search.dispatch.rpc.RpcResourcePool", rpcResourcePool.getClassId().stringValue()); + assertEquals("j1/component/dispatcher." + cluster + "/rpcresourcepool." + cluster, rpcResourcePool.getConfigId()); + dispatchConfigBuilder = new DispatchConfig.Builder(); + model.getConfig(dispatchConfigBuilder, rpcResourcePool.getConfigId()); + assertEquals(host, dispatchConfigBuilder.build().node(0).host()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index b6180ab78b9..cdfd9fab194 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -104,33 +104,41 @@ public class VespaModelTester { /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, 0, retiredHostNames); + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0, retiredHostNames); + } + + /** Creates a model which uses 0 as start index */ + public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, String ... retiredHostNames) { + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, 0, retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, startIndexForClusters, retiredHostNames); + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, startIndexForClusters, retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(zone, services, failOnOutOfCapacity, 0, retiredHostNames); + return createModel(zone, services, failOnOutOfCapacity, false, 0, retiredHostNames); } /** * Creates a model using the hosts already added to this * * @param services the services xml string + * @param useMaxResources false to use the minmal resources (when given a range), true to use max * @param failOnOutOfCapacity whether we should get an exception when not enough hosts of the requested flavor * is available or if we should just silently receive a smaller allocation * @return the resulting model */ - public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { + public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, boolean useMaxResources, + int startIndexForClusters, String ... retiredHostNames) { VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; - HostProvisioner provisioner = hosted ? - new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) : + HostProvisioner provisioner = hosted ? + new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, useMaxResources, + startIndexForClusters, retiredHostNames) : new SingleNodeProvisioner(); TestProperties properties = new TestProperties() diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index ee6fc60ba46..df62a3bff07 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -44,14 +44,14 @@ public class ApplicationPackageUtils { } public static List<String> generateSearchDefinition(String name) { - return generateSearchDefinitions(name); + return generateSchemas(name); } - public static List<String> generateSearchDefinitions(String ... sdNames) { - return generateSearchDefinitions(Arrays.asList(sdNames)); + public static List<String> generateSchemas(String ... sdNames) { + return generateSchemas(Arrays.asList(sdNames)); } - public static List<String> generateSearchDefinitions(List<String> sdNames) { + public static List<String> generateSchemas(List<String> sdNames) { List<String> sds = new ArrayList<>(); int i = 0; for (String sdName : sdNames) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java index 814ec008285..70ce588bec1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java @@ -31,8 +31,8 @@ public class VespaModelCreatorWithMockPkg { this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).build()); } - public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> searchDefinitions) { - this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withSearchDefinitions(searchDefinitions).build()); + public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> schemas) { + this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withSchemas(schemas).build()); } public VespaModelCreatorWithMockPkg(ApplicationPackage appPkg) { diff --git a/config-model/src/test/schema-test-files/services-hosted.xml b/config-model/src/test/schema-test-files/services-hosted.xml index 07839239c81..71a07926240 100644 --- a/config-model/src/test/schema-test-files/services-hosted.xml +++ b/config-model/src/test/schema-test-files/services-hosted.xml @@ -7,7 +7,7 @@ </admin> <container id="container1" version="1.0"> - <nodes count="5" required="true"> + <nodes count="[5,7]" required="true"> <resources vcpu="1.2" memory="10Gb" disk="0.3 TB"/> </nodes> </container> @@ -27,4 +27,11 @@ </nodes> </content> + <content id="ml" version="1.0"> + <redundancy>2</redundancy> + <nodes count="[10,20]" flavor="large" groups="[1,3]"> + <resources vcpu="[3.0, 4]" memory="[32000.0Mb, 33Gb]" disk="[300 Gb, 1Tb]"/> + </nodes> + </content> + </services> diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 2bbd98f72ac..e7ea2683e3f 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -15,15 +15,29 @@ <slobrok hostalias="rtc-1" /> </slobroks> <metrics> - <consumer id="my-consumer"> + + <consumer id="cloudwatch-hosted"> <metric-set id="my-set" /> <metric id="my-metric"/> <metric id="my-metric2" display-name="my-metric3"/> <metric display-name="my-metric4" id="my-metric4.avg"/> + <cloudwatch region="us-east1" namespace="my-namespace"> + <credentials access-key-name="my-access-key" secret-key-name="my-secret-key" /> + </cloudwatch> </consumer> - <consumer id="my-consumer2"> - <metric-set id="my-set2" /> + + <consumer id="cloudwatch-self-hosted-with-default-auth"> + <metric-set id="public" /> + <cloudwatch region="us-east1" namespace="namespace_legal.chars:/#1" /> </consumer> + + <consumer id="cloudwatch-self-hosted-with-profile"> + <metric id="my-custom-metric" /> + <cloudwatch region="us-east1" namespace="another-namespace"> + <shared-credentials file="/user/.aws/credentials" profile="profile-in-credentials-file" /> + </cloudwatch> + </consumer> + </metrics> <logforwarding> <splunk deployment-server="foo:8989" client-name="foobar" splunk-home="/opt/splunk" phone-home-interval="900"/> @@ -76,7 +90,6 @@ <exclude> <binding>http//*/foo/*</binding> </exclude> - <application>my-app</application> <vespa-domain>vespa.vespa.cd</vespa-domain> </access-control> @@ -119,6 +132,13 @@ <certificate-file>/foo/cert</certificate-file> <ca-certificates-file>/foo/cacerts</ca-certificates-file> <client-authentication>want</client-authentication> + <cipher-suites> + TLS_AES_128_GCM_SHA256, + TLS_AES_256_GCM_SHA384, + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + </cipher-suites> + <protocols>TLSv1.2,TLSv1.3</protocols> </ssl> </server> <server port="4083" id="sslProvider"> |