diff options
| author | Harald Musum <musum@verizonmedia.com> | 2020-03-18 08:59:30 +0100 |
|---|---|---|
| committer | Harald Musum <musum@verizonmedia.com> | 2020-03-18 08:59:30 +0100 |
| commit | 5f3fb0019486caeee422bd13a23c33c0833c7267 (patch) | |
| tree | 9e20b2972e3d66d19b2ad7534f1dd325a6317dc0 | |
| parent | f1a0e0f4d94b66f1d62b79dac2940bc1ef7e66c5 (diff) | |
| parent | 8259eb143d2d1dc54a869c0742d00b62da8de3b0 (diff) | |
Merge branch 'master' into hmusum/builder-for-clusterspec
143 files changed, 764 insertions, 631 deletions
diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java index b200244a21c..87e82a3fdeb 100644 --- a/application/src/main/java/com/yahoo/application/Application.java +++ b/application/src/main/java/com/yahoo/application/Application.java @@ -254,13 +254,13 @@ public final class Application implements AutoCloseable { * @throws java.io.IOException e.g.if file not found */ public Builder documentType(String name, String searchDefinition) throws IOException { - Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.SD_NAME_SUFFIX); + Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.SD_NAME_SUFFIX); createFile(path, searchDefinition); return this; } public Builder expressionInclude(String name, String searchDefinition) throws IOException { - Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX); + Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX); createFile(path, searchDefinition); return this; } @@ -271,7 +271,7 @@ public final class Application implements AutoCloseable { * @throws java.io.IOException e.g.if file not found */ public Builder rankExpression(String name, String rankExpressionContent) throws IOException { - Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX); + Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX); createFile(path, rankExpressionContent); return this; } diff --git a/application/src/main/java/com/yahoo/application/ApplicationBuilder.java b/application/src/main/java/com/yahoo/application/ApplicationBuilder.java index 9d7cf2c8673..ac64ae188e5 100644 --- a/application/src/main/java/com/yahoo/application/ApplicationBuilder.java +++ b/application/src/main/java/com/yahoo/application/ApplicationBuilder.java @@ -37,13 +37,13 @@ public class ApplicationBuilder { } public ApplicationBuilder documentType(String name, String searchDefinition) throws IOException { - Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.SD_NAME_SUFFIX); + Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.SD_NAME_SUFFIX); createFile(path, searchDefinition); return this; } public ApplicationBuilder rankExpression(String name, String rankExpressionContent) throws IOException { - Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX); + Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX); createFile(path, rankExpressionContent); return this; } diff --git a/application/src/test/java/com/yahoo/application/ApplicationBuilderTest.java b/application/src/test/java/com/yahoo/application/ApplicationBuilderTest.java index a4f566e2a71..a0c2ec9983b 100644 --- a/application/src/test/java/com/yahoo/application/ApplicationBuilderTest.java +++ b/application/src/test/java/com/yahoo/application/ApplicationBuilderTest.java @@ -32,9 +32,10 @@ public class ApplicationBuilderTest { @Test public void query_profile_can_be_added() throws Exception { withApplicationBuilder(builder -> { - builder.queryProfile("MyProfile", "<query-profile id=\"MyProfile\">" + // - "<field name=\"message\">Hello world!</field>" + // - "</query-profile>"); + builder.queryProfile("MyProfile", + "<query-profile id=\"MyProfile\">" + + "<field name=\"message\">Hello world!</field>" + + "</query-profile>"); assertTrue(Files.exists(builder.getPath().resolve("search/query-profiles/MyProfile.xml"))); }); @@ -44,7 +45,7 @@ public class ApplicationBuilderTest { public void rank_expression_can_be_added() throws Exception { withApplicationBuilder(builder -> { builder.rankExpression("myExpression", "content"); - assertTrue(Files.exists(builder.getPath().resolve("searchdefinitions/myExpression.expression"))); + assertTrue(Files.exists(builder.getPath().resolve("schemas/myExpression.expression"))); }); } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java index 32054b1ad9a..2b7aa5ab6e1 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java @@ -59,8 +59,7 @@ public class AppSubDirs { return configDefs.first; } - public File searchdefinitions() { - return searchdefinitions.first; - } + @Deprecated // Remove after March 2020 + public File searchdefinitions() { return searchdefinitions.first; } } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java index 74ade9d8e14..b0c4f74f9f6 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java @@ -35,20 +35,11 @@ public class ApplicationPackageXmlFilesValidator { return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), vespaVersion); } - @SuppressWarnings("deprecation") public void checkApplication() throws IOException { validate(validators.servicesXmlValidator(), servicesFileName()); validateOptional(validators.hostsXmlValidator(), FilesApplicationPackage.HOSTS); validateOptional(validators.deploymentXmlValidator(), FilesApplicationPackage.DEPLOYMENT_FILE.getName()); validateOptional(validators.validationOverridesXmlValidator(), FilesApplicationPackage.VALIDATION_OVERRIDES.getName()); - - if (appDirs.searchdefinitions().exists()) { - if (FilesApplicationPackage.getSearchDefinitionFiles(appDirs.root()).isEmpty()) { - throw new IllegalArgumentException("Application package in " + appDirs.root() + - " must contain at least one search definition (.sd) file when directory searchdefinitions/ exists."); - } - } - validateRouting(appDirs.routingtables); } @@ -71,7 +62,6 @@ public class ApplicationPackageXmlFilesValidator { validator.validate(appDirs.file(filename)); } - @SuppressWarnings("deprecation") private String servicesFileName() { String servicesFile = FilesApplicationPackage.SERVICES; if ( ! appDirs.file(servicesFile).exists()) { diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 8c3ccdc4c0b..e93246f49e7 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -49,6 +49,7 @@ import java.security.MessageDigest; import java.util.*; import java.util.jar.JarFile; import java.util.logging.Logger; +import java.util.stream.Stream; import static com.yahoo.text.Lowercase.toLowerCase; @@ -336,77 +337,73 @@ public class FilesApplicationPackage implements ApplicationPackage { public static Map<String, String> allSdsFromDocprocBundlesAndClasspath(File appDir) throws IOException { File dpChains = new File(appDir, ApplicationPackage.COMPONENT_DIR); if (!dpChains.exists() || !dpChains.isDirectory()) return Collections.emptyMap(); - List<String> usedNames = new ArrayList<>(); - Map<String, String> ret = new LinkedHashMap<>(); + Set<String> usedNames = new HashSet<>(); + Map<String, String> schemas = new LinkedHashMap<>(); // try classpath first - allSdsOnClassPath(usedNames, ret); + allSdsOnClassPath(usedNames, schemas); - for (File bundle : dpChains.listFiles(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.endsWith(".jar"); - }})) { + for (File bundle : dpChains.listFiles((File dir, String name) -> name.endsWith(".jar"))) { for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", new JarFile(bundle)).entrySet()) { String sdName = entry.getKey(); if (usedNames.contains(sdName)) { - throw new IllegalArgumentException("The search definition name '"+sdName+"' used in bundle '"+ - bundle.getName()+"' is already used in classpath or previous bundle."); + throw new IllegalArgumentException("The search definition name '" + sdName + "' used in bundle '"+ + bundle.getName()+ "' is already used in classpath or previous bundle."); } usedNames.add(sdName); String sdPayload = entry.getValue(); - ret.put(sdName, sdPayload); + schemas.put(sdName, sdPayload); } } - return ret; + return schemas; } - private static void allSdsOnClassPath(List<String> usedNames, Map<String, String> ret) throws IOException { - Enumeration<java.net.URL> resources = FilesApplicationPackage.class.getClassLoader().getResources(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); - - while(resources.hasMoreElements()) { - URL resource = resources.nextElement(); - - String protocol = resource.getProtocol(); + private static void allSdsOnClassPath(Set<String> usedNames, Map<String, String> schemas) { + ClassLoader cl = FilesApplicationPackage.class.getClassLoader(); + Stream<URL> resources = Stream.concat(cl.resources(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()), + cl.resources(ApplicationPackage.SCHEMAS_DIR.getRelative())); + resources.forEach(resource -> addSchemaFrom(resource, schemas, usedNames)); + } - if ("file".equals(protocol)) { - File file; - try { - file = new File(resource.toURI()); - } catch (URISyntaxException e) { - continue; - } - // only interested in directories - if (file.isDirectory()) { - List<File> sdFiles = getSearchDefinitionFiles(file); - for (File sdFile : sdFiles) { - String sdName = sdFile.getName(); + private static void addSchemaFrom(URL resource, Map<String, String> schemas, Set<String> usedNames) { + try { + switch (resource.getProtocol()) { + case "file": + File file = new File(resource.toURI()); + if (file.isDirectory()) { + List<File> sdFiles = getSearchDefinitionFiles(file); + for (File sdFile : sdFiles) { + String sdName = sdFile.getName(); + if (usedNames.contains(sdName)) { + throw new IllegalArgumentException("The search definition name '" + sdName + + "' found in classpath already used earlier in classpath."); + } + usedNames.add(sdName); + String contents = IOUtils.readAll(new FileReader(sdFile)); + schemas.put(sdFile.getName(), contents); + } + } + break; + case "jar": + JarURLConnection jarConnection = (JarURLConnection) resource.openConnection(); + JarFile jarFile = jarConnection.getJarFile(); + for (Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", jarFile).entrySet()) { + String sdName = entry.getKey(); if (usedNames.contains(sdName)) { - throw new IllegalArgumentException("The search definition name '"+sdName+ - "' found in classpath already used earlier in classpath."); + throw new IllegalArgumentException("The search definitions name '" + sdName + + "' used in bundle '" + jarFile.getName() + "' " + + "is already used in classpath or previous bundle."); } usedNames.add(sdName); - String contents = IOUtils.readAll(new FileReader(sdFile)); - ret.put(sdFile.getName(), contents); - } - } - } - else if ("jar".equals(protocol)) { - JarURLConnection jarConnection = (JarURLConnection) resource.openConnection(); - JarFile jarFile = jarConnection.getJarFile(); - for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", jarFile).entrySet()) { - String sdName = entry.getKey(); - if (usedNames.contains(sdName)) { - throw new IllegalArgumentException("The search definitions name '"+sdName+ - "' used in bundle '"+jarFile.getName()+"' " + - "is already used in classpath or previous bundle."); + String sdPayload = entry.getValue(); + schemas.put(sdName, sdPayload); } - usedNames.add(sdName); - String sdPayload = entry.getValue(); - ret.put(sdName, sdPayload); - } + break; } } + catch (IOException | URISyntaxException e) { + throw new IllegalArgumentException("Could not read schema from '" + resource + "'", e); + } } /** @@ -462,11 +459,7 @@ public class FilesApplicationPackage implements ApplicationPackage { if (! configDefsDir.isDirectory()) return; log.log(LogLevel.DEBUG, "Getting all config definitions from '" + configDefsDir + "'"); - for (File def : configDefsDir.listFiles( - new FilenameFilter() { @Override public boolean accept(File dir, String name) { // TODO: Fix - return name.matches(".*\\.def");}})) { - - log.log(LogLevel.DEBUG, "Processing config definition '" + def + "'"); + for (File def : configDefsDir.listFiles((File dir, String name) -> name.matches(".*\\.def"))) { String[] nv = def.getName().split("\\.def"); ConfigDefinitionKey key; try { @@ -521,30 +514,25 @@ public class FilesApplicationPackage implements ApplicationPackage { } } - //Only intended for DeployProcessor, others should use the member version static List<File> getSearchDefinitionFiles(File appDir) { - //The dot is escaped later in this method: - assert (ApplicationPackage.SD_NAME_SUFFIX.charAt(0) == '.'); + List<File> schemaFiles = new ArrayList<>(); + + File sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + if (sdDir.isDirectory()) + schemaFiles.addAll(Arrays.asList(sdDir.listFiles((dir, name) -> name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX)))); - List<File> ret = new ArrayList<>(); - File sdDir; + sdDir = new File(appDir, ApplicationPackage.SCHEMAS_DIR.getRelative()); + if (sdDir.isDirectory()) + schemaFiles.addAll(Arrays.asList(sdDir.listFiles((dir, name) -> name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX)))); - sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); - if (!sdDir.isDirectory()) { - return ret; - } - ret.addAll(Arrays.asList( - sdDir.listFiles( - new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX);}}))); - return ret; + return schemaFiles; } public List<File> getSearchDefinitionFiles() { return getSearchDefinitionFiles(appDir); } - //Only for use by deploy processor + // Only for use by deploy processor public static List<Component> getComponents(File appDir) { List<Component> components = new ArrayList<>(); for (Bundle bundle : Bundle.getBundles(new File(appDir, ApplicationPackage.COMPONENT_DIR))) { @@ -622,6 +610,7 @@ public class FilesApplicationPackage implements ApplicationPackage { public Bundle getBundle() { return bundle; } + } // class Component /** @@ -643,12 +632,16 @@ public class FilesApplicationPackage implements ApplicationPackage { } private File expressionFileNameToFile(String name) { - File expressionFile = new File(name); - if (expressionFile.isAbsolute()) { + if (new File(name).isAbsolute()) throw new IllegalArgumentException("Absolute path to ranking expression file is not allowed: " + name); + + File sdDir = new File(appDir, ApplicationPackage.SCHEMAS_DIR.getRelative()); + File expressionFile = new File(sdDir, name); + if ( ! expressionFile.exists()) { + sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); + expressionFile = new File(sdDir, name); } - File sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); - return new File(sdDir, name); + return expressionFile; } @Override @@ -700,9 +693,8 @@ public class FilesApplicationPackage implements ApplicationPackage { ! name.equals(HOSTS) && ! name.equals(CONFIG_DEFINITIONS_DIR)); preprocessXML(new File(preprocessedDir, SERVICES), getServicesFile(), zone); - if (getHostsFile().exists()) { + if (getHostsFile().exists()) preprocessXML(new File(preprocessedDir, HOSTS), getHostsFile(), zone); - } FilesApplicationPackage preprocessed = FilesApplicationPackage.fromFile(preprocessedDir, includeSourceFiles); preprocessed.copyUserDefsIntoApplication(); return preprocessed; @@ -742,11 +734,12 @@ public class FilesApplicationPackage implements ApplicationPackage { /** * Adds the given path to the digest, or does nothing if path is neither file nor dir + * * @param path path to add to message digest * @param suffix only files with this suffix are considered * @param digest the {link @MessageDigest} to add the file paths to * @param recursive whether to recursively find children in the paths - * @param fullPathNames Whether to include the full paths in checksum or only the names + * @param fullPathNames whether to include the full paths in checksum or only the names * @throws java.io.IOException if adding path to digest fails when reading files from path */ private static void addPathToDigest(File path, String suffix, MessageDigest digest, boolean recursive, boolean fullPathNames) throws IOException { @@ -773,16 +766,17 @@ public class FilesApplicationPackage implements ApplicationPackage { } private static final int MD5_BUFFER_SIZE = 65536; + private static void addToDigest(InputStream is, MessageDigest digest) throws IOException { - if (is==null) return; + if (is == null) return; byte[] buffer = new byte[MD5_BUFFER_SIZE]; int i; do { - i=is.read(buffer); + i = is.read(buffer); if (i > 0) { digest.update(buffer, 0, i); } - } while(i!=-1); + } while(i != -1); } /** diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index e1e3d6f610c..ada3119f5bb 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -137,6 +137,7 @@ "fields": [ "public static final java.lang.String HOSTS", "public static final java.lang.String SERVICES", + "public static final com.yahoo.path.Path SCHEMAS_DIR", "public static final com.yahoo.path.Path SEARCH_DEFINITIONS_DIR", "public static final java.lang.String COMPONENT_DIR", "public static final java.lang.String SEARCHCHAINS_DIR", @@ -882,7 +883,8 @@ "public abstract boolean useBucketSpaceMetric()", "public boolean useNewAthenzFilter()", "public boolean usePhraseSegmenting()", - "public java.lang.String proxyProtocol()" + "public java.lang.String proxyProtocol()", + "public java.util.Optional athenzDomain()" ], "fields": [] }, diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index db3d391d19b..174f9bb54d7 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -36,8 +36,6 @@ import java.util.jar.JarFile; * The class hides detail as to whether the source is local files or ZooKeeper * data in config server. * - * Anyone wanting to access application data should use this interface. - * * @author Vegard Havdal */ public interface ApplicationPackage { @@ -47,7 +45,8 @@ public interface ApplicationPackage { String HOSTS = "hosts.xml"; String SERVICES = "services.xml"; - Path SEARCH_DEFINITIONS_DIR = Path.fromString("searchdefinitions"); + Path SCHEMAS_DIR = Path.fromString("schemas"); + Path SEARCH_DEFINITIONS_DIR = Path.fromString("searchdefinitions"); // Legacy addition to schemas String COMPONENT_DIR = "components"; String SEARCHCHAINS_DIR = "search/chains"; String DOCPROCCHAINS_DIR = "docproc/chains"; @@ -168,7 +167,7 @@ public interface ApplicationPackage { } /** - * Returns inforamtion about a file + * Returns information about a file * * @param relativePath the relative path of the file within this application package. * @return information abut the file, returned whether or not the file exists @@ -206,23 +205,26 @@ public interface ApplicationPackage { Reader getRankingExpression(String name); /** - * Returns the name-payload pairs of any sd files under path/searchdefinitions/ in the given jar bundle - * @param bundle The jar file, which will be closed afterwards by this method. - * @param path For example 'complex/' + * Returns the name-payload pairs of any sd files under path/schemas and path/searchdefinitions/ + * in the given jar bundle. + * + * @param bundle the jar file, which will be closed afterwards by this method + * @param path for example 'complex/' * @return map of the SD payloads * @throws IOException if it is reading sd files fails */ static Map<String, String> getBundleSdFiles(String path, JarFile bundle) throws IOException { - Map<String,String> ret = new LinkedHashMap<>(); + Map<String, String> schemas = new LinkedHashMap<>(); for (Enumeration<JarEntry> e = bundle.entries(); e.hasMoreElements();) { - JarEntry je=e.nextElement(); - if (je.getName().startsWith(path+SEARCH_DEFINITIONS_DIR+"/") && je.getName().endsWith(SD_NAME_SUFFIX)) { - String contents = IOUtils.readAll(new InputStreamReader(bundle.getInputStream(je))); - ret.put(getFileName(je), contents); + JarEntry entry = e.nextElement(); + if ((entry.getName().startsWith(path + SCHEMAS_DIR + "/") || entry.getName().startsWith(path + SEARCH_DEFINITIONS_DIR + "/")) + && entry.getName().endsWith(SD_NAME_SUFFIX)) { + String contents = IOUtils.readAll(new InputStreamReader(bundle.getInputStream(entry))); + schemas.put(getFileName(entry), contents); } } bundle.close(); - return ret; + return schemas; } /** diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 39d1f8096b6..7fcde1b5e6b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; @@ -65,6 +66,7 @@ public interface ModelContext { default boolean useNewAthenzFilter() { return false; } default boolean usePhraseSegmenting() { return false; } default String proxyProtocol() { return "https-only"; } + default Optional<AthenzDomain> athenzDomain() { return Optional.empty(); } } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java index e66a7e1ef7e..930c3c94907 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; * Interface for those wanting to be notified about changes to the SuperModel. */ public interface SuperModelListener { + /** * Application has been activated: Either deployed the first time, * internally redeployed, or externally triggered redeploy. @@ -22,7 +23,7 @@ public interface SuperModelListener { * Invoked once all applications that were supposed to be deployed on bootstrap * have been activated (and the respective {@link #applicationActivated(SuperModel, ApplicationInfo) * applicationActivated} have been invoked). The SuperModel is then said to be "complete". - * @param superModel */ void notifyOfCompleteness(SuperModel superModel); + } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java index 140cb3001a0..b9ad830090c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java @@ -8,7 +8,6 @@ import java.util.Optional; /** * @author Ulf Lilleengen - * @since 5.1 */ public interface ConfigDefinitionStore { diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 074b19d7238..696ce4195eb 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -12,10 +12,10 @@ import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -36,9 +36,8 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.container.search.QueryProfilesBuilder; import com.yahoo.vespa.model.container.search.SemanticRuleBuilder; import com.yahoo.vespa.model.container.search.SemanticRules; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.Reader; @@ -62,7 +61,7 @@ public class DeployState implements ConfigDefinitionStore { private final DeployLogger logger; private final FileRegistry fileRegistry; private final DocumentModel documentModel; - private final List<SearchDefinition> searchDefinitions; + private final List<NamedSchema> schemas; private final ApplicationPackage applicationPackage; private final Optional<ConfigDefinitionRepo> configDefinitionRepo; private final Optional<ApplicationPackage> permanentApplicationPackage; @@ -122,7 +121,7 @@ public class DeployState implements ConfigDefinitionStore { this.previousModel = previousModel; this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault; this.provisioner = hostProvisioner.orElse(getDefaultModelHostProvisioner(applicationPackage)); - this.searchDefinitions = searchDocumentModel.getSearchDefinitions(); + this.schemas = searchDocumentModel.getSchemas(); this.documentModel = searchDocumentModel.getDocumentModel(); this.permanentApplicationPackage = permanentApplicationPackage; this.configDefinitionRepo = configDefinitionRepo; @@ -209,8 +208,8 @@ public class DeployState implements ConfigDefinitionStore { return applicationPackage; } - public List<SearchDefinition> getSearchDefinitions() { - return searchDefinitions; + public List<NamedSchema> getSchemas() { + return schemas; } public DocumentModel getDocumentModel() { @@ -427,20 +426,18 @@ public class DeployState implements ConfigDefinitionStore { for (NamedReader reader : readers) { try { String readerName = reader.getName(); - String searchName = builder.importReader(reader, readerName, logger); + String topLevelName = builder.importReader(reader, readerName, logger); String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX); - names.put(searchName, sdName); - if ( ! sdName.equals(searchName)) { - throw new IllegalArgumentException("Search definition file name ('" + sdName + "') and name of " + - "search element ('" + searchName + + names.put(topLevelName, sdName); + if ( ! sdName.equals(topLevelName)) { + throw new IllegalArgumentException("Schema definition file name ('" + sdName + "') and name of " + + "top level element ('" + topLevelName + "') are not equal for file '" + readerName + "'"); } } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse search definition file '" + - getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e); + throw new IllegalArgumentException("Could not parse sd file '" + reader.getName() + "'", e); } catch (IOException e) { - throw new IllegalArgumentException("Could not read search definition file '" + - getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e); + throw new IllegalArgumentException("Could not read sd file '" + reader.getName() + "'", e); } finally { closeIgnoreException(reader.getReader()); } @@ -449,10 +446,6 @@ public class DeployState implements ConfigDefinitionStore { return SearchDocumentModel.fromBuilderAndNames(builder, names); } - private String getSearchDefinitionRelativePath(String name) { - return ApplicationPackage.SEARCH_DEFINITIONS_DIR + File.separator + name; - } - private static String stripSuffix(String nodeName, String postfix) { assert (nodeName.endsWith(postfix)); return nodeName.substring(0, nodeName.length() - postfix.length()); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java index cdd4f6f8e8a..9b9729dddb3 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java @@ -3,7 +3,7 @@ package com.yahoo.config.model.deploy; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.vespa.documentmodel.DocumentModel; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; import java.util.ArrayList; import java.util.List; @@ -13,16 +13,15 @@ import java.util.Map; * Internal helper class to retrieve document model and search definitions. * * @author Ulf Lilleengen - * @since 5.1 */ public class SearchDocumentModel { private final DocumentModel documentModel; - private final List<SearchDefinition> searchDefinitions; + private final List<NamedSchema> schemas; - public SearchDocumentModel(DocumentModel documentModel, List<SearchDefinition> searchDefinitions) { + public SearchDocumentModel(DocumentModel documentModel, List<NamedSchema> schemas) { this.documentModel = documentModel; - this.searchDefinitions = searchDefinitions; + this.schemas = schemas; } @@ -30,22 +29,22 @@ public class SearchDocumentModel { return documentModel; } - public List<SearchDefinition> getSearchDefinitions() { - return searchDefinitions; + public List<NamedSchema> getSchemas() { + return schemas; } public static SearchDocumentModel fromBuilderAndNames(SearchBuilder builder, Map<String, String> names) { - List<SearchDefinition> ret = new ArrayList<>(); + List<NamedSchema> ret = new ArrayList<>(); for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) { - ret.add(new SearchDefinition(names.get(search.getName()), search)); + ret.add(new NamedSchema(names.get(search.getName()), search)); } return new SearchDocumentModel(builder.getModel(), ret); } public static SearchDocumentModel fromBuilder(SearchBuilder builder) { - List<SearchDefinition> ret = new ArrayList<>(); + List<NamedSchema> ret = new ArrayList<>(); for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) { - ret.add(new SearchDefinition(search.getName(), search)); + ret.add(new NamedSchema(search.getName(), search)); } return new SearchDocumentModel(builder.getModel(), ret); } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index eb61bda83a6..10649df88e1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -55,23 +55,23 @@ public class MockApplicationPackage implements ApplicationPackage { private final File root; private final String hostsS; private final String servicesS; - private final List<String> searchDefinitions; - private final String searchDefinitionDir; + private final List<String> schemas; + private final String schemaDir; private final Optional<String> deploymentSpec; private final Optional<String> validationOverrides; private final boolean failOnValidateXml; private final QueryProfileRegistry queryProfileRegistry; private final ApplicationMetaData applicationMetaData; - protected MockApplicationPackage(File root, String hosts, String services, List<String> searchDefinitions, - String searchDefinitionDir, + protected MockApplicationPackage(File root, String hosts, String services, List<String> schemas, + String schemaDir, String deploymentSpec, String validationOverrides, boolean failOnValidateXml, String queryProfile, String queryProfileType) { this.root = root; this.hostsS = hosts; this.servicesS = services; - this.searchDefinitions = searchDefinitions; - this.searchDefinitionDir = searchDefinitionDir; + this.schemas = schemas; + this.schemaDir = schemaDir; this.deploymentSpec = Optional.ofNullable(deploymentSpec); this.validationOverrides = Optional.ofNullable(validationOverrides); this.failOnValidateXml = failOnValidateXml; @@ -108,7 +108,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public Reader getHosts() { - if (hostsS==null) return null; + if (hostsS == null) return null; return new StringReader(hostsS); } @@ -118,7 +118,7 @@ public class MockApplicationPackage implements ApplicationPackage { SearchBuilder searchBuilder = new SearchBuilder(this, new RankProfileRegistry(), queryProfileRegistry); - for (String sd : searchDefinitions) { + for (String sd : schemas) { try { String name = searchBuilder.importString(sd); readers.add(new NamedReader(name + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd))); @@ -184,7 +184,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public Reader getRankingExpression(String name) { - File expressionFile = new File(searchDefinitionDir, name); + File expressionFile = new File(schemaDir, name); try { return IOUtils.createReader(expressionFile, "utf-8"); } @@ -200,9 +200,9 @@ public class MockApplicationPackage implements ApplicationPackage { public static ApplicationPackage fromSearchDefinitionDirectory(String dir) { return new MockApplicationPackage.Builder() - .withEmptyHosts() - .withEmptyServices() - .withSearchDefinitionDir(dir).build(); + .withEmptyHosts() + .withEmptyServices() + .withSchemaDir(dir).build(); } public static class Builder { @@ -210,8 +210,8 @@ public class MockApplicationPackage implements ApplicationPackage { private File root = new File("nonexisting"); private String hosts = null; private String services = null; - private List<String> searchDefinitions = Collections.emptyList(); - private String searchDefinitionDir = null; + private List<String> schemas = Collections.emptyList(); + private String schemaDir = null; private String deploymentSpec = null; private String validationOverrides = null; private boolean failOnValidateXml = false; @@ -245,17 +245,17 @@ public class MockApplicationPackage implements ApplicationPackage { } public Builder withSearchDefinition(String searchDefinition) { - this.searchDefinitions = Collections.singletonList(searchDefinition); + this.schemas = Collections.singletonList(searchDefinition); return this; } - public Builder withSearchDefinitions(List<String> searchDefinition) { - this.searchDefinitions = Collections.unmodifiableList(searchDefinition); + public Builder withSchemas(List<String> searchDefinition) { + this.schemas = Collections.unmodifiableList(searchDefinition); return this; } - public Builder withSearchDefinitionDir(String searchDefinitionDir) { - this.searchDefinitionDir = searchDefinitionDir; + public Builder withSchemaDir(String schemaDir) { + this.schemaDir = schemaDir; return this; } @@ -285,7 +285,7 @@ public class MockApplicationPackage implements ApplicationPackage { } public ApplicationPackage build() { - return new MockApplicationPackage(root, hosts, services, searchDefinitions, searchDefinitionDir, + return new MockApplicationPackage(root, hosts, services, schemas, schemaDir, deploymentSpec, validationOverrides, failOnValidateXml, queryProfile, queryProfileType); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java index 14f8a0a9d37..0d0da71bd0f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java @@ -25,8 +25,8 @@ public class DocumentReferenceResolver { private final Map<String, Search> searchMapping; - public DocumentReferenceResolver(List<Search> searchDefinitions) { - this.searchMapping = createDocumentNameToSearchMapping(searchDefinitions); + public DocumentReferenceResolver(List<Search> schemas) { + this.searchMapping = createDocumentNameToSearchMapping(schemas); } public void resolveReferences(SDDocumentType documentType) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java index ee4fc41f2f9..25cdd1e08cd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java @@ -2,9 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.TemporaryImportedFields; -import java.util.Collections; import java.util.List; /** @@ -13,17 +11,17 @@ import java.util.List; */ public class ImportedFieldsEnumerator { - private final List<Search> searchDefinitions; + private final List<Search> schemas; - public ImportedFieldsEnumerator(List<Search> searchDefinitions) { - this.searchDefinitions = searchDefinitions; + public ImportedFieldsEnumerator(List<Search> schemas) { + this.schemas = schemas; } public void enumerateImportedFields(SDDocumentType documentType) { - var search = this.searchDefinitions.stream() - .filter(s -> s.getDocument() != null) - .filter(s -> s.getDocument().getName().equals(documentType.getName())) - .findFirst(); + var search = this.schemas.stream() + .filter(s -> s.getDocument() != null) + .filter(s -> s.getDocument().getName().equals(documentType.getName())) + .findFirst(); if (search.isEmpty()) { return; // No imported fields present. } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index f90a7e4f6cd..0ab8a2308a4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -288,7 +288,7 @@ public class Search implements ImmutableSearch { /** * Adds an extra field of this search definition not contained in a document * - * @param field to add to the searchdefinitions list of external fields. + * @param field to add to the schemas list of external fields */ public void addExtraField(SDField field) { if (fields.containsKey(field.getName())) { @@ -383,7 +383,7 @@ public class Search implements ImmutableSearch { * Consolidates a set of index settings for the same index into one * * @param indices The list of indexes to consolidate. - * @return The consolidated index + * @return the consolidated index */ private Index consolidateIndices(List<Index> indices) { Index first = indices.get(0); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index f3e6c128a67..001e6a9a407 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -532,6 +532,15 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.sum")); metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.count")); metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.max")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.sum")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.count")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.max")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.sum")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.count")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.max")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.sum")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.count")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.max")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.sum")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index f9762ce58fa..43c1a88b0a1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -33,7 +33,7 @@ public class ComplexAttributeFieldsValidator extends Validator { continue; } SearchCluster searchCluster = (SearchCluster) cluster; - for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) { + for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) { validateComplexFields(searchCluster.getClusterName(), spec.getSearchDefinition().getSearch()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java index 907418ea9f0..9568ea5c27c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -9,7 +9,7 @@ import com.yahoo.path.Path; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; import java.io.FileNotFoundException; @@ -47,7 +47,7 @@ public class RankingConstantsValidator extends Validator { ApplicationPackage applicationPackage = deployState.getApplicationPackage(); ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Invalid constant tensor file(s):"); - for (SearchDefinition sd : deployState.getSearchDefinitions()) { + for (NamedSchema sd : deployState.getSchemas()) { for (RankingConstant rc : sd.getSearch().rankingConstants().asMap().values()) { try { validateRankingConstant(rc, applicationPackage); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java index 85ba75639eb..031ce0dbdd4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -15,7 +15,7 @@ import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.AbstractSearchCluster; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; import java.util.List; @@ -34,7 +34,7 @@ public class SearchDataTypeValidator extends Validator { if (cluster.isStreaming()) { continue; } - for (AbstractSearchCluster.SearchDefinitionSpec spec : cluster.getLocalSDS()) { + for (AbstractSearchCluster.SchemaSpec spec : cluster.getLocalSDS()) { SDDocumentType docType = spec.getSearchDefinition().getSearch().getDocument(); if (docType == null) { continue; @@ -44,7 +44,7 @@ public class SearchDataTypeValidator extends Validator { } } - private void validateDocument(AbstractSearchCluster cluster, SearchDefinition def, SDDocumentType doc) { + private void validateDocument(AbstractSearchCluster cluster, NamedSchema def, SDDocumentType doc) { for (SDDocumentType child : doc.getTypes()) { validateDocument(cluster, def, child); } @@ -84,7 +84,7 @@ public class SearchDataTypeValidator extends Validator { } } - private void disallowIndexingOfMaps(AbstractSearchCluster cluster, SearchDefinition def, Field field) { + private void disallowIndexingOfMaps(AbstractSearchCluster cluster, NamedSchema def, Field field) { DataType fieldType = field.getDataType(); if ((fieldType instanceof MapDataType) && (((SDField) field).doesIndexing())) { throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' cannot be indexed for search " + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 3fb4c5d6b6b..37657bea0be 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -39,7 +39,6 @@ public final class AccessControl { public static final class Builder { private String domain; - private String applicationId; private boolean readEnabled = false; private boolean writeEnabled = true; private final Set<String> excludeBindings = new LinkedHashSet<>(); @@ -47,9 +46,8 @@ public final class AccessControl { private Collection<Servlet> servlets = Collections.emptyList(); private final DeployLogger logger; - public Builder(String domain, String applicationId, DeployLogger logger) { + public Builder(String domain, DeployLogger logger) { this.domain = domain; - this.applicationId = applicationId; this.logger = logger; } @@ -79,13 +77,12 @@ public final class AccessControl { } public AccessControl build() { - return new AccessControl(domain, applicationId, writeEnabled, readEnabled, + return new AccessControl(domain, writeEnabled, readEnabled, excludeBindings, servlets, handlers, logger); } } public final String domain; - public final String applicationId; public final boolean readEnabled; public final boolean writeEnabled; private final Set<String> excludedBindings; @@ -94,7 +91,6 @@ public final class AccessControl { private final DeployLogger logger; private AccessControl(String domain, - String applicationId, boolean writeEnabled, boolean readEnabled, Set<String> excludedBindings, @@ -102,7 +98,6 @@ public final class AccessControl { Collection<Handler<?>> handlers, DeployLogger logger) { this.domain = domain; - this.applicationId = applicationId; this.readEnabled = readEnabled; this.writeEnabled = writeEnabled; this.excludedBindings = Collections.unmodifiableSet(excludedBindings); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index 8e78b2c7064..9e0b8ad7424 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -6,8 +6,6 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; @@ -63,10 +61,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> } private AccessControl buildAccessControl(DeployState deployState, AbstractConfigProducer ancestor, Element accessControlElem) { - String application = XmlHelper.getOptionalChildValue(accessControlElem, "application") - .orElse(getDeployedApplicationId(deployState, ancestor).value()); - - AccessControl.Builder builder = new AccessControl.Builder(accessControlElem.getAttribute("domain"), application, deployState.getDeployLogger()); + AccessControl.Builder builder = new AccessControl.Builder(accessControlElem.getAttribute("domain"), deployState.getDeployLogger()); getContainerCluster(ancestor).ifPresent(cluster -> { builder.setHandlers(cluster.getHandlers()); @@ -87,15 +82,6 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> return builder.build(); } - /** - * Returns the id of the deployed application, or the default value if not explicitly set (self-hosted). - */ - private static ApplicationName getDeployedApplicationId(DeployState deployState, AbstractConfigProducer ancestor) { - return getContainerCluster(ancestor) - .map(cluster -> deployState.getProperties().applicationId().application()) - .orElse(ApplicationId.defaultId().application()); - } - private static Optional<ApplicationContainerCluster> getContainerCluster(AbstractConfigProducer configProducer) { AbstractConfigProducer currentProducer = configProducer; while (! ApplicationContainerCluster.class.isAssignableFrom(currentProducer.getClass())) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index fa5fa4bd227..4c4b1ca7f82 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -8,7 +8,6 @@ import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ContainerSubsystem; import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; @@ -132,7 +131,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> AbstractSearchCluster sys = findClusterWithId(searchClusters, i); QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder(). name(sys.getClusterName()); - for (AbstractSearchCluster.SearchDefinitionSpec spec : sys.getLocalSDS()) { + for (AbstractSearchCluster.SchemaSpec spec : sys.getLocalSDS()) { scB.searchdef(spec.getSearchDefinition().getName()); } scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java index e05b2d27e09..4ecc666a9f2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java @@ -118,7 +118,7 @@ public class LocalProvider extends Provider implements public List<String> getDocumentTypes() { List<String> documentTypes = new ArrayList<>(); - for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) { + for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) { documentTypes.add(spec.getSearchDefinition().getSearch().getDocument().getName()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 91c5cbe1d75..6148cbfd77f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -535,7 +535,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addNodesFromXml(ApplicationContainerCluster cluster, Element containerElement, ConfigModelContext context) { Element nodesElement = XML.getChild(containerElement, "nodes"); - if (nodesElement == null) { + if (nodesElement == null) { // default single node on localhost ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa()); HostResource host = allocateSingleNodeHost(cluster, log, containerElement, context); node.setHostResource(host); @@ -621,16 +621,22 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private HostResource allocateSingleNodeHost(ApplicationContainerCluster cluster, DeployLogger logger, Element containerElement, ConfigModelContext context) { DeployState deployState = context.getDeployState(); HostSystem hostSystem = cluster.hostSystem(); - if (deployState.isHosted()) { // request 1 node - ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName())) - .vespaVersion(deployState.getWantedNodeVespaVersion()) - .dockerImageRepo(deployState.getWantedDockerImageRepo()) - .build(); - Capacity capacity = Capacity.fromCount(1, - Optional.empty(), - false, - !deployState.getProperties().isBootstrap()); - return hostSystem.allocateHosts(clusterSpec, capacity, 1, logger).keySet().iterator().next(); + if (deployState.isHosted()) { + Optional<HostResource> singleContentHost = getHostResourceFromContentClusters(cluster, containerElement, context); + if (singleContentHost.isPresent()) { // there is a content cluster; put the container on its first node + return singleContentHost.get(); + } + else { // request 1 node + ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName())) + .vespaVersion(deployState.getWantedNodeVespaVersion()) + .dockerImageRepo(deployState.getWantedDockerImageRepo()) + .build(); + Capacity capacity = Capacity.fromCount(1, + Optional.empty(), + false, + ! deployState.getProperties().isBootstrap()); + return hostSystem.allocateHosts(clusterSpec, capacity, 1, logger).keySet().iterator().next(); + } } else { return hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); } @@ -675,6 +681,43 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return createNodesFromHosts(context.getDeployLogger(), hosts, cluster); } + /** + * This is used in case we are on hosted Vespa and no nodes tag is supplied: + * If there are content clusters this will pick the first host in the first cluster as the container node. + * If there are no content clusters this will return empty (such that the node can be created by the container here). + */ + private Optional<HostResource> getHostResourceFromContentClusters(ApplicationContainerCluster cluster, Element containersElement, ConfigModelContext context) { + Optional<Element> services = servicesRootOf(containersElement); + if ( ! services.isPresent()) + return Optional.empty(); + List<Element> contentServices = XML.getChildren(services.get(), "content"); + if ( contentServices.isEmpty() ) return Optional.empty(); + Element contentNodesElementOrNull = XML.getChild(contentServices.get(0), "nodes"); + + NodesSpecification nodesSpec; + if (contentNodesElementOrNull == null) + nodesSpec = NodesSpecification.nonDedicated(1, context); + else + nodesSpec = NodesSpecification.from(new ModelElement(contentNodesElementOrNull), context); + + Map<HostResource, ClusterMembership> hosts = + StorageGroup.provisionHosts(nodesSpec, + contentServices.get(0).getAttribute("id"), + cluster.getRoot().hostSystem(), + context.getDeployLogger()); + return Optional.of(hosts.keySet().iterator().next()); + } + + /** Returns the services element above the given Element, or empty if there is no services element */ + private Optional<Element> servicesRootOf(Element element) { + Node parent = element.getParentNode(); + if (parent == null) return Optional.empty(); + if ( ! (parent instanceof Element)) return Optional.empty(); + Element parentElement = (Element)parent; + if (parentElement.getTagName().equals("services")) return Optional.of(parentElement); + return servicesRootOf(parentElement); + } + private List<ApplicationContainer> createNodesFromHosts(DeployLogger deployLogger, Map<HostResource, ClusterMembership> hosts, ApplicationContainerCluster cluster) { List<ApplicationContainer> nodes = new ArrayList<>(); for (Map.Entry<HostResource, ClusterMembership> entry : hosts.entrySet()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index fa822df652f..fcaba66ef69 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -16,8 +16,8 @@ import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SearchCluster; -import com.yahoo.vespa.model.search.SearchDefinition; -import com.yahoo.vespa.model.search.SearchDefinitionXMLHandler; +import com.yahoo.vespa.model.search.NamedSchema; +import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchNode; import com.yahoo.vespa.model.search.StreamingSearchCluster; import com.yahoo.vespa.model.search.TransactionLogServer; @@ -136,7 +136,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) { - List<ModelElement> indexedDefs = getIndexedSearchDefinitions(clusterElem); + List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); if (!indexedDefs.isEmpty()) { IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, deployState); isc.setRoutingSelector(clusterElem.childAsString("documents.selection")); @@ -150,7 +150,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot } } - private List<ModelElement> getIndexedSearchDefinitions(ModelElement clusterElem) { + private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) { List<ModelElement> indexedDefs = new ArrayList<>(); ModelElement docElem = clusterElem.child("documents"); if (docElem == null) { @@ -188,28 +188,28 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot } private void addSearchCluster(DeployState deployState, SearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) { - addSearchDefinitions(deployState, documentDefs, cluster); + addSchemas(deployState, documentDefs, cluster); if (queryTimeout != null) { cluster.setQueryTimeout(queryTimeout); } cluster.defaultDocumentsConfig(); - cluster.deriveSearchDefinitions(deployState); + cluster.deriveSchemas(deployState); addCluster(cluster); } - private void addSearchDefinitions(DeployState deployState, List<ModelElement> searchDefs, AbstractSearchCluster sc) { + private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, AbstractSearchCluster sc) { for (ModelElement e : searchDefs) { - SearchDefinitionXMLHandler searchDefinitionXMLHandler = new SearchDefinitionXMLHandler(e); - SearchDefinition searchDefinition = - searchDefinitionXMLHandler.getResponsibleSearchDefinition(deployState.getSearchDefinitions()); + SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e); + NamedSchema searchDefinition = + schemaDefinitionXMLHandler.getResponsibleSearchDefinition(deployState.getSchemas()); if (searchDefinition == null) throw new RuntimeException("Search definition parsing error or file does not exist: '" + - searchDefinitionXMLHandler.getName() + "'"); + schemaDefinitionXMLHandler.getName() + "'"); // TODO: remove explicit building of user configs when the complete content model is built using builders. - sc.getLocalSDS().add(new AbstractSearchCluster.SearchDefinitionSpec(searchDefinition, - UserConfigBuilder.build(e.getXml(), deployState, deployState.getDeployLogger()))); + sc.getLocalSDS().add(new AbstractSearchCluster.SchemaSpec(searchDefinition, + UserConfigBuilder.build(e.getXml(), deployState, deployState.getDeployLogger()))); //need to get the document names from this sdfile sc.addDocumentNames(searchDefinition); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index 8a88e720bed..fe6c6c52e2d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -29,7 +29,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer protected int index; private Double visibilityDelay = 0.0; private List<String> documentNames = new ArrayList<>(); - private List<SearchDefinitionSpec> localSDS = new LinkedList<>(); + private List<SchemaSpec> localSDS = new LinkedList<>(); public AbstractSearchCluster(AbstractConfigProducer parent, String clusterName, int index) { super(parent, "cluster." + clusterName); @@ -38,11 +38,11 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer } public void prepareToDistributeFiles(List<SearchNode> backends) { - for (SearchDefinitionSpec sds : localSDS) + for (SchemaSpec sds : localSDS) sds.getSearchDefinition().getSearch().rankingConstants().sendTo(backends); } - public void addDocumentNames(SearchDefinition searchDefinition) { + public void addDocumentNames(NamedSchema searchDefinition) { String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName(); documentNames.add(dName); } @@ -50,7 +50,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer /** Returns a List with document names used in this search cluster */ public List<String> getDocumentNames() { return documentNames; } - public List<SearchDefinitionSpec> getLocalSDS() { + public List<SchemaSpec> getLocalSDS() { return localSDS; } @@ -107,18 +107,17 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer } } - public static final class SearchDefinitionSpec { + public static final class SchemaSpec { - private final SearchDefinition searchDefinition; + private final NamedSchema searchDefinition; private final UserConfigRepo userConfigRepo; - public SearchDefinitionSpec(SearchDefinition searchDefinition, - UserConfigRepo userConfigRepo) { + public SchemaSpec(NamedSchema searchDefinition, UserConfigRepo userConfigRepo) { this.searchDefinition = searchDefinition; this.userConfigRepo = userConfigRepo; } - public SearchDefinition getSearchDefinition() { + public NamedSchema getSearchDefinition() { return searchDefinition; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 3c45b29b1e2..9746c50450e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -195,9 +195,10 @@ public class IndexedSearchCluster extends SearchCluster routingSelector = sb.toString(); } } + @Override - protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, DeployState deployState) { - for (SearchDefinitionSpec spec : localSearches) { + protected void deriveAllSchemas(List<SchemaSpec> localSearches, DeployState deployState) { + for (SchemaSpec spec : localSearches) { com.yahoo.searchdefinition.Search search = spec.getSearchDefinition().getSearch(); if ( ! (search instanceof DocumentOnlySearch)) { DocumentDatabase db = new DocumentDatabase(this, search.getName(), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java index 860f89792e2..ba81073709e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java @@ -8,7 +8,8 @@ import java.util.Collection; /** * @author Tony Vaagenes */ -public class SearchDefinition { +// TODO: This class is quite pointless +public class NamedSchema { private final Search search; private final String name; @@ -23,15 +24,15 @@ public class SearchDefinition { return name; } - public SearchDefinition(String name, Search search) { + public NamedSchema(String name, Search search) { this.name = name; this.search = search; } //Find search definition from a collection with the name specified - public static SearchDefinition findByName(final String searchDefinitionName, Collection<SearchDefinition> searchDefinitions) { - for (SearchDefinition candidate : searchDefinitions) { - if (candidate.getName().equals(searchDefinitionName) ) + public static NamedSchema findByName(String schemaName, Collection<NamedSchema> schemas) { + for (NamedSchema candidate : schemas) { + if (candidate.getName().equals(schemaName) ) return candidate; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java index 1054253e3f0..b505b5e681c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java @@ -7,15 +7,15 @@ import java.io.Serializable; import java.util.List; /** - * Represents a single searchdefinition file. + * Represents a single schema file. * * @author arnej27959 */ -public class SearchDefinitionXMLHandler implements Serializable { +public class SchemaDefinitionXMLHandler implements Serializable { private String sdName; - public SearchDefinitionXMLHandler(ModelElement elem) { + public SchemaDefinitionXMLHandler(ModelElement elem) { sdName = elem.stringAttribute("name"); if (sdName == null) { sdName = elem.stringAttribute("type"); @@ -24,8 +24,8 @@ public class SearchDefinitionXMLHandler implements Serializable { public String getName() { return sdName; } - public SearchDefinition getResponsibleSearchDefinition(List<SearchDefinition> searchDefinitions) { - return SearchDefinition.findByName( getName(), searchDefinitions ); + public NamedSchema getResponsibleSearchDefinition(List<NamedSchema> schemas) { + return NamedSchema.findByName(getName(), schemas ); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 321564bfec1..0139e949c7a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -41,8 +41,8 @@ public abstract class SearchCluster extends AbstractSearchCluster * Also stores the document names contained in the search * definitions. */ - public void deriveSearchDefinitions(DeployState deployState) { - deriveAllSearchDefinitions(getLocalSDS(), deployState); + public void deriveSchemas(DeployState deployState) { + deriveAllSchemas(getLocalSDS(), deployState); } @Override @@ -132,7 +132,7 @@ public abstract class SearchCluster extends AbstractSearchCluster return false; } - protected abstract void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, DeployState deployState); + protected abstract void deriveAllSchemas(List<SchemaSpec> localSearches, DeployState deployState); public abstract void defaultDocumentsConfig(); public abstract DerivedConfiguration getSdConfig(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index d668adea116..28e7b3eb37a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -91,7 +91,7 @@ public class StreamingSearchCluster extends SearchCluster implements } @Override - protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> local, DeployState deployState) { + protected void deriveAllSchemas(List<SchemaSpec> local, DeployState deployState) { if (local.size() == 1) { deriveSingleSearchDefinition(local.get(0).getSearchDefinition().getSearch(), deployState); } else if (local.size() > 1){ diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 0cfd13f2a9c..2ee0d870d0e 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -197,6 +197,7 @@ TOKEN : < NL: "\n" > | < ANNOTATION: "annotation" > | < ANNOTATIONREFERENCE: "annotationreference" > +| < SCHEMA: "schema" > | < SEARCH: "search" > | < DIVERSITY: "diversity" > | < MIN_GROUPS: "min-groups" > @@ -403,38 +404,38 @@ Search search(DocumentTypeManager docMan, String dir) : Search search; } { - (<NL>)* (search = rootSearch(dir) | search = rootDocument(dir)) + (<NL>)* (search = rootSchema(dir) | search = rootDocument(dir)) { return search; } } /** - * This rule consumes a proper search block. This and rootDocument() are the only rules that should ever consume + * This rule consumes a proper schema block. This and rootDocument() are the only rules that should ever consume * trailing newline tokens. * - * @param dir The directory containing the file being parsed. - * @return The search definition object. + * @param dir the directory containing the file being parsed. + * @return the schema definition object. */ -Search rootSearch(String dir) : +Search rootSchema(String dir) : { String name; Search search; } { - ( <SEARCH> name = identifier() { search = new Search(name, app); + ( ( <SCHEMA> | <SEARCH> ) name = identifier() { search = new Search(name, app); rankProfileRegistry.add(new DefaultRankProfile(search, rankProfileRegistry)); rankProfileRegistry.add(new UnrankedRankProfile(search, rankProfileRegistry));} - lbrace() (rootSearchItem(search) (<NL>)*)* <RBRACE> (<NL>)* <EOF>) + lbrace() (rootSchemaItem(search) (<NL>)*)* <RBRACE> (<NL>)* <EOF>) { return search; } } /** - * Consumes an element of a search block. This and rootSearch() are the only rules that should ever consume + * Consumes an element of a schema block. This and rootSearch() are the only rules that should ever consume * trailing newline tokens. * * @param search The search object to modify. * @return Null. */ -Object rootSearchItem(Search search) : { } +Object rootSchemaItem(Search search) : { } { ( document(search) | documentSummary(search) @@ -452,10 +453,10 @@ Object rootSearchItem(Search search) : { } } /** - * Consumes a search definition that contains only documents to be used for inheritance, etc. + * Consumes a schema definition that contains only documents to be used for inheritance, etc. * - * @param dir The directory containing the file being parsed. - * @return The search definition object. + * @param dir the directory containing the file being parsed. + * @return the schema definition object. */ Search rootDocument(String dir) : { @@ -481,7 +482,7 @@ Object rootDocumentItem(Search search) : { } /** * Consumes a use-document statement. This currently does nothing. * - * @param search The search object to modify. + * @param search the search object to modify. */ void useDocument(Search search) : { } { @@ -491,7 +492,7 @@ void useDocument(Search search) : { } /** * Consumes a document element. The name defaults to the search's name, but may be set. * - * @param search The search object to add content to. + * @param search the search object to add content to. */ void document(Search search) : { @@ -510,7 +511,7 @@ void document(Search search) : /** * Consumes a document element, explicitly named * - * @param search The search object to add content to. + * @param search the search object to add content to. */ void namedDocument(Search search) : { @@ -2617,6 +2618,7 @@ String identifier() : { } | <REFERENCE> | <REMOVEIFZERO> | <RERANKCOUNT> + | <SCHEMA> | <SEARCH> | <SECONDARY> | <SECONDPHASE> diff --git a/config-model/src/main/resources/schema/container.rnc b/config-model/src/main/resources/schema/container.rnc index fde3eb4dd5a..b9cb5e1fae6 100644 --- a/config-model/src/main/resources/schema/container.rnc +++ b/config-model/src/main/resources/schema/container.rnc @@ -27,7 +27,6 @@ AccessControl = element access-control { attribute read { string "true" | string "false" }? & attribute write { string "true" | string "false" }? & element vespa-domain { xsd:NCName }? & # TODO Remove after end of March 2020 - element application { xsd:NCName }? & element exclude { Binding+ }? diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd index e61b64dc0ed..a8c801b1421 100644 --- a/config-model/src/test/examples/simple.sd +++ b/config-model/src/test/examples/simple.sd @@ -3,7 +3,7 @@ # You can get a reasonable configuration by only configuring # a document # ...this has become less and less simple over time actually -search simple { +schema simple { document simple { diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index cb1577417b4..fe82f2406f2 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -20,7 +20,7 @@ import com.yahoo.searchdefinition.DocumentOnlySearch; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; import org.junit.After; import org.junit.Rule; import org.junit.Test; @@ -31,7 +31,6 @@ import java.io.File; import java.io.IOException; import java.io.Reader; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -60,9 +59,9 @@ public class ApplicationDeployTest { public void testVespaModel() throws SAXException, IOException { ApplicationPackageTester tester = ApplicationPackageTester.create(TESTDIR + "app1"); VespaModel model = new VespaModel(tester.app()); - List<SearchDefinition> searchDefinitions = tester.getSearchDefinitions(); - assertEquals(searchDefinitions.size(), 5); - for (SearchDefinition searchDefinition : searchDefinitions) { + List<NamedSchema> schemas = tester.getSchemas(); + assertEquals(schemas.size(), 5); + for (NamedSchema searchDefinition : schemas) { Search s = searchDefinition.getSearch(); switch (s.getName()) { case "music": @@ -72,21 +71,18 @@ public class ApplicationDeployTest { break; case "product": assertTrue(s instanceof DocumentOnlySearch); - assertEquals(s.getDocument().getField("title").getDataType(), DataType.STRING); + assertEquals(DataType.STRING, s.getDocument().getField("title").getDataType()); break; default: fail(); } } - File[] truth = new File[]{new File(TESTSDDIR + "laptop.sd"), - new File(TESTSDDIR + "music.sd"), - new File(TESTSDDIR + "pc.sd"), - new File(TESTSDDIR + "product.sd"), - new File(TESTSDDIR + "sock.sd")}; - Arrays.sort(truth); - List<File> appSdFiles = tester.app().getSearchDefinitionFiles(); - Collections.sort(appSdFiles); - assertEquals(appSdFiles, Arrays.asList(truth)); + assertEquals(Set.of(new File(TESTSDDIR + "laptop.sd"), + new File(TESTSDDIR + "music.sd"), + new File(TESTSDDIR + "pc.sd"), + new File(TESTSDDIR + "product.sd"), + new File(TESTSDDIR + "sock.sd")), + new HashSet<>(tester.app().getSearchDefinitionFiles())); List<FilesApplicationPackage.Component> components = tester.app().getComponents(); assertEquals(1, components.size()); @@ -103,7 +99,7 @@ public class ApplicationDeployTest { // Check that getFilename works ArrayList<String> sdFileNames = new ArrayList<>(); - for (SearchDefinition sd : searchDefinitions) + for (NamedSchema sd : schemas) sdFileNames.add(sd.getFilename()); Collections.sort(sdFileNames); assertEquals("laptop.sd", sdFileNames.get(0)); @@ -190,11 +186,11 @@ public class ApplicationDeployTest { File tmpDir = tmpFolder.getRoot(); IOUtils.copyDirectory(new File(TESTDIR, "app1"), tmpDir); ApplicationPackageTester tester = ApplicationPackageTester.create(tmpDir.getAbsolutePath()); - assertEquals(5, tester.getSearchDefinitions().size()); - File sdDir = new File(tmpDir, "searchdefinitions"); + assertEquals(5, tester.getSchemas().size()); + File sdDir = new File(tmpDir, "schemas"); File sd = new File(sdDir, "testfoo.sd"); IOUtils.writeFile(sd, "search testfoo { document testfoo { field bar type string { } } }", false); - assertEquals(6, tester.getSearchDefinitions().size()); + assertEquals(6, tester.getSchemas().size()); } @Test @@ -300,6 +296,16 @@ public class ApplicationDeployTest { @Test public void testGetJarEntryName() { + JarEntry e = new JarEntry("/schemas/foo.sd"); + assertEquals(ApplicationPackage.getFileName(e), "foo.sd"); + e = new JarEntry("bar"); + assertEquals(ApplicationPackage.getFileName(e), "bar"); + e = new JarEntry(""); + assertEquals(ApplicationPackage.getFileName(e), ""); + } + + @Test + public void testGetJarEntryNameForLegacyPath() { JarEntry e = new JarEntry("/searchdefinitions/foo.sd"); assertEquals(ApplicationPackage.getFileName(e), "foo.sd"); e = new JarEntry("bar"); diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java index 87b6efa83d6..8e7d5aadb36 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.search.SearchDefinition; +import com.yahoo.vespa.model.search.NamedSchema; import java.io.File; import java.io.IOException; @@ -39,8 +39,8 @@ public class ApplicationPackageTester { public FilesApplicationPackage app() { return applicationPackage; } - public List<SearchDefinition> getSearchDefinitions() { - return new DeployState.Builder().applicationPackage(app()).build().getSearchDefinitions(); + public List<NamedSchema> getSchemas() { + return new DeployState.Builder().applicationPackage(app()).build().getSchemas(); } public static ApplicationPackageTester create(String applicationPackageDir) { diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index f303ea9a42d..d215fdbb7a0 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1374,7 +1374,7 @@ public class ModelProvisioningTest { } @Test - public void testNoNodeTagMeans1NodePerCluster() { + public void testNoNodeTagMeans1Node() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + @@ -1389,9 +1389,9 @@ public class ModelProvisioningTest { " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(2); + tester.addHosts(1); VespaModel model = tester.createModel(services, true); - assertEquals(2, model.getRoot().hostSystem().getHosts().size()); + assertEquals(1, model.getRoot().hostSystem().getHosts().size()); assertEquals(1, model.getAdmin().getSlobroks().size()); assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); diff --git a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java index 94602d5201a..127c121197b 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.document.test; import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import org.junit.Test; @@ -17,7 +17,7 @@ import static org.junit.Assert.*; * @author Thomas Gundersen * @author bratseth */ -public class SDDocumentTypeTestCase extends SearchDefinitionTestCase { +public class SDDocumentTypeTestCase extends SchemaTestCase { // Verify that we can register and retrieve fields. @Test diff --git a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java index 7dcbd92655b..b3109c3c2e4 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java @@ -2,7 +2,7 @@ package com.yahoo.document.test; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import org.junit.Test; @@ -11,7 +11,7 @@ import static org.junit.Assert.fail; /** * @author Thomas Gundersen */ -public class SDFieldTestCase extends SearchDefinitionTestCase { +public class SDFieldTestCase extends SchemaTestCase { @Test public void testIdSettingConflict() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java index 846166ae93c..6a40778c9c4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java @@ -18,7 +18,7 @@ import static org.junit.Assert.assertTrue; * * @author bratseth */ -public class ArraysTestCase extends SearchDefinitionTestCase { +public class ArraysTestCase extends SchemaTestCase { @Test public void testArrayImporting() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java index cfd02c22b89..af1e061b7fa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java @@ -16,7 +16,7 @@ import static org.junit.Assert.assertTrue; * * @author Einar M R Rosenvinge */ -public class ArraysWeightedSetsTestCase extends SearchDefinitionTestCase { +public class ArraysWeightedSetsTestCase extends SchemaTestCase { @Test public void testArrayWeightedSetsImporting() throws java.io.IOException, com.yahoo.searchdefinition.parser.ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index 084cbcfdfc0..83cad4cf266 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -25,7 +25,7 @@ import static org.junit.Assert.*; * * @author bratseth */ -public class AttributeSettingsTestCase extends SearchDefinitionTestCase { +public class AttributeSettingsTestCase extends SchemaTestCase { @Rule public final ExpectedException exceptionRule = ExpectedException.none(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java index 8ccb1ed969a..3bb464c5fa5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java @@ -14,7 +14,7 @@ import static org.junit.Assert.assertEquals; * * @author bratseth */ -public class CommentTestCase extends SearchDefinitionTestCase { +public class CommentTestCase extends SchemaTestCase { @Test public void testComments() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java index be3bae05c5b..1c7b3e19663 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertSame; /** * @author Einar M R Rosenvinge */ -public class FieldOfTypeDocumentTestCase extends SearchDefinitionTestCase { +public class FieldOfTypeDocumentTestCase extends SchemaTestCase { @Test public void testDocument() throws IOException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 519828497fe..4453f327bb4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -18,7 +18,7 @@ import static org.junit.Assert.fail; /** * @author bratseth */ -public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionTestCase { +public class IncorrectRankingExpressionFileRefTestCase extends SchemaTestCase { @Test public void testIncorrectRef() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java index c145c0e5634..91ab5e2b5df 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java @@ -11,7 +11,7 @@ import static org.junit.Assert.fail; * * @author bratseth */ -public class IncorrectSummaryTypesTestCase extends SearchDefinitionTestCase { +public class IncorrectSummaryTypesTestCase extends SchemaTestCase { @Test public void testImportingIncorrect() throws ParseException { try { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java index 2cfb542d06b..f992d5ee0ba 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java @@ -15,7 +15,7 @@ import static org.junit.Assert.assertEquals; * * @author bratseth */ -public class IndexSettingsTestCase extends SearchDefinitionTestCase { +public class IndexSettingsTestCase extends SchemaTestCase { @Test public void testStemmingSettings() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java index 21ba3a5e80a..70119ad42f9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java @@ -11,7 +11,7 @@ import static org.junit.Assert.assertNotNull; * * @author frodelu */ -public class IndexingParsingTestCase extends SearchDefinitionTestCase { +public class IndexingParsingTestCase extends SchemaTestCase { @Test public void requireThatIndexingExpressionsCanBeParsed() throws Exception { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java index df9f5778614..5721dbf06e8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java @@ -11,7 +11,7 @@ import java.io.IOException; * * @author bratseth */ -public class MultipleSummariesTestCase extends SearchDefinitionTestCase { +public class MultipleSummariesTestCase extends SchemaTestCase { @Test public void testArrayImporting() throws IOException, ParseException { SearchBuilder.buildFromFile("src/test/examples/multiplesummaries.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java index d2360453976..47b6905c677 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java @@ -14,7 +14,7 @@ import static org.junit.Assert.fail; * * @author Lars Christian Jensen */ -public class NameFieldCheckTestCase extends SearchDefinitionTestCase { +public class NameFieldCheckTestCase extends SchemaTestCase { @Test public void testNameField() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java index 5ac37bf0a3a..64527e7f7a5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java @@ -13,7 +13,7 @@ import static org.junit.Assert.assertTrue; * * @author bratseth */ -public class OutsideTestCase extends SearchDefinitionTestCase { +public class OutsideTestCase extends SchemaTestCase { @Test public void testOutsideIndex() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index 5ba508e3ef3..e2f2c1fd407 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -34,7 +34,7 @@ import static org.junit.Assert.assertTrue; * * @author bratseth */ -public class RankProfileTestCase extends SearchDefinitionTestCase { +public class RankProfileTestCase extends SchemaTestCase { @Test public void testRankProfileInheritance() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java index f4666f7fb3b..3fe2861de0c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java @@ -13,7 +13,7 @@ import static org.junit.Assert.assertEquals; /** * @author bratseth */ -public class RankPropertiesTestCase extends SearchDefinitionTestCase { +public class RankPropertiesTestCase extends SchemaTestCase { @Test public void testRankPropertyInheritance() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index 51508414205..d84d967a184 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -17,7 +17,7 @@ import static org.junit.Assert.*; /** * @author bratseth */ -public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase { +public class RankingExpressionConstantsTestCase extends SchemaTestCase { @Test public void testConstants() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index 58e62353e5c..e0679eb5175 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -17,7 +17,7 @@ import static org.junit.Assert.assertTrue; /** * @author bratseth */ -public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase { +public class RankingExpressionInliningTestCase extends SchemaTestCase { @Test public void testFunctionInliningPreserveArithmeticOrdering() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 3d842be129f..5c1134f928c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; /** * @author lesters */ -public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase { +public class RankingExpressionShadowingTestCase extends SchemaTestCase { @Test public void testBasicFunctionShadowing() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index c5027af2a0c..c1fe5e42dfa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -15,7 +15,7 @@ import static org.junit.Assert.fail; /** * @author bratseth */ -public class RankingExpressionValidationTestCase extends SearchDefinitionTestCase { +public class RankingExpressionValidationTestCase extends SchemaTestCase { @Test public void testInvalidExpressionProducesException() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java index 5a5fc1cc312..1a939d71937 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java @@ -11,7 +11,7 @@ import static org.junit.Assert.assertNotNull; /** * @author bratseth */ -public class ReservedWordsAsFieldNamesTestCase extends SearchDefinitionTestCase { +public class ReservedWordsAsFieldNamesTestCase extends SchemaTestCase { @Test public void testIt() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java index fd4bb393c49..0ae39b7f8b6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java @@ -15,7 +15,7 @@ import static org.junit.Assert.*; * * @author hmusum */ -public class SearchDefinitionsParsingTestCase extends SearchDefinitionTestCase { +public class SchemaParsingTestCase extends SchemaTestCase { @Test public void requireThatIndexingExpressionsCanBeParsed() throws Exception { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java index ba6da8792fa..7f3ea7d14bc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java @@ -10,7 +10,7 @@ import java.io.IOException; import static helpers.CompareConfigTestHelper.assertSerializedConfigEquals; import static helpers.CompareConfigTestHelper.assertSerializedConfigFileEquals; -public abstract class SearchDefinitionTestCase { +public abstract class SchemaTestCase { protected static void assertConfigFile(String filename, String cfg) throws IOException { assertSerializedConfigFileEquals(filename, cfg); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java index 66ff1877994..018703153ac 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; -import com.yahoo.document.Document; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.RankType; @@ -31,7 +30,7 @@ import static org.junit.Assert.fail; * * @author bratseth */ -public class SearchImporterTestCase extends SearchDefinitionTestCase { +public class SearchImporterTestCase extends SchemaTestCase { @Test @SuppressWarnings("deprecation") diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java index 9b27d338ced..e5b8ec85d75 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java @@ -16,7 +16,7 @@ import static org.junit.Assert.assertNull; * * @author bratseth */ -public class StemmingSettingTestCase extends SearchDefinitionTestCase { +public class StemmingSettingTestCase extends SchemaTestCase { @Test public void testStemmingSettings() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java index 001ad64e2da..77df5b391dc 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java @@ -14,7 +14,7 @@ import static org.junit.Assert.fail; * * @author bratseth */ -public class StructTestCase extends SearchDefinitionTestCase { +public class StructTestCase extends SchemaTestCase { @Test public void testStruct() throws IOException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index 7e0eb01f611..a345cabe909 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -7,7 +7,7 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.vespa.configmodel.producers.DocumentManager; @@ -21,7 +21,7 @@ import java.io.IOException; * * @author bratseth */ -public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase { +public abstract class AbstractExportingTestCase extends SchemaTestCase { private static final String tempDir = "temp/"; private static final String searchDefRoot = "src/test/derived/"; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java index 79ec3027c20..80a92a5b5ec 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -20,7 +20,7 @@ import static org.junit.Assert.assertFalse; * * @author bratseth */ -public class AttributeListTestCase extends SearchDefinitionTestCase { +public class AttributeListTestCase extends SchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java index b47a268d95a..07762fc6937 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -16,7 +16,7 @@ import static org.junit.Assert.assertEquals; * * @author vegardh */ -public class CasingTestCase extends SearchDefinitionTestCase { +public class CasingTestCase extends SchemaTestCase { @Test public void testCasing() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java index 1209da6d64b..8b09a4efd57 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.document.DataType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import org.junit.Test; import java.util.List; @@ -16,7 +16,7 @@ import static org.junit.Assert.assertEquals; * * @author bratseth */ -public class DeriverTestCase extends SearchDefinitionTestCase { +public class DeriverTestCase extends SchemaTestCase { @Test public void testDeriveDocManager() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index aaac1631722..47862a2611b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; @@ -17,7 +17,7 @@ import org.junit.Test; * * @author bratseth */ -public class EmptyRankProfileTestCase extends SearchDefinitionTestCase { +public class EmptyRankProfileTestCase extends SchemaTestCase { @Test public void testDeriving() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java index c8ba5168e1c..69c247b94d4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.RankType; import org.junit.Test; @@ -14,7 +14,7 @@ import static org.junit.Assert.*; * * @author geirst */ -public class NativeRankTypeDefinitionsTestCase extends SearchDefinitionTestCase { +public class NativeRankTypeDefinitionsTestCase extends SchemaTestCase { @Test public void testTables() { assertEquals(NativeTable.Type.FIRST_OCCURRENCE.getName(), "firstOccurrenceTable"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java index b770024ebf1..0c677456a87 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.searchdefinition.DocumentReference; import com.yahoo.searchdefinition.DocumentReferences; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.TemporarySDField; @@ -26,47 +26,47 @@ import static org.junit.Assert.assertEquals; * @author bratseth * @author bjorncs */ -public class SearchOrdererTestCase extends SearchDefinitionTestCase { +public class SearchOrdererTestCase extends SchemaTestCase { - private static Map<String, Search> createSearchDefinitions() { - Map<String, Search> searchDefinitions = new HashMap<>(); + private static Map<String, Search> createSchemas() { + Map<String, Search> schemas = new HashMap<>(); - Search grandParent = createSearchDefinition("grandParent", searchDefinitions); + Search grandParent = createSchema("grandParent", schemas); - Search mother = createSearchDefinition("mother", searchDefinitions); + Search mother = createSchema("mother", schemas); inherit(mother, grandParent); - Search father = createSearchDefinition("father", searchDefinitions); + Search father = createSchema("father", schemas); inherit(father, grandParent); createDocumentReference(father, mother, "wife_ref"); - Search daugther = createSearchDefinition("daughter", searchDefinitions); + Search daugther = createSchema("daughter", schemas); inherit(daugther, father); inherit(daugther, mother); - Search son = createSearchDefinition("son", searchDefinitions); + Search son = createSchema("son", schemas); inherit(son, father); inherit(son, mother); - Search product = createSearchDefinition("product", searchDefinitions); + Search product = createSchema("product", schemas); - Search pc = createSearchDefinition("pc", searchDefinitions); + Search pc = createSchema("pc", schemas); inherit(pc, product); - Search pcAccessory = createSearchDefinition("accessory-pc", searchDefinitions); + Search pcAccessory = createSchema("accessory-pc", schemas); inherit(pcAccessory, product); createDocumentReference(pcAccessory, pc, "pc_ref"); - createSearchDefinition("alone", searchDefinitions); + createSchema("alone", schemas); - return searchDefinitions; + return schemas; } - private static Search createSearchDefinition(String name, Map<String, Search> searchDefinitions) { + private static Search createSchema(String name, Map<String, Search> schemas) { Search search = new Search(name, null); SDDocumentType document = new SDDocumentType(name); document.setDocumentReferences(new DocumentReferences(emptyMap())); search.addDocument(document); - searchDefinitions.put(search.getName(), search); + schemas.put(search.getName(), search); return search; } @@ -75,13 +75,13 @@ public class SearchOrdererTestCase extends SearchDefinitionTestCase { } private static void assertOrder(List<String> expectedSearchOrder, List<String> inputNames) { - Map<String, Search> searchDefinitions = createSearchDefinitions(); - List<Search> inputSearchDefinitions = inputNames.stream() - .map(searchDefinitions::get) + Map<String, Search> schemas = createSchemas(); + List<Search> inputSchemas = inputNames.stream() + .map(schemas::get) .map(Objects::requireNonNull) .collect(toList()); List<String> actualSearchOrder = new SearchOrderer() - .order(inputSearchDefinitions) + .order(inputSchemas) .stream() .map(Search::getName) .collect(toList()); @@ -104,31 +104,37 @@ public class SearchOrdererTestCase extends SearchDefinitionTestCase { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("grandParent", "mother", "father", "daughter", "son", "product", "pc", "alone")); } + @Test public void testOneLevelReordering() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("grandParent", "daughter", "son", "mother", "father", "pc", "product", "alone")); } + @Test public void testMultiLevelReordering() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("daughter", "son", "mother", "father", "grandParent", "pc", "product", "alone")); } + @Test public void testAloneIsKeptInPlaceWithMultiLevelReordering() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("alone", "daughter", "son", "mother", "father", "grandParent", "pc", "product")); } + @Test public void testPartialMultiLevelReordering() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("daughter", "grandParent", "mother", "son", "father", "product", "pc", "alone")); } + @Test public void testMultilevelReorderingAccrossHierarchies() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"), Arrays.asList("daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone")); } + @Test public void referees_are_ordered_before_referrer() { assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "accessory-pc", "son"), diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index f85f9994e04..07d7405b1db 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -25,7 +25,7 @@ import static org.junit.Assert.assertTrue; * * @author bratseth */ -public class SummaryMapTestCase extends SearchDefinitionTestCase { +public class SummaryMapTestCase extends SchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/simple.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java index b82620b1cf5..afbc9f52f6b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -20,7 +20,7 @@ import static org.junit.Assert.assertNull; * * @author bratseth */ -public class SummaryTestCase extends SearchDefinitionTestCase { +public class SummaryTestCase extends SchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index 24575df8c91..c03e915aa8b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.document.DataType; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.processing.Processing; @@ -20,7 +20,7 @@ import static org.junit.Assert.assertFalse; * * @author bratseth */ -public class TypeConversionTestCase extends SearchDefinitionTestCase { +public class TypeConversionTestCase extends SchemaTestCase { /** Tests that exact-string stuff is not spilled over to the default index */ @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java index 7236ccbc117..35ce4dff730 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -17,7 +17,7 @@ import static org.junit.Assert.assertFalse; * @author vegardh * */ -public class AttributesExactMatchTestCase extends SearchDefinitionTestCase { +public class AttributesExactMatchTestCase extends SchemaTestCase { @Test public void testAttributesExactMatch() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java index ac3ba1d98d9..9a4357c5d65 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; @@ -18,7 +18,7 @@ import static org.junit.Assert.fail; /** * @author Mathias Mølster Lidal */ -public class BoldingTestCase extends SearchDefinitionTestCase { +public class BoldingTestCase extends SchemaTestCase { @Test public void testBoldingNonString() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java index 620cee49ac4..809ccdb3a3a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; @@ -18,7 +18,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -public class ImplicitSearchFieldsTestCase extends SearchDefinitionTestCase { +public class ImplicitSearchFieldsTestCase extends SchemaTestCase { @Test public void testRequireThatExtraFieldsAreIncluded() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java index f2d81414b5a..c9ea57c5b9b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.document.*; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; @@ -13,7 +13,7 @@ import org.junit.Test; import java.io.IOException; import static org.junit.Assert.*; -public class ImplicitStructTypesTestCase extends SearchDefinitionTestCase { +public class ImplicitStructTypesTestCase extends SchemaTestCase { @Test public void testRequireThatImplicitStructsAreCreated() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java index 7acbf67772a..ae00e4f3079 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.documentmodel.DocumentSummary; import org.junit.Test; @@ -13,7 +13,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -public class ImplicitSummaryFieldsTestCase extends SearchDefinitionTestCase { +public class ImplicitSummaryFieldsTestCase extends SchemaTestCase { @Test public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java index d313c2391fd..7863c544b60 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java @@ -7,7 +7,7 @@ import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; @@ -28,7 +28,7 @@ import static org.junit.Assert.assertEquals; /** * @author Simon Thoresen Hult */ -public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { +public class IndexingScriptRewriterTestCase extends SchemaTestCase { @Test public void testSetLanguageRewriting() { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java index cac50354dc2..fcf1c39f5b4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -20,7 +20,7 @@ import static org.junit.Assert.assertTrue; /** * @author baldersheim */ -public class IntegerIndex2AttributeTestCase extends SearchDefinitionTestCase { +public class IntegerIndex2AttributeTestCase extends SchemaTestCase { @Test public void testIntegerIndex2Attribute() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java index 385d1df90ad..c792d3bf40b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; @@ -21,7 +21,7 @@ import static org.junit.Assert.fail; /** * @author bratseth */ -public class NGramTestCase extends SearchDefinitionTestCase { +public class NGramTestCase extends SchemaTestCase { @Test public void testNGram() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java index 0d6334a5223..4ab56f809c9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -14,7 +14,7 @@ import java.io.IOException; * @author vegardh * */ -public class RankModifierTestCase extends SearchDefinitionTestCase { +public class RankModifierTestCase extends SchemaTestCase { @Test public void testLiteral() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java index d740884d3e5..502fc4472bc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java @@ -7,7 +7,7 @@ import com.yahoo.searchdefinition.RankProfile.RankProperty; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -16,7 +16,7 @@ import java.util.List; import static org.junit.Assert.fail; -public class RankPropertyVariablesTestCase extends SearchDefinitionTestCase { +public class RankPropertyVariablesTestCase extends SchemaTestCase { @Test public void testRankPropVariables() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 96fa59a77cc..b3eda9b7e13 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -19,7 +19,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; -public class RankingExpressionsTestCase extends SearchDefinitionTestCase { +public class RankingExpressionsTestCase extends SchemaTestCase { @Test public void testFunctions() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java index dbcfc8c202d..ca8744a07bb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -6,7 +6,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; @@ -16,7 +16,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTestCase { +public class SummaryFieldsMustHaveValidSourceTestCase extends SchemaTestCase { @Test public void requireThatInvalidSourceIsCaught() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index f90320ad686..8308b638497 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -13,7 +13,7 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; @@ -25,7 +25,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -public class TensorTransformTestCase extends SearchDefinitionTestCase { +public class TensorTransformTestCase extends SchemaTestCase { @Test public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java index ef6bc57223d..957b5c55889 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -12,7 +12,7 @@ import java.io.IOException; import static org.junit.Assert.assertNotNull; /** @author bratseth */ -public class WeightedSetSummaryToTestCase extends SearchDefinitionTestCase { +public class WeightedSetSummaryToTestCase extends SchemaTestCase { @Test public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java index d0ee0523489..9144ad411b2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; @@ -12,7 +12,7 @@ import java.io.IOException; /** * Utility functions for testing generated configs for reference/imported fields. */ -public abstract class AbstractReferenceFieldTestCase extends SearchDefinitionTestCase { +public abstract class AbstractReferenceFieldTestCase extends SchemaTestCase { private static String TEST_FOLDER = "src/test/configmodel/types/references/"; diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java index f5034d0530b..91152648b10 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.SearchDefinitionTestCase; +import com.yahoo.searchdefinition.SchemaTestCase; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; @@ -13,7 +13,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -public class DocumentModelBuilderTestCase extends SearchDefinitionTestCase { +public class DocumentModelBuilderTestCase extends SchemaTestCase { @Test public void testDocumentManagerSimple() throws IOException, ParseException { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index cd67e432b8f..4b23a7e2e71 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -52,7 +52,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { @Before public void setup() { - sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); } @Test @@ -457,7 +457,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { private VespaModel createVespaModel(String servicesXml, boolean isHosted) throws IOException, SAXException { ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() .withServices(servicesXml) - .withSearchDefinitions(sds) + .withSchemas(sds) .build(); // Need to create VespaModel to make deploy properties have effect DeployLogger logger = new DeployLoggerStub(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java index f7b6d7b25e1..09e4b377085 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java @@ -65,7 +65,7 @@ public class ValidationTester { Instant now = LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant(); ApplicationPackage newApp = new MockApplicationPackage.Builder() .withServices(services) - .withSearchDefinitions(ImmutableList.of(MUSIC_SEARCHDEFINITION, BOOK_SEARCHDEFINITION)) + .withSchemas(ImmutableList.of(MUSIC_SEARCHDEFINITION, BOOK_SEARCHDEFINITION)) .withValidationOverrides(validationOverrides) .build(); VespaModelCreatorWithMockPkg newModelCreator = new VespaModelCreatorWithMockPkg(newApp); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index bb209e58e24..81b7f870a10 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -201,7 +201,7 @@ public class ConfigValueChangeValidatorTest { " </engine>\n" + " </content>\n" + "</services>", - createSearchDefinitions(docTypes) + createSchemas(docTypes) ).create(); } @@ -213,7 +213,7 @@ public class ConfigValueChangeValidatorTest { "</documents>"; } - private static List<String> createSearchDefinitions(List<String> docTypes) { + private static List<String> createSchemas(List<String> docTypes) { return docTypes.stream() .map(type -> "search " + type + " { document " + type + " { } }") .collect(Collectors.toList()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java index 43ad1bc0e8a..80127ac6854 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; -import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; import org.junit.Test; import java.time.Instant; @@ -41,7 +41,7 @@ public class IndexedSearchClusterChangeValidatorTest { public static VespaModel newOneDocModel(String sdContent) { return new ApplicationPackageBuilder(). addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")). - addSearchDefinition(new SearchDefinitionBuilder(). + addSchemas(new SchemaBuilder(). name("d1").content(sdContent).build()).buildCreator().create(); } @@ -52,9 +52,9 @@ public class IndexedSearchClusterChangeValidatorTest { public static VespaModel newTwoDocModel(String d1Content, String d2Content) { return new ApplicationPackageBuilder(). addCluster(new ContentClusterBuilder().name("foo").docTypes("d1", "d2")). - addSearchDefinition(new SearchDefinitionBuilder(). + addSchemas(new SchemaBuilder(). name("d1").content(d1Content).build()). - addSearchDefinition(new SearchDefinitionBuilder(). + addSchemas(new SchemaBuilder(). name("d2").content(d2Content).build()). buildCreator().create(); } @@ -67,9 +67,9 @@ public class IndexedSearchClusterChangeValidatorTest { return new ApplicationPackageBuilder(). addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")). addCluster(new ContentClusterBuilder().name("bar").docTypes("d2")). - addSearchDefinition(new SearchDefinitionBuilder(). + addSchemas(new SchemaBuilder(). name("d1").content(d1Content).build()). - addSearchDefinition(new SearchDefinitionBuilder(). + addSchemas(new SchemaBuilder(). name("d2").content(d2Content).build()). buildCreator().create(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java index c5fee3efa99..8edbc964bfb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java @@ -8,7 +8,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.DocType; -import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; import org.junit.Test; import java.time.Instant; @@ -40,7 +40,7 @@ public class StreamingSearchClusterChangeValidatorTest { public static VespaModel createOneDocModel(String sdContent) { return new ApplicationPackageBuilder() .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1")))) - .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(sdContent).build()) + .addSchemas(new SchemaBuilder().name("d1").content(sdContent).build()) .buildCreator().create(); } @@ -51,8 +51,8 @@ public class StreamingSearchClusterChangeValidatorTest { public static VespaModel createTwoDocModel(String d1Content, String d2Content) { return new ApplicationPackageBuilder() .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1"), DocType.streaming("d2")))) - .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(d1Content).build()) - .addSearchDefinition(new SearchDefinitionBuilder().name("d2").content(d2Content).build()) + .addSchemas(new SchemaBuilder().name("d1").content(d1Content).build()) + .addSchemas(new SchemaBuilder().name("d2").content(d2Content).build()) .buildCreator().create(); } @@ -64,8 +64,8 @@ public class StreamingSearchClusterChangeValidatorTest { return new ApplicationPackageBuilder() .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1")))) .addCluster(new ContentClusterBuilder().name("bar").docTypes(Arrays.asList(DocType.streaming("d2")))) - .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(d1Content).build()) - .addSearchDefinition(new SearchDefinitionBuilder().name("d2").content(d2Content).build()) + .addSchemas(new SchemaBuilder().name("d1").content(d1Content).build()) + .addSchemas(new SchemaBuilder().name("d2").content(d2Content).build()) .buildCreator().create(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java index 3827da08679..20ff9afd530 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java @@ -6,7 +6,7 @@ import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeActi import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; -import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; import com.yahoo.vespa.model.search.DocumentDatabase; import java.util.Arrays; @@ -34,7 +34,7 @@ public abstract class ContentClusterFixture { private static ContentCluster createCluster(String sdContent) throws Exception { return new ContentClusterBuilder().build( ContentClusterUtils.createMockRoot( - Arrays.asList(new SearchDefinitionBuilder().content(sdContent).build()))); + Arrays.asList(new SchemaBuilder().content(sdContent).build()))); } protected DocumentDatabase currentDb() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 31a2a6496d7..b4fe4175707 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -236,8 +236,8 @@ public class ContentBuilderTest extends DomBuilderTest { } @Test - public void requireThatContentStreamingHandlesMultipleSearchDefinitions() { - final String musicClusterId = "music-cluster-id"; + public void requireThatContentStreamingHandlesMultipleSchemas() { + String musicClusterId = "music-cluster-id"; ContentCluster cluster = createContentWithBooksToo( "<content version='1.0' id='" + musicClusterId + "'>" + @@ -825,8 +825,8 @@ public class ContentBuilderTest extends DomBuilderTest { VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withHosts(getHosts()) .withServices(combined) - .withSearchDefinitions(Arrays.asList(MockApplicationPackage.MUSIC_SEARCHDEFINITION, - MockApplicationPackage.BOOK_SEARCHDEFINITION)) + .withSchemas(Arrays.asList(MockApplicationPackage.MUSIC_SEARCHDEFINITION, + MockApplicationPackage.BOOK_SEARCHDEFINITION)) .build()) .create(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index ffcafecda52..b1e63628852 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -64,9 +64,7 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( " <http>", " <filtering>", - " <access-control domain='my-domain'>", - " <application>my-app</application>", - " </access-control>", + " <access-control domain='my-domain'/>", " </filtering>", " </http>"); @@ -75,7 +73,6 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { AccessControl accessControl = http.getAccessControl().get(); assertEquals("Wrong domain.", "my-domain", accessControl.domain); - assertEquals("Wrong application.", "my-app", accessControl.applicationId); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index 1f0b0188681..ac915d8a939 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -190,7 +190,7 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { private VespaModel getVespaModelWithMusic(String hosts, String services) { - return new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create(); + return new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSchemas("music")).create(); } private String hostsXml() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 305bcc1d7d5..b08cc92d20c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -26,7 +26,7 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; -import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; import com.yahoo.vespa.model.routing.DocumentProtocol; import com.yahoo.vespa.model.routing.Routing; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; @@ -218,7 +218,7 @@ public class ContentClusterTest extends ContentBaseTest { "\n" + "</services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); VespaModel model = (new VespaModelCreatorWithMockPkg(null, xml, sds)).create(); assertEquals(2, model.getContentClusters().get("bar").getDocumentDefinitions().size()); ContainerCluster cluster = model.getAdmin().getClusterControllers(); @@ -259,7 +259,7 @@ public class ContentClusterTest extends ContentBaseTest { " </services>"; DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(properties); - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1"); return (new VespaModelCreatorWithMockPkg(null, services, sds)).create(deployStateBuilder); } @Test @@ -301,7 +301,7 @@ public class ContentClusterTest extends ContentBaseTest { "\n" + "</services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); assertTrue(model.getContentClusters().get("bar").getPersistence() instanceof ProtonEngine.Factory); @@ -340,7 +340,7 @@ public class ContentClusterTest extends ContentBaseTest { " </content>\n" + "</services>\n"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); try{ new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); assertTrue("Deploying without redundancy should fail", false); @@ -697,7 +697,7 @@ public class ContentClusterTest extends ContentBaseTest { "</services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); { @@ -818,7 +818,7 @@ public class ContentClusterTest extends ContentBaseTest { " </group>" + "</content>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("true"); + List<String> sds = ApplicationPackageUtils.generateSchemas("true"); new VespaModelCreatorWithMockPkg(null, xml, sds).create(); } @@ -865,7 +865,7 @@ public class ContentClusterTest extends ContentBaseTest { " </group>" + "</content>" + "</services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("bunnies", "hares", "rabbits"); + List<String> sds = ApplicationPackageUtils.generateSchemas("bunnies", "hares", "rabbits"); return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); } @@ -912,8 +912,8 @@ public class ContentClusterTest extends ContentBaseTest { DeployState.Builder deployStateBuilder = new DeployState.Builder() .zone(zone) .properties(new TestProperties().setHostedVespa(true)); - List<String> searchDefinitions = SearchDefinitionBuilder.createSearchDefinitions("test"); - MockRoot root = ContentClusterUtils.createMockRoot(searchDefinitions, deployStateBuilder); + List<String> schemas = SchemaBuilder.createSchemas("test"); + MockRoot root = ContentClusterUtils.createMockRoot(schemas, deployStateBuilder); ContentCluster cluster = ContentClusterUtils.createCluster(clusterXml, root); root.freezeModelTopology(); cluster.validate(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 98fa179b219..3415044b088 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.DocType; -import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; import org.junit.Test; import java.util.ArrayList; @@ -17,7 +17,7 @@ import java.util.List; import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; -import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions; +import static com.yahoo.vespa.model.content.utils.SchemaBuilder.createSchemas; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -38,7 +38,7 @@ public class ContentSearchClusterTest { private static ContentCluster createClusterWithTwoDocumentType() throws Exception { return createCluster(new ContentClusterBuilder().docTypes("foo", "bar").getXml(), - createSearchDefinitions("foo", "bar")); + createSchemas("foo", "bar")); } private static ContentCluster createClusterWithGlobalType() throws Exception { @@ -55,7 +55,7 @@ public class ContentSearchClusterTest { "<node distribution-key='1' hostalias='mockhost'/>", "</group>")); String clusterXml = builder.getXml(); - return createCluster(clusterXml, createSearchDefinitions(docTypes)); + return createCluster(clusterXml, createSchemas(docTypes)); } private static ContentClusterBuilder createClusterBuilderWithGlobalType() { @@ -127,18 +127,19 @@ public class ContentSearchClusterTest { } private static ContentCluster createClusterWithThreeDocumentTypes() throws Exception { - List<String> searchDefinitions = new ArrayList<>(); - searchDefinitions.add(new SearchDefinitionBuilder().name("a") - .content(joinLines("field ref_to_b type reference<b> { indexing: attribute }", - "field ref_to_c type reference<c> { indexing: attribute }")).build()); - searchDefinitions.add(new SearchDefinitionBuilder().name("b") - .content("field ref_to_c type reference<c> { indexing: attribute }").build()); - searchDefinitions.add(new SearchDefinitionBuilder().name("c").build()); - return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList( - DocType.index("a"), - DocType.indexGlobal("b"), - DocType.indexGlobal("c"))).getXml(), - searchDefinitions); + List<String> schemas = new ArrayList<>(); + schemas.add(new SchemaBuilder().name("a") + .content(joinLines("field ref_to_b type reference<b> { indexing: attribute }", + "field ref_to_c type reference<c> { indexing: attribute }")) + .build()); + schemas.add(new SchemaBuilder().name("b") + .content("field ref_to_c type reference<c> { indexing: attribute }") + .build()); + schemas.add(new SchemaBuilder().name("c").build()); + return createCluster(new ContentClusterBuilder().docTypes(List.of(DocType.index("a"), + DocType.indexGlobal("b"), + DocType.indexGlobal("c"))).getXml(), + schemas); } private static BucketspacesConfig getBucketspacesConfig(ContentCluster cluster) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index f36ef6c3ba3..365dc74274d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -24,7 +24,7 @@ public class DistributorTest { ContentCluster parseCluster(String xml) { try { - List<String> searchDefs = ApplicationPackageUtils.generateSearchDefinitions("music", "movies", "bunnies"); + List<String> searchDefs = ApplicationPackageUtils.generateSchemas("music", "movies", "bunnies"); MockRoot root = ContentClusterUtils.createMockRoot(searchDefs); return ContentClusterUtils.createCluster(xml, root); } catch (Exception e) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java index 992edf6b1bb..51d0afc1f93 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java @@ -50,7 +50,7 @@ public class GenericConfigTest { @Before public void getVespaModel() { - model = (new VespaModelCreatorWithMockPkg(ContentBaseTest.getHosts(), servicesXml(), ApplicationPackageUtils.generateSearchDefinitions("type1"))).create(); + model = (new VespaModelCreatorWithMockPkg(ContentBaseTest.getHosts(), servicesXml(), ApplicationPackageUtils.generateSchemas("type1"))).create(); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index ecf8f100288..504c3d9ba9c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -112,12 +112,12 @@ public class IndexedTest extends ContentBaseTest { } private VespaModelCreatorWithMockPkg getIndexedVespaModelCreator() { - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2", "type3"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2", "type3"); return new VespaModelCreatorWithMockPkg(getHosts(), createProtonIndexedVespaServices(Arrays.asList("type1", "type2", "type3")), sds); } private VespaModel getStreamingVespaModel() { - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1"); return new VespaModelCreatorWithMockPkg(getHosts(), createProtonStreamingVespaServices(Arrays.asList("type1")), sds).create(); } @@ -229,7 +229,7 @@ public class IndexedTest extends ContentBaseTest { " </content>\n" + " </services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("docstorebench"); + List<String> sds = ApplicationPackageUtils.generateSchemas("docstorebench"); VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create(); ProtonConfig.Builder pb = new ProtonConfig.Builder(); model.getConfig(pb, "docstore/search/cluster.docstore/0"); @@ -252,7 +252,7 @@ public class IndexedTest extends ContentBaseTest { " </content>" + "</services>"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("index_me", "store_me"); + List<String> sds = ApplicationPackageUtils.generateSchemas("index_me", "store_me"); VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create(); ProtonConfig.Builder pb = new ProtonConfig.Builder(); model.getConfig(pb, "docstore/search/cluster.docstore/0"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index 55d070d7247..177b86c953e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -164,7 +164,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { " </container>\n" + "</services>\n"; - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("music", "title", "artist"); + List<String> sds = ApplicationPackageUtils.generateSchemas("music", "title", "artist"); VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create(); assertIndexing(model, new DocprocClusterSpec("dokprok")); @@ -448,7 +448,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { } private VespaModel getIndexedSearchVespaModel(String xml) { - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("music", "album", "artist"); + List<String> sds = ApplicationPackageUtils.generateSchemas("music", "album", "artist"); return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java index c0ddd49069d..e099476ebb6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java @@ -49,7 +49,7 @@ public class StorageContentTest extends ContentBaseTest { } private VespaModel getStorageVespaModel(String cluster1docs, String cluster2docs) { - List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2", "type3"); + List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2", "type3"); return new VespaModelCreatorWithMockPkg(getHosts(), createStorageVespaServices(cluster1docs, cluster2docs), sds).create(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 62221c206fd..852844fe451 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -176,7 +176,7 @@ public class ClusterTest { " </tuning>", " </content>", "</services>")) - .withSearchDefinitions(ApplicationPackageUtils.generateSearchDefinition("my_document")) + .withSchemas(ApplicationPackageUtils.generateSearchDefinition("my_document")) .build(); List<Content> contents = new TestDriver().buildModel(app).getConfigModels(Content.class); assertEquals(1, contents.size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java index 1b7f3e9b14d..5fc213fad1d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java @@ -14,7 +14,7 @@ import java.util.List; public class ApplicationPackageBuilder { private List<ContentClusterBuilder> contentClusters = new ArrayList<>(); - private List<String> searchDefinitions = new ArrayList<>(); + private List<String> schemas = new ArrayList<>(); public ApplicationPackageBuilder() { } @@ -24,13 +24,13 @@ public class ApplicationPackageBuilder { return this; } - public ApplicationPackageBuilder addSearchDefinition(String searchDefinition) { - searchDefinitions.add(searchDefinition); + public ApplicationPackageBuilder addSchemas(String schemas) { + this.schemas.add(schemas); return this; } public VespaModelCreatorWithMockPkg buildCreator() { - return new VespaModelCreatorWithMockPkg(null, getServices(), searchDefinitions); + return new VespaModelCreatorWithMockPkg(null, getServices(), schemas); } private String getServices() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index 62d2bc51830..db9153fcf23 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -30,31 +30,31 @@ import java.util.Optional; public class ContentClusterUtils { public static MockRoot createMockRoot(String[] hosts) { - return createMockRoot(hosts, SearchDefinitionBuilder.createSearchDefinitions("test")); + return createMockRoot(hosts, SchemaBuilder.createSchemas("test")); } - private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions) { - return createMockRoot(provisioner, searchDefinitions, new DeployState.Builder()); + private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> schemas) { + return createMockRoot(provisioner, schemas, new DeployState.Builder()); } - private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions, DeployState.Builder deployStateBuilder) { - ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSearchDefinitions(searchDefinitions).build(); + private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> schemas, DeployState.Builder deployStateBuilder) { + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSchemas(schemas).build(); DeployState deployState = deployStateBuilder.applicationPackage(applicationPackage) .modelHostProvisioner(provisioner) .build(); return new MockRoot("", deployState); } - public static MockRoot createMockRoot(String[] hosts, List<String> searchDefinitions) { - return createMockRoot(new InMemoryProvisioner(true, hosts), searchDefinitions); + public static MockRoot createMockRoot(String[] hosts, List<String> schemas) { + return createMockRoot(new InMemoryProvisioner(true, hosts), schemas); } - public static MockRoot createMockRoot(List<String> searchDefinitions) { - return createMockRoot(new SingleNodeProvisioner(), searchDefinitions); + public static MockRoot createMockRoot(List<String> schemas) { + return createMockRoot(new SingleNodeProvisioner(), schemas); } - public static MockRoot createMockRoot(List<String> searchDefinitions, DeployState.Builder deployStateBuilder) { - return createMockRoot(new SingleNodeProvisioner(), searchDefinitions, deployStateBuilder); + public static MockRoot createMockRoot(List<String> schemas, DeployState.Builder deployStateBuilder) { + return createMockRoot(new SingleNodeProvisioner(), schemas, deployStateBuilder); } public static ContentCluster createCluster(String clusterXml, MockRoot root) { @@ -63,13 +63,13 @@ public class ContentClusterUtils { new FileDistributionConfigProducer(root, new MockFileRegistry(), null), root.getDeployState().isHosted()); ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(), - null,null, root, null); + null,null, root, null); return new ContentCluster.Builder(admin).build(Collections.emptyList(), context, doc.getDocumentElement()); } - public static ContentCluster createCluster(String clusterXml, List<String> searchDefinitions) throws Exception { - MockRoot root = createMockRoot(searchDefinitions); + public static ContentCluster createCluster(String clusterXml, List<String> schemas) throws Exception { + MockRoot root = createMockRoot(schemas); ContentCluster cluster = createCluster(clusterXml, root); root.freezeModelTopology(); cluster.validate(); @@ -77,7 +77,7 @@ public class ContentClusterUtils { } public static ContentCluster createCluster(String clusterXml) throws Exception { - return createCluster(clusterXml, SearchDefinitionBuilder.createSearchDefinitions("test")); + return createCluster(clusterXml, SchemaBuilder.createSchemas("test")); } public static String createClusterXml(String groupXml, int redundancy, int searchableCopies) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java index c622bcfaec5..c18dac17064 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java @@ -12,20 +12,20 @@ import static com.yahoo.config.model.test.TestUtil.joinLines; * * @author geirst */ -public class SearchDefinitionBuilder { +public class SchemaBuilder { private String name = "test"; private String content = ""; - public SearchDefinitionBuilder() { + public SchemaBuilder() { } - public SearchDefinitionBuilder name(String name) { + public SchemaBuilder name(String name) { this.name = name; return this; } - public SearchDefinitionBuilder content(String content) { + public SchemaBuilder content(String content) { this.content = content; return this; } @@ -38,10 +38,10 @@ public class SearchDefinitionBuilder { "}"); } - public static List<String> createSearchDefinitions(String ... docTypes) { + public static List<String> createSchemas(String ... docTypes) { return Arrays.asList(docTypes) .stream() - .map(type -> new SearchDefinitionBuilder().name(type).build()) + .map(type -> new SchemaBuilder().name(type).build()) .collect(Collectors.toList()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 177e741937d..97417f5a522 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -90,7 +90,7 @@ public class DocumentDatabaseTestCase { private void assertSingleSD(String mode) { final List<String> sds = Arrays.asList("type1"); VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); assertEquals(1, indexedSearchCluster.getDocumentDbs().size()); @@ -111,7 +111,7 @@ public class DocumentDatabaseTestCase { sds.add(nameAndMode.getType()); } return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); } @Test @@ -211,10 +211,10 @@ public class DocumentDatabaseTestCase { } @Test - public void requireThatWeCanHaveMultipleSearchDefinitions() { - final List<String> sds = Arrays.asList("type1", "type2", "type3"); + public void testMultipleSchemas() { + List<String> sds = List.of("type1", "type2", "type3"); VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); String type1Id = "test/search/cluster.test/type1"; @@ -264,7 +264,7 @@ public class DocumentDatabaseTestCase { public void requireThatRelevantConfigIsAvailableForClusterSearcher() { final List<String> sds = Arrays.asList("type1", "type2"); VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); String searcherId = "container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher"; { // documentdb-info config @@ -325,7 +325,7 @@ public class DocumentDatabaseTestCase { private void assertDocumentDBConfigAvailableForStreaming(String mode) { final List<String> sds = Arrays.asList("type"); VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type"); assertEquals(1, dcfg.documentdb().size()); @@ -343,7 +343,7 @@ public class DocumentDatabaseTestCase { List<String> documentDBConfigIds, Map<String, List<String>> expectedAttributesMap) { VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + ApplicationPackageUtils.generateSchemas(sds)).create(); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); ProtonConfig proton = getProtonCfg(contentSearchCluster); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java index 70e307e1748..ea4b3db5ebb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java @@ -127,7 +127,7 @@ public class SearchClusterTest { " </content>" + "</services>"; - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create(); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSchemas("music")).create(); ContainerCluster containerCluster1 = (ContainerCluster)model.getConfigProducer("j1").get(); assertFalse(containerCluster1.getSearch().getChains().localProviders().isEmpty()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index ee6fc60ba46..df62a3bff07 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -44,14 +44,14 @@ public class ApplicationPackageUtils { } public static List<String> generateSearchDefinition(String name) { - return generateSearchDefinitions(name); + return generateSchemas(name); } - public static List<String> generateSearchDefinitions(String ... sdNames) { - return generateSearchDefinitions(Arrays.asList(sdNames)); + public static List<String> generateSchemas(String ... sdNames) { + return generateSchemas(Arrays.asList(sdNames)); } - public static List<String> generateSearchDefinitions(List<String> sdNames) { + public static List<String> generateSchemas(List<String> sdNames) { List<String> sds = new ArrayList<>(); int i = 0; for (String sdName : sdNames) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java index 814ec008285..70ce588bec1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java @@ -31,8 +31,8 @@ public class VespaModelCreatorWithMockPkg { this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).build()); } - public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> searchDefinitions) { - this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withSearchDefinitions(searchDefinitions).build()); + public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> schemas) { + this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withSchemas(schemas).build()); } public VespaModelCreatorWithMockPkg(ApplicationPackage appPkg) { diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 253af585c73..e7ea2683e3f 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -90,7 +90,6 @@ <exclude> <binding>http//*/foo/*</binding> </exclude> - <application>my-app</application> <vespa-domain>vespa.vespa.cd</vespa-domain> </access-control> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 55a1482cde8..930bdaadcea 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -14,6 +14,7 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.FetchVector; @@ -144,6 +145,7 @@ public class ModelContextImpl implements ModelContext { private final boolean useNewAthenzFilter; private final boolean usePhraseSegmenting; private final String proxyProtocol; + private final Optional<AthenzDomain> athenzDomain; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -157,7 +159,8 @@ public class ModelContextImpl implements ModelContext { boolean isBootstrap, boolean isFirstTimeDeployment, FlagSource flagSource, - Optional<EndpointCertificateSecrets> endpointCertificateSecrets) { + Optional<EndpointCertificateSecrets> endpointCertificateSecrets, + Optional<AthenzDomain> athenzDomain) { this.applicationId = applicationId; this.multitenant = multitenantFromConfig || hostedVespa || Boolean.getBoolean("multitenant"); this.configServerSpecs = configServerSpecs; @@ -182,6 +185,7 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.proxyProtocol = Flags.PROXY_PROTOCOL.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.athenzDomain = athenzDomain; } @Override @@ -244,6 +248,9 @@ public class ModelContextImpl implements ModelContext { @Override public String proxyProtocol() { return proxyProtocol; } + + @Override + public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 13ef19f5f5d..09f5178cf6b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -87,17 +87,16 @@ public class ZooKeeperClient { /** Sets the app id and attempts to set up zookeeper. The app id must be ordered for purge to work OK. */ private void createZooKeeperNodes() { - if (!configCurator.exists(rootPath.getAbsolute())) { + if ( ! configCurator.exists(rootPath.getAbsolute())) configCurator.createNode(rootPath.getAbsolute()); - } - for (String subPath : Arrays.asList( - ConfigCurator.DEFCONFIGS_ZK_SUBPATH, - ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH, - ConfigCurator.USERAPP_ZK_SUBPATH, - ZKApplicationPackage.fileRegistryNode)) { - // TODO The replaceFirst below is hackish. - configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(), subPath.replaceFirst("/", "")); + for (String subPath : Arrays.asList(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, + ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH, + ConfigCurator.USERAPP_ZK_SUBPATH, + ZKApplicationPackage.fileRegistryNode)) { + // TODO: The replaceFirst below is hackish. + configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(), + subPath.replaceFirst("/", "")); } } @@ -108,7 +107,6 @@ public class ZooKeeperClient { */ void write(ApplicationPackage app) { logFine("Feeding application config into ZooKeeper"); - // gives lots and lots of debug output: // BasicConfigurator.configure(); try { logFine("Feeding user def files into ZooKeeper"); writeUserDefs(app); @@ -121,43 +119,34 @@ public class ZooKeeperClient { write(app.getMetaData()); } catch (Exception e) { throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" + - "Please ensure that cloudconfig_server is started on the config server node(s), " + - "and check the vespa log for configserver errors. ", e); + "Please ensure that cloudconfig_server is started on the config server node(s), " + + "and check the vespa log for configserver errors. ", e); } } private void writeSearchDefinitions(ApplicationPackage app) throws IOException { Collection<NamedReader> sds = app.getSearchDefinitions(); - if (sds.isEmpty()) { - return; - } + if (sds.isEmpty()) return; + + // TODO: Change to SCHEMAS_DIR after March 2020 Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR); configCurator.createNode(zkPath.getAbsolute()); - // Ensures that ranking expressions and other files are also fed. + // Ensures that ranking expressions and other files are also written writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); + writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, false); for (NamedReader sd : sds) { - String name = sd.getName(); - Reader reader = sd.getReader(); - String data = com.yahoo.io.IOUtils.readAll(reader); - reader.close(); - configCurator.putData(zkPath.getAbsolute(), name, data); + configCurator.putData(zkPath.getAbsolute(), sd.getName(), com.yahoo.io.IOUtils.readAll(sd.getReader())); + sd.getReader().close(); } } /** * Puts some of the application package files into ZK - see write(app). * - * @param app The application package to use as input. - * @throws java.io.IOException if not able to write to Zookeeper + * @param app the application package to use as input. + * @throws java.io.IOException if not able to write to Zookeeper */ private void writeSomeOf(ApplicationPackage app) throws IOException { - ApplicationFile.PathFilter srFilter = new ApplicationFile.PathFilter() { - @Override - public boolean accept(Path path) { - return path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX); - } - }; - // Copy app package files and subdirs into zk // TODO: We should have a way of doing this which doesn't require repeating all the content writeFile(app.getFile(Path.fromString(ApplicationPackage.SERVICES)), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); @@ -169,7 +158,8 @@ public class ZooKeeperClient { getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); writeDir(app.getFile(ApplicationPackage.RULES_DIR), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.RULES_DIR), - srFilter, true); + (path) -> path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX), + true); writeDir(app.getFile(ApplicationPackage.QUERY_PROFILES_DIR), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.QUERY_PROFILES_DIR), xmlFilter, true); @@ -194,20 +184,12 @@ public class ZooKeeperClient { } private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { - writeDir(file, zooKeeperAppPath, new ApplicationFile.PathFilter() { - @Override - public boolean accept(Path path) { - return true; - } - }, recurse); + writeDir(file, zooKeeperAppPath, (__) -> true, recurse); } private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException { - if (!dir.isDirectory()) { - logger.log(LogLevel.FINE, dir.getPath().getAbsolute()+" is not a directory. Not feeding the files into ZooKeeper."); - return; - } - for (ApplicationFile file: listFiles(dir, filenameFilter)) { + if ( ! dir.isDirectory()) return; + for (ApplicationFile file : listFiles(dir, filenameFilter)) { String name = file.getPath().getName(); if (name.startsWith(".")) continue; //.svn , .git ... if ("CVS".equals(name)) continue; @@ -223,7 +205,8 @@ public class ZooKeeperClient { } /** - * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)} with a slightly different semantic. Never filter out directories. + * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)} + * with slightly different semantics: Never filter out directories. */ private List<ApplicationFile> listFiles(ApplicationFile dir, ApplicationFile.PathFilter filter) { List<ApplicationFile> rawList = dir.listFiles(); @@ -243,9 +226,8 @@ public class ZooKeeperClient { } private void writeFile(ApplicationFile file, Path zkPath) throws IOException { - if (!file.exists()) { - return; - } + if ( ! file.exists()) return; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (InputStream inputStream = file.createInputStream()) { inputStream.transferTo(baos); @@ -292,8 +274,8 @@ public class ZooKeeperClient { String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); configCurator.putData(getZooKeeperAppPath(null).append(ZKApplicationPackage.fileRegistryNode).getAbsolute(), - vespaVersion.toFullString(), - exportedRegistry); + vespaVersion.toFullString(), + exportedRegistry); } /** @@ -343,9 +325,8 @@ public class ZooKeeperClient { } public void write(AllocatedHosts hosts) throws IOException { - configCurator.putData( - rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), - AllocatedHostsSerializer.toJson(hosts)); + configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), + AllocatedHostsSerializer.toJson(hosts)); } public void write(Map<Version, FileRegistry> fileRegistryMap) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 70faf3ff36f..46a0c44674b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -140,7 +140,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { flagSource, new EndpointCertificateMetadataStore(curator, TenantRepository.getTenantPath(tenant)) .readEndpointCertificateMetadata(applicationId) - .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets)); + .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets), + zkClient.readAthenzDomain()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index e217bb39b39..6a671648b27 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.slime.Slime; @@ -36,6 +37,7 @@ public final class PrepareParams { static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName"; static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata"; static final String DOCKER_IMAGE_REPOSITORY = "dockerImageRepository"; + static final String ATHENZ_DOMAIN = "athenzDomain"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -48,12 +50,13 @@ public final class PrepareParams { private final Optional<String> tlsSecretsKeyName; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private final Optional<String> dockerImageRepository; + private final Optional<AthenzDomain> athenzDomain; private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, - Optional<String> dockerImageRepository) { + Optional<String> dockerImageRepository, Optional<AthenzDomain> athenzDomain) { this.timeoutBudget = timeoutBudget; this.applicationId = applicationId; this.ignoreValidationErrors = ignoreValidationErrors; @@ -65,6 +68,7 @@ public final class PrepareParams { this.tlsSecretsKeyName = tlsSecretsKeyName; this.endpointCertificateMetadata = endpointCertificateMetadata; this.dockerImageRepository = dockerImageRepository; + this.athenzDomain = athenzDomain; } public static class Builder { @@ -80,6 +84,7 @@ public final class PrepareParams { private Optional<String> tlsSecretsKeyName = Optional.empty(); private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty(); private Optional<String> dockerImageRepository = Optional.empty(); + private Optional<AthenzDomain> athenzDomain = Optional.empty(); public Builder() { } @@ -153,10 +158,15 @@ public final class PrepareParams { return this; } + public Builder athenzDomain(String athenzDomain) { + this.athenzDomain = Optional.ofNullable(athenzDomain).map(AthenzDomain::from); + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, - endpointCertificateMetadata, dockerImageRepository); + endpointCertificateMetadata, dockerImageRepository, athenzDomain); } } @@ -172,6 +182,7 @@ public final class PrepareParams { .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME)) .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME)) .dockerImageRepository(request.getProperty(DOCKER_IMAGE_REPOSITORY)) + .athenzDomain(request.getProperty(ATHENZ_DOMAIN)) .build(); } @@ -237,4 +248,6 @@ public final class PrepareParams { return dockerImageRepository; } + public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index c65af076e54..b88fdc90316 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -17,6 +17,7 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; @@ -152,6 +153,7 @@ public class SessionPreparer { private final EndpointCertificateRetriever endpointCertificateRetriever; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; + private final Optional<AthenzDomain> athenzDomain; private ApplicationPackage applicationPackage; private List<PreparedModelsBuilder.PreparedModelResult> modelResultList; @@ -182,6 +184,7 @@ public class SessionPreparer { .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); this.endpointsSet = getEndpoints(params.containerEndpoints()); + this.athenzDomain = params.athenzDomain(); this.properties = new ModelContextImpl.Properties(params.getApplicationId(), configserverConfig.multitenant(), @@ -195,7 +198,8 @@ public class SessionPreparer { params.isBootstrap(), ! currentActiveApplicationSet.isPresent(), context.getFlagSource(), - endpointCertificateSecrets); + endpointCertificateSecrets, + athenzDomain); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, @@ -247,7 +251,8 @@ public class SessionPreparer { vespaVersion, logger, prepareResult.getFileRegistries(), - prepareResult.allocatedHosts()); + prepareResult.allocatedHosts(), + athenzDomain); checkTimeout("write state to zookeeper"); } @@ -290,13 +295,15 @@ public class SessionPreparer { Version vespaVersion, DeployLogger deployLogger, Map<Version, FileRegistry> fileRegistryMap, - AllocatedHosts allocatedHosts) { + AllocatedHosts allocatedHosts, + Optional<AthenzDomain> athenzDomain) { ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger); try { zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); zooKeeperClient.writeApplicationId(applicationId); zooKeeperClient.writeVespaVersion(vespaVersion); zooKeeperClient.writeDockerImageRepository(dockerImageRepository); + zooKeeperClient.writeAthenzDomain(athenzDomain); } catch (RuntimeException | IOException e) { zkDeployer.cleanup(); throw new RuntimeException("Error preparing session", e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 4c64bdd380f..44cf3cec1b7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -5,23 +5,24 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.transaction.Transaction; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; -import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.text.Utf8; -import com.yahoo.config.provision.ApplicationId; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.UserConfigDefinitionRepo; import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -42,6 +43,7 @@ public class SessionZooKeeperClient { private static final String VERSION_PATH = "version"; private static final String CREATE_TIME_PATH = "createTime"; private static final String DOCKER_IMAGE_REPOSITORY_PATH = "dockerImageRepository"; + private static final String ATHENZ_DOMAIN = "athenzDomain"; private final Curator curator; private final ConfigCurator configCurator; private final Path sessionPath; @@ -170,6 +172,10 @@ public class SessionZooKeeperClient { return sessionPath.append(DOCKER_IMAGE_REPOSITORY_PATH).getAbsolute(); } + private String athenzDomainPath() { + return sessionPath.append(ATHENZ_DOMAIN).getAbsolute(); + } + public void writeVespaVersion(Version version) { configCurator.putData(versionPath(), version.toString()); } @@ -221,6 +227,17 @@ public class SessionZooKeeperClient { return transaction; } + public void writeAthenzDomain(Optional<AthenzDomain> athenzDomain) { + athenzDomain.ifPresent(domain -> configCurator.putData(athenzDomainPath(), domain.toString())); + } + + public Optional<AthenzDomain> readAthenzDomain() { + if ( ! configCurator.exists(athenzDomainPath())) return Optional.empty(); + return Optional.ofNullable(configCurator.getData(athenzDomainPath())) + .filter(domain -> ! domain.isBlank()) + .map(AthenzDomain::from); + } + /** * Create necessary paths atomically for a new session. * diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java index 3ae678969eb..5f180bdaee1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java @@ -9,14 +9,12 @@ import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; /** * Responsible for providing data from an application subtree in zookeeper. * (i.e. /config/v2/tenants/x/session/<session id for an application>/). * - * Takes care of - * - * * @author Tony Vaagenes */ public class ZKApplication { @@ -84,6 +82,10 @@ public class ZKApplication { return reader(data); } + Optional<Reader> getOptionalDataReader(String path, String node) { + return Optional.ofNullable(getData(path, node)).map(data -> reader(data)); + } + public String getData(String path, String node) { try { return zk.getData(getFullPath(path), node); @@ -181,10 +183,9 @@ public class ZKApplication { } Reader getDataReader(String path) { - final String data = getData(path); - if (data == null) { + String data = getData(path); + if (data == null) throw new IllegalArgumentException("No node for " + getFullPath(path) + " exists"); - } return reader(data); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index bcb958c4b58..c7ec2657996 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -139,13 +139,16 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public List<NamedReader> searchDefinitionContents() { - List<NamedReader> ret = new ArrayList<>(); - for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR)) { - if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) { - ret.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, sd)))); - } + List<NamedReader> schemas = new ArrayList<>(); + for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) { + if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) + schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd)))); } - return ret; + for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) { + if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) + schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd)))); + } + return schemas; } @Override @@ -176,7 +179,7 @@ public class ZKApplicationPackage implements ApplicationPackage { try { return zkApplication.getDataReader(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, def); } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not retrieve config definition " + def + ".", e); + throw new IllegalArgumentException("Could not retrieve config definition " + def, e); } } @@ -264,7 +267,10 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public Reader getRankingExpression(String name) { - return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, name); + Optional<Reader> reader = zkApplication.getOptionalDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name); + if (reader.isPresent()) + return reader.get(); + return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, name); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index 339c676000b..c3124fd3ed6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -59,7 +59,8 @@ public class ModelContextImplTest { false, false, flagSource, - null), + null, + Optional.empty()), Optional.empty(), Optional.empty(), new Version(7), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 97d90d8b6b6..d730b21762d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -60,7 +60,7 @@ public class ZKApplicationPackageTest { assertTrue(Pattern.compile(".*<alias>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches()); assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches()); DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build(); - assertEquals(deployState.getSearchDefinitions().size(), 5); + assertEquals(deployState.getSchemas().size(), 5); assertEquals(zkApp.searchDefinitionContents().size(), 5); assertEquals(IOUtils.readAll(zkApp.getRankingExpression("foo.expression")), "foo()+1\n"); assertEquals(zkApp.getFiles(Path.fromString(""), "xml").size(), 3); diff --git a/container-search/src/main/java/com/yahoo/prelude/Location.java b/container-search/src/main/java/com/yahoo/prelude/Location.java index 37284bd6bcc..908bf835e3c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Location.java +++ b/container-search/src/main/java/com/yahoo/prelude/Location.java @@ -126,8 +126,8 @@ public class Location { if (ns < -90.1 || ns > +90.1) { throw new IllegalArgumentException("n/s location must be in range [-90,+90]"); } - if (radius_in_degrees < 0 || radius_in_degrees > 180.0) { - throw new IllegalArgumentException("radius must be in range [0,180] degrees, approximately upto 20000km"); + if (radius_in_degrees < 0) { + pr = 512 * 1024 * 1024; } x = px; y = py; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java index 43717ecf6cd..37561d3a0f5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java @@ -147,7 +147,9 @@ public class PosSearcher extends Searcher { String radius = query.properties().getString(posRadius); int radiusUnits; if (radius == null) { - radiusUnits = 5000; + double radiuskm = 50.0; + double radiusdegrees = radiuskm * km2deg; + radiusUnits = (int)(radiusdegrees * 1000000); } else if (radius.endsWith("km")) { double radiuskm = Double.valueOf(radius.substring(0, radius.length()-2)); double radiusdegrees = radiuskm * km2deg; diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java index e2973f8ed65..aa3fa53119e 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java @@ -118,6 +118,12 @@ public class PosSearcherTestCase { q.properties().set("pos.units", "udeg"); doSearch(searcher, q, 0, 10); assertEquals("(2,0,0,18026,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + + q = new Query(); + q.properties().set("pos.ll", "N0;E0"); + q.properties().set("pos.radius", "-1"); + doSearch(searcher, q, 0, 10); + assertEquals("(2,0,0,536870912,0,1,0,4294967295)", q.getRanking().getLocation().toString()); } /** @@ -128,13 +134,13 @@ public class PosSearcherTestCase { Query q = new Query(); q.properties().set("pos.xy", "22500;22500"); doSearch(searcher, q, 0, 10); - assertEquals("(2,22500,22500,5000,0,1,0)", q.getRanking().getLocation().toString()); + assertEquals("(2,22500,22500,450668,0,1,0)", q.getRanking().getLocation().toString()); q = new Query(); q.properties().set("pos.xy", "22500;22500"); q.properties().set("pos.units", "unknown"); doSearch(searcher, q, 0, 10); - assertEquals("(2,22500,22500,5000,0,1,0)", q.getRanking().getLocation().toString()); + assertEquals("(2,22500,22500,450668,0,1,0)", q.getRanking().getLocation().toString()); } @Test diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index ee2e292ade9..0afe9347341 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -4,6 +4,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -26,11 +27,13 @@ public class DeploymentData { private final Set<ContainerEndpoint> containerEndpoints; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private final Optional<DockerImage> dockerImageRepo; + private final Optional<AthenzDomain> athenzDomain; public DeploymentData(ApplicationId instance, ZoneId zone, byte[] applicationPackage, Version platform, Set<ContainerEndpoint> containerEndpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, - Optional<DockerImage> dockerImageRepo) { + Optional<DockerImage> dockerImageRepo, + Optional<AthenzDomain> athenzDomain) { this.instance = requireNonNull(instance); this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); @@ -38,6 +41,7 @@ public class DeploymentData { this.containerEndpoints = requireNonNull(containerEndpoints); this.endpointCertificateMetadata = requireNonNull(endpointCertificateMetadata); this.dockerImageRepo = requireNonNull(dockerImageRepo); + this.athenzDomain = athenzDomain; } public ApplicationId instance() { @@ -67,4 +71,8 @@ public class DeploymentData { public Optional<DockerImage> dockerImageRepo() { return dockerImageRepo; } + + public Optional<AthenzDomain> athenzDomain() { + return athenzDomain; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index cb6d02b4b77..e5e92117849 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -498,9 +498,14 @@ public class ApplicationController { .value()) .filter(s -> !s.isBlank()) .map(DockerImage::fromString); + + Optional<AthenzDomain> domain = controller.tenants().get(application.tenant()) + .filter(tenant-> tenant instanceof AthenzTenant) + .map(tenant -> ((AthenzTenant)tenant).domain()); + ConfigServer.PreparedApplication preparedApplication = configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, - endpoints, endpointCertificateMetadata, dockerImageRepo)); + endpoints, endpointCertificateMetadata, dockerImageRepo, domain)); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index dbf6cddce83..a49dec4c0f2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -305,8 +305,6 @@ public class DockerOperationsImpl implements DockerOperations { boolean isMain = context.zone().getSystemName() == SystemName.cd || context.zone().getSystemName() == SystemName.main; if (isMain && context.nodeType() == NodeType.tenant) command.withSharedVolume(Paths.get("/var/zpe"), context.pathInNodeUnderVespaHome("var/zpe")); - - command.withSharedVolume(Paths.get("/dev/log"), Paths.get("/dev/log")); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index 6ae086146c4..62041b924b9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.util.logging.Level; @@ -41,7 +42,8 @@ public class NodeMetricsDbMaintainer extends Maintainer { catch (Exception e) { // TODO: Don't warn if this only happens occasionally if (warnings++ < maxWarningsPerInvocation) - log.log(Level.WARNING, "Could not update metrics for " + application, e); + log.log(Level.WARNING, "Could not update metrics for " + application, + Exceptions.toMessageString(e)); } } nodeMetricsDb.gc(nodeRepository().clock()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index 675e3400722..db99f8544f1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -92,6 +92,7 @@ public class Rebalancer extends Maintainer { for (Node node : allNodes.nodeType(NodeType.tenant).state(Node.State.active)) { if (node.parentHostname().isEmpty()) continue; if (node.allocation().get().owner().instance().isTester()) continue; + if (node.allocation().get().owner().application().value().equals("lsbe-dictionaries")) continue; // TODO: Remove for (Node toHost : allNodes.nodeType(NodeType.host).state(NodePrioritizer.ALLOCATABLE_HOST_STATES)) { if (toHost.hostname().equals(node.parentHostname().get())) continue; if ( ! capacity.freeCapacityOf(toHost).satisfies(node.flavor().resources())) continue; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index c0664086498..94f8fb29b9a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -31,12 +31,12 @@ public class CapacityPolicies { this.isUsingAdvertisedResources = zone.cloud().value().equals("aws"); } - public int decideSize(Capacity capacity, ClusterSpec.Type clusterType, ApplicationId application) { + public int decideSize(Capacity capacity, ClusterSpec cluster, ApplicationId application) { int requestedNodes = capacity.nodeCount(); if (application.instance().isTester()) return 1; - ensureRedundancy(requestedNodes, clusterType, capacity.canFail()); + ensureRedundancy(requestedNodes, cluster, capacity.canFail()); if (capacity.isRequired()) return requestedNodes; @@ -105,12 +105,12 @@ public class CapacityPolicies { * * @throws IllegalArgumentException if only one node is requested and we can fail */ - private void ensureRedundancy(int nodeCount, ClusterSpec.Type clusterType, boolean canFail) { + private void ensureRedundancy(int nodeCount, ClusterSpec cluster, boolean canFail) { if (canFail && nodeCount == 1 && - requiresRedundancy(clusterType) && + requiresRedundancy(cluster.type()) && zone.environment().isProduction()) - throw new IllegalArgumentException("Deployments to prod require at least 2 nodes per cluster for redundancy"); + throw new IllegalArgumentException("Deployments to prod require at least 2 nodes per cluster for redundancy. Not fulfilled for " + cluster); } private static boolean requiresRedundancy(ClusterSpec.Type clusterType) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 07cf297a314..7a49ad9c44d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -89,7 +89,7 @@ public class NodeRepositoryProvisioner implements Provisioner { NodeSpec requestedNodes; Optional<NodeResources> resources = requestedCapacity.nodeResources(); if ( requestedCapacity.type() == NodeType.tenant) { - int nodeCount = capacityPolicies.decideSize(requestedCapacity, cluster.type(), application); + int nodeCount = capacityPolicies.decideSize(requestedCapacity, cluster, application); if (zone.environment().isManuallyDeployed() && nodeCount < requestedCapacity.nodeCount()) logger.log(Level.INFO, "Requested " + requestedCapacity.nodeCount() + " nodes for " + cluster + ", downscaling to " + nodeCount + " nodes in " + zone.environment()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 1ea0af67b5e..1f9d7ce5126 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -733,10 +733,10 @@ public class ProvisioningTest { allHosts.addAll(content1); Function<Integer, Capacity> capacity = count -> Capacity.fromCount(count, Optional.empty(), required, true); - int expectedContainer0Size = tester.capacityPolicies().decideSize(capacity.apply(container0Size), containerCluster0.type(), application); - int expectedContainer1Size = tester.capacityPolicies().decideSize(capacity.apply(container1Size), containerCluster1.type(), application); - int expectedContent0Size = tester.capacityPolicies().decideSize(capacity.apply(content0Size), contentCluster0.type(), application); - int expectedContent1Size = tester.capacityPolicies().decideSize(capacity.apply(content1Size), contentCluster1.type(), application); + int expectedContainer0Size = tester.capacityPolicies().decideSize(capacity.apply(container0Size), containerCluster0, application); + int expectedContainer1Size = tester.capacityPolicies().decideSize(capacity.apply(container1Size), containerCluster1, application); + int expectedContent0Size = tester.capacityPolicies().decideSize(capacity.apply(content0Size), contentCluster0, application); + int expectedContent1Size = tester.capacityPolicies().decideSize(capacity.apply(content1Size), contentCluster1, application); assertEquals("Hosts in each group cluster is disjunct and the total number of unretired nodes is correct", expectedContainer0Size + expectedContainer1Size + expectedContent0Size + expectedContent1Size, diff --git a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp index f90ed81cd7a..461bfd9ef30 100644 --- a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp +++ b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp @@ -135,7 +135,7 @@ TEST("require that interval refs are reused for identical data.") { PredicateIntervalStore store; auto ref = store.insert<Interval>({{0x00010001}, {0x0002ffff}}); ASSERT_TRUE(ref.valid()); - ASSERT_EQUAL(0x02000040u, ref.ref()); + ASSERT_EQUAL(0x02000001u, ref.ref()); auto ref2 = store.insert<Interval>({{0x00010001}, {0x0002ffff}}); EXPECT_EQUAL(ref.ref(), ref2.ref()); diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 1604a39eaf3..a6c4ac61ba2 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -243,9 +243,11 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_sel // Doc 3 is also added to level 1 add_document(3, 1); expect_entry_point(3, 1); + // Doc 3 is closest to 1 and they are linked. + // Doc 3 is NOT linked to 2, since that is closer to 1 also. expect_level_0(1, {2, 3}); - expect_level_0(2, {1, 3}); - expect_levels(3, {{1, 2}, {}}); + expect_level_0(2, {1}); + expect_levels(3, {{1}, {}}); // Doc 4 is closest to 1 and they are linked. // Doc 4 is NOT linked to 3 as the distance between 4 and 3 is greater than the distance between 3 and 1. @@ -253,8 +255,8 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_sel add_document(4); expect_entry_point(3, 1); expect_level_0(1, {2, 3, 4}); - expect_level_0(2, {1, 3}); - expect_levels(3, {{1, 2}, {}}); + expect_level_0(2, {1}); + expect_levels(3, {{1}, {}}); expect_level_0(4, {1}); // Doc 5 is closest to 2 and they are linked. @@ -262,8 +264,8 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_sel add_document(5); expect_entry_point(3, 1); expect_level_0(1, {2, 3, 4}); - expect_level_0(2, {1, 3, 5}); - expect_levels(3, {{1, 2}, {}}); + expect_level_0(2, {1, 5}); + expect_levels(3, {{1}, {}}); expect_level_0(4, {1}); expect_level_0(5, {2}); @@ -273,8 +275,8 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_sel add_document(6, 2); expect_entry_point(6, 2); expect_level_0(1, {2, 3, 4}); - expect_level_0(2, {1, 3, 5, 6}); - expect_levels(3, {{1, 2}, {6}}); + expect_level_0(2, {1, 5, 6}); + expect_levels(3, {{1}, {6}}); expect_level_0(4, {1}); expect_level_0(5, {2, 6}); expect_levels(6, {{2, 5}, {3}, {}}); @@ -285,12 +287,24 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_sel add_document(7); expect_entry_point(6, 2); expect_level_0(1, {2, 3, 4}); - expect_level_0(2, {1, 3, 5, 6}); - expect_levels(3, {{1, 2, 7}, {6}}); + expect_level_0(2, {1, 5, 6}); + expect_levels(3, {{1, 7}, {6}}); expect_level_0(4, {1}); expect_level_0(5, {2, 6}); expect_levels(6, {{2, 5, 7}, {3}, {}}); expect_level_0(7, {3, 6}); + + // removing 1, its neighbors {2,3,4} will try to + // link together, but since 2 already has enough links + // only 3 and 4 will become neighbors: + remove_document(1); + expect_entry_point(6, 2); + expect_level_0(2, {5, 6}); + expect_levels(3, {{4, 7}, {6}}); + expect_level_0(4, {3}); + expect_level_0(5, {2, 6}); + expect_levels(6, {{2, 5, 7}, {3}, {}}); + expect_level_0(7, {3, 6}); } TEST_F(HnswIndexTest, manual_insert) @@ -328,7 +342,7 @@ TEST_F(HnswIndexTest, manual_insert) TEST_F(HnswIndexTest, memory_is_reclaimed_when_doing_changes_to_graph) { - init(true); + init(false); add_document(1); add_document(3); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 09608a9abbe..9795330b061 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -28,6 +28,18 @@ bool has_link_to(vespalib::ConstArrayRef<uint32_t> links, uint32_t id) { return false; } +struct PairDist { + uint32_t id_first; + uint32_t id_second; + double distance; + PairDist(uint32_t i1, uint32_t i2, double d) + : id_first(i1), id_second(i2), distance(d) + {} +}; +bool operator< (const PairDist &a, const PairDist &b) { + return (a.distance < b.distance); +} + } search::datastore::ArrayStoreConfig @@ -99,7 +111,7 @@ HnswIndex::set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& li } bool -HnswIndex::have_closer_distance(HnswCandidate candidate, const LinkArray& result) const +HnswIndex::have_closer_distance(HnswCandidate candidate, const LinkArrayRef& result) const { for (uint32_t result_docid : result) { double dist = calc_distance(candidate.docid, result_docid); @@ -130,7 +142,6 @@ HnswIndex::SelectResult HnswIndex::select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const { SelectResult result; - bool need_filtering = neighbors.size() > max_links; NearestPriQ nearest; for (const auto& entry : neighbors) { nearest.push(entry); @@ -138,7 +149,7 @@ HnswIndex::select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint while (!nearest.empty()) { auto candidate = nearest.top(); nearest.pop(); - if (need_filtering && have_closer_distance(candidate, result.used)) { + if (have_closer_distance(candidate, result.used)) { result.unused.push_back(candidate.docid); continue; } @@ -189,9 +200,7 @@ HnswIndex::connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint3 set_link_array(docid, level, neighbors); for (uint32_t neighbor_docid : neighbors) { auto old_links = get_link_array(neighbor_docid, level); - LinkArray new_links(old_links.begin(), old_links.end()); - new_links.push_back(docid); - set_link_array(neighbor_docid, level, new_links); + add_link_to(neighbor_docid, level, old_links, docid); } for (uint32_t neighbor_docid : neighbors) { shrink_if_needed(neighbor_docid, level); @@ -335,6 +344,32 @@ HnswIndex::add_document(uint32_t docid) } void +HnswIndex::mutual_reconnect(const LinkArrayRef &cluster, uint32_t level) +{ + std::vector<PairDist> pairs; + for (uint32_t i = 0; i + 1 < cluster.size(); ++i) { + uint32_t n_id_1 = cluster[i]; + LinkArrayRef n_list_1 = get_link_array(n_id_1, level); + for (uint32_t j = i + 1; j < cluster.size(); ++j) { + uint32_t n_id_2 = cluster[j]; + if (has_link_to(n_list_1, n_id_2)) continue; + pairs.emplace_back(n_id_1, n_id_2, calc_distance(n_id_1, n_id_2)); + } + } + std::sort(pairs.begin(), pairs.end()); + for (const PairDist & pair : pairs) { + LinkArrayRef old_links_1 = get_link_array(pair.id_first, level); + if (old_links_1.size() >= _cfg.max_links_on_inserts()) continue; + + LinkArrayRef old_links_2 = get_link_array(pair.id_second, level); + if (old_links_2.size() >= _cfg.max_links_on_inserts()) continue; + + add_link_to(pair.id_first, level, old_links_1, pair.id_second); + add_link_to(pair.id_second, level, old_links_2, pair.id_first); + } +} + +void HnswIndex::remove_document(uint32_t docid) { bool need_new_entrypoint = (docid == _entry_docid); @@ -350,6 +385,7 @@ HnswIndex::remove_document(uint32_t docid) } remove_link_to(neighbor_id, docid, level); } + mutual_reconnect(my_links, level); set_link_array(docid, level, empty); } if (need_new_entrypoint) { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 4066316a991..7c0d650ab2f 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -101,6 +101,11 @@ protected: LevelArrayRef get_level_array(uint32_t docid) const; LinkArrayRef get_link_array(uint32_t docid, uint32_t level) const; void set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links); + void add_link_to(uint32_t docid, uint32_t level, const LinkArrayRef& old_links, uint32_t new_link) { + LinkArray new_links(old_links.begin(), old_links.end()); + new_links.push_back(new_link); + set_link_array(docid, level, new_links); + } /** * Returns true if the distance between the candidate and a node in the current result @@ -109,7 +114,7 @@ protected: * where the candidate is located. * Used by select_neighbors_heuristic(). */ - bool have_closer_distance(HnswCandidate candidate, const LinkArray& curr_result) const; + bool have_closer_distance(HnswCandidate candidate, const LinkArrayRef& curr_result) const; struct SelectResult { LinkArray used; LinkArray unused; @@ -119,6 +124,7 @@ protected: SelectResult select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_links) const; void shrink_if_needed(uint32_t docid, uint32_t level); void connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level); + void mutual_reconnect(const LinkArrayRef &cluster, uint32_t level); void remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level); inline TypedCells get_vector(uint32_t docid) const { diff --git a/vespalib/src/vespa/vespalib/datastore/entryref.h b/vespalib/src/vespa/vespalib/datastore/entryref.h index c65e788637f..d49b1ab9071 100644 --- a/vespalib/src/vespa/vespalib/datastore/entryref.h +++ b/vespalib/src/vespa/vespalib/datastore/entryref.h @@ -32,9 +32,8 @@ public: EntryRefT() : EntryRef() {} EntryRefT(size_t offset_, uint32_t bufferId_); EntryRefT(const EntryRef & ref_) : EntryRef(ref_.ref()) {} - uint32_t hash() const { return offset() + (bufferId() << OffsetBits); } - size_t offset() const { return _ref >> BufferBits; } - uint32_t bufferId() const { return _ref & (numBuffers() - 1); } + size_t offset() const { return _ref & (offsetSize() - 1); } + uint32_t bufferId() const { return _ref >> OffsetBits; } static size_t offsetSize() { return 1ul << OffsetBits; } static uint32_t numBuffers() { return 1 << BufferBits; } static size_t align(size_t val) { return val; } diff --git a/vespalib/src/vespa/vespalib/datastore/entryref.hpp b/vespalib/src/vespa/vespalib/datastore/entryref.hpp index 6e8b94f8989..b7aafb30fdd 100644 --- a/vespalib/src/vespa/vespalib/datastore/entryref.hpp +++ b/vespalib/src/vespa/vespalib/datastore/entryref.hpp @@ -9,7 +9,7 @@ namespace search::datastore { template <uint32_t OffsetBits, uint32_t BufferBits> EntryRefT<OffsetBits, BufferBits>::EntryRefT(size_t offset_, uint32_t bufferId_) : - EntryRef((offset_ << BufferBits) + bufferId_) + EntryRef((bufferId_ << OffsetBits) + offset_) { ASSERT_ONCE_OR_LOG(offset_ < offsetSize(), "EntryRefT.offset_overflow", 10000); ASSERT_ONCE_OR_LOG(bufferId_ < numBuffers(), "EntryRefT.bufferId_overflow", 10000); |