diff options
343 files changed, 5214 insertions, 1670 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 873e4c5fad7..e1e3219a5d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,6 +66,7 @@ add_subdirectory(container-messagebus) add_subdirectory(container-search) add_subdirectory(container-search-gui) add_subdirectory(container-search-and-docproc) +add_subdirectory(cloud-tenant-cd) add_subdirectory(clustercontroller-apps) add_subdirectory(clustercontroller-apputil) add_subdirectory(clustercontroller-utils) @@ -127,11 +128,13 @@ add_subdirectory(storageframework) add_subdirectory(storageserver) add_subdirectory(statistics) add_subdirectory(streamingvisitors) +add_subdirectory(tenant-cd-api) add_subdirectory(vbench) add_subdirectory(vdslib) add_subdirectory(vdstestlib) add_subdirectory(vespa-athenz) add_subdirectory(vespa-http-client) +add_subdirectory(vespa-osgi-testrunner) add_subdirectory(vespa-testrunner-components) add_subdirectory(vespa_feed_perf) add_subdirectory(vespa_jersey2) @@ -160,3 +163,6 @@ __create_module_targets(TARGETS "module") # Create module targets with name ${MODULE}+test depending on every test target defined within that module __create_module_targets(TEST_TARGETS "test") + +# Create module targets with name ${MODULE}+source depending on every source target defined within that module +__create_module_targets(SOURCE_TARGETS "source") diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractAssembleBundleMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractAssembleBundleMojo.java new file mode 100644 index 00000000000..622cb9a49b0 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractAssembleBundleMojo.java @@ -0,0 +1,96 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.mojo; + +import com.yahoo.container.plugin.util.Files; +import com.yahoo.container.plugin.util.JarFiles; +import org.apache.maven.archiver.MavenArchiveConfiguration; +import org.apache.maven.archiver.MavenArchiver; +import org.apache.maven.artifact.Artifact; +import org.apache.maven.execution.MavenSession; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; +import org.codehaus.plexus.archiver.jar.JarArchiver; + +import java.io.File; +import java.nio.channels.Channels; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Enumeration; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.zip.ZipEntry; + +/** + * @author bjorncs + */ +abstract class AbstractAssembleBundleMojo extends AbstractMojo { + + @Parameter(defaultValue = "${project}") + MavenProject project; + + @Parameter(defaultValue = "${session}", readonly = true, required = true) + MavenSession session; + + @Parameter + MavenArchiveConfiguration archiveConfiguration = new MavenArchiveConfiguration(); + + void addDirectory(JarArchiver jarArchiver, Path directory) { + if (java.nio.file.Files.isDirectory(directory)) { + jarArchiver.addDirectory(directory.toFile()); + } + } + + void createArchive(JarArchiver jarArchiver, Path jarFile, Path manifestFile) throws MojoExecutionException { + archiveConfiguration.setForced(true); // force recreating the archive + archiveConfiguration.setManifestFile(manifestFile.toFile()); + MavenArchiver mavenArchiver = new MavenArchiver(); + mavenArchiver.setArchiver(jarArchiver); + mavenArchiver.setOutputFile(jarFile.toFile()); + try { + mavenArchiver.createArchive(session, project, archiveConfiguration); + } catch (Exception e) { + throw new MojoExecutionException("Error creating archive " + jarFile.toFile().getName(), e); + } + } + + void addArtifacts(JarArchiver jarArchiver, Collection<Artifact> artifacts) { + for (Artifact artifact : artifacts) { + if ("jar".equals(artifact.getType())) { + jarArchiver.addFile(artifact.getFile(), "dependencies/" + artifact.getFile().getName()); + copyConfigDefinitions(artifact.getFile(), jarArchiver); + } else { + getLog().warn("Unknown artifact type " + artifact.getType()); + } + } + } + + private void copyConfigDefinitions(File file, JarArchiver jarArchiver) { + JarFiles.withJarFile(file, jarFile -> { + for (Enumeration<JarEntry> entries = jarFile.entries(); entries.hasMoreElements();) { + JarEntry entry = entries.nextElement(); + String name = entry.getName(); + if (name.startsWith("configdefinitions/") && name.endsWith(".def")) { + copyConfigDefinition(jarFile, entry, jarArchiver); + } + } + return null; + }); + } + + private void copyConfigDefinition(JarFile jarFile, ZipEntry entry, JarArchiver jarArchiver) { + JarFiles.withInputStream(jarFile, entry, input -> { + String defPath = entry.getName().replace("/", File.separator); + File destinationFile = new File(project.getBuild().getOutputDirectory(), defPath); + destinationFile.getParentFile().mkdirs(); + + Files.withFileOutputStream(destinationFile, output -> { + output.getChannel().transferFrom(Channels.newChannel(input), 0, Long.MAX_VALUE); + return null; + }); + jarArchiver.addFile(destinationFile, entry.getName()); + return null; + }); + } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractGenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractGenerateOsgiManifestMojo.java new file mode 100644 index 00000000000..d648d9ca258 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractGenerateOsgiManifestMojo.java @@ -0,0 +1,210 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.mojo; + +import com.yahoo.container.plugin.classanalysis.Analyze; +import com.yahoo.container.plugin.classanalysis.ClassFileMetaData; +import com.yahoo.container.plugin.classanalysis.ExportPackageAnnotation; +import com.yahoo.container.plugin.classanalysis.PackageTally; +import com.yahoo.container.plugin.osgi.ExportPackageParser; +import com.yahoo.container.plugin.osgi.ExportPackages; +import com.yahoo.container.plugin.osgi.ImportPackages; +import com.yahoo.container.plugin.util.Strings; +import org.apache.maven.artifact.Artifact; +import org.apache.maven.artifact.versioning.ArtifactVersion; +import org.apache.maven.artifact.versioning.DefaultArtifactVersion; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.container.plugin.util.IO.withFileOutputStream; +import static java.util.stream.Collectors.toList; + +/** + * @author bjorncs + */ +abstract class AbstractGenerateOsgiManifestMojo extends AbstractMojo { + + @Parameter(defaultValue = "${project}") + MavenProject project; + + /** + * If set to true, the artifact's version is used as default package version for ExportPackages. + * Packages from included (compile scoped) artifacts will use the version for their own artifact. + * If the package is exported with an explicit version in package-info.java, that version will be + * used regardless of this parameter. + */ + @Parameter(alias = "UseArtifactVersionForExportPackages", defaultValue = "false") + boolean useArtifactVersionForExportPackages; + + @Parameter(alias = "Bundle-Version", defaultValue = "${project.version}") + String bundleVersion; + + // TODO: default should include groupId, but that will require a lot of changes both by us and users. + @Parameter(alias = "Bundle-SymbolicName", defaultValue = "${project.artifactId}") + String bundleSymbolicName; + + @Parameter(alias = "Import-Package") + String importPackage; + + Map<String, String> generateManifestContent( + Collection<Artifact> jarArtifactsToInclude, + Map<String, ImportPackages.Import> calculatedImports, + PackageTally pluginPackageTally) { + + Map<String, Optional<String>> manualImports = getManualImports(); + for (String packageName : manualImports.keySet()) { + calculatedImports.remove(packageName); + } + Collection<ImportPackages.Import> imports = calculatedImports.values(); + + Map<String, String> ret = new HashMap<>(); + String importPackage = Stream.concat(manualImports.entrySet().stream().map(e -> asOsgiImport(e.getKey(), e.getValue())), + imports.stream().map(ImportPackages.Import::asOsgiImport)).sorted() + .collect(Collectors.joining(",")); + + String exportPackage = osgiExportPackages(pluginPackageTally.exportedPackages()).stream().sorted() + .collect(Collectors.joining(",")); + + ret.put("Created-By", "vespa container maven plugin"); + ret.put("Bundle-ManifestVersion", "2"); + ret.put("Bundle-Name", project.getName()); + ret.put("Bundle-SymbolicName", bundleSymbolicName); + ret.put("Bundle-Version", asBundleVersion(bundleVersion)); + ret.put("Bundle-Vendor", "Yahoo!"); + addIfNotEmpty(ret, "Bundle-ClassPath", bundleClassPath(jarArtifactsToInclude)); + addIfNotEmpty(ret, "Import-Package", importPackage); + addIfNotEmpty(ret, "Export-Package", exportPackage); + + return ret; + } + + PackageTally definedPackages(Collection<Artifact> jarArtifacts) { + List<PackageTally> tallies = new ArrayList<>(); + for (var artifact : jarArtifacts) { + try { + tallies.add(definedPackages(new JarFile(artifact.getFile()), artifactVersionOrNull(artifact.getVersion()))); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return PackageTally.combine(tallies); + } + + ArtifactVersion artifactVersionOrNull(String version) { + return useArtifactVersionForExportPackages ? new DefaultArtifactVersion(version) : null; + } + + static void createManifestFile(Path outputDirectory, Map<String, String> manifestContent) { + Manifest manifest = new Manifest(); + Attributes mainAttributes = manifest.getMainAttributes(); + + mainAttributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifestContent.forEach(mainAttributes::putValue); + + withFileOutputStream(outputDirectory.resolve(JarFile.MANIFEST_NAME).toFile(), out -> { + manifest.write(out); + return null; + }); + } + + static void addIfNotEmpty(Map<String, String> map, String key, String value) { + if (value != null && ! value.isEmpty()) { + map.put(key, value); + } + } + + private Collection<String> osgiExportPackages(Map<String, ExportPackageAnnotation> exportedPackages) { + return exportedPackages.entrySet().stream().map(entry -> entry.getKey() + ";version=" + entry.getValue().osgiVersion()) + .collect(Collectors.toList()); + } + + private static String asOsgiImport(String packageName, Optional<String> version) { + return version + .map(s -> packageName + ";version=" + ("\"" + s + "\"")) + .orElse(packageName); + } + + private static String bundleClassPath(Collection<Artifact> artifactsToInclude) { + return Stream.concat(Stream.of("."), artifactsToInclude.stream() + .map(artifact -> "dependencies/" + artifact.getFile().getName())) + .collect(Collectors.joining(",")); + } + + private Map<String, Optional<String>> getManualImports() { + try { + if (importPackage == null || importPackage.isBlank()) return Map.of(); + Map<String, Optional<String>> ret = new HashMap<>(); + List<ExportPackages.Export> imports = ExportPackageParser.parseExports(importPackage); + for (ExportPackages.Export imp : imports) { + Optional<String> version = getVersionThrowOthers(imp.getParameters()); + imp.getPackageNames().forEach(pn -> ret.put(pn, version)); + } + return ret; + } catch (Exception e) { + throw new RuntimeException("Error in Import-Package:" + importPackage, e); + } + } + + private static String asBundleVersion(String projectVersion) { + if (projectVersion == null) { + throw new IllegalArgumentException("Missing project version."); + } + + String[] parts = projectVersion.split("-", 2); + List<String> numericPart = Stream.of(parts[0].split("\\.")).map(s -> Strings.replaceEmptyString(s, "0")).limit(3) + .collect(toList()); + while (numericPart.size() < 3) { + numericPart.add("0"); + } + + return String.join(".", numericPart); + } + + private static Optional<String> getVersionThrowOthers(List<ExportPackages.Parameter> parameters) { + if (parameters.size() == 1 && "version".equals(parameters.get(0).getName())) { + return Optional.of(parameters.get(0).getValue()); + } else if (parameters.size() == 0) { + return Optional.empty(); + } else { + List<String> paramNames = parameters.stream().map(ExportPackages.Parameter::getName).collect(Collectors.toList()); + throw new RuntimeException("A single, optional version parameter expected, but got " + paramNames); + } + } + + private static PackageTally definedPackages(JarFile jarFile, ArtifactVersion version) throws MojoExecutionException { + List<ClassFileMetaData> analyzedClasses = new ArrayList<>(); + for (Enumeration<JarEntry> entries = jarFile.entries(); entries.hasMoreElements();) { + JarEntry entry = entries.nextElement(); + if (! entry.isDirectory() && entry.getName().endsWith(".class")) { + analyzedClasses.add(analyzeClass(jarFile, entry, version)); + } + } + return PackageTally.fromAnalyzedClassFiles(analyzedClasses); + } + + private static ClassFileMetaData analyzeClass(JarFile jarFile, JarEntry entry, ArtifactVersion version) throws MojoExecutionException { + try { + return Analyze.analyzeClass(jarFile.getInputStream(entry), version); + } catch (Exception e) { + throw new MojoExecutionException( + String.format("While analyzing the class '%s' in jar file '%s'", entry.getName(), jarFile.getName()), e); + } + } + +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/Artifacts.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/Artifacts.java index a0d0e143724..bc6a970140d 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/Artifacts.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/Artifacts.java @@ -18,7 +18,10 @@ class Artifacts { private final List<Artifact> jarArtifactsProvided; private final List<Artifact> nonJarArtifacts; - private ArtifactSet(List<Artifact> jarArtifactsToInclude, List<Artifact> jarArtifactsProvided, List<Artifact> nonJarArtifacts) { + private ArtifactSet( + List<Artifact> jarArtifactsToInclude, + List<Artifact> jarArtifactsProvided, + List<Artifact> nonJarArtifacts) { this.jarArtifactsToInclude = jarArtifactsToInclude; this.jarArtifactsProvided = jarArtifactsProvided; this.nonJarArtifacts = nonJarArtifacts; @@ -37,19 +40,24 @@ class Artifacts { } } - static ArtifactSet getArtifacts(MavenProject project) { + static ArtifactSet getArtifacts(MavenProject project) { return getArtifacts(project, false, null); } + static ArtifactSet getArtifacts(MavenProject project, boolean includeTestArtifacts, String testProvidedConfig) { + TestProvidedArtifacts testProvidedArtifacts = TestProvidedArtifacts.from(project.getArtifactMap(), testProvidedConfig); List<Artifact> jarArtifactsToInclude = new ArrayList<>(); List<Artifact> jarArtifactsProvided = new ArrayList<>(); List<Artifact> nonJarArtifactsToInclude = new ArrayList<>(); List<Artifact> nonJarArtifactsProvided = new ArrayList<>(); - for (Artifact artifact : project.getArtifacts()) { if ("jar".equals(artifact.getType())) { - if (Artifact.SCOPE_COMPILE.equals(artifact.getScope())) { + if (includeTestArtifacts && testProvidedArtifacts.isTestProvided(artifact)) { + jarArtifactsProvided.add(artifact); + } else if (Artifact.SCOPE_COMPILE.equals(artifact.getScope())) { jarArtifactsToInclude.add(artifact); } else if (Artifact.SCOPE_PROVIDED.equals(artifact.getScope())) { jarArtifactsProvided.add(artifact); + } else if (includeTestArtifacts && Artifact.SCOPE_TEST.equals(artifact.getScope())) { + jarArtifactsToInclude.add(artifact); } } else { if (Artifact.SCOPE_COMPILE.equals(artifact.getScope())) { @@ -64,6 +72,6 @@ class Artifacts { } static Collection<Artifact> getArtifactsToInclude(MavenProject project) { - return getArtifacts(project).getJarArtifactsToInclude(); + return getArtifacts(project, false, null).getJarArtifactsToInclude(); } } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.java index 13d73d58a97..bed7610e82f 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.java @@ -1,53 +1,36 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.plugin.mojo; -import com.yahoo.container.plugin.util.Files; -import com.yahoo.container.plugin.util.JarFiles; -import org.apache.maven.archiver.MavenArchiveConfiguration; -import org.apache.maven.archiver.MavenArchiver; -import org.apache.maven.execution.MavenSession; import org.apache.maven.model.Build; -import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.Component; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; import org.apache.maven.plugins.annotations.ResolutionScope; -import org.apache.maven.project.MavenProject; import org.apache.maven.project.MavenProjectHelper; import org.codehaus.plexus.archiver.jar.JarArchiver; import java.io.File; -import java.nio.channels.Channels; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.EnumMap; -import java.util.Enumeration; import java.util.Map; -import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.zip.ZipEntry; /** * @author Tony Vaagenes - * @author ollivir + * @author Olli Virtanen + * @author bjorncs */ @Mojo(name = "assemble-container-plugin", requiresDependencyResolution = ResolutionScope.COMPILE, threadSafe = true) -public class AssembleContainerPluginMojo extends AbstractMojo { +public class AssembleContainerPluginMojo extends AbstractAssembleBundleMojo { private static enum Dependencies { WITH, WITHOUT } - @Parameter(defaultValue = "${project}") - private MavenProject project = null; - @Component private MavenProjectHelper projectHelper; - @Parameter(defaultValue = "${session}", readonly = true, required = true) - private MavenSession session = null; - - @Parameter - private MavenArchiveConfiguration archiveConfiguration = new MavenArchiveConfiguration(); - @Parameter(alias = "UseCommonAssemblyIds", defaultValue = "false") private boolean useCommonAssemblyIds = false; @@ -74,19 +57,17 @@ public class AssembleContainerPluginMojo extends AbstractMojo { jarFiles.put(dep, jarFileInBuildDirectory(build().getFinalName(), suffix)); }); - // force recreating the archive - archiveConfiguration.setForced(true); - archiveConfiguration.setManifestFile(new File(new File(build().getOutputDirectory()), JarFile.MANIFEST_NAME)); + Path manifestFile = Paths.get(build().getOutputDirectory(), JarFile.MANIFEST_NAME); JarArchiver jarWithoutDependencies = new JarArchiver(); - addClassesDirectory(jarWithoutDependencies); - createArchive(jarFiles.get(Dependencies.WITHOUT), jarWithoutDependencies); + addDirectory(jarWithoutDependencies, Paths.get(build().getOutputDirectory())); + createArchive(jarWithoutDependencies, jarFiles.get(Dependencies.WITHOUT).toPath(), manifestFile); project.getArtifact().setFile(jarFiles.get(Dependencies.WITHOUT)); JarArchiver jarWithDependencies = new JarArchiver(); - addClassesDirectory(jarWithDependencies); - addDependencies(jarWithDependencies); - createArchive(jarFiles.get(Dependencies.WITH), jarWithDependencies); + addDirectory(jarWithDependencies, Paths.get(build().getOutputDirectory())); + addArtifacts(jarWithDependencies, Artifacts.getArtifactsToInclude(project)); + createArchive(jarWithDependencies, jarFiles.get(Dependencies.WITH).toPath(), manifestFile); if (attachBundleArtifact) { projectHelper.attachArtifact(project, @@ -96,68 +77,6 @@ public class AssembleContainerPluginMojo extends AbstractMojo { } } - private File jarFileInBuildDirectory(String name, String suffix) { - return new File(build().getDirectory(), name + suffix); - } - - private void addClassesDirectory(JarArchiver jarArchiver) { - File classesDirectory = new File(build().getOutputDirectory()); - if (classesDirectory.isDirectory()) { - jarArchiver.addDirectory(classesDirectory); - } - } - - private void createArchive(File jarFile, JarArchiver jarArchiver) throws MojoExecutionException { - MavenArchiver mavenArchiver = new MavenArchiver(); - mavenArchiver.setArchiver(jarArchiver); - mavenArchiver.setOutputFile(jarFile); - try { - mavenArchiver.createArchive(session, project, archiveConfiguration); - } catch (Exception e) { - throw new MojoExecutionException("Error creating archive " + jarFile.getName(), e); - } - } - - private void addDependencies(JarArchiver jarArchiver) { - Artifacts.getArtifactsToInclude(project).forEach(artifact -> { - if ("jar".equals(artifact.getType())) { - jarArchiver.addFile(artifact.getFile(), "dependencies/" + artifact.getFile().getName()); - copyConfigDefinitions(artifact.getFile(), jarArchiver); - } else { - getLog().warn("Unkown artifact type " + artifact.getType()); - } - }); - } - - private void copyConfigDefinitions(File file, JarArchiver jarArchiver) { - JarFiles.withJarFile(file, jarFile -> { - for (Enumeration<JarEntry> entries = jarFile.entries(); entries.hasMoreElements();) { - JarEntry entry = entries.nextElement(); - String name = entry.getName(); - if (name.startsWith("configdefinitions/") && name.endsWith(".def")) { - copyConfigDefinition(jarFile, entry, jarArchiver); - } - } - return null; - }); - } - - private void copyConfigDefinition(JarFile jarFile, ZipEntry entry, JarArchiver jarArchiver) { - JarFiles.withInputStream(jarFile, entry, input -> { - String defPath = entry.getName().replace("/", File.separator); - File destinationFile = new File(build().getOutputDirectory(), defPath); - destinationFile.getParentFile().mkdirs(); - - Files.withFileOutputStream(destinationFile, output -> { - output.getChannel().transferFrom(Channels.newChannel(input), 0, Long.MAX_VALUE); - return null; - }); - jarArchiver.addFile(destinationFile, entry.getName()); - return null; - }); - } - - private Build build() { - return project.getBuild(); - } + private File jarFileInBuildDirectory(String name, String suffix) { return new File(build().getDirectory(), name + suffix); } + private Build build() { return project.getBuild(); } } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleTestBundleMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleTestBundleMojo.java new file mode 100644 index 00000000000..ab827275f53 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleTestBundleMojo.java @@ -0,0 +1,38 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.mojo; + +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.codehaus.plexus.archiver.jar.JarArchiver; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import static com.yahoo.container.plugin.mojo.TestBundleUtils.archiveFile; +import static com.yahoo.container.plugin.mojo.TestBundleUtils.manifestFile; + +/** + * @author bjorncs + */ +@Mojo(name = "assemble-test-bundle", requiresDependencyResolution = ResolutionScope.TEST, threadSafe = true) +public class AssembleTestBundleMojo extends AbstractAssembleBundleMojo { + + @Parameter + private String testProvidedArtifacts; + + @Override + public void execute() throws MojoExecutionException { + Artifacts.ArtifactSet artifacts = Artifacts.getArtifacts(project, true, testProvidedArtifacts); + JarArchiver archiver = new JarArchiver(); + addDirectory(archiver, Paths.get(project.getBuild().getOutputDirectory())); + addDirectory(archiver, Paths.get(project.getBuild().getTestOutputDirectory())); + addArtifacts(archiver, artifacts.getJarArtifactsToInclude()); + Path archiveFile = archiveFile(project); + createArchive(archiver, archiveFile, manifestFile(project)); + project.getArtifact().setFile(archiveFile.toFile()); + } + + +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java index 41420b38360..3020184fd35 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java @@ -4,36 +4,23 @@ package com.yahoo.container.plugin.mojo; import com.google.common.collect.Sets; import com.yahoo.container.plugin.classanalysis.Analyze; import com.yahoo.container.plugin.classanalysis.ClassFileMetaData; -import com.yahoo.container.plugin.classanalysis.ExportPackageAnnotation; import com.yahoo.container.plugin.classanalysis.PackageTally; -import com.yahoo.container.plugin.osgi.ExportPackageParser; import com.yahoo.container.plugin.osgi.ExportPackages; import com.yahoo.container.plugin.osgi.ExportPackages.Export; import com.yahoo.container.plugin.osgi.ImportPackages.Import; -import com.yahoo.container.plugin.util.Strings; import org.apache.maven.artifact.Artifact; -import org.apache.maven.artifact.versioning.ArtifactVersion; -import org.apache.maven.artifact.versioning.DefaultArtifactVersion; -import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; import org.apache.maven.plugins.annotations.ResolutionScope; -import org.apache.maven.project.MavenProject; import java.io.File; -import java.util.ArrayList; +import java.nio.file.Paths; import java.util.Collection; -import java.util.Enumeration; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.jar.Attributes; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import java.util.jar.Manifest; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -41,7 +28,6 @@ import static com.yahoo.container.plugin.bundle.AnalyzeBundle.exportedPackagesAg import static com.yahoo.container.plugin.osgi.ExportPackages.exportsByPackageName; import static com.yahoo.container.plugin.osgi.ImportPackages.calculateImports; import static com.yahoo.container.plugin.util.Files.allDescendantFiles; -import static com.yahoo.container.plugin.util.IO.withFileOutputStream; /** @@ -49,19 +35,7 @@ import static com.yahoo.container.plugin.util.IO.withFileOutputStream; * @author ollivir */ @Mojo(name = "generate-osgi-manifest", requiresDependencyResolution = ResolutionScope.TEST, threadSafe = true) -public class GenerateOsgiManifestMojo extends AbstractMojo { - - @Parameter(defaultValue = "${project}") - private MavenProject project = null; - - /** - * If set to true, the artifact's version is used as default package version for ExportPackages. - * Packages from included (compile scoped) artifacts will use the version for their own artifact. - * If the package is exported with an explicit version in package-info.java, that version will be - * used regardless of this parameter. - */ - @Parameter(alias = "UseArtifactVersionForExportPackages", defaultValue = "false") - private boolean useArtifactVersionForExportPackages; +public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo { @Parameter private String discApplicationClass = null; @@ -69,22 +43,12 @@ public class GenerateOsgiManifestMojo extends AbstractMojo { @Parameter private String discPreInstallBundle = null; - @Parameter(alias = "Bundle-Version", defaultValue = "${project.version}") - private String bundleVersion = null; - - // TODO: default should include groupId, but that will require a lot of changes both by us and users. - @Parameter(alias = "Bundle-SymbolicName", defaultValue = "${project.artifactId}") - private String bundleSymbolicName = null; - @Parameter(alias = "Bundle-Activator") private String bundleActivator = null; @Parameter(alias = "X-JDisc-Privileged-Activator") private String jdiscPrivilegedActivator = null; - @Parameter(alias = "Import-Package") - private String importPackage = null; - @Parameter(alias = "WebInfUrl") private String webInfUrl = null; @@ -127,19 +91,25 @@ public class GenerateOsgiManifestMojo extends AbstractMojo { includedPackages.definedPackages(), exportsByPackageName(exportedPackagesFromProvidedJars)); - Map<String, Optional<String>> manualImports = emptyToNone(importPackage).map(GenerateOsgiManifestMojo::getManualImports) - .orElseGet(HashMap::new); - for (String packageName : manualImports.keySet()) { - calculatedImports.remove(packageName); - } - createManifestFile(new File(project.getBuild().getOutputDirectory()), manifestContent(project, - artifactSet.getJarArtifactsToInclude(), manualImports, calculatedImports.values(), includedPackages)); + + Map<String, String> manifestContent = generateManifestContent(artifactSet.getJarArtifactsToInclude(), calculatedImports, includedPackages); + addAdditionalManifestProperties(manifestContent); + createManifestFile(Paths.get(project.getBuild().getOutputDirectory()), manifestContent); } catch (Exception e) { throw new MojoExecutionException("Failed generating osgi manifest", e); } } + private void addAdditionalManifestProperties(Map<String, String> manifestContent) { + addIfNotEmpty(manifestContent, "Bundle-Activator", bundleActivator); + addIfNotEmpty(manifestContent, "X-JDisc-Privileged-Activator", jdiscPrivilegedActivator); + addIfNotEmpty(manifestContent, "Main-Class", mainClass); + addIfNotEmpty(manifestContent, "X-JDisc-Application", discApplicationClass); + addIfNotEmpty(manifestContent, "X-JDisc-Preinstall-Bundle", trimWhitespace(Optional.ofNullable(discPreInstallBundle))); + addIfNotEmpty(manifestContent, "WebInfUrl", webInfUrl); + } + private void logDebugPackageSets(List<Export> exportedPackagesFromProvidedJars, PackageTally includedPackages) { if (getLog().isDebugEnabled()) { getLog().debug("Referenced packages = " + includedPackages.referencedPackages()); @@ -197,101 +167,10 @@ public class GenerateOsgiManifestMojo extends AbstractMojo { } } - private Collection<String> osgiExportPackages(Map<String, ExportPackageAnnotation> exportedPackages) { - return exportedPackages.entrySet().stream().map(entry -> entry.getKey() + ";version=" + entry.getValue().osgiVersion()) - .collect(Collectors.toList()); - } - private static String trimWhitespace(Optional<String> lines) { return Stream.of(lines.orElse("").split(",")).map(String::trim).collect(Collectors.joining(",")); } - private Map<String, String> manifestContent(MavenProject project, Collection<Artifact> jarArtifactsToInclude, - Map<String, Optional<String>> manualImports, Collection<Import> imports, PackageTally pluginPackageTally) { - Map<String, String> ret = new HashMap<>(); - String importPackage = Stream.concat(manualImports.entrySet().stream().map(e -> asOsgiImport(e.getKey(), e.getValue())), - imports.stream().map(Import::asOsgiImport)).sorted() - .collect(Collectors.joining(",")); - - String exportPackage = osgiExportPackages(pluginPackageTally.exportedPackages()).stream().sorted() - .collect(Collectors.joining(",")); - - ret.put("Created-By", "vespa container maven plugin"); - ret.put("Bundle-ManifestVersion", "2"); - addIfNotEmpty(ret, "Bundle-Name", project.getName()); - addIfNotEmpty(ret, "Bundle-SymbolicName", bundleSymbolicName); - addIfNotEmpty(ret, "Bundle-Version", asBundleVersion(bundleVersion)); - ret.put("Bundle-Vendor", "Yahoo!"); - addIfNotEmpty(ret, "Bundle-ClassPath", bundleClassPath(jarArtifactsToInclude)); - addIfNotEmpty(ret, "Bundle-Activator", bundleActivator); - addIfNotEmpty(ret, "X-JDisc-Privileged-Activator", jdiscPrivilegedActivator); - addIfNotEmpty(ret, "Main-Class", mainClass); - addIfNotEmpty(ret, "X-JDisc-Application", discApplicationClass); - addIfNotEmpty(ret, "X-JDisc-Preinstall-Bundle", trimWhitespace(Optional.ofNullable(discPreInstallBundle))); - addIfNotEmpty(ret, "WebInfUrl", webInfUrl); - addIfNotEmpty(ret, "Import-Package", importPackage); - addIfNotEmpty(ret, "Export-Package", exportPackage); - - return ret; - } - - private static void addIfNotEmpty(Map<String, String> map, String key, String value) { - if (value != null && ! value.isEmpty()) { - map.put(key, value); - } - } - - private static String asOsgiImport(String packageName, Optional<String> version) { - return version.map(s -> packageName + ";version=" + quote(s)).orElse(packageName); - } - - private static String quote(String s) { - return "\"" + s + "\""; - } - - private static void createManifestFile(File outputDirectory, Map<String, String> manifestContent) { - Manifest manifest = toManifest(manifestContent); - - withFileOutputStream(new File(outputDirectory, JarFile.MANIFEST_NAME), outputStream -> { - manifest.write(outputStream); - return null; - }); - } - - private static Manifest toManifest(Map<String, String> manifestContent) { - Manifest manifest = new Manifest(); - Attributes mainAttributes = manifest.getMainAttributes(); - - mainAttributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); - manifestContent.forEach(mainAttributes::putValue); - - return manifest; - } - - private static String bundleClassPath(Collection<Artifact> artifactsToInclude) { - return Stream.concat(Stream.of("."), artifactsToInclude.stream().map(GenerateOsgiManifestMojo::dependencyPath)) - .collect(Collectors.joining(",")); - } - - private static String dependencyPath(Artifact artifact) { - return "dependencies/" + artifact.getFile().getName(); - } - - private static String asBundleVersion(String projectVersion) { - if (projectVersion == null) { - throw new IllegalArgumentException("Missing project version."); - } - - String[] parts = projectVersion.split("-", 2); - List<String> numericPart = Stream.of(parts[0].split("\\.")).map(s -> Strings.replaceEmptyString(s, "0")).limit(3) - .collect(Collectors.toList()); - while (numericPart.size() < 3) { - numericPart.add("0"); - } - - return String.join(".", numericPart); - } - private void warnOnUnsupportedArtifacts(Collection<Artifact> nonJarArtifacts) { List<Artifact> unsupportedArtifacts = nonJarArtifacts.stream().filter(a -> ! a.getType().equals("pom")) .collect(Collectors.toList()); @@ -301,10 +180,6 @@ public class GenerateOsgiManifestMojo extends AbstractMojo { artifact.getId(), artifact.getType()))); } - private ArtifactVersion artifactVersionOrNull(String version) { - return useArtifactVersionForExportPackages ? new DefaultArtifactVersion(version) : null; - } - private PackageTally getProjectClassesTally() { File outputDirectory = new File(project.getBuild().getOutputDirectory()); @@ -315,71 +190,4 @@ public class GenerateOsgiManifestMojo extends AbstractMojo { return PackageTally.fromAnalyzedClassFiles(analyzedClasses); } - - private PackageTally definedPackages(Collection<Artifact> jarArtifacts) { - List<PackageTally> tallies = new ArrayList<>(); - for (var artifact : jarArtifacts) { - try { - tallies.add(definedPackages(new JarFile(artifact.getFile()), artifactVersionOrNull(artifact.getVersion()))); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - return PackageTally.combine(tallies); - } - - private static PackageTally definedPackages(JarFile jarFile, ArtifactVersion version) throws MojoExecutionException { - List<ClassFileMetaData> analyzedClasses = new ArrayList<>(); - for (Enumeration<JarEntry> entries = jarFile.entries(); entries.hasMoreElements();) { - JarEntry entry = entries.nextElement(); - if (! entry.isDirectory() && entry.getName().endsWith(".class")) { - analyzedClasses.add(analyzeClass(jarFile, entry, version)); - } - } - return PackageTally.fromAnalyzedClassFiles(analyzedClasses); - } - - private static ClassFileMetaData analyzeClass(JarFile jarFile, JarEntry entry, ArtifactVersion version) throws MojoExecutionException { - try { - return Analyze.analyzeClass(jarFile.getInputStream(entry), version); - } catch (Exception e) { - throw new MojoExecutionException( - String.format("While analyzing the class '%s' in jar file '%s'", entry.getName(), jarFile.getName()), e); - } - } - - private static Map<String, Optional<String>> getManualImports(String importPackage) { - try { - Map<String, Optional<String>> ret = new HashMap<>(); - List<Export> imports = parseImportPackages(importPackage); - for (Export imp : imports) { - Optional<String> version = getVersionThrowOthers(imp.getParameters()); - imp.getPackageNames().forEach(pn -> ret.put(pn, version)); - } - - return ret; - } catch (Exception e) { - throw new RuntimeException("Error in Import-Package:" + importPackage, e); - } - } - - private static Optional<String> getVersionThrowOthers(List<ExportPackages.Parameter> parameters) { - if (parameters.size() == 1 && "version".equals(parameters.get(0).getName())) { - return Optional.of(parameters.get(0).getValue()); - } else if (parameters.size() == 0) { - return Optional.empty(); - } else { - List<String> paramNames = parameters.stream().map(ExportPackages.Parameter::getName).collect(Collectors.toList()); - throw new RuntimeException("A single, optional version parameter expected, but got " + paramNames); - } - } - - private static List<Export> parseImportPackages(String importPackages) { - return ExportPackageParser.parseExports(importPackages); - } - - private static Optional<String> emptyToNone(String str) { - return Optional.ofNullable(str).map(String::trim).filter(s -> ! s.isEmpty()); - } - } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateTestBundleOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateTestBundleOsgiManifestMojo.java new file mode 100644 index 00000000000..811aff87b7e --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateTestBundleOsgiManifestMojo.java @@ -0,0 +1,80 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.mojo; + +import com.yahoo.container.plugin.classanalysis.Analyze; +import com.yahoo.container.plugin.classanalysis.ClassFileMetaData; +import com.yahoo.container.plugin.classanalysis.PackageTally; +import com.yahoo.container.plugin.osgi.ExportPackages.Export; +import com.yahoo.container.plugin.osgi.ImportPackages; +import org.apache.maven.artifact.Artifact; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static com.yahoo.container.plugin.bundle.AnalyzeBundle.exportedPackagesAggregated; +import static com.yahoo.container.plugin.mojo.TestBundleUtils.outputDirectory; +import static com.yahoo.container.plugin.osgi.ExportPackages.exportsByPackageName; +import static com.yahoo.container.plugin.osgi.ImportPackages.calculateImports; +import static com.yahoo.container.plugin.util.Files.allDescendantFiles; +import static java.util.stream.Collectors.toList; + +/** + * @author bjorncs + */ +@Mojo(name = "generate-test-bundle-osgi-manifest", requiresDependencyResolution = ResolutionScope.TEST, threadSafe = true) +public class GenerateTestBundleOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo { + + @Parameter + private String testProvidedArtifacts; + + public void execute() throws MojoExecutionException { + try { + Artifacts.ArtifactSet artifactSet = Artifacts.getArtifacts(project, true, testProvidedArtifacts); + + List<File> providedJars = artifactSet.getJarArtifactsProvided().stream() + .map(Artifact::getFile) + .collect(toList()); + + List<Export> exportedPackagesFromProvidedJars = exportedPackagesAggregated(providedJars); + + PackageTally projectPackages = getProjectMainAndTestClassesTally(); + + PackageTally jarArtifactsToInclude = definedPackages(artifactSet.getJarArtifactsToInclude()); + + PackageTally includedPackages = projectPackages.combine(jarArtifactsToInclude); + + Map<String, ImportPackages.Import> calculatedImports = calculateImports(includedPackages.referencedPackages(), + includedPackages.definedPackages(), + exportsByPackageName(exportedPackagesFromProvidedJars)); + + Map<String, String> manifestContent = generateManifestContent(artifactSet.getJarArtifactsToInclude(), calculatedImports, includedPackages); + addAdditionalManifestProperties(manifestContent); + createManifestFile(outputDirectory(project), manifestContent); + + } catch (Exception e) { + throw new MojoExecutionException("Failed generating osgi manifest", e); + } + } + + private void addAdditionalManifestProperties(Map<String, String> manifestContent) { + manifestContent.put("X-JDisc-Test-Bundle-Version", "1.0"); + } + + private PackageTally getProjectMainAndTestClassesTally() { + List<ClassFileMetaData> analyzedClasses = + Stream.concat( + allDescendantFiles(new File(project.getBuild().getOutputDirectory())), + allDescendantFiles(new File(project.getBuild().getTestOutputDirectory()))) + .filter(file -> file.getName().endsWith(".class")) + .map(classFile -> Analyze.analyzeClass(classFile, null)) + .collect(toList()); + return PackageTally.fromAnalyzedClassFiles(analyzedClasses); + } + +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/TestBundleUtils.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/TestBundleUtils.java new file mode 100644 index 00000000000..9a3fc89bbd5 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/TestBundleUtils.java @@ -0,0 +1,25 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.mojo; + +import org.apache.maven.project.MavenProject; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.jar.JarFile; + +/** + * @author bjorncs + */ +class TestBundleUtils { + private TestBundleUtils() {} + + static Path outputDirectory(MavenProject project) { return targetDirectory(project).resolve("test-bundle/"); } + + static Path manifestFile(MavenProject project) { return outputDirectory(project).resolve(JarFile.MANIFEST_NAME); } + + static Path archiveFile(MavenProject project) { + return targetDirectory(project).resolve(project.getBuild().getFinalName() + "-tests.jar"); + } + + private static Path targetDirectory(MavenProject project) { return Paths.get(project.getBuild().getDirectory()); } +} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/TestProvidedArtifacts.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/TestProvidedArtifacts.java new file mode 100644 index 00000000000..6f9e305f8d9 --- /dev/null +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/TestProvidedArtifacts.java @@ -0,0 +1,73 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.mojo; + +import org.apache.maven.artifact.Artifact; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; + +/** + * Determines the test dependencies that are provided by the Vespa/JDisc test runtime based on the resolved dependency graph and a config string. + * "Test provided" dependencies are treated as "provided" scope dependencies when building a test bundle. + * + * @author bjorncs + */ +class TestProvidedArtifacts { + + private final List<Artifact> artifacts; + + private TestProvidedArtifacts(List<Artifact> artifacts) { this.artifacts = artifacts; } + + boolean isTestProvided(Artifact artifact) { return artifacts.contains(artifact); } + + static TestProvidedArtifacts from(Map<String, Artifact> artifacts, String configString) { + if (configString == null || configString.isBlank()) return new TestProvidedArtifacts(List.of()); + return new TestProvidedArtifacts(getTestProvidedArtifacts(artifacts, configString)); + } + + private static List<Artifact> getTestProvidedArtifacts(Map<String, Artifact> artifacts, String configString) { + List<String> testProvidedArtifactStringIds = toTestProvidedArtifactStringIds(configString); + List<Artifact> testProvidedArtifacts = new ArrayList<>(); + for (Artifact artifact : artifacts.values()) { + boolean hasTestProvidedArtifactAsParent = + dependencyTrail(artifact, artifacts) + .anyMatch(parent -> testProvidedArtifactStringIds.contains(toArtifactStringId(parent))); + boolean isBlacklisted = testProvidedArtifactStringIds.contains(toBlacklistedArtifactStringId(artifact)); + if (hasTestProvidedArtifactAsParent && !isBlacklisted) { + testProvidedArtifacts.add(artifact); + } + } + return testProvidedArtifacts; + } + + private static List<String> toTestProvidedArtifactStringIds(String commaSeparatedString) { + if (commaSeparatedString == null || commaSeparatedString.isBlank()) return List.of(); + return Arrays.stream(commaSeparatedString.split(",")) + .map(String::strip) + .filter(s -> !s.isBlank()) + .collect(toList()); + } + + private static Stream<Artifact> dependencyTrail(Artifact artifact, Map<String, Artifact> otherArtifacts) { + return artifact.getDependencyTrail().stream() + .map(parentId -> otherArtifacts.get(stripVersionAndScope(parentId))) + .filter(Objects::nonNull); + } + + private static String stripVersionAndScope(String fullArtifactIdentifier) { + int firstDelimiter = fullArtifactIdentifier.indexOf(':'); + int secondDelimiter = fullArtifactIdentifier.indexOf(':', firstDelimiter + 1); + return fullArtifactIdentifier.substring(0, secondDelimiter); + } + + private static String toArtifactStringId(Artifact artifact) { return artifact.getGroupId() + ":" + artifact.getArtifactId(); } + + private static String toBlacklistedArtifactStringId(Artifact artifact) { return "!" + toArtifactStringId(artifact); } + +} diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/mojo/TestProvidedArtifactsTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/mojo/TestProvidedArtifactsTest.java new file mode 100644 index 00000000000..b60bce794f5 --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/mojo/TestProvidedArtifactsTest.java @@ -0,0 +1,68 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.mojo; + + +import org.apache.maven.artifact.Artifact; +import org.apache.maven.artifact.DefaultArtifact; +import org.apache.maven.artifact.handler.DefaultArtifactHandler; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author bjorncs + */ +public class TestProvidedArtifactsTest { + + private static final String GROUP_ID = "com.test"; + + @Test + public void findsAllTestProvidedDependencies() { + Map<String, Artifact> artifacts = new TreeMap<>(); + Artifact a = createArtifact(artifacts, "a"); + Artifact aa = createArtifact(artifacts, "a-a", "a"); + Artifact ab = createArtifact(artifacts, "a-b", "a"); + Artifact aaa = createArtifact(artifacts, "a-a-a", "a", "a-a"); + Artifact b = createArtifact(artifacts, "b"); + Artifact ba = createArtifact(artifacts, "b-a", "b"); + Artifact c = createArtifact(artifacts, "c"); + + String configString = "com.test:a,com.test:b-a,!com.test:a-b"; + TestProvidedArtifacts testProvidedArtifacts = TestProvidedArtifacts.from(artifacts, configString); + + assertTrue(testProvidedArtifacts.isTestProvided(a)); + assertTrue(testProvidedArtifacts.isTestProvided(aa)); + assertFalse(testProvidedArtifacts.isTestProvided(ab)); + assertTrue(testProvidedArtifacts.isTestProvided(aaa)); + assertFalse(testProvidedArtifacts.isTestProvided(b)); + assertTrue(testProvidedArtifacts.isTestProvided(ba)); + assertFalse(testProvidedArtifacts.isTestProvided(c)); + } + + private static Artifact createArtifact(Map<String, Artifact> artifacts, String artifactId, String... dependents) { + Artifact artifact = createArtifact(artifactId, dependents); + artifacts.put(simpleId(artifactId), artifact); + return artifact; + } + + private static Artifact createArtifact(String artifactId, String... dependents) { + Artifact artifact = new DefaultArtifact(GROUP_ID, artifactId, "1.0", "test", "jar", "deploy", new DefaultArtifactHandler("jar")); + List<String> dependencyTrail = new ArrayList<>(); + dependencyTrail.add(fullId("bundle-plugin")); + Arrays.stream(dependents).forEach(dependent -> dependencyTrail.add(fullId(dependent))); + dependencyTrail.add(fullId(artifactId)); + artifact.setDependencyTrail(dependencyTrail); + return artifact; + } + + private static String fullId(String artifactId) { return simpleId(artifactId) + ":1.0:compile"; } + private static String simpleId(String artifactId) { return GROUP_ID + ":" + artifactId; } + +}
\ No newline at end of file diff --git a/cloud-tenant-cd/CMakeLists.txt b/cloud-tenant-cd/CMakeLists.txt new file mode 100644 index 00000000000..2d30b9c4611 --- /dev/null +++ b/cloud-tenant-cd/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(cloud-tenant-cd) diff --git a/cloud-tenant-cd/pom.xml b/cloud-tenant-cd/pom.xml index c771e2dd1c3..b4670e6e83f 100644 --- a/cloud-tenant-cd/pom.xml +++ b/cloud-tenant-cd/pom.xml @@ -78,9 +78,7 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> <configuration> - <attachBundleArtifact>true</attachBundleArtifact> - <bundleClassifierName>deploy</bundleClassifierName> - <useCommonAssemblyIds>false</useCommonAssemblyIds> + <useCommonAssemblyIds>true</useCommonAssemblyIds> </configuration> </plugin> <plugin> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 210f54de6a6..3e421f9ba05 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -248,8 +248,6 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce ib.hnsw.enabled(true); ib.hnsw.maxlinkspernode(params.maxLinksPerNode()); ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert()); - var dm = AttributesConfig.Attribute.Index.Hnsw.Distancemetric.Enum.valueOf(dma.toString()); - ib.hnsw.distancemetric(dm); ib.hnsw.multithreadedindexing(params.multiThreadedIndexing()); aaB.index(ib); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index 49d396327bf..f7f2259bd58 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -39,7 +39,7 @@ import java.util.Set; */ public final class Attribute implements Cloneable, Serializable { - public enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES } + public enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES, INNERPRODUCT } // Remember to change hashCode and equals when you add new fields diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index 51751b2e247..d5b22988e36 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -58,14 +58,13 @@ public class ExactMatch extends Processor { field.addQueryCommand("word"); } else { // exact String exactTerminator = DEFAULT_EXACT_TERMINATOR; - if (field.getMatching().getExactMatchTerminator() != null && - ! field.getMatching().getExactMatchTerminator().equals("")) - { + if (field.getMatching().getExactMatchTerminator() != null + && ! field.getMatching().getExactMatchTerminator().equals("")) { exactTerminator = field.getMatching().getExactMatchTerminator(); } else { warn(search, field, - "With 'exact' matching, an exact-terminator is needed (using \"" - + exactTerminator +"\" as terminator)"); + "With 'exact' matching, an exact-terminator is needed " + + "(using '" + exactTerminator +"' as terminator)"); } field.addQueryCommand("exact " + exactTerminator); @@ -103,6 +102,7 @@ public class ExactMatch extends Processor { } return exp; } + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 20f2bfe6636..86b6ab8a25c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -117,7 +117,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC addHttpHandler(PrometheusHandler.class, PrometheusHandler.V1_PATH); addHttpHandler(YamasHandler.class, YamasHandler.V1_PATH); - addHttpHandler(ApplicationMetricsHandler.class, ApplicationMetricsHandler.V1_PATH); + addHttpHandler(ApplicationMetricsHandler.class, ApplicationMetricsHandler.METRICS_V1_PATH); addMetricsProxyComponent(ApplicationMetricsRetriever.class); addTelegrafComponents(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index b9fc9643ac3..941870e980b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -471,7 +471,10 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.max")); metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.sum")); metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.count")); - metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); + metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate")); // TODO: Consider remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.max")); + metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.queries.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doomed_queries.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doom_factor.min")); @@ -493,7 +496,10 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.rate")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.rate")); // TODO: Consider remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.max")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.sum")); + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.limited_queries.rate")); return metrics; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index c8908495c0a..234387f5a6b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -124,8 +124,14 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } private void addTestrunnerComponentsIfTester(DeployState deployState) { - if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) + if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) { addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-testrunner-components-jar-with-dependencies.jar"))); + addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-osgi-testrunner-jar-with-dependencies.jar"))); + addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/tenant-cd-api-jar-with-dependencies.jar"))); + if(deployState.zone().system().isPublic()) { + addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/cloud-tenant-cd-jar-with-dependencies.jar"))); + } + } } public void setModelEvaluation(ContainerModelEvaluation modelEvaluation) { @@ -214,7 +220,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat @Override public void getConfig(MetricsProxyApiConfig.Builder builder) { builder.metricsPort(MetricsProxyContainer.BASEPORT) - .metricsApiPath(ApplicationMetricsHandler.VALUES_PATH); + .metricsApiPath(ApplicationMetricsHandler.METRICS_VALUES_PATH) + .prometheusApiPath(ApplicationMetricsHandler.PROMETHEUS_VALUES_PATH); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index ab491367510..7374242e94b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -52,6 +52,9 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro @Override public void getConfig(ServerConfig.Builder builder) { + builder.metric(new ServerConfig.Metric.Builder().monitoringHandlerPaths( + List.of("/state/v1", "/status.html") + )); } static ComponentModel providerComponentModel(final ComponentId parentId, String className) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java index 07b87a41b2f..1d5d2420c8f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java @@ -69,12 +69,10 @@ public class DocumentSelectionBuilder { String globalSelection = elem.stringAttribute("selection"); if (globalSelection != null) { validateSelectionExpression(globalSelection, null); - StringBuilder global = new StringBuilder(); - global.append('(').append(globalSelection).append(") AND (") - .append(sb.toString()).append(')'); - return global.toString(); + return "(" + globalSelection + ") AND (" + sb + ")"; } } return sb.toString(); } + } diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg index 9cea76174c7..c356fcb37ce 100644 --- a/config-model/src/test/derived/hnsw_index/attributes.cfg +++ b/config-model/src/test/derived/hnsw_index/attributes.cfg @@ -22,7 +22,7 @@ attribute[].imported false attribute[].distancemetric ANGULAR attribute[].index.hnsw.enabled true attribute[].index.hnsw.maxlinkspernode 32 -attribute[].index.hnsw.distancemetric ANGULAR +attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.neighborstoexploreatinsert 300 attribute[].index.hnsw.multithreadedindexing true attribute[].name "t2" diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 739ae055ec7..03c05af1145 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -4,16 +4,19 @@ package com.yahoo.vespa.model.container; import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.RoutingProviderConfig; +import com.yahoo.config.FileReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.container.BundlesConfig; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.Host; @@ -24,13 +27,19 @@ import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import org.hamcrest.CoreMatchers; import org.junit.Test; +import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; +import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; /** * @author Simon Thoresen Hult @@ -259,6 +268,41 @@ public class ContainerClusterTest { assertFalse(config.enabled()); } + @Test + public void requireThatBundlesForTesterApplicationAreInstalled() { + List<String> expectedOnpremBundles = + List.of("vespa-testrunner-components-jar-with-dependencies.jar", + "vespa-osgi-testrunner-jar-with-dependencies.jar", + "tenant-cd-api-jar-with-dependencies.jar"); + verifyTesterApplicationInstalledBundles(Zone.defaultZone(), expectedOnpremBundles); + + List<String> expectedPublicBundles = new ArrayList<>(expectedOnpremBundles); + expectedPublicBundles.add("cloud-tenant-cd-jar-with-dependencies.jar"); + Zone publicZone = new Zone(SystemName.PublicCd, Environment.dev, RegionName.defaultName()); + verifyTesterApplicationInstalledBundles(publicZone, expectedPublicBundles); + + } + + private void verifyTesterApplicationInstalledBundles(Zone zone, List<String> expectedBundleNames) { + ApplicationId appId = ApplicationId.from("tenant", "application", "instance-t"); + DeployState state = new DeployState.Builder().properties( + new TestProperties() + .setHostedVespa(true) + .setApplicationId(appId)) + .zone(zone).build(); + MockRoot root = new MockRoot("foo", state); + ApplicationContainerCluster cluster = new ApplicationContainerCluster(root, "container0", "container1", state); + BundlesConfig.Builder bundleBuilder = new BundlesConfig.Builder(); + cluster.getConfig(bundleBuilder); + List<String> installedBundles = bundleBuilder.build().bundle().stream().map(FileReference::value).collect(Collectors.toList()); + + assertEquals(expectedBundleNames.size(), installedBundles.size()); + assertThat(installedBundles, containsInAnyOrder( + expectedBundleNames.stream().map(CoreMatchers::endsWith).collect(Collectors.toList()) + )); + } + + private static void addContainer(DeployLogger deployLogger, ApplicationContainerCluster cluster, String name, String hostName) { ApplicationContainer container = new ApplicationContainer(cluster, name, 0, cluster.isHostedVespa()); container.setHostResource(new HostResource(new Host(null, hostName))); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Cloud.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Cloud.java index 24ec4a7ab70..0899a6f1007 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Cloud.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Cloud.java @@ -36,7 +36,7 @@ public class Cloud { return dynamicProvisioning; } - /** Returns wheter this allows different applications to share the same host */ + /** Returns whether this allows different applications to share the same host */ public boolean allowHostSharing() { return allowHostSharing; } diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index a091fbc573c..3609f00a4ce 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -33,13 +33,13 @@ attribute[].imported bool default=false # The distance metric to use for nearest neighbor search. # Is only used when the attribute is a 1-dimensional indexed tensor. -attribute[].distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES } default=EUCLIDEAN +attribute[].distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES, INNERPRODUCT } default=EUCLIDEAN # Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. attribute[].index.hnsw.enabled bool default=false attribute[].index.hnsw.maxlinkspernode int default=16 +attribute[].index.hnsw.neighborstoexploreatinsert int default=200 # Deprecated: Remove on Vespa 8 or before when possible. attribute[].index.hnsw.distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES } default=EUCLIDEAN -attribute[].index.hnsw.neighborstoexploreatinsert int default=200 # Whether multi-threaded indexing is enabled for this hnsw index. attribute[].index.hnsw.multithreadedindexing bool default=false diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 88a637e3ecb..26051c7ff9d 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -63,7 +63,6 @@ sleepTimeWhenRedeployingFails long default=30 # Features (to be overridden in configserver-config.xml if needed) buildMinimalSetOfConfigModels bool default=true -throwIfBootstrappingTenantRepoFails bool default=true # Unused, remove in Vespa 8 throwIfActiveSessionCannotBeLoaded bool default=true diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index becf01c191c..0557fa6e552 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -59,6 +59,7 @@ import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.orchestrator.Orchestrator; @@ -389,11 +390,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } NestedTransaction transaction = new NestedTransaction(); - transaction.add(new ContainerEndpointsCache(tenant.getPath(), tenant.getCurator()).delete(applicationId)); // TODO: Not unit tested + Curator curator = tenantRepository.getCurator(); + transaction.add(new ContainerEndpointsCache(tenant.getPath(), curator).delete(applicationId)); // TODO: Not unit tested // Delete any application roles - transaction.add(new ApplicationRolesStore(tenant.getCurator(), tenant.getPath()).delete(applicationId)); + transaction.add(new ApplicationRolesStore(curator, tenant.getPath()).delete(applicationId)); // Delete endpoint certificates - transaction.add(new EndpointCertificateMetadataStore(tenant.getCurator(), tenant.getPath()).delete(applicationId)); + transaction.add(new EndpointCertificateMetadataStore(curator, tenant.getPath()).delete(applicationId)); // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server // this is done asynchronously in application maintenance by the node repository) transaction.add(tenantApplications.createDeleteTransaction(applicationId)); @@ -481,6 +483,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return getLocalSession(tenant, sessionId).getApplicationFile(Path.fromString(path), mode); } + public Tenant getTenant(ApplicationId applicationId) { + return tenantRepository.getTenant(applicationId.tenant()); + } + private Application getApplication(ApplicationId applicationId) { return getApplication(applicationId, Optional.empty()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java index 41119077b28..12353c0e7e3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java @@ -96,4 +96,12 @@ public final class ApplicationSet { return new ArrayList<>(applications.values()); } + public List<Version> getAllVersions(ApplicationId applicationId) { + return applications.values().stream() + .filter(application -> application.getId().equals(applicationId)) + .map(Application::getVespaVersion) + .sorted() + .collect(Collectors.toList()); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 3c909730902..d1861184c24 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -15,13 +15,16 @@ import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingMatch; import com.yahoo.jdisc.application.UriPattern; +import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.http.ContentHandler; import com.yahoo.vespa.config.server.http.ContentRequest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; import com.yahoo.vespa.config.server.http.NotFoundException; +import com.yahoo.vespa.config.server.tenant.Tenant; import java.io.IOException; import java.time.Duration; @@ -151,7 +154,11 @@ public class ApplicationHandler extends HttpHandler { } } - return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId)); + Tenant tenant = applicationRepository.getTenant(applicationId); + Optional<ApplicationSet> applicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, applicationId); + return new GetApplicationResponse(Response.Status.OK, + applicationRepository.getApplicationGeneration(applicationId), + applicationSet.get().getAllVersions(applicationId)); } @Override @@ -309,9 +316,11 @@ public class ApplicationHandler extends HttpHandler { } private static class GetApplicationResponse extends JSONResponse { - GetApplicationResponse(int status, long generation) { + GetApplicationResponse(int status, long generation, List<Version> modelVersions) { super(status); object.setLong("generation", generation); + Cursor modelVersionArray = object.setArray("modelVersions"); + modelVersions.forEach(version -> modelVersionArray.addString(version.toFullString())); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java index b3c67859098..d3865f287cd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java @@ -4,8 +4,8 @@ package com.yahoo.vespa.config.server.maintenance; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.curator.Curator; +import java.time.Clock; import java.time.Duration; -import java.time.Instant; /** * Removes unused tenants (has no applications and was created more than 7 days ago) @@ -14,19 +14,19 @@ import java.time.Instant; */ public class TenantsMaintainer extends ConfigServerMaintainer { - private final Duration ttlForUnusedTenant; + static final Duration defaultTtlForUnusedTenant = Duration.ofDays(7); - TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { - this(applicationRepository, curator, interval, Duration.ofDays(7)); - } + private final Duration ttlForUnusedTenant; + private final Clock clock; - private TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, Duration ttlForUnusedTenant) { + TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, Clock clock) { super(applicationRepository, curator, interval, interval); - this.ttlForUnusedTenant = ttlForUnusedTenant; + this.ttlForUnusedTenant = defaultTtlForUnusedTenant; + this.clock = clock; } @Override protected void maintain() { - applicationRepository.deleteUnusedTenants(ttlForUnusedTenant, Instant.now()); + applicationRepository.deleteUnusedTenants(ttlForUnusedTenant, clock.instant()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 3cad8def541..a6c12f49a8e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -39,7 +39,7 @@ import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.MetricUpdaterFactory; import com.yahoo.vespa.config.server.rpc.security.RpcAuthorizer; -import com.yahoo.vespa.config.server.tenant.TenantHandlerProvider; +import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantListener; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.filedistribution.FileDownloader; @@ -83,7 +83,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { private static final int JRT_RPC_TRANSPORT_THREADS = threadsToUse(); private final Supervisor supervisor = new Supervisor(new Transport(JRT_RPC_TRANSPORT_THREADS)); - private Spec spec; + private final Spec spec; private final boolean useRequestVersion; private final boolean hostedVespa; private final boolean canReturnEmptySentinelConfig; @@ -93,7 +93,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { private final DelayedConfigResponses delayedConfigResponses; private final HostRegistry<TenantName> hostRegistry; - private final Map<TenantName, TenantHandlerProvider> tenantProviders = new ConcurrentHashMap<>(); + private final Map<TenantName, Tenant> tenants = new ConcurrentHashMap<>(); private final Map<ApplicationId, ApplicationState> applicationStateMap = new ConcurrentHashMap<>(); private final SuperModelRequestHandler superModelRequestHandler; private final MetricUpdater metrics; @@ -395,7 +395,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { TenantName tenant = optionalTenant.orElse(TenantName.defaultName()); // perhaps needed for non-hosted? Optional<RequestHandler> requestHandler = getRequestHandler(tenant); if (requestHandler.isEmpty()) { - String msg = TenantRepository.logPre(tenant) + "Unable to find request handler for tenant. Requested from host '" + request.getClientHostName() + "'"; + String msg = TenantRepository.logPre(tenant) + "Unable to find request handler for tenant '" + tenant + + "'. Request from host '" + request.getClientHostName() + "'"; metrics.incUnknownHostRequests(); trace.trace(TRACELEVEL, msg); log.log(Level.WARNING, msg); @@ -410,8 +411,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } Optional<RequestHandler> getRequestHandler(TenantName tenant) { - return Optional.ofNullable(tenantProviders.get(tenant)) - .map(TenantHandlerProvider::getRequestHandler); + return Optional.ofNullable(tenants.get(tenant)) + .map(Tenant::getRequestHandler); } void delayResponse(JRTServerConfigRequest request, GetConfigContext context) { @@ -421,7 +422,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Override public void onTenantDelete(TenantName tenant) { log.log(Level.FINE, TenantRepository.logPre(tenant)+"Tenant deleted, removing request handler and cleaning host registry"); - tenantProviders.remove(tenant); + tenants.remove(tenant); hostRegistry.removeHostsForKey(tenant); } @@ -432,9 +433,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } @Override - public void onTenantCreate(TenantName tenant, TenantHandlerProvider tenantHandlerProvider) { - log.log(Level.FINE, TenantRepository.logPre(tenant)+"Tenant created, adding request handler"); - tenantProviders.put(tenant, tenantHandlerProvider); + public void onTenantCreate(Tenant tenant) { + tenants.put(tenant.getName(), tenant); } /** Returns true only after all tenants are loaded */ @@ -455,7 +455,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { return useRequestVersion; } - class ChunkedFileReceiver implements FileServer.Receiver { + static class ChunkedFileReceiver implements FileServer.Receiver { Target target; ChunkedFileReceiver(Target target) { this.target = target; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 2110e6476b6..2da93b7f243 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -41,6 +41,8 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -323,13 +325,15 @@ public class SessionRepository { loadSessionIfActive(remoteSession); addRemoteSession(remoteSession); Optional<LocalSession> localSession = Optional.empty(); - if (distributeApplicationPackage.value()) { + if (distributeApplicationPackage()) localSession = createLocalSessionUsingDistributedApplicationPackage(sessionId); - localSession.ifPresent(this::addSession); - } addWatcher(sessionId, fileCache, remoteSession, localSession); } + private boolean distributeApplicationPackage() { + return distributeApplicationPackage.value(); + } + private void sessionRemoved(long sessionId) { SessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); if (watcher != null) watcher.close(); @@ -462,7 +466,7 @@ public class SessionRepository { LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() session.setApplicationId(existingApplicationId); - if (distributeApplicationPackage.value() && existingSession.getApplicationPackageReference() != null) { + if (distributeApplicationPackage() && existingSession.getApplicationPackageReference() != null) { session.setApplicationPackageReference(existingSession.getApplicationPackageReference()); } session.setVespaVersion(existingSession.getVespaVersion()); @@ -504,7 +508,7 @@ public class SessionRepository { long sessionId, Optional<Long> currentlyActiveSessionId, boolean internalRedeploy) throws IOException { File userApplicationDir = getSessionAppDir(sessionId); - IOUtils.copyDirectory(applicationFile, userApplicationDir); + copyApp(applicationFile, userApplicationDir); ApplicationPackage applicationPackage = createApplication(applicationFile, userApplicationDir, applicationId, @@ -515,6 +519,20 @@ public class SessionRepository { return applicationPackage; } + private void copyApp(File sourceDir, File destinationDir) throws IOException { + if (destinationDir.exists()) + throw new RuntimeException("Destination dir " + destinationDir + " already exists"); + if (! sourceDir.isDirectory()) + throw new IllegalArgumentException(sourceDir.getAbsolutePath() + " is not a directory"); + + // Copy app atomically: Copy to a temp dir and move to destination + java.nio.file.Path tempDestinationDir = Files.createTempDirectory(destinationDir.getParentFile().toPath(), "app-package"); + log.log(Level.FINE, "Copying dir " + sourceDir.getAbsolutePath() + " to " + tempDestinationDir.toFile().getAbsolutePath()); + IOUtils.copyDirectory(sourceDir, tempDestinationDir.toFile()); + log.log(Level.FINE, "Moving " + tempDestinationDir + " to " + destinationDir.getAbsolutePath()); + Files.move(tempDestinationDir, destinationDir.toPath(), StandardCopyOption.ATOMIC_MOVE); + } + /** * Returns a new session instance for the given session id. */ @@ -526,7 +544,8 @@ public class SessionRepository { } /** - * Returns a new session instance for the given session id. + * Returns a new local session for the given session id if it does not already exist. + * Will also add the session to the local session cache if necessary */ public Optional<LocalSession> createLocalSessionUsingDistributedApplicationPackage(long sessionId) { if (applicationRepo.hasLocalSession(sessionId)) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java index f182a28ab70..e3f4854105a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; import java.util.Optional; @@ -17,6 +17,7 @@ import java.util.logging.Logger; * The session must be in the session repo. * * @author Vegard Havdal + * @author hmusum */ public class SessionStateWatcher { @@ -42,8 +43,8 @@ public class SessionStateWatcher { this.remoteSession = remoteSession; this.localSession = localSession; this.metrics = metrics; - this.fileCache.start(); this.fileCache.addListener(this::nodeChanged); + this.fileCache.start(); this.zkWatcherExecutor = zkWatcherExecutor; this.sessionRepository = sessionRepository; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java index bf0601bf2f1..f8299bdba1a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java @@ -6,11 +6,8 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.session.SessionRepository; -import com.yahoo.vespa.curator.Curator; -import org.apache.zookeeper.data.Stat; import java.time.Instant; -import java.util.Optional; /** * Contains all tenant-level components for a single tenant, dealing with editing sessions and @@ -19,7 +16,7 @@ import java.util.Optional; * @author vegardh * @author Ulf Lilleengen */ -public class Tenant implements TenantHandlerProvider { +public class Tenant { static final String SESSIONS = "sessions"; static final String APPLICATIONS = "applications"; @@ -29,19 +26,20 @@ public class Tenant implements TenantHandlerProvider { private final SessionRepository sessionRepository; private final TenantApplications applicationRepo; private final RequestHandler requestHandler; - private final Curator curator; - - Tenant(TenantName name, - SessionRepository sessionRepository, - RequestHandler requestHandler, - TenantApplications applicationRepo, - Curator curator) { + private final Instant created; + + // Protected due to being subclassed in a system test + protected Tenant(TenantName name, + SessionRepository sessionRepository, + RequestHandler requestHandler, + TenantApplications applicationRepo, + Instant created) { this.name = name; this.path = TenantRepository.getTenantPath(name); this.requestHandler = requestHandler; this.sessionRepository = sessionRepository; this.applicationRepo = applicationRepo; - this.curator = curator; + this.created = created; } /** @@ -72,16 +70,8 @@ public class Tenant implements TenantHandlerProvider { return applicationRepo; } - public Curator getCurator() { - return curator; - } - public Instant getCreatedTime() { - Optional<Stat> stat = curator.getStat(path); - if (stat.isPresent()) - return Instant.ofEpochMilli(stat.get().getCtime()); - else - return Instant.now(); + return created; } @Override @@ -101,16 +91,11 @@ public class Tenant implements TenantHandlerProvider { /** * Closes any watchers, thread pools that may react to changes in tenant state, * and removes any session data in filesystem and zookeeper. - * Called by watchers as a reaction to {@link #delete()}. + * Called by watchers as a reaction to deleting a tenant. */ void close() { - applicationRepo.close(); // Closes watchers. - sessionRepository.close(); // Closes watchers, clears memory, and deletes local files and ZK session state. - } - - /** Deletes the tenant tree from ZooKeeper (application and session status for the tenant) and triggers {@link #close()}. */ - void delete() { - curator.delete(path); // Deletes tenant ZK tree: applications and sessions. + applicationRepo.close(); // Closes watchers. + sessionRepository.close(); // Closes watchers, clears memory, and deletes local files and ZK session state. } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantHandlerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantHandlerProvider.java deleted file mode 100644 index 5b49bd69758..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantHandlerProvider.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.vespa.config.server.RequestHandler; - -/** - * Represents something that can provide a request handler for a tenant. - * - * @author Ulf Lilleengen - */ -public interface TenantHandlerProvider { - - RequestHandler getRequestHandler(); - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantListener.java index ea407ba917a..252ca2b97d7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantListener.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantListener.java @@ -2,22 +2,19 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.tenant.TenantHandlerProvider; /** * Interface for something that listens for created and deleted tenants. * * @author Ulf Lilleengen - * @since 5.8 */ public interface TenantListener { /** * Called whenever a new tenant is created. * - * @param tenant name of newly created tenant. - * @param provider provider of request and reload handlers for new tenant. + * @param tenant newly created tenant. */ - void onTenantCreate(TenantName tenant, TenantHandlerProvider provider); + void onTenantCreate(Tenant tenant); /** * Called whenever a tenant is deleted. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 2c9d9bf239c..4316f03272a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -20,8 +20,10 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.state.ConnectionState; import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.Stat; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -82,7 +84,6 @@ public class TenantRepository { private final ExecutorService bootstrapExecutor; private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1); private final Optional<Curator.DirectoryCache> directoryCache; - private final boolean throwExceptionIfBootstrappingFails; /** * Creates a new tenant repository @@ -105,7 +106,6 @@ public class TenantRepository { this.componentRegistry = componentRegistry; ConfigserverConfig configserverConfig = componentRegistry.getConfigserverConfig(); this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders()); - this.throwExceptionIfBootstrappingFails = configserverConfig.throwIfBootstrappingTenantRepoFails(); this.curator = componentRegistry.getCurator(); metricUpdater = componentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); this.tenantListeners.add(componentRegistry.getTenantListener()); @@ -148,7 +148,7 @@ public class TenantRepository { public synchronized void addTenant(TenantName tenantName, RequestHandler requestHandler, ReloadHandler reloadHandler) { writeTenantPath(tenantName); - createTenant(tenantName, requestHandler, reloadHandler); + createTenant(tenantName, componentRegistry.getClock().instant(), requestHandler, reloadHandler); } private static Set<TenantName> readTenantsFromZooKeeper(Curator curator) { @@ -164,14 +164,14 @@ public class TenantRepository { zkWatcherExecutor.execute(tenantName, () -> closeTenant(tenantName)); for (TenantName tenantName : allTenants) if ( ! tenants.containsKey(tenantName)) - zkWatcherExecutor.execute(tenantName, () -> createTenant(tenantName)); + zkWatcherExecutor.execute(tenantName, () -> bootstrapTenant(tenantName)); metricUpdater.setTenants(tenants.size()); } private void bootstrapTenants() { // Keep track of tenants created Map<TenantName, Future<?>> futures = new HashMap<>(); - readTenantsFromZooKeeper(curator).forEach(t -> futures.put(t, bootstrapExecutor.submit(() -> createTenant(t)))); + readTenantsFromZooKeeper(curator).forEach(t -> futures.put(t, bootstrapExecutor.submit(() -> bootstrapTenant(t)))); // Wait for all tenants to be created Set<TenantName> failed = new HashSet<>(); @@ -187,7 +187,7 @@ public class TenantRepository { } } - if (failed.size() > 0 && throwExceptionIfBootstrappingFails) + if (failed.size() > 0) throw new RuntimeException("Could not create all tenants when bootstrapping, failed to create: " + failed); metricUpdater.setTenants(tenants.size()); @@ -199,12 +199,21 @@ public class TenantRepository { } } - protected void createTenant(TenantName tenantName) { - createTenant(tenantName, null, null); + // Use when bootstrapping an existing tenant based on ZooKeeper data + protected void bootstrapTenant(TenantName tenantName) { + createTenant(tenantName, readCreatedTimeFromZooKeeper(tenantName), null, null); + } + + public Instant readCreatedTimeFromZooKeeper(TenantName tenantName) { + Optional<Stat> stat = curator.getStat(getTenantPath(tenantName)); + if (stat.isPresent()) + return Instant.ofEpochMilli(stat.get().getCtime()); + else + return componentRegistry.getClock().instant(); } // Creates tenant and all its dependencies. This also includes loading active applications - private void createTenant(TenantName tenantName, RequestHandler requestHandler, ReloadHandler reloadHandler) { + private void createTenant(TenantName tenantName, Instant created, RequestHandler requestHandler, ReloadHandler reloadHandler) { if (tenants.containsKey(tenantName)) return; TenantApplications applicationRepo = @@ -225,9 +234,8 @@ public class TenantRepository { applicationRepo, reloadHandler, componentRegistry.getFlagSource(), componentRegistry.getSessionPreparer()); - log.log(Level.INFO, "Creating tenant '" + tenantName + "'"); - Tenant tenant = new Tenant(tenantName, sessionRepository, requestHandler, applicationRepo, - componentRegistry.getCurator()); + log.log(Level.INFO, "Adding tenant '" + tenantName + "'" + ", created " + created); + Tenant tenant = new Tenant(tenantName, sessionRepository, requestHandler, applicationRepo, created); notifyNewTenant(tenant); tenants.putIfAbsent(tenantName, tenant); } @@ -247,7 +255,7 @@ public class TenantRepository { private void notifyNewTenant(Tenant tenant) { for (TenantListener listener : tenantListeners) { - listener.onTenantCreate(tenant.getName(), tenant); + listener.onTenantCreate(tenant); } } @@ -303,7 +311,9 @@ public class TenantRepository { throw new IllegalArgumentException("Deleting '" + name + "' failed, tenant does not exist"); log.log(Level.INFO, "Deleting tenant '" + name + "'"); - tenants.get(name).delete(); + // Deletes the tenant tree from ZooKeeper (application and session status for the tenant) + // and triggers Tenant.close(). + curator.delete(tenants.get(name).getPath()); } private synchronized void closeTenant(TenantName name) { @@ -316,11 +326,6 @@ public class TenantRepository { tenant.close(); } - // For unit testing - String tenantZkPath(TenantName tenant) { - return getTenantPath(tenant).getAbsolute(); - } - /** * A helper to format a log preamble for messages with a tenant and app id * @param app the app @@ -446,4 +451,6 @@ public class TenantRepository { return locksPath.append(tenantName.value()); } + public Curator getCurator() { return curator; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index c7ec2657996..685856d5cf8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -42,7 +42,7 @@ import java.util.Optional; */ public class ZKApplicationPackage implements ApplicationPackage { - private ZKApplication zkApplication; + private final ZKApplication zkApplication; private final Map<Version, PreGeneratedFileRegistry> fileRegistryMap = new HashMap<>(); private final Optional<AllocatedHosts> allocatedHosts; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 8745a9bf596..c6d2c6b6438 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -63,7 +63,6 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -101,7 +100,7 @@ public class ApplicationRepositoryTest { private final static TenantName tenant1 = TenantName.from("test1"); private final static TenantName tenant2 = TenantName.from("test2"); private final static TenantName tenant3 = TenantName.from("test3"); - private final static Clock clock = Clock.systemUTC(); + private final static ManualClock clock = new ManualClock(Instant.now()); private ApplicationRepository applicationRepository; private TenantRepository tenantRepository; @@ -133,6 +132,7 @@ public class ApplicationRepositoryTest { .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build()) .flagSource(flagSource) + .clock(clock) .build(); tenantRepository = new TenantRepository(componentRegistry, false); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); @@ -245,29 +245,6 @@ public class ApplicationRepositoryTest { } @Test - public void deleteUnusedTenants() { - // Set clock to epoch plus hour, as mock curator will always return epoch as creation time - Instant now = ManualClock.at("1970-01-01T01:00:00"); - - // 3 tenants exist, tenant1 and tenant2 has applications deployed: - deployApp(testApp); - deployApp(testApp, new PrepareParams.Builder().applicationId(applicationId(tenant2)).build()); - - // Should not be deleted, not old enough - Duration ttlForUnusedTenant = Duration.ofHours(1); - assertTrue(applicationRepository.deleteUnusedTenants(ttlForUnusedTenant, now).isEmpty()); - // Should be deleted - ttlForUnusedTenant = Duration.ofMillis(1); - assertEquals(tenant3, applicationRepository.deleteUnusedTenants(ttlForUnusedTenant, now).iterator().next()); - - // Delete app used by tenant1, tenant2 still has an application - applicationRepository.delete(applicationId()); - Set<TenantName> tenantsDeleted = applicationRepository.deleteUnusedTenants(Duration.ofMillis(1), now); - assertTrue(tenantsDeleted.contains(tenant1)); - assertFalse(tenantsDeleted.contains(tenant2)); - } - - @Test public void deleteUnusedFileReferences() throws IOException { File fileReferencesDir = temporaryFolder.newFolder(); @@ -372,7 +349,6 @@ public class ApplicationRepositoryTest { @Test public void testDeletingInactiveSessions() throws IOException { - ManualClock clock = new ManualClock(Instant.now()); ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) @@ -455,7 +431,7 @@ public class ApplicationRepositoryTest { var prepareParams = new PrepareParams.Builder().applicationId(applicationId) .applicationRoles(ApplicationRoles.fromString("hostRole","containerRole")).build(); deployApp(testApp, prepareParams); - var approlesStore = new ApplicationRolesStore(tenant.getCurator(), tenant.getPath()); + var approlesStore = new ApplicationRolesStore(tenantRepository.getCurator(), tenant.getPath()); var appRoles = approlesStore.readApplicationRoles(applicationId); // App roles present after deploy diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index 2e0466c27b1..f03550c0a80 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -170,6 +170,11 @@ public class TestComponentRegistry implements GlobalComponentRegistry { return this; } + public Builder configDefinitionRepo(ConfigDefinitionRepo configDefinitionRepo) { + this.defRepo = configDefinitionRepo; + return this; + } + public TestComponentRegistry build() { final PermanentApplicationPackage permApp = this.permanentApplicationPackage .orElse(new PermanentApplicationPackage(configserverConfig)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestConfigDefinitionRepo.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestConfigDefinitionRepo.java index d472f64d228..5878b250bc8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestConfigDefinitionRepo.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestConfigDefinitionRepo.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -20,6 +21,8 @@ public class TestConfigDefinitionRepo implements ConfigDefinitionRepo { new ConfigDefinition(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_SCHEMA)); repo.put(new ConfigDefinitionKey(LbServicesConfig.CONFIG_DEF_NAME, LbServicesConfig.CONFIG_DEF_NAMESPACE), new ConfigDefinition(LbServicesConfig.CONFIG_DEF_NAME, LbServicesConfig.CONFIG_DEF_SCHEMA)); + repo.put(new ConfigDefinitionKey(SentinelConfig.CONFIG_DEF_NAME, SentinelConfig.CONFIG_DEF_NAMESPACE), + new ConfigDefinition(SentinelConfig.CONFIG_DEF_NAME, SentinelConfig.CONFIG_DEF_SCHEMA)); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationSetTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationSetTest.java index cf1e00674cb..f5d661e265a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationSetTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationSetTest.java @@ -6,6 +6,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.Set; import com.yahoo.config.provision.ApplicationId; import com.yahoo.component.Version; @@ -21,8 +22,8 @@ import static org.junit.Assert.assertEquals; public class ApplicationSetTest { private ApplicationSet applicationSet; - private List<Version> vespaVersions = new ArrayList<>(); - private List<Application> applications = new ArrayList<>(); + private final List<Version> vespaVersions = new ArrayList<>(); + private final List<Application> applications = new ArrayList<>(); @Before public void setUp() { @@ -50,6 +51,13 @@ public class ApplicationSetTest { applicationSet.getForVersionOrLatest(Optional.of(vespaVersions.get(1)), Instant.now()); } + @Test + public void testGetAllVersions() { + applicationSet = ApplicationSet.fromList(applications); + assertEquals(List.of(Version.fromString("1.2.3"), Version.fromString("1.2.4"), Version.fromString("1.2.5")), + applicationSet.getAllVersions(ApplicationId.defaultId())); + } + private Application createApplication(Version version) { return new Application(new ModelStub(), null, 0, false, version, MetricUpdater.createTestUpdater(), ApplicationId.defaultId()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 089b662b797..32ce2c6f509 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -35,7 +35,7 @@ public class HttpGetConfigHandlerTest { @Before public void setUp() { - mockRequestHandler = new MockRequestHandler(); + mockRequestHandler = new MockRequestHandler(ApplicationId.defaultId()); mockRequestHandler.setAllConfigs(new HashSet<>() {{ add(new ConfigKey<>("bar", "myid", "foo")); }} ); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java index 9113978d58b..dea9196c949 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.ConfigKey; @@ -30,7 +31,7 @@ public class HttpListConfigsHandlerTest { @Before public void setUp() { - mockRequestHandler = new MockRequestHandler(); + mockRequestHandler = new MockRequestHandler(ApplicationId.defaultId()); mockRequestHandler.setAllConfigs(new HashSet<>() {{ add(new ConfigKey<>("bar", "conf/id/", "foo")); }} ); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 06e404bee32..3438fbecc59 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.config.server.http.v2; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; +import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -19,11 +21,13 @@ import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.application.HttpProxy; import com.yahoo.vespa.config.server.application.OrchestratorMock; +import com.yahoo.vespa.config.server.deploy.DeployTester; import com.yahoo.vespa.config.server.deploy.InfraDeployerProvider; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.StaticResponse; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; @@ -40,8 +44,10 @@ import java.io.InputStream; import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Clock; +import java.util.List; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -66,6 +72,7 @@ public class ApplicationHandlerTest { private final static NullMetric metric = new NullMetric(); private final static ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); private static final MockLogRetriever logRetriever = new MockLogRetriever(); + private static final Version vespaVersion = Version.fromString("7.8.9"); private TenantRepository tenantRepository; private ApplicationRepository applicationRepository; @@ -75,7 +82,11 @@ public class ApplicationHandlerTest { @Before public void setup() { - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().provisioner(provisioner).build(); + List<ModelFactory> modelFactories = List.of(DeployTester.createModelFactory(vespaVersion)); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .provisioner(provisioner) + .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) + .build(); tenantRepository = new TenantRepository(componentRegistry, false); tenantRepository.addTenant(mytenantName); provisioner = new SessionHandlerTest.MockProvisioner(); @@ -149,9 +160,14 @@ public class ApplicationHandlerTest { @Test public void testGet() throws Exception { - long sessionId = applicationRepository.deploy(testApp, prepareParams(applicationId)).sessionId(); - assertApplicationGeneration(applicationId, Zone.defaultZone(), sessionId, true); - assertApplicationGeneration(applicationId, Zone.defaultZone(), sessionId, false); + PrepareParams prepareParams = new PrepareParams.Builder() + .applicationId(applicationId) + .vespaVersion(vespaVersion) + .build(); + long sessionId = applicationRepository.deploy(testApp, prepareParams).sessionId(); + + assertApplicationResponse(applicationId, Zone.defaultZone(), sessionId, true, vespaVersion); + assertApplicationResponse(applicationId, Zone.defaultZone(), sessionId, false, vespaVersion); } @Test @@ -196,7 +212,7 @@ public class ApplicationHandlerTest { when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1"))) .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>")); - HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET)); HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>...</html>"); } @@ -229,7 +245,7 @@ public class ApplicationHandlerTest { String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/logs?from=100&to=200"; ApplicationHandler mockHandler = createApplicationHandler(); - HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET)); assertEquals(200, response.getStatus()); assertEquals("log line", getRenderedString(response)); @@ -240,7 +256,7 @@ public class ApplicationHandlerTest { applicationRepository.deploy(testApp, prepareParams(applicationId)); String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/status"; ApplicationHandler mockHandler = createApplicationHandler(); - HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET)); assertEquals(200, response.getStatus()); assertEquals("OK", getRenderedString(response)); } @@ -251,7 +267,7 @@ public class ApplicationHandlerTest { String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/log?after=1234"; ApplicationHandler mockHandler = createApplicationHandler(); - HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET)); assertEquals(200, response.getStatus()); assertEquals("log", getRenderedString(response)); } @@ -273,7 +289,7 @@ public class ApplicationHandlerTest { applicationRepository.deploy(testApp, prepareParams(applicationId)); String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/ready"; ApplicationHandler mockHandler = createApplicationHandler(); - HttpRequest testRequest = HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET); + HttpRequest testRequest = HttpRequest.createTestRequest(url, GET); HttpResponse response = mockHandler.handle(testRequest); assertEquals(200, response.getStatus()); } @@ -315,13 +331,14 @@ public class ApplicationHandlerTest { } } - private void assertApplicationGeneration(ApplicationId applicationId, Zone zone, long expectedGeneration, boolean fullAppIdInUrl) throws IOException { - assertApplicationGeneration(toUrlPath(applicationId, zone, fullAppIdInUrl), expectedGeneration); + private void assertApplicationResponse(ApplicationId applicationId, Zone zone, long expectedGeneration, + boolean fullAppIdInUrl, Version expectedVersion) throws IOException { + assertApplicationResponse(toUrlPath(applicationId, zone, fullAppIdInUrl), expectedGeneration, expectedVersion); } private void assertSuspended(boolean expectedValue, ApplicationId application, Zone zone) throws IOException { String restartUrl = toUrlPath(application, zone, true) + "/suspended"; - HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, GET)); HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "{\"suspended\":" + expectedValue + "}"); } @@ -332,24 +349,28 @@ public class ApplicationHandlerTest { return url; } - private void assertApplicationGeneration(String url, long expectedGeneration) throws IOException { - HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); - HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "{\"generation\":" + expectedGeneration + "}"); + private void assertApplicationResponse(String url, long expectedGeneration, Version expectedVersion) throws IOException { + HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(url, GET)); + assertEquals(200, response.getStatus()); + String renderedString = SessionHandlerTest.getRenderedString(response); + assertEquals("{\"generation\":" + expectedGeneration + ",\"modelVersions\":[\"" + expectedVersion.toFullString() + "\"]}", renderedString); } private void assertApplicationExists(ApplicationId applicationId, Zone zone) throws IOException { - String tenantName = applicationId == null ? null : applicationId.tenant().value(); - String expected = applicationId == null ? "[]" : "[\"http://myhost:14000/application/v2/tenant/" + tenantName + "/application/" + applicationId.application().value() + - "/environment/" + zone.environment().value() + - "/region/" + zone.region().value() + - "/instance/" + applicationId.instance().value() + "\"]"; + String tenantName = applicationId.tenant().value(); + String expected = "[\"http://myhost:14000/application/v2/tenant/" + + tenantName + "/application/" + applicationId.application().value() + + "/environment/" + zone.environment().value() + + "/region/" + zone.region().value() + + "/instance/" + applicationId.instance().value() + "\"]"; ListApplicationsHandler listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), tenantRepository, Zone.defaultZone()); - ListApplicationsHandlerTest.assertResponse(listApplicationsHandler, "http://myhost:14000/application/v2/tenant/" + tenantName + "/application/", + ListApplicationsHandlerTest.assertResponse(listApplicationsHandler, + "http://myhost:14000/application/v2/tenant/" + tenantName + "/application/", Response.Status.OK, expected, - com.yahoo.jdisc.http.HttpRequest.Method.GET); + GET); } private void restart(ApplicationId application, Zone zone) throws IOException { @@ -360,13 +381,13 @@ public class ApplicationHandlerTest { private void converge(ApplicationId application, Zone zone) throws IOException { String convergeUrl = toUrlPath(application, zone, true) + "/serviceconverge"; - HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(convergeUrl, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(convergeUrl, GET)); HandlerTest.assertHttpStatusCodeAndMessage(response, 200, ""); } private HttpResponse fileDistributionStatus(ApplicationId application, Zone zone) { String restartUrl = toUrlPath(application, zone, true) + "/filedistributionstatus"; - return createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + return createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, GET)); } private static class MockStateApiFactory implements ConfigConvergenceChecker.StateApiFactory { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index 46a17795acf..97789caeb4b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -41,7 +41,7 @@ public class HttpGetConfigHandlerTest { @Before public void setUp() { - mockRequestHandler = new MockRequestHandler(); + mockRequestHandler = new MockRequestHandler(ApplicationId.defaultId()); mockRequestHandler.setAllConfigs(new HashSet<>() {{ add(new ConfigKey<>("bar", "myid", "foo")); }} ); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java index e8484ad10fe..d91d41173b2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; @@ -39,7 +40,7 @@ public class HttpListConfigsHandlerTest { @Before public void setUp() { - mockRequestHandler = new MockRequestHandler(); + mockRequestHandler = new MockRequestHandler(ApplicationId.defaultId()); mockRequestHandler.setAllConfigs(new HashSet<>() {{ add(new ConfigKey<>("bar", "conf/id", "foo")); }} ); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index c2489a1d3e9..4b37638cb50 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -10,15 +10,20 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import java.time.Clock; + class MaintainerTester { private final Curator curator; private final TenantRepository tenantRepository; private final ApplicationRepository applicationRepository; - MaintainerTester() { + MaintainerTester(Clock clock) { curator = new MockCurator(); - GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); + GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .clock(clock) + .build(); tenantRepository = new TenantRepository(componentRegistry, false); applicationRepository = new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java index fdc6ffeacf0..53326a89293 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -20,7 +21,8 @@ public class TenantsMaintainerTest { @Test public void deleteTenantWithNoApplications() { - MaintainerTester tester = new MaintainerTester(); + ManualClock clock = new ManualClock("2020-06-01T00:00:00"); + MaintainerTester tester = new MaintainerTester(clock); TenantRepository tenantRepository = tester.tenantRepository(); ApplicationRepository applicationRepository = tester.applicationRepository(); File applicationPackage = new File("src/test/apps/app"); @@ -36,7 +38,8 @@ public class TenantsMaintainerTest { assertNotNull(tenantRepository.getTenant(shouldBeDeleted)); assertNotNull(tenantRepository.getTenant(shouldNotBeDeleted)); - new TenantsMaintainer(applicationRepository, tester.curator(), Duration.ofDays(1)).run(); + clock.advance(TenantsMaintainer.defaultTtlForUnusedTenant.plus(Duration.ofDays(1))); + new TenantsMaintainer(applicationRepository, tester.curator(), Duration.ofDays(1), clock).run(); tenantRepository.updateTenants(); // One tenant should now have been deleted @@ -59,4 +62,5 @@ public class TenantsMaintainerTest { private ApplicationId applicationId(TenantName tenantName) { return ApplicationId.from(tenantName, ApplicationName.from("foo"), InstanceName.defaultName()); } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java index 21b85e0d09c..e800faf78a4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.RequestHandler; -import com.yahoo.vespa.config.server.tenant.TenantHandlerProvider; import java.util.Collections; import java.util.HashSet; @@ -24,18 +23,14 @@ import java.util.Set; * * @author Ulf Lilleengen */ -public class MockRequestHandler implements RequestHandler, ReloadHandler, TenantHandlerProvider { +public class MockRequestHandler implements RequestHandler, ReloadHandler { private Set<ConfigKey<?>> allConfigs = new HashSet<>(); public Map<ApplicationId, ConfigResponse> responses = new LinkedHashMap<>(); - private final boolean pretendToHaveLoadedAnyApplication; + private final ApplicationId applicationId; - public MockRequestHandler() { - this(false); - } - - public MockRequestHandler(boolean pretendToHaveLoadedAnyApplication) { - this.pretendToHaveLoadedAnyApplication = pretendToHaveLoadedAnyApplication; + public MockRequestHandler(ApplicationId applicationId) { + this.applicationId = applicationId; } @Override @@ -82,13 +77,12 @@ public class MockRequestHandler implements RequestHandler, ReloadHandler, Tenant @Override public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { - if (pretendToHaveLoadedAnyApplication) return true; return responses.containsKey(appId); } @Override public ApplicationId resolveApplicationId(String hostName) { - return ApplicationId.defaultId(); + return applicationId; } @Override @@ -96,9 +90,6 @@ public class MockRequestHandler implements RequestHandler, ReloadHandler, Tenant return Set.of(); } - @Override - public RequestHandler getRequestHandler() { - return this; - } + public RequestHandler getRequestHandler() { return this; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java index d923f4c1856..9f514d9996f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.provision.TenantName; import com.yahoo.component.Version; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; @@ -12,7 +11,6 @@ import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.rpc.security.NoopRpcAuthorizer; -import com.yahoo.vespa.config.server.tenant.MockTenantProvider; import java.io.File; import java.time.Duration; @@ -39,20 +37,15 @@ public class MockRpc extends RpcServer { public volatile JRTServerConfigRequest latestRequest = null; - public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication, File tempDir) { - super(createConfig(port), null, Metrics.createTestMetrics(), - new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(tempDir), new NoopRpcAuthorizer(), new RpcRequestHandlerProvider()); - if (createDefaultTenant) { - onTenantCreate(TenantName.from("default"), new MockTenantProvider(pretendToHaveLoadedAnyApplication)); - } - } - - public MockRpc(int port, boolean createDefaultTenant, File tempDir) { - this(port, createDefaultTenant, true, tempDir); - } - public MockRpc(int port, File tempDir) { - this(port, true, tempDir); + super(createConfig(port), + null, + Metrics.createTestMetrics(), + new HostRegistries(), + new ConfigRequestHostLivenessTracker(), + new FileServer(tempDir), + new NoopRpcAuthorizer(), + new RpcRequestHandlerProvider()); } private static ConfigserverConfig createConfig(int port) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index 0b33de2a42c..ae6bd5feeab 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -8,6 +8,8 @@ import com.yahoo.component.Version; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.jrt.Request; import com.yahoo.vespa.config.ConfigKey; @@ -16,22 +18,23 @@ import com.yahoo.vespa.config.ConfigPayloadApplier; import com.yahoo.vespa.config.ErrorCode; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.protocol.CompressionType; -import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; -import com.yahoo.vespa.config.protocol.SlimeConfigResponse; import com.yahoo.vespa.config.protocol.Trace; +import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.model.VespaModel; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; +import java.io.File; import java.io.IOException; import java.util.Optional; @@ -47,12 +50,21 @@ import static org.junit.Assert.assertTrue; */ public class RpcServerTest { + private static final TenantName tenantName = TenantName.from("testTenant"); + private static final ApplicationId applicationId = + ApplicationId.from(tenantName, ApplicationName.defaultName(), InstanceName.defaultName()); + private final static File testApp = new File("src/test/resources/deploy/validapp"); + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Test public void testRpcServer() throws IOException, SAXException, InterruptedException { - try (RpcTester tester = new RpcTester(temporaryFolder)) { + try (RpcTester tester = new RpcTester(applicationId, temporaryFolder)) { + ApplicationRepository applicationRepository = tester.applicationRepository(); + applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId).build()); + TenantApplications applicationRepo = tester.tenant().getApplicationRepo(); + applicationRepo.reloadConfig(applicationRepository.getActiveSession(applicationId).ensureApplicationLoaded()); testPrintStatistics(tester); testGetConfig(tester); testEnabled(tester); @@ -62,7 +74,7 @@ public class RpcServerTest { } private void testApplicationNotLoadedErrorWhenAppDeleted(RpcTester tester) throws InterruptedException, IOException { - tester.rpcServer().onTenantDelete(TenantName.defaultName()); + tester.rpcServer().onTenantDelete(tenantName); tester.rpcServer().onTenantsLoaded(); JRTClientConfigRequest clientReq = createSimpleRequest(); tester.performRequest(clientReq.getRequest()); @@ -79,8 +91,8 @@ public class RpcServerTest { @Test public void testEmptySentinelConfigWhenAppDeletedOnHostedVespa() throws IOException, InterruptedException { ConfigserverConfig.Builder configBuilder = new ConfigserverConfig.Builder().canReturnEmptySentinelConfig(true); - try (RpcTester tester = new RpcTester(temporaryFolder, configBuilder)) { - tester.rpcServer().onTenantDelete(TenantName.defaultName()); + try (RpcTester tester = new RpcTester(applicationId, temporaryFolder, configBuilder)) { + tester.rpcServer().onTenantDelete(tenantName); tester.rpcServer().onTenantsLoaded(); JRTClientConfigRequest clientReq = createSentinelRequest(); @@ -119,7 +131,7 @@ public class RpcServerTest { false, new Version(1, 2, 3), MetricUpdater.createTestUpdater(), - ApplicationId.defaultId()); + applicationId); ApplicationSet appSet = ApplicationSet.fromSingle(app); tester.rpcServer().configActivated(appSet); ConfigKey<?> key = new ConfigKey<>(LbServicesConfig.class, "*"); @@ -141,29 +153,16 @@ public class RpcServerTest { private void testGetConfig(RpcTester tester) { ConfigKey<?> key = new ConfigKey<>(SimpletypesConfig.class, "brim"); JRTClientConfigRequest req = createRequest(new RawConfig(key, SimpletypesConfig.getDefMd5())); - ((MockRequestHandler)tester.tenantProvider().getRequestHandler()).responses.put(ApplicationId.defaultId(), createResponse()); assertTrue(req.validateParameters()); tester.performRequest(req.getRequest()); assertThat(req.errorCode(), is(0)); assertTrue(req.validateResponse()); - assertTrue(req.responseIsInternalRedeploy()); ConfigPayload payload = ConfigPayload.fromUtf8Array(req.getNewPayload().getData()); assertNotNull(payload); SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); new ConfigPayloadApplier<>(builder).applyPayload(payload); SimpletypesConfig config = new SimpletypesConfig(builder); - assertThat(config.intval(), is(123)); - } - - private ConfigResponse createResponse() { - SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); - builder.intval(123); - SimpletypesConfig responseConfig = new SimpletypesConfig(builder); - ConfigPayload responsePayload = ConfigPayload.fromInstance(responseConfig); - return SlimeConfigResponse.fromConfigPayload(responsePayload, - 3L, - true, /* internalRedeploy */ - ConfigUtils.getMd5(responsePayload)); + assertThat(config.intval(), is(0)); } private void testPrintStatistics(RpcTester tester) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java index ad42e90db82..868846674df 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -12,23 +13,28 @@ import com.yahoo.jrt.Transport; import com.yahoo.net.HostName; import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.GenerationCounter; +import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MemoryGenerationCounter; import com.yahoo.vespa.config.server.PortRangeAllocator; import com.yahoo.vespa.config.server.SuperModelManager; import com.yahoo.vespa.config.server.SuperModelRequestHandler; +import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TestConfigDefinitionRepo; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.filedistribution.FileServer; import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker; import com.yahoo.vespa.config.server.host.HostRegistries; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.rpc.security.NoopRpcAuthorizer; -import com.yahoo.vespa.config.server.tenant.MockTenantProvider; -import com.yahoo.vespa.config.server.tenant.TenantHandlerProvider; +import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.After; import org.junit.rules.TemporaryFolder; import java.io.IOException; +import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -50,32 +56,46 @@ public class RpcTester implements AutoCloseable { private final ManualClock clock = new ManualClock(Instant.ofEpochMilli(100)); private final String myHostname = HostName.getLocalhost(); private final HostLivenessTracker hostLivenessTracker = new ConfigRequestHostLivenessTracker(clock); - private final MockTenantProvider tenantProvider; private final GenerationCounter generationCounter; private final Spec spec; private RpcServer rpcServer; private Thread t; private Supervisor sup; + private final ApplicationId applicationId; + private final TenantName tenantName; + private final TenantRepository tenantRepository; - private List<Integer> allocatedPorts; - + private final ApplicationRepository applicationRepository; + private final List<Integer> allocatedPorts = new ArrayList<>(); private final TemporaryFolder temporaryFolder; private final ConfigserverConfig configserverConfig; - RpcTester(TemporaryFolder temporaryFolder) throws InterruptedException, IOException { - this(temporaryFolder, new ConfigserverConfig.Builder()); + RpcTester(ApplicationId applicationId, TemporaryFolder temporaryFolder) throws InterruptedException, IOException { + this(applicationId, temporaryFolder, new ConfigserverConfig.Builder()); } - RpcTester(TemporaryFolder temporaryFolder, ConfigserverConfig.Builder configBuilder) throws InterruptedException, IOException { + RpcTester(ApplicationId applicationId, TemporaryFolder temporaryFolder, ConfigserverConfig.Builder configBuilder) throws InterruptedException, IOException { this.temporaryFolder = temporaryFolder; - allocatedPorts = new ArrayList<>(); + this.applicationId = applicationId; + this.tenantName = applicationId.tenant(); int port = allocatePort(); spec = createSpec(port); - tenantProvider = new MockTenantProvider(); - generationCounter = new MemoryGenerationCounter(); - configBuilder.rpcport(port); + configBuilder.rpcport(port) + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()); configserverConfig = new ConfigserverConfig(configBuilder); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .configDefinitionRepo(new TestConfigDefinitionRepo()) + .configServerConfig(configserverConfig) + .build(); + tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(tenantName); + applicationRepository = new ApplicationRepository(tenantRepository, + new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), + Clock.systemUTC()); + generationCounter = new MemoryGenerationCounter(); createAndStartRpcServer(); assertFalse(hostLivenessTracker.lastRequestFrom(myHostname).isPresent()); } @@ -93,6 +113,9 @@ public class RpcTester implements AutoCloseable { } void createAndStartRpcServer() throws IOException { + HostRegistries hostRegistries = new HostRegistries(); + hostRegistries.createApplicationHostRegistry(tenantName).update(applicationId, List.of("localhost")); + hostRegistries.getTenantHostRegistry().update(tenantName, List.of("localhost")); rpcServer = new RpcServer(configserverConfig, new SuperModelRequestHandler(new TestConfigDefinitionRepo(), configserverConfig, @@ -101,11 +124,13 @@ public class RpcTester implements AutoCloseable { Zone.defaultZone() , generationCounter, new InMemoryFlagSource())), - Metrics.createTestMetrics(), new HostRegistries(), - hostLivenessTracker, new FileServer(temporaryFolder.newFolder()), + Metrics.createTestMetrics(), + hostRegistries, + hostLivenessTracker, + new FileServer(temporaryFolder.newFolder()), new NoopRpcAuthorizer(), new RpcRequestHandlerProvider()); - rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider); + rpcServer.onTenantCreate(tenantRepository.getTenant(tenantName)); t = new Thread(rpcServer); t.start(); sup = new Supervisor(new Transport()); @@ -148,8 +173,8 @@ public class RpcTester implements AutoCloseable { return rpcServer; } - TenantHandlerProvider tenantProvider() { - return tenantProvider; - } + Tenant tenant() { return tenantRepository.getTenant(tenantName); } + + public ApplicationRepository applicationRepository() { return applicationRepository; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/MockTenantListener.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/MockTenantListener.java index 109a2220e0d..0ad8e43f066 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/MockTenantListener.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/MockTenantListener.java @@ -7,16 +7,13 @@ import com.yahoo.config.provision.TenantName; * @author Ulf Lilleengen */ public class MockTenantListener implements TenantListener { + TenantName tenantCreatedName; - TenantHandlerProvider provider; TenantName tenantDeletedName; boolean tenantsLoaded; @Override - public void onTenantCreate(TenantName tenantName, TenantHandlerProvider provider) { - this.tenantCreatedName = tenantName; - this.provider = provider; - } + public void onTenantCreate(Tenant tenant) { this.tenantCreatedName = tenant.getName(); } @Override public void onTenantDelete(TenantName tenantName) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/MockTenantProvider.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/MockTenantProvider.java deleted file mode 100644 index 4f839fbd811..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/MockTenantProvider.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.vespa.config.server.RequestHandler; -import com.yahoo.vespa.config.server.rpc.MockRequestHandler; - -/** - * @author Ulf Lilleengen - */ -public class MockTenantProvider implements TenantHandlerProvider { - - private final MockRequestHandler requestHandler; - - public MockTenantProvider() { - this(false); - } - - public MockTenantProvider(boolean pretendToHaveLoadedAnyApplication) { - this.requestHandler = new MockRequestHandler(pretendToHaveLoadedAnyApplication); - } - - @Override - public RequestHandler getRequestHandler() { return requestHandler; } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 104b94b274f..8d0285ac12b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -170,15 +170,10 @@ public class TenantRepositoryTest { public void testFailingBootstrap() throws IOException { tenantRepository.close(); // stop using the one setup in Before method - // No exception if config is false - boolean throwIfBootstrappingTenantRepoFails = false; - new FailingDuringBootstrapTenantRepository(createComponentRegistry(throwIfBootstrappingTenantRepoFails)); - // Should get exception if config is true - throwIfBootstrappingTenantRepoFails = true; expectedException.expect(RuntimeException.class); expectedException.expectMessage("Could not create all tenants when bootstrapping, failed to create: [default]"); - new FailingDuringBootstrapTenantRepository(createComponentRegistry(throwIfBootstrappingTenantRepoFails)); + new FailingDuringBootstrapTenantRepository(createComponentRegistry()); } private List<String> readZKChildren(String path) throws Exception { @@ -186,16 +181,16 @@ public class TenantRepositoryTest { } private void assertZooKeeperTenantPathExists(TenantName tenantName) throws Exception { - assertNotNull(globalComponentRegistry.getCurator().framework().checkExists().forPath(tenantRepository.tenantZkPath(tenantName))); + assertNotNull(globalComponentRegistry.getCurator().framework() + .checkExists().forPath(TenantRepository.getTenantPath(tenantName).getAbsolute())); } - private GlobalComponentRegistry createComponentRegistry(boolean throwIfBootstrappingTenantRepoFails) throws IOException { + private GlobalComponentRegistry createComponentRegistry() throws IOException { return new TestComponentRegistry.Builder() .curator(new MockCurator()) .configServerConfig(new ConfigserverConfig(new ConfigserverConfig.Builder() - .throwIfBootstrappingTenantRepoFails(throwIfBootstrappingTenantRepoFails) - .configDefinitionsDir(temporaryFolder.newFolder("configdefs" + throwIfBootstrappingTenantRepoFails).getAbsolutePath()) - .configServerDBDir(temporaryFolder.newFolder("configserverdb" + throwIfBootstrappingTenantRepoFails).getAbsolutePath()))) + .configDefinitionsDir(temporaryFolder.newFolder("configdefs").getAbsolutePath()) + .configServerDBDir(temporaryFolder.newFolder("configserverdb").getAbsolutePath()))) .zone(new Zone(SystemName.cd, Environment.prod, RegionName.from("foo"))) .build(); } @@ -207,7 +202,7 @@ public class TenantRepositoryTest { } @Override - public void createTenant(TenantName tenantName) { + public void bootstrapTenant(TenantName tenantName) { throw new RuntimeException("Failed to create: " + tenantName); } } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 1d6e1a0893d..f3149ed4998 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -11,6 +11,7 @@ import java.io.OutputStream; import java.time.Instant; import java.util.Optional; import java.util.concurrent.Executor; +import java.util.logging.Level; public class LogHandler extends ThreadedHttpRequestHandler { @@ -37,7 +38,12 @@ public class LogHandler extends ThreadedHttpRequestHandler { return new HttpResponse(200) { @Override public void render(OutputStream outputStream) { - logReader.writeLogs(outputStream, from, to); + try { + logReader.writeLogs(outputStream, from, to); + } + catch (Throwable t) { + log.log(Level.WARNING, "Failed reading logs from " + from + " to " + to, t); + } } }; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java new file mode 100644 index 00000000000..23b7a62ffa3 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java @@ -0,0 +1,66 @@ +package com.yahoo.container.handler.metrics; + +import ai.vespa.util.http.VespaHttpClientBuilder; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.Path; +import com.yahoo.restapi.StringResponse; +import com.yahoo.vespa.jdk8compat.List; +import com.yahoo.yolean.Exceptions; +import java.io.IOException; +import java.net.URI; +import java.util.Optional; +import java.util.concurrent.Executor; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.BasicResponseHandler; +import org.apache.http.impl.client.CloseableHttpClient; + +import static com.yahoo.container.handler.metrics.MetricsV2Handler.consumerQuery; +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; + +public class PrometheusV1Handler extends HttpHandlerBase{ + + public static final String V1_PATH = "/prometheus/v1"; + static final String VALUES_PATH = V1_PATH + "/values"; + + private static final int HTTP_CONNECT_TIMEOUT = 5000; + private static final int HTTP_SOCKET_TIMEOUT = 30000; + + private final String metricsProxyUri; + private final HttpClient httpClient = createHttpClient(); + + protected PrometheusV1Handler(Executor executor, + MetricsProxyApiConfig config) { + super(executor); + metricsProxyUri = "http://localhost:" + config.metricsPort() + config.prometheusApiPath(); + } + + @Override + protected Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) { + if (apiPath.matches(V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); + if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer)); + return Optional.empty(); + } + + private HttpResponse valuesResponse(String consumer) { + try { + String uri = metricsProxyUri + consumerQuery(consumer); + String prometheusText = httpClient.execute(new HttpGet(uri), new BasicResponseHandler()); + return new StringResponse(prometheusText); + } catch (IOException e) { + log.warning("Unable to retrieve metrics from " + metricsProxyUri + ": " + Exceptions.toMessageString(e)); + return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + + private static CloseableHttpClient createHttpClient() { + return VespaHttpClientBuilder.create() + .setUserAgent("application-prometheus-receiver") + .setDefaultRequestConfig(RequestConfig.custom() + .setConnectTimeout(HTTP_CONNECT_TIMEOUT) + .setSocketTimeout(HTTP_SOCKET_TIMEOUT) + .build()) + .build(); + } +} diff --git a/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def b/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def index 3e5b973e3f3..d2b85cc1df7 100644 --- a/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def +++ b/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def @@ -4,3 +4,5 @@ namespace=container.handler.metrics metricsPort int metricsApiPath string + +prometheusApiPath string diff --git a/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java index b57814e50aa..9020ed91026 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java @@ -60,6 +60,7 @@ public class MetricsV2HandlerTest { new MetricsProxyApiConfig.Builder() .metricsPort(wireMockRule.port()) .metricsApiPath(MOCK_METRICS_PATH) + .prometheusApiPath("Not/In/Use") .build()); testDriver = new RequestHandlerTestDriver(handler); } @@ -132,7 +133,7 @@ public class MetricsV2HandlerTest { } } - private static String getFileContents(String filename) { + static String getFileContents(String filename) { InputStream in = MetricsV2HandlerTest.class.getClassLoader().getResourceAsStream(filename); if (in == null) { throw new RuntimeException("File not found: " + filename); diff --git a/container-core/src/test/java/com/yahoo/container/handler/metrics/PrometheusV1HandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/metrics/PrometheusV1HandlerTest.java new file mode 100644 index 00000000000..a0e8c131c2b --- /dev/null +++ b/container-core/src/test/java/com/yahoo/container/handler/metrics/PrometheusV1HandlerTest.java @@ -0,0 +1,119 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.metrics; + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.stream.Collectors; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import java.util.concurrent.Executors; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static com.yahoo.container.handler.metrics.MetricsV2Handler.consumerQuery; +import static com.yahoo.container.handler.metrics.MetricsV2HandlerTest.getFileContents; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author gjoranv + */ +public class PrometheusV1HandlerTest { + + private static final String URI_BASE = "http://localhost"; + + private static final String V1_URI = URI_BASE + PrometheusV1Handler.V1_PATH; + private static final String VALUES_URI = URI_BASE + PrometheusV1Handler.VALUES_PATH; + + // Mock applicationmetrics api + private static final String MOCK_METRICS_PATH = "/node0"; + + private static final String TEST_FILE = "application-prometheus.txt"; + private static final String RESPONSE = getFileContents(TEST_FILE); + private static final String CPU_METRIC = "cpu"; + private static final String REPLACED_CPU_METRIC = "replaced_cpu"; + private static final String CUSTOM_CONSUMER = "custom-consumer"; + + private static RequestHandlerTestDriver testDriver; + + @Rule + public WireMockRule wireMockRule = new WireMockRule(options().dynamicPort()); + + @Before + public void setup() { + setupWireMock(); + var handler = new PrometheusV1Handler(Executors.newSingleThreadExecutor(), + new MetricsProxyApiConfig.Builder() + .prometheusApiPath(MOCK_METRICS_PATH) + .metricsPort(wireMockRule.port()) + .metricsApiPath("/Not/In/Use") + .build()); + testDriver = new RequestHandlerTestDriver(handler); + + } + + private void setupWireMock() { + wireMockRule.stubFor(get(urlPathEqualTo(MOCK_METRICS_PATH)) + .willReturn(aResponse().withBody(RESPONSE))); + + // Add a slightly different response for a custom consumer. + String myConsumerResponse = RESPONSE.replaceAll(CPU_METRIC, REPLACED_CPU_METRIC); + wireMockRule.stubFor(get(urlPathEqualTo(MOCK_METRICS_PATH)) + .withQueryParam("consumer", equalTo(CUSTOM_CONSUMER)) + .willReturn(aResponse().withBody(myConsumerResponse))); + } + + @Test + public void v1_response_contains_values_uri() throws Exception { + String response = testDriver.sendRequest(V1_URI).readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("resources")); + + JSONArray resources = root.getJSONArray("resources"); + assertEquals(1, resources.length()); + + JSONObject valuesUri = resources.getJSONObject(0); + assertEquals(VALUES_URI, valuesUri.getString("url")); + } + + @Ignore + @Test + public void visually_inspect_values_response() { + String response = testDriver.sendRequest(VALUES_URI).readAll(); + System.out.println(response); + } + + @Test + public void invalid_path_yields_error_response() throws Exception { + String response = testDriver.sendRequest(V1_URI + "/invalid").readAll(); + JSONObject root = new JSONObject(response); + assertTrue(root.has("error")); + assertTrue(root.getString("error" ).startsWith("No content")); + } + + @Test + public void values_response_is_equal_to_test_file() { + String response = testDriver.sendRequest(VALUES_URI).readAll(); + assertEquals(RESPONSE, response); + } + + @Test + public void consumer_is_propagated_to_metrics_proxy_api() { + String response = testDriver.sendRequest(VALUES_URI + consumerQuery(CUSTOM_CONSUMER)).readAll(); + + assertTrue(response.contains(REPLACED_CPU_METRIC)); + } +} diff --git a/container-core/src/test/resources/application-prometheus.txt b/container-core/src/test/resources/application-prometheus.txt new file mode 100644 index 00000000000..c184737bf67 --- /dev/null +++ b/container-core/src/test/resources/application-prometheus.txt @@ -0,0 +1,178 @@ +# HELP memory_virt +# TYPE memory_virt untyped +memory_virt{metrictype="system",instance="container-clustercontroller",clustername="cluster-controllers",vespaVersion="7.242.24",vespa_service="vespa_container_clustercontroller",} 1.322737664E9 1593416625000 +memory_virt{metrictype="system",instance="distributor",vespaVersion="7.242.24",vespa_service="vespa_distributor",} 3.01477888E8 1593416625000 +memory_virt{metrictype="system",instance="logd",vespaVersion="7.242.24",vespa_service="vespa_logd",} 1.10235648E8 1593416625000 +memory_virt{metrictype="system",instance="container",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 3.973951488E9 1593416625000 +memory_virt{metrictype="system",instance="configserver",vespaVersion="7.242.24",vespa_service="vespa_configserver",} 0.0 1593416625000 +memory_virt{metrictype="system",instance="slobrok",vespaVersion="7.242.24",vespa_service="vespa_slobrok",} 1.00839424E8 1593416625000 +memory_virt{metrictype="system",instance="metricsproxy-container",clustername="metrics",vespaVersion="7.242.24",vespa_service="vespa_metricsproxy_container",} 1.327828992E9 1593416625000 +memory_virt{metrictype="system",instance="logserver",vespaVersion="7.242.24",vespa_service="vespa_logserver",} 8.62457856E8 1593416625000 +memory_virt{metrictype="system",instance="searchnode",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 4.51248128E8 1593416625000 +memory_virt{metrictype="system",instance="config-sentinel",vespaVersion="7.242.24",vespa_service="vespa_config_sentinel",} 0.0 1593416625000 +# HELP memory_rss +# TYPE memory_rss untyped +memory_rss{metrictype="system",instance="container-clustercontroller",clustername="cluster-controllers",vespaVersion="7.242.24",vespa_service="vespa_container_clustercontroller",} 2.59342336E8 1593416625000 +memory_rss{metrictype="system",instance="distributor",vespaVersion="7.242.24",vespa_service="vespa_distributor",} 1.13975296E8 1593416625000 +memory_rss{metrictype="system",instance="logd",vespaVersion="7.242.24",vespa_service="vespa_logd",} 1.5110144E7 1593416625000 +memory_rss{metrictype="system",instance="container",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 1.900249088E9 1593416625000 +memory_rss{metrictype="system",instance="configserver",vespaVersion="7.242.24",vespa_service="vespa_configserver",} 0.0 1593416625000 +memory_rss{metrictype="system",instance="slobrok",vespaVersion="7.242.24",vespa_service="vespa_slobrok",} 1.3324288E7 1593416625000 +memory_rss{metrictype="system",instance="metricsproxy-container",clustername="metrics",vespaVersion="7.242.24",vespa_service="vespa_metricsproxy_container",} 3.64470272E8 1593416625000 +memory_rss{metrictype="system",instance="logserver",vespaVersion="7.242.24",vespa_service="vespa_logserver",} 7.0815744E7 1593416625000 +memory_rss{metrictype="system",instance="searchnode",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 2.04427264E8 1593416625000 +memory_rss{metrictype="system",instance="config-sentinel",vespaVersion="7.242.24",vespa_service="vespa_config_sentinel",} 0.0 1593416625000 +# HELP cpu +# TYPE cpu untyped +cpu{metrictype="system",instance="container-clustercontroller",clustername="cluster-controllers",vespaVersion="7.242.24",vespa_service="vespa_container_clustercontroller",} 1.7821054565496968 1593416625000 +cpu{metrictype="system",instance="distributor",vespaVersion="7.242.24",vespa_service="vespa_distributor",} 23.38783758956458 1593416625000 +cpu{metrictype="system",instance="logd",vespaVersion="7.242.24",vespa_service="vespa_logd",} 1.028844387286423 1593416625000 +cpu{metrictype="system",instance="container",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 64.119051993386 1593416625000 +cpu{metrictype="system",instance="configserver",vespaVersion="7.242.24",vespa_service="vespa_configserver",} 0.0 1593416625000 +cpu{metrictype="system",instance="slobrok",vespaVersion="7.242.24",vespa_service="vespa_slobrok",} 1.8923387837589565 1593416625000 +cpu{metrictype="system",instance="metricsproxy-container",clustername="metrics",vespaVersion="7.242.24",vespa_service="vespa_metricsproxy_container",} 7.697960683446628 1593416625000 +cpu{metrictype="system",instance="logserver",vespaVersion="7.242.24",vespa_service="vespa_logserver",} 0.4409333088370384 1593416625000 +cpu{metrictype="system",instance="searchnode",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 51.589197133933496 1593416625000 +cpu{metrictype="system",instance="config-sentinel",vespaVersion="7.242.24",vespa_service="vespa_config_sentinel",} 0.0 1593416625000 +# HELP jdisc_gc_ms_average +# TYPE jdisc_gc_ms_average untyped +jdisc_gc_ms_average{metrictype="standard",instance="container-clustercontroller",gcName="G1OldGeneration",clustername="cluster-controllers",vespaVersion="7.242.24",vespa_service="vespa_container_clustercontroller",} 0.0 1593416625000 +jdisc_gc_ms_average{metrictype="standard",instance="container-clustercontroller",gcName="G1YoungGeneration",clustername="cluster-controllers",vespaVersion="7.242.24",vespa_service="vespa_container_clustercontroller",} 5.666666666666667 1593416625000 +jdisc_gc_ms_average{metrictype="standard",instance="container",gcName="G1YoungGeneration",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 148.5 1593416625000 +jdisc_gc_ms_average{metrictype="standard",instance="container",gcName="G1OldGeneration",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 0.0 1593416625000 +jdisc_gc_ms_average{metrictype="standard",instance="configserver",gcName="G1OldGeneration",vespaVersion="7.242.24",vespa_service="vespa_configserver",} 0.0 1593416625000 +jdisc_gc_ms_average{metrictype="standard",instance="configserver",gcName="G1YoungGeneration",vespaVersion="7.242.24",vespa_service="vespa_configserver",} 7.5 1593416625000 +jdisc_gc_ms_average{metrictype="standard",instance="metricsproxy-container",gcName="G1OldGeneration",clustername="metrics",vespaVersion="7.242.24",vespa_service="vespa_metricsproxy_container",} 0.0 1593416625000 +jdisc_gc_ms_average{metrictype="standard",instance="metricsproxy-container",gcName="G1YoungGeneration",clustername="metrics",vespaVersion="7.242.24",vespa_service="vespa_metricsproxy_container",} 172.83333333333334 1593416625000 +# HELP mem_heap_free_average +# TYPE mem_heap_free_average untyped +mem_heap_free_average{metrictype="standard",instance="container-clustercontroller",clustername="cluster-controllers",vespaVersion="7.242.24",vespa_service="vespa_container_clustercontroller",} 1.6804585333333332E7 1593416625000 +mem_heap_free_average{metrictype="standard",instance="container",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 1.176528152E9 1593416625000 +mem_heap_free_average{metrictype="standard",instance="configserver",vespaVersion="7.242.24",vespa_service="vespa_configserver",} 8.277591866666667E7 1593416625000 +mem_heap_free_average{metrictype="standard",instance="metricsproxy-container",clustername="metrics",vespaVersion="7.242.24",vespa_service="vespa_metricsproxy_container",} 5.7865174666666664E7 1593416625000 +# HELP http_status_2xx_rate +# TYPE http_status_2xx_rate untyped +http_status_2xx_rate{metrictype="standard",instance="container-clustercontroller",scheme="http",httpMethod="GET",clustername="cluster-controllers",vespaVersion="7.242.24",vespa_service="vespa_container_clustercontroller",} 0.10011513240226261 1593416625000 +http_status_2xx_rate{metrictype="standard",instance="container",scheme="http",httpMethod="GET",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 0.10011513240226261 1593416625000 +http_status_2xx_rate{metrictype="standard",instance="container",scheme="http",httpMethod="POST",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 131.35105371176854 1593416625000 +http_status_2xx_rate{metrictype="standard",instance="configserver",scheme="http",httpMethod="POST",vespaVersion="7.242.24",vespa_service="vespa_configserver",} 0.0 1593416625000 +http_status_2xx_rate{metrictype="standard",instance="configserver",scheme="http",httpMethod="PUT",vespaVersion="7.242.24",vespa_service="vespa_configserver",} 0.0 1593416625000 +http_status_2xx_rate{metrictype="standard",instance="configserver",scheme="http",httpMethod="GET",vespaVersion="7.242.24",vespa_service="vespa_configserver",} 0.10011513240226261 1593416625000 +http_status_2xx_rate{metrictype="standard",instance="metricsproxy-container",scheme="http",httpMethod="GET",clustername="metrics",vespaVersion="7.242.24",vespa_service="vespa_metricsproxy_container",} 0.20023026480452522 1593416625000 +# HELP serverActiveThreads_average +# TYPE serverActiveThreads_average untyped +serverActiveThreads_average{metrictype="standard",instance="container-clustercontroller",threadpool="default-pool",clustername="cluster-controllers",vespaVersion="7.242.24",vespa_service="vespa_container_clustercontroller",} 0.0 1593416625000 +serverActiveThreads_average{metrictype="standard",instance="container",threadpool="default-pool",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 0.5293132328308208 1593416625000 +serverActiveThreads_average{metrictype="standard",instance="configserver",threadpool="default-pool",vespaVersion="7.242.24",vespa_service="vespa_configserver",} 0.0 1593416625000 +serverActiveThreads_average{metrictype="standard",instance="metricsproxy-container",threadpool="default-pool",clustername="metrics",vespaVersion="7.242.24",vespa_service="vespa_metricsproxy_container",} 0.11912751677852348 1593416625000 +# HELP vespa_container_clustercontroller_status status of service +# TYPE vespa_container_clustercontroller_status untyped +vespa_container_clustercontroller_status 1.0 1593416625000 +# HELP vespa_distributor_status status of service +# TYPE vespa_distributor_status untyped +vespa_distributor_status 1.0 1593416625000 +# HELP vespa_logd_status status of service +# TYPE vespa_logd_status untyped +vespa_logd_status 1.0 1593416625000 +# HELP feed_operations_rate +# TYPE feed_operations_rate untyped +feed_operations_rate{metrictype="standard",instance="container",api="vespa.http.server",operation="PUT",status="OK",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 15.951677762760507 1593416625000 +# HELP degraded_queries_rate +# TYPE degraded_queries_rate untyped +degraded_queries_rate{metrictype="standard",instance="container",chain="vespa",reason="timeout",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 0.0 1593416625000 +# HELP query_latency_average +# TYPE query_latency_average untyped +query_latency_average{metrictype="standard",instance="container",chain="vespa",endpoint="vespa:8080",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 5.821542940320233 1593416625000 +# HELP query_latency_99percentile +# TYPE query_latency_99percentile untyped +query_latency_99percentile{metrictype="standard",instance="container",chain="vespa",endpoint="vespa:8080",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 20.109375 1593416625000 +# HELP query_latency_95percentile +# TYPE query_latency_95percentile untyped +query_latency_95percentile{metrictype="standard",instance="container",chain="vespa",endpoint="vespa:8080",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 11.046875 1593416625000 +# HELP totalhits_per_query_average +# TYPE totalhits_per_query_average untyped +totalhits_per_query_average{metrictype="standard",instance="container",chain="vespa",endpoint="vespa:8080",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 23109.089956331878 1593416625000 +# HELP hits_per_query_average +# TYPE hits_per_query_average untyped +hits_per_query_average{metrictype="standard",instance="container",chain="vespa",endpoint="vespa:8080",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 10.0 1593416625000 +# HELP queries_rate +# TYPE queries_rate untyped +queries_rate{metrictype="standard",instance="container",chain="vespa",endpoint="vespa:8080",clustername="container",vespaVersion="7.242.24",vespa_service="vespa_container",} 57.33259915569572 1593416625000 +# HELP vespa_container_status status of service +# TYPE vespa_container_status untyped +vespa_container_status 1.0 1593416625000 +# HELP vespa_configserver_status status of service +# TYPE vespa_configserver_status untyped +vespa_configserver_status 1.0 1593416625000 +# HELP vespa_slobrok_status status of service +# TYPE vespa_slobrok_status untyped +vespa_slobrok_status 1.0 1593416625000 +# HELP vespa_metricsproxy_container_status status of service +# TYPE vespa_metricsproxy_container_status untyped +vespa_metricsproxy_container_status 1.0 1593416625000 +# HELP vespa_logserver_status status of service +# TYPE vespa_logserver_status untyped +vespa_logserver_status 1.0 1593416625000 +# HELP content_proton_documentdb_matching_docs_reranked_rate +# TYPE content_proton_documentdb_matching_docs_reranked_rate untyped +content_proton_documentdb_matching_docs_reranked_rate{metrictype="standard",instance="searchnode",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.0 1593416625000 +# HELP content_proton_documentdb_memory_usage_allocated_bytes_last +# TYPE content_proton_documentdb_memory_usage_allocated_bytes_last untyped +content_proton_documentdb_memory_usage_allocated_bytes_last{metrictype="standard",instance="searchnode",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 2.7138485E7 1593416625000 +# HELP content_proton_transactionlog_disk_usage_last +# TYPE content_proton_transactionlog_disk_usage_last untyped +content_proton_transactionlog_disk_usage_last{metrictype="standard",instance="searchnode",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 1.8301504E7 1593416625000 +# HELP content_proton_documentdb_matching_rank_profile_rerank_time_average +# TYPE content_proton_documentdb_matching_rank_profile_rerank_time_average untyped +content_proton_documentdb_matching_rank_profile_rerank_time_average{metrictype="standard",instance="searchnode",rankProfile="unranked",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.0 1593416625000 +content_proton_documentdb_matching_rank_profile_rerank_time_average{metrictype="standard",instance="searchnode",rankProfile="default",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.0 1593416625000 +content_proton_documentdb_matching_rank_profile_rerank_time_average{metrictype="standard",instance="searchnode",rankProfile="rank_albums",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.0 1593416625000 +# HELP content_proton_documentdb_matching_rank_profile_query_latency_average +# TYPE content_proton_documentdb_matching_rank_profile_query_latency_average untyped +content_proton_documentdb_matching_rank_profile_query_latency_average{metrictype="standard",instance="searchnode",rankProfile="unranked",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.0 1593416625000 +content_proton_documentdb_matching_rank_profile_query_latency_average{metrictype="standard",instance="searchnode",rankProfile="default",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.004089901414609053 1593416625000 +content_proton_documentdb_matching_rank_profile_query_latency_average{metrictype="standard",instance="searchnode",rankProfile="rank_albums",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.0 1593416625000 +# HELP content_proton_documentdb_matching_rank_profile_query_setup_time_average +# TYPE content_proton_documentdb_matching_rank_profile_query_setup_time_average untyped +content_proton_documentdb_matching_rank_profile_query_setup_time_average{metrictype="standard",instance="searchnode",rankProfile="unranked",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.0 1593416625000 +content_proton_documentdb_matching_rank_profile_query_setup_time_average{metrictype="standard",instance="searchnode",rankProfile="default",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 1.617217592592592E-4 1593416625000 +content_proton_documentdb_matching_rank_profile_query_setup_time_average{metrictype="standard",instance="searchnode",rankProfile="rank_albums",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.0 1593416625000 +# HELP content_proton_documentdb_matching_docs_matched_rate +# TYPE content_proton_documentdb_matching_docs_matched_rate untyped +content_proton_documentdb_matching_docs_matched_rate{metrictype="standard",instance="searchnode",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 1459709.916666 1593416625000 +# HELP content_proton_documentdb_documents_active_last +# TYPE content_proton_documentdb_documents_active_last untyped +content_proton_documentdb_documents_active_last{metrictype="standard",instance="searchnode",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 48968.0 1593416625000 +# HELP content_proton_documentdb_documents_ready_last +# TYPE content_proton_documentdb_documents_ready_last untyped +content_proton_documentdb_documents_ready_last{metrictype="standard",instance="searchnode",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 48968.0 1593416625000 +# HELP content_proton_documentdb_documents_total_last +# TYPE content_proton_documentdb_documents_total_last untyped +content_proton_documentdb_documents_total_last{metrictype="standard",instance="searchnode",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 48968.0 1593416625000 +# HELP content_proton_documentdb_disk_usage_last +# TYPE content_proton_documentdb_disk_usage_last untyped +content_proton_documentdb_disk_usage_last{metrictype="standard",instance="searchnode",documenttype="music",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 4872789.0 1593416625000 +# HELP content_proton_resource_usage_disk_average +# TYPE content_proton_resource_usage_disk_average untyped +content_proton_resource_usage_disk_average{metrictype="standard",instance="searchnode",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.1484413210467096 1593416625000 +# HELP content_proton_resource_usage_memory_average +# TYPE content_proton_resource_usage_memory_average untyped +content_proton_resource_usage_memory_average{metrictype="standard",instance="searchnode",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.02216534368263088 1593416625000 +# HELP content_proton_resource_usage_feeding_blocked_last +# TYPE content_proton_resource_usage_feeding_blocked_last untyped +content_proton_resource_usage_feeding_blocked_last{metrictype="standard",instance="searchnode",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.0 1593416625000 +# HELP content_proton_search_protocol_query_latency_average +# TYPE content_proton_search_protocol_query_latency_average untyped +content_proton_search_protocol_query_latency_average{metrictype="standard",instance="searchnode",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 0.004598581322356573 1593416625000 +# HELP content_proton_search_protocol_docsum_latency_average +# TYPE content_proton_search_protocol_docsum_latency_average untyped +content_proton_search_protocol_docsum_latency_average{metrictype="standard",instance="searchnode",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 5.904414458451241E-4 1593416625000 +# HELP content_proton_search_protocol_docsum_requested_documents_rate +# TYPE content_proton_search_protocol_docsum_requested_documents_rate untyped +content_proton_search_protocol_docsum_requested_documents_rate{metrictype="standard",instance="searchnode",clustername="music",vespaVersion="7.242.24",vespa_service="vespa_searchnode",} 647.833333 1593416625000 +# HELP vespa_searchnode_status status of service +# TYPE vespa_searchnode_status untyped +vespa_searchnode_status 1.0 1593416625000 +# HELP vespa_config_sentinel_status status of service +# TYPE vespa_config_sentinel_status untyped +vespa_config_sentinel_status 1.0 1593416625000 diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java index 542f1393852..9b34fd7d62b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java @@ -19,16 +19,13 @@ public class PhraseSegmentItem extends IndexedSegmentItem { /** Whether this was explicitly written as a phrase using quotes by the user */ private boolean explicit = false; - /** - * Creates a phrase containing the same words and state (as pertinent) as - * the given SegmentAndItem. - */ - public PhraseSegmentItem(AndSegmentItem segAnd) { - super(segAnd.getRawWord(), segAnd.stringValue(), segAnd.isFromQuery(), segAnd.isStemmed(), segAnd.getOrigin()); - if (segAnd.getItemCount() > 0) { - WordItem w = (WordItem) segAnd.getItem(0); + /** Creates a phrase containing the same words and state (as pertinent) as the given SegmentAndItem. */ + public PhraseSegmentItem(AndSegmentItem andSegment) { + super(andSegment.getRawWord(), andSegment.stringValue(), andSegment.isFromQuery(), andSegment.isStemmed(), andSegment.getOrigin()); + if (andSegment.getItemCount() > 0) { + WordItem w = (WordItem) andSegment.getItem(0); setIndexName(w.getIndexName()); - for (Iterator<Item> i = segAnd.getItemIterator(); i.hasNext();) { + for (Iterator<Item> i = andSegment.getItemIterator(); i.hasNext();) { WordItem word = (WordItem) i.next(); addWordItem(word); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index cd8579be7f0..902be7e15dd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -326,6 +326,7 @@ public abstract class AbstractParser implements CustomParser { * * @param indexName the index name which preceeded this token, or null if none * @param token the token to segment + * @param quoted whether this segment is within quoted text * @return the resulting item */ // TODO: The segmenting stuff is a mess now, this will fix it: @@ -341,7 +342,7 @@ public abstract class AbstractParser implements CustomParser { // This can be solved by making the segment method language independent by // always producing a query item containing the token text and resolve it to a WordItem or // SegmentItem after parsing and language detection. - protected Item segment(String indexName, Token token) { + protected Item segment(String indexName, Token token, boolean quoted) { String normalizedToken = normalize(token.toString()); if (token.isSpecial()) { @@ -361,12 +362,13 @@ public abstract class AbstractParser implements CustomParser { if (segments.size() == 0) { return null; } + if (segments.size() == 1) { return new WordItem(segments.get(0), "", true, token.substring); } CompositeItem composite; - if (indexFacts.getIndex(indexName).getPhraseSegmenting()) { + if (indexFacts.getIndex(indexName).getPhraseSegmenting() || quoted) { composite = new PhraseSegmentItem(token.toString(), normalizedToken, true, false, token.substring); } else { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java index 6d4401aca04..12f63276269 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java @@ -23,8 +23,7 @@ public class PhraseParser extends AbstractParser { /** * Ignores everything but words and numbers * - * @return a phrase item if several words/numbers was found, - * a word item if only one was found + * @return a phrase item if several words/numbers was found, a word item if only one was found */ private Item forcedPhrase() { Item firstWord = null; @@ -38,7 +37,7 @@ public class PhraseParser extends AbstractParser { } // Note, this depends on segment never creating AndItems when quoted // (the second argument) is true. - Item newWord = segment(null, token); + Item newWord = segment(null, token, true); if (firstWord == null) { // First pass firstWord = newWord; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index 9ba6c1a8101..76ea7fb11a8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -406,13 +406,18 @@ abstract class StructuredParser extends AbstractParser { } } - /** Words for phrases also permits numerals as words */ - private Item phraseWord(String indexName, boolean insidePhrase) { + /** + * Words for phrases also permits numerals as words + * + * @param quoted whether we are consuming text within quoted + * @param insidePhrase whether we are consuming additional items for an existing phrase + */ + private Item phraseWord(String indexName, boolean quoted, boolean insidePhrase) { int position = tokens.getPosition(); Item item = null; try { - item = word(indexName); + item = word(indexName, quoted); if (item == null && tokens.currentIs(NUMBER)) { Token t = tokens.next(); @@ -434,10 +439,12 @@ abstract class StructuredParser extends AbstractParser { /** * Returns a WordItem if this is a non CJK query, - * a WordItem or PhraseSegmentItem if this is a CJK query, + * a WordItem or SegmentItem if this is a CJK query, * null if the current item is not a word + * + * @param quoted whether this token is inside quotes */ - private Item word(String indexName) { + private Item word(String indexName, boolean quoted) { int position = tokens.getPosition(); Item item = null; @@ -452,7 +459,7 @@ abstract class StructuredParser extends AbstractParser { if (submodes.url) { item = new WordItem(word, true); } else { - item = segment(indexName, word); + item = segment(indexName, word, quoted); } if (submodes.url || submodes.site) { @@ -539,7 +546,7 @@ abstract class StructuredParser extends AbstractParser { quoted = !quoted; } - Item word = phraseWord(indexName, (firstWord != null) || (composite != null)); + Item word = phraseWord(indexName, quoted, (firstWord != null) || (composite != null)); if (word == null) { if (tokens.skipMultiple(QUOTE)) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java index ae8c289a5b0..785477d6df7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java @@ -56,26 +56,31 @@ public class CJKSearcher extends Searcher { AndItem replacement = new AndItem(); for (ListIterator<Item> i = ((CompositeItem) root).getItemIterator(); i.hasNext();) { Item item = i.next(); - if (item instanceof WordItem) replacement.addItem(item); - else if (item instanceof PhraseSegmentItem) { + if (item instanceof WordItem) + replacement.addItem(item); + else if (item instanceof PhraseSegmentItem) replacement.addItem(new AndSegmentItem((PhraseSegmentItem) item)); - } - else replacement.addItem(item); // should never run, but hey... just convert and hope it's OK :) + else + replacement.addItem(item); // should never get here } return replacement; - } else if (root instanceof PhraseSegmentItem) { + } + else if (root instanceof PhraseSegmentItem) { PhraseSegmentItem asSegment = (PhraseSegmentItem) root; - if (asSegment.isExplicit() || hasOverlappingTokens(asSegment)) return root; - else return new AndSegmentItem(asSegment); - } else if (root instanceof SegmentItem) { + if (asSegment.isExplicit() || hasOverlappingTokens(asSegment)) + return root; + else + return new AndSegmentItem(asSegment); + } + else if (root instanceof SegmentItem) { return root; // avoid descending into AndSegmentItems and similar - } else if (root instanceof CompositeItem) { + } + else if (root instanceof CompositeItem) { for (ListIterator<Item> i = ((CompositeItem) root).getItemIterator(); i.hasNext();) { Item item = i.next(); Item transformedItem = transform(item); - if (item != transformedItem) { + if (item != transformedItem) i.set(transformedItem); - } } return root; } @@ -96,8 +101,7 @@ public class CJKSearcher extends Searcher { * We have overlapping tokens (see * com.yahoo.prelude.querytransform.test.CJKSearcherTestCase * .testCjkQueryWithOverlappingTokens and ParseTestCase for an explanation) - * if the sum of length of tokens is greater than the lenght of the original - * word + * if the sum of length of tokens is greater than the length of the original word */ private boolean hasOverlappingTokens(PhraseSegmentItem segments) { int segmentsLength=0; diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index c1db7d73561..80b2f845e84 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -1679,8 +1679,8 @@ public class ParseTestCase { // "first" "second" and "third" are segments in the test language Item item = tester.parseQuery("name:\"firstsecondthird\"", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertTrue(item instanceof AndSegmentItem); - AndSegmentItem segment = (AndSegmentItem) item; + assertTrue(item instanceof PhraseSegmentItem); + PhraseSegmentItem segment = (PhraseSegmentItem) item; assertEquals(3, segment.getItemCount()); assertEquals("name:first", segment.getItem(0).toString()); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java index 271e145087b..795c7cfef20 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java @@ -3,6 +3,8 @@ package com.yahoo.search.dispatch; import org.junit.Test; +import java.util.Locale; + import static org.junit.Assert.assertEquals; public class TopKEstimatorTest { @@ -150,13 +152,13 @@ public class TopKEstimatorTest { StringBuilder sb = new StringBuilder(); sb.append(String.format("Prob/Hits:")); for (double p : P) { - sb.append(String.format(" %1.10f", p)); + sb.append(String.format(Locale.ENGLISH, " %1.10f", p)); } sb.append("\n"); for (int k : K) { - sb.append(String.format("%9d:", k)); + sb.append(String.format(Locale.ENGLISH, "%9d:", k)); for (double p : P) { - sb.append(String.format("%13.3f", (double)(estimator.estimateK(k, n, p)*n) / k)); + sb.append(String.format(Locale.ENGLISH, "%13.3f", (double)(estimator.estimateK(k, n, p)*n) / k)); } sb.append("\n"); } diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index c831ee29631..3a67245e912 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -1037,7 +1037,7 @@ public class QueryTestCase { IndexModel indexModel = new IndexModel(test); query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); - assertEquals("myfield:\"it s fine\"", query.getModel().getQueryTree().toString()); + assertEquals("myfield:\"'it s' fine\"", query.getModel().getQueryTree().toString()); } @Test diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index 0b5f2538892..a522e26a46d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.vespa.hosted.controller.api.integration.aws.ApplicationRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; @@ -77,6 +77,6 @@ public interface ServiceRegistry { SystemMonitor systemMonitor(); - PlanController planController(); + BillingController billingController(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java new file mode 100644 index 00000000000..bd9568fe891 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public interface BillingController { + + PlanId getPlan(TenantName tenant); + + /** + * Returns true if plan was changed + */ + boolean setPlan(TenantName tenant, PlanId planId, boolean hasApplications); + + Invoice.Id createInvoiceForPeriod(TenantName tenant, ZonedDateTime startTime, ZonedDateTime endTime, String agent); + + Invoice createUncommittedInvoice(TenantName tenant, LocalDate until); + + Map<TenantName, Invoice> createUncommittedInvoices(LocalDate until); + + List<Invoice.LineItem> getUnusedLineItems(TenantName tenant); + + Optional<PaymentInstrument> getDefaultInstrument(TenantName tenant); + + String createClientToken(String tenant, String userId); + + boolean deleteInstrument(TenantName tenant, String userId, String instrumentId); + + void updateInvoiceStatus(Invoice.Id invoiceId, String agent, String status); + + void addLineItem(TenantName tenant, String description, BigDecimal amount, String agent); + + void deleteLineItem(String lineItemId); + + boolean setActivePaymentInstrument(InstrumentOwner paymentInstrument); + + InstrumentList listInstruments(TenantName tenant, String userId); + + List<Invoice> getInvoices(TenantName tenant); + +}
\ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CostCalculator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CostCalculator.java deleted file mode 100644 index 628beec8450..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CostCalculator.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.billing; - -import com.yahoo.config.provision.NodeResources; -import com.yahoo.vespa.hosted.controller.api.integration.resource.CostInfo; - - -/** - * @author ogronnesby - */ -public interface CostCalculator { - - /** Calculate the cost for the given usage */ - CostInfo calculate(ResourceUsage usage); - - /** Estimate the cost for the given resources */ - double calculate(NodeResources resources); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InstrumentList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InstrumentList.java new file mode 100644 index 00000000000..f26261cd157 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InstrumentList.java @@ -0,0 +1,39 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author olaa + */ +public class InstrumentList { + + private String activeInstrumentId; + private List<PaymentInstrument> instruments; + + + public InstrumentList(List<PaymentInstrument> instruments) { + this.instruments = instruments; + } + + public void setActiveInstrumentId(String activeInstrumentId) { + this.activeInstrumentId = activeInstrumentId; + } + + public void addInstrument(PaymentInstrument instrument) { + instruments.add(instrument); + } + + public void addInstruments(List<PaymentInstrument> instruments) { + instruments.addAll(instruments); + } + + public String getActiveInstrumentId() { + return activeInstrumentId; + } + + public List<PaymentInstrument> getInstruments() { + return instruments; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InstrumentOwner.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InstrumentOwner.java new file mode 100644 index 00000000000..45e06b11b2a --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InstrumentOwner.java @@ -0,0 +1,67 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import com.yahoo.config.provision.TenantName; + +import java.util.Objects; + +/** + * @author olaa + */ +public class InstrumentOwner { + + private final TenantName tenantName; + private final String userId; + private final String paymentInstrumentId; + private final boolean isDefault; + + public InstrumentOwner(TenantName tenantName, String userId, String paymentInstrumentId, boolean isDefault) { + this.tenantName = tenantName; + this.userId = userId; + this.paymentInstrumentId = paymentInstrumentId; + this.isDefault = isDefault; + } + + public TenantName getTenantName() { + return tenantName; + } + + public String getUserId() { + return userId; + } + + public String getPaymentInstrumentId() { + return paymentInstrumentId; + } + + public boolean isDefault() { + return isDefault; + } + + @Override + public String toString() { + return String.format( + "Tenant: %s\nCusomer ID: %s\nPayment Instrument ID: %s\nIs default: %s", + tenantName.value(), + userId, + paymentInstrumentId, + isDefault + ); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InstrumentOwner other = (InstrumentOwner) o; + return this.tenantName.equals(other.getTenantName()) && + this.userId.equals(other.getUserId()) && + this.paymentInstrumentId.equals(other.getPaymentInstrumentId()) && + this.isDefault() == other.isDefault(); + } + + @Override + public int hashCode() { + return Objects.hash(tenantName, userId, paymentInstrumentId, isDefault); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Invoice.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Invoice.java new file mode 100644 index 00000000000..31388d24e2e --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Invoice.java @@ -0,0 +1,266 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; + +import java.math.BigDecimal; +import java.time.ZonedDateTime; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.UUID; + + +/** + * An Invoice is an identifier with a status (with history) and line items. A line item is the meat and + * potatoes of the content of the invoice, and are a history of items. Most line items are connected to + * a given deployment in Vespa Cloud, but they can also be manually added to e.g. give a discount or represent + * support. + * <p> + * All line items have a Plan associated with them - which was used to map from utilization to an actual price. + * <p> + * The invoice has a status history, but only the latest status is exposed through this API. + * + * @author ogronnesby + */ +public class Invoice { + private static final BigDecimal SCALED_ZERO = new BigDecimal("0.00"); + + private final Id id; + private final List<LineItem> lineItems; + private final StatusHistory statusHistory; + private final ZonedDateTime startTime; + private final ZonedDateTime endTime; + + public Invoice(Id id, StatusHistory statusHistory, List<LineItem> lineItems, ZonedDateTime startTime, ZonedDateTime endTime) { + this.id = id; + this.lineItems = List.copyOf(lineItems); + this.statusHistory = statusHistory; + this.startTime = startTime; + this.endTime = endTime; + } + + public Id id() { + return id; + } + + public String status() { + return statusHistory.current(); + } + + public StatusHistory statusHistory() { + return statusHistory; + } + + public List<LineItem> lineItems() { + return lineItems; + } + + public ZonedDateTime getStartTime() { + return startTime; + } + + public ZonedDateTime getEndTime() { + return endTime; + } + + public BigDecimal sum() { + return lineItems.stream().map(LineItem::amount).reduce(SCALED_ZERO, BigDecimal::add); + } + + public static final class Id { + private final String value; + + public static Id of(String value) { + Objects.requireNonNull(value); + return new Id(value); + } + + public static Id generate() { + var id = UUID.randomUUID().toString(); + return new Id(id); + } + + private Id(String value) { + this.value = value; + } + + public String value() { + return value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Id invoiceId = (Id) o; + return value.equals(invoiceId.value); + } + + @Override + public int hashCode() { + return Objects.hash(value); + } + + @Override + public String toString() { + return "InvoiceId{" + + "value='" + value + '\'' + + '}'; + } + } + + /** + * Represents a chargeable line on an invoice. + */ + public static class LineItem { + private final String id; + private final String description; + private final BigDecimal amount; + private final String plan; + private final String agent; + private final ZonedDateTime addedAt; + private final Optional<ZonedDateTime> startedAt; + private final Optional<ZonedDateTime> endedAt; + private final Optional<ApplicationId> applicationId; + private final Optional<ZoneId> zoneId; + + public LineItem(String id, String description, BigDecimal amount, String plan, String agent, ZonedDateTime addedAt, ZonedDateTime startedAt, ZonedDateTime endedAt, ApplicationId applicationId, ZoneId zoneId) { + this.id = id; + this.description = description; + this.amount = amount; + this.plan = plan; + this.agent = agent; + this.addedAt = addedAt; + this.startedAt = Optional.ofNullable(startedAt); + this.endedAt = Optional.ofNullable(endedAt); + + if (applicationId == null && zoneId != null) + throw new IllegalArgumentException("Must supply applicationId if zoneId is supplied"); + + this.applicationId = Optional.ofNullable(applicationId); + this.zoneId = Optional.ofNullable(zoneId); + } + + public LineItem(String id, String description, BigDecimal amount, String plan, String agent, ZonedDateTime addedAt) { + this(id, description, amount, plan, agent, addedAt, null, null, null, null); + } + + /** The opaque ID of this */ + public String id() { + return id; + } + + /** The string description of this - used for display purposes */ + public String description() { + return description; + } + + /** The dollar amount of this */ + public BigDecimal amount() { + return SCALED_ZERO.add(amount); + } + + /** The plan used to calculate amount of this */ + public String plan() { + return plan; + } + + /** Who created this line item */ + public String agent() { + return agent; + } + + /** When was this line item added */ + public ZonedDateTime addedAt() { + return addedAt; + } + + /** What time period is this line item for - time start */ + public Optional<ZonedDateTime> startedAt() { + return startedAt; + } + + /** What time period is this line item for - time end */ + public Optional<ZonedDateTime> endedAt() { + return endedAt; + } + + /** Optionally - what application is this line item about */ + public Optional<ApplicationId> applicationId() { + return applicationId; + } + + /** Optionally - what zone deployment is this line item about */ + public Optional<ZoneId> zoneId() { + return zoneId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LineItem lineItem = (LineItem) o; + return id.equals(lineItem.id) && + description.equals(lineItem.description) && + amount.equals(lineItem.amount) && + plan.equals(lineItem.plan) && + agent.equals(lineItem.agent) && + addedAt.equals(lineItem.addedAt) && + startedAt.equals(lineItem.startedAt) && + endedAt.equals(lineItem.endedAt) && + applicationId.equals(lineItem.applicationId) && + zoneId.equals(lineItem.zoneId); + } + + @Override + public int hashCode() { + return Objects.hash(id, description, amount, plan, agent, addedAt, startedAt, endedAt, applicationId, zoneId); + } + + @Override + public String toString() { + return "LineItem{" + + "id='" + id + '\'' + + ", description='" + description + '\'' + + ", amount=" + amount + + ", plan='" + plan + '\'' + + ", agent='" + agent + '\'' + + ", addedAt=" + addedAt + + ", startedAt=" + startedAt + + ", endedAt=" + endedAt + + ", applicationId=" + applicationId + + ", zoneId=" + zoneId + + '}'; + } + } + + public static class StatusHistory { + SortedMap<ZonedDateTime, String> history; + + public StatusHistory(SortedMap<ZonedDateTime, String> history) { + this.history = history; + } + + public static StatusHistory open() { + return new StatusHistory( + new TreeMap<>(Map.of(ZonedDateTime.now(), "OPEN")) + ); + } + + public String current() { + return history.get(history.lastKey()); + } + + public SortedMap<ZonedDateTime, String> getHistory() { + return history; + } + + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java new file mode 100644 index 00000000000..a4c25e301ba --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java @@ -0,0 +1,144 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * @author olaa + */ +public class MockBillingController implements BillingController { + + Map<TenantName, PlanId> plans = new HashMap<>(); + Map<TenantName, PaymentInstrument> activeInstruments = new HashMap<>(); + Map<TenantName, List<Invoice>> committedInvoices = new HashMap<>(); + Map<TenantName, Invoice> uncommittedInvoices = new HashMap<>(); + Map<TenantName, List<Invoice.LineItem>> unusedLineItems = new HashMap<>(); + + @Override + public PlanId getPlan(TenantName tenant) { + return plans.get(tenant); + } + + @Override + public boolean setPlan(TenantName tenant, PlanId planId, boolean hasApplications) { + plans.put(tenant, planId); + return true; + } + + @Override + public Invoice.Id createInvoiceForPeriod(TenantName tenant, ZonedDateTime startTime, ZonedDateTime endTime, String agent) { + var invoiceId = Invoice.Id.of("id-123"); + committedInvoices.computeIfAbsent(tenant, l -> new ArrayList<>()) + .add(new Invoice( + invoiceId, + Invoice.StatusHistory.open(), + List.of(), + startTime, + endTime + )); + return invoiceId; + } + + @Override + public Invoice createUncommittedInvoice(TenantName tenant, LocalDate until) { + return uncommittedInvoices.get(tenant); + } + + @Override + public Map<TenantName, Invoice> createUncommittedInvoices(LocalDate until) { + return uncommittedInvoices; + } + + @Override + public List<Invoice.LineItem> getUnusedLineItems(TenantName tenant) { + return unusedLineItems.getOrDefault(tenant, List.of()); + } + + @Override + public Optional<PaymentInstrument> getDefaultInstrument(TenantName tenant) { + return Optional.ofNullable(activeInstruments.get(tenant)); + } + + @Override + public String createClientToken(String tenant, String userId) { + return "some-token"; + } + + @Override + public boolean deleteInstrument(TenantName tenant, String userId, String instrumentId) { + activeInstruments.remove(tenant); + return true; + } + + @Override + public void updateInvoiceStatus(Invoice.Id invoiceId, String agent, String status) { + committedInvoices.values().stream() + .flatMap(List::stream) + .filter(invoice -> invoiceId.equals(invoice.id())) + .forEach(invoice -> invoice.statusHistory().history.put(ZonedDateTime.now(), status)); + } + + @Override + public void addLineItem(TenantName tenant, String description, BigDecimal amount, String agent) { + unusedLineItems.computeIfAbsent(tenant, l -> new ArrayList<>()) + .add(new Invoice.LineItem( + "line-item-id", + description, + amount, + "some-plan", + agent, + ZonedDateTime.now() + )); + } + + @Override + public void deleteLineItem(String lineItemId) { + + } + + @Override + public boolean setActivePaymentInstrument(InstrumentOwner paymentInstrument) { + var instrumentId = paymentInstrument.getPaymentInstrumentId(); + activeInstruments.put(paymentInstrument.getTenantName(), createInstrument(instrumentId)); + return true; + } + + @Override + public InstrumentList listInstruments(TenantName tenant, String userId) { + return null; + } + + @Override + public List<Invoice> getInvoices(TenantName tenant) { + return committedInvoices.getOrDefault(tenant, List.of()); + } + + private PaymentInstrument createInstrument(String id) { + return new PaymentInstrument(id, + "name", + "displayText", + "brand", + "type", + "endingWith", + "expiryDate" + ); + } + + public void addInvoice(TenantName tenantName, Invoice invoice, boolean committed) { + if (committed) + committedInvoices.computeIfAbsent(tenantName, i -> new ArrayList<>()) + .add(invoice); + else + uncommittedInvoices.put(tenantName, invoice); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PaymentInstrument.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PaymentInstrument.java new file mode 100644 index 00000000000..7b8d36f3d4f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PaymentInstrument.java @@ -0,0 +1,51 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +/** + * @author olaa + */ +public class PaymentInstrument { + + private final String id; + private final String nameOnCard; + private final String displayText; + private final String brand; + private final String type; + private final String endingWith; + private final String expiryDate; + + + public PaymentInstrument(String id, String nameOnCard, String displayText, String brand, String type, String endingWith, String expiryDate) { + this.id = id; + this.nameOnCard = nameOnCard; + this.displayText = displayText; + this.brand = brand; + this.type = type; + this.endingWith = endingWith; + this.expiryDate = expiryDate; + } + + public String getId() { + return id; + } + + public String getNameOnCard() { + return nameOnCard; + } + + public String getDisplayText() { + return displayText; + } + + public String getBrand() { + return brand; + } + + public String getType() { + return type; + } + + public String getEndingWith() { + return endingWith; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Plan.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Plan.java deleted file mode 100644 index 75a88136c45..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Plan.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.billing; - -/** - * A Plan decides two different things: - * - * - How to map from usage to a sum of money that is owed. - * - Limits on how much resources can be used. - * - * @author ogronnesby - */ -public interface Plan { - - /** The ID of the plan as used in APIs and storage systems */ - String id(); - - /** The calculator used to calculate a bill for usage */ - CostCalculator calculator(); - - /** The quota limits associated with the plan */ - Object quota(); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanController.java deleted file mode 100644 index f13c251d212..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanController.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.billing; - -import com.yahoo.config.provision.TenantName; - -public interface PlanController { - - Plan getPlan(TenantName tenant); - -}
\ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanId.java new file mode 100644 index 00000000000..68a897c904f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanId.java @@ -0,0 +1,43 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import java.util.Objects; + +/** + * @author olaa + */ +public class PlanId { + + private final String value; + + public PlanId(String value) { + if (value.isBlank()) + throw new IllegalArgumentException("Id must be non-blank."); + this.value = value; + } + + public static PlanId from(String value) { + return new PlanId(value); + } + + public String value() { return value; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PlanId id = (PlanId) o; + return Objects.equals(value, id.value); + } + + @Override + public int hashCode() { + return Objects.hash(value); + } + + @Override + public String toString() { + return "plan '" + value + "'"; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/ResourceUsage.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/ResourceUsage.java deleted file mode 100644 index cbfd2b6ff50..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/ResourceUsage.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.billing; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; - -import java.math.BigDecimal; - -/** - * @author olaa - */ -public class ResourceUsage { - - private final ApplicationId applicationId; - private final ZoneId zoneId; - private final Plan plan; - private final BigDecimal cpuMillis; - private final BigDecimal memoryMillis; - private final BigDecimal diskMillis; - - public ResourceUsage(ApplicationId applicationId, ZoneId zoneId, Plan plan, - BigDecimal cpuMillis, BigDecimal memoryMillis, BigDecimal diskMillis) { - this.applicationId = applicationId; - this.zoneId = zoneId; - this.cpuMillis = cpuMillis; - this.memoryMillis = memoryMillis; - this.diskMillis = diskMillis; - this.plan = plan; - } - - public ApplicationId getApplicationId() { - return applicationId; - } - - public ZoneId getZoneId() { - return zoneId; - } - - public BigDecimal getCpuMillis() { - return cpuMillis; - } - - public BigDecimal getMemoryMillis() { - return memoryMillis; - } - - public BigDecimal getDiskMillis() { - return diskMillis; - } - - public Plan getPlan() { - return plan; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTarget.java index b92eb80df80..41723dbdea6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTarget.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTarget.java @@ -1,73 +1,68 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.dns; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.zone.ZoneId; import java.util.Objects; /** - * Represents the target of an ALIAS record. + * The target of an {@link Record.Type#ALIAS} record. Contains record fields unique to aliases. * * @author mpolden */ -public class AliasTarget { +public abstract class AliasTarget { private final HostName name; private final String dnsZone; - private final ZoneId zone; + private final String id; - public AliasTarget(HostName name, String dnsZone, ZoneId zone) { + public AliasTarget(HostName name, String dnsZone, String id) { this.name = Objects.requireNonNull(name, "name must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); - this.zone = Objects.requireNonNull(zone, "zone must be non-null"); + this.id = Objects.requireNonNull(id, "id must be non-null"); } - /** DNS name of this */ - public HostName name() { - return name; + /** A unique identifier of this record within the ALIAS record group */ + public String id() { + return id; } - /** DNS zone of this */ - public String dnsZone() { - return dnsZone; + /** DNS name this points to */ + public final HostName name() { + return name; } - /** The zone where this exists */ - public ZoneId zone() { - return zone; + /** The DNS zone this belongs to */ + public final String dnsZone() { + return dnsZone; } /** Returns the fields in this encoded as record data */ - public RecordData asData() { - return RecordData.from(name.value() + "/" + dnsZone + "/" + zone.value()); - } + public abstract RecordData pack(); @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - AliasTarget that = (AliasTarget) o; - return name.equals(that.name); + AliasTarget alias = (AliasTarget) o; + return name.equals(alias.name) && + dnsZone.equals(alias.dnsZone) && + id.equals(alias.id); } @Override public int hashCode() { - return Objects.hash(name); - } - - @Override - public String toString() { - return String.format("rotation target %s [zone: %s] in %s", name, dnsZone, zone); + return Objects.hash(name, dnsZone, id); } - public static AliasTarget from(RecordData data) { - var parts = data.asString().split("/"); - if (parts.length != 3) { - throw new IllegalArgumentException("Expected data to be on format [hostname]/[DNS zone]/[zone], but got " + - data.asString()); + /** Unpack target from given record data */ + public static AliasTarget unpack(RecordData data) { + String[] parts = data.asString().split("/"); + switch (parts[0]) { + case "latency": return LatencyAliasTarget.unpack(data); + case "weighted": return WeightedAliasTarget.unpack(data); } - return new AliasTarget(HostName.from(parts[0]), parts[1], ZoneId.from(parts[2])); + throw new IllegalArgumentException("Unknown alias type '" + parts[0] + "'"); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/LatencyAliasTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/LatencyAliasTarget.java new file mode 100644 index 00000000000..7bd43ff1dcb --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/LatencyAliasTarget.java @@ -0,0 +1,60 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.dns; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneId; + +import java.util.Objects; + +/** + * An implementation of {@link AliasTarget} that uses latency-based routing. + * + * @author mpolden + */ +public class LatencyAliasTarget extends AliasTarget { + + private final ZoneId zone; + + public LatencyAliasTarget(HostName name, String dnsZone, ZoneId zone) { + super(name, dnsZone, zone.value()); + this.zone = Objects.requireNonNull(zone); + } + + /** The zone this record points to */ + public ZoneId zone() { + return zone; + } + + @Override + public RecordData pack() { + return RecordData.from("latency/" + name().value() + "/" + dnsZone() + "/" + id()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + LatencyAliasTarget that = (LatencyAliasTarget) o; + return zone.equals(that.zone); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), zone); + } + + /** Unpack latency alias from given record data */ + public static LatencyAliasTarget unpack(RecordData data) { + var parts = data.asString().split("/"); + if (parts.length != 4) { + throw new IllegalArgumentException("Expected data to be on format type/name/DNS-zone/zone-id, but got " + + data.asString()); + } + if (!"latency".equals(parts[0])) { + throw new IllegalArgumentException("Unexpected type '" + parts[0] + "'"); + } + return new LatencyAliasTarget(HostName.from(parts[1]), parts[2], ZoneId.from(parts[3])); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index 59324079e6f..e8688b17347 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.dns; @@ -43,7 +43,7 @@ public class MemoryNameService implements NameService { public List<Record> createAlias(RecordName name, Set<AliasTarget> targets) { var records = targets.stream() .sorted((a, b) -> Comparator.comparing(AliasTarget::name).compare(a, b)) - .map(target -> new Record(Record.Type.ALIAS, name, target.asData())) + .map(d -> new Record(Record.Type.ALIAS, name, d.pack())) .collect(Collectors.toList()); // Satisfy idempotency contract of interface records.stream() @@ -80,7 +80,7 @@ public class MemoryNameService implements NameService { if (record.type() == type) { if (type == Record.Type.ALIAS) { // Unpack ALIAS record and compare FQDN of data part - return RecordData.fqdn(AliasTarget.from(record.data()).name().value()) + return RecordData.fqdn(AliasTarget.unpack(record.data()).name().value()) .equals(data); } return record.data().equals(data); @@ -111,7 +111,7 @@ public class MemoryNameService implements NameService { } /** - * Returns whether record r1 and r2 can co-exist in a name service. This attempts to enforce the same constraints as + * Returns whether record r1 and r2 are in conflict. This attempts to enforce the same constraints a * most real name services. */ private static boolean conflicts(Record r1, Record r2) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java index 903ea250935..9f2fd887482 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.dns; import java.util.List; @@ -21,18 +21,19 @@ public interface NameService { Record createCname(RecordName name, RecordData canonicalName); /** - * Create a non-standard ALIAS record pointing to given targets. Implementations of this can be expected to be + * Create a non-standard ALIAS record pointing to given targets. Implementations of this are expected to be * idempotent * - * @param targets Targets that should be resolved by this alias. pointing to given targets. - * @return The created records. One for each target. + * @param targets Targets that should be resolved by this name. + * @return The created records. One per target. */ List<Record> createAlias(RecordName name, Set<AliasTarget> targets); /** * Create a new TXT record containing the provided data. * @param name Name of the created record - * @param txtRecords TXT data values for the record, each consisting of one or more space-separated <em>double-quoted</em> strings: "string1" "string2" + * @param txtRecords TXT data values for the record, each consisting of one or more space-separated double-quoted + * strings: "string1" "string2" * @return The created records */ List<Record> createTxtRecords(RecordName name, List<RecordData> txtRecords); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java new file mode 100644 index 00000000000..9d741cb2dbc --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java @@ -0,0 +1,64 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.dns; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneId; + +import java.util.Objects; + +/** + * An implementation of {@link AliasTarget} where is requests are answered based on the weight assigned to the + * record, as a proportion of the total weight for all records having the same DNS name. + * + * The portion of received traffic is calculated as follows: (record weight / sum of the weights of all records). + * + * @author mpolden + */ +public class WeightedAliasTarget extends AliasTarget { + + private final long weight; + + public WeightedAliasTarget(HostName name, String dnsZone, ZoneId zone, long weight) { + super(name, dnsZone, zone.value()); + this.weight = weight; + } + + /** The weight of this target */ + public long weight() { + return weight; + } + + @Override + public RecordData pack() { + return RecordData.from("weighted/" + name().value() + "/" + dnsZone() + "/" + id() + "/" + weight); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + WeightedAliasTarget that = (WeightedAliasTarget) o; + return weight == that.weight; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), weight); + } + + /** Unpack weighted alias from given record data */ + public static WeightedAliasTarget unpack(RecordData data) { + var parts = data.asString().split("/"); + if (parts.length != 5) { + throw new IllegalArgumentException("Expected data to be on format type/name/DNS-zone/zone-id/weight, " + + "but got " + data.asString()); + } + if (!"weighted".equals(parts[0])) { + throw new IllegalArgumentException("Unexpected type '" + parts[0] + "'"); + } + return new WeightedAliasTarget(HostName.from(parts[1]), parts[2], ZoneId.from(parts[3]), + Long.parseLong(parts[4])); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 2fdf442dbe0..aaddd3811bc 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -72,6 +72,10 @@ enum PathGroup { PathPrefix.api, "/billing/v1/tenant/{tenant}/instrument/{*}"), + billingPlan(Matcher.tenant, + PathPrefix.api, + "/billing/v1/tenant/{tenant}/plan/{*}"), + billingList(Matcher.tenant, PathPrefix.api, "/billing/v1/tenant/{tenant}/billing/{*}"), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index 83adba6f59b..548ad0af484 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -158,6 +158,11 @@ enum Policy { .on(PathGroup.billingToken) .in(SystemName.PublicCd)), + /** Ability to update tenant payment instrument */ + planUpdate(Privilege.grant(Action.update) + .on(PathGroup.billingPlan) + .in(SystemName.PublicCd)), + /** Read the generated bills */ billingInformationRead(Privilege.grant(Action.read) .on(PathGroup.billingList) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index b9d534019db..801661f454e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -67,6 +67,7 @@ public enum RoleDefinition { Policy.paymentInstrumentUpdate, Policy.paymentInstrumentDelete, Policy.paymentInstrumentCreate, + Policy.planUpdate, Policy.billingInformationRead), /** Headless — the application specific role identified by deployment keys for production */ diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTargetTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTargetTest.java new file mode 100644 index 00000000000..7169222fe26 --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTargetTest.java @@ -0,0 +1,39 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.dns; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneId; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author mpolden + */ +public class AliasTargetTest { + + @Test + public void packing() { + List<AliasTarget> tests = List.of( + new LatencyAliasTarget(HostName.from("foo.example.com"), "dns-zone-1", ZoneId.from("prod.us-north-1")), + new WeightedAliasTarget(HostName.from("bar.example.com"), "dns-zone-2", ZoneId.from("prod.us-north-2"), 50) + ); + for (var target : tests) { + AliasTarget unpacked = AliasTarget.unpack(target.pack()); + assertEquals(target, unpacked); + } + + List<RecordData> invalidData = List.of(RecordData.from(""), RecordData.from("foobar")); + for (var data : invalidData) { + try { + AliasTarget.unpack(data); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) { + } + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index c441188b1be..98169ba3196 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -264,6 +264,7 @@ public class RoutingController { private boolean canRouteDirectlyTo(DeploymentId deploymentId, Application application) { if (controller.system().isPublic()) return true; // Public always supports direct routing if (controller.system().isCd()) return true; // CD deploys directly so we cannot enforce all requirements below + if(deploymentId.zoneId().environment().isManuallyDeployed()) return true; // Manually deployed zones does not include any use cases where direct routing is not supported // Check Athenz service presence. The test framework uses this identity when sending requests to the // deployment's container(s). diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java index 9dd02735638..1697c05b8fb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; @@ -41,7 +41,7 @@ public class CreateRecords implements NameServiceRequest { public void dispatchTo(NameService nameService) { switch (type) { case ALIAS: - var targets = records.stream().map(Record::data).map(AliasTarget::from).collect(Collectors.toSet()); + var targets = records.stream().map(Record::data).map(AliasTarget::unpack).collect(Collectors.toSet()); nameService.createAlias(name, targets); break; case TXT: diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index 459378def6c..b8692233b2d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.vespa.curator.Lock; @@ -45,8 +45,8 @@ public class NameServiceForwarder { /** Create or update an ALIAS record with given name and targets */ public void createAlias(RecordName name, Set<AliasTarget> targets, NameServiceQueue.Priority priority) { - var records = targets.stream().map(AliasTarget::asData) - .map(data -> new Record(Record.Type.ALIAS, name, data)) + var records = targets.stream() + .map(target -> new Record(Record.Type.ALIAS, name, target.pack())) .collect(Collectors.toList()); forward(new CreateRecords(records), priority); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java index a8dcaa5740c..245747a882f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java @@ -1,7 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -11,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationState; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.util.LinkedHashMap; @@ -69,11 +69,11 @@ public class RotationStatusUpdater extends ControllerMaintainer { try { pool.awaitTermination(30, TimeUnit.SECONDS); if (lastException.get() != null) { - log.log(Level.WARNING, String.format("Failed to get global routing status of %d/%d applications. Retrying in %s. Last error: ", + log.log(Level.WARNING, String.format("Failed to get global routing status of %d/%d applications. Retrying in %s. Last error: %s", failures.get(), attempts.get(), - interval()), - lastException.get()); + interval(), + Exceptions.toMessageString(lastException.get()))); } } catch (InterruptedException e) { throw new RuntimeException(e); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java index 6c271ed0470..0fe6f7e0bfb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java @@ -2,9 +2,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; @@ -19,11 +16,8 @@ import java.time.Duration; */ public class SystemRoutingPolicyMaintainer extends ControllerMaintainer { - private final BooleanFlag featureFlag; - public SystemRoutingPolicyMaintainer(Controller controller, Duration interval) { super(controller, interval); - this.featureFlag = Flags.CONFIGSERVER_PROVISION_LB.bindTo(controller.flagSource()); } @Override @@ -31,7 +25,6 @@ public class SystemRoutingPolicyMaintainer extends ControllerMaintainer { for (var zone : controller().zoneRegistry().zones().all().ids()) { for (var application : SystemApplication.values()) { if (!application.hasEndpoint()) continue; - if (!featureFlag.with(FetchVector.Dimension.ZONE_ID, zone.value()).value()) continue; controller().routing().policies().refresh(application.id(), DeploymentSpec.empty, zone); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java new file mode 100644 index 00000000000..ccbee15d2c5 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -0,0 +1,396 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.billing; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.JacksonJsonResponse; +import com.yahoo.restapi.MessageResponse; +import com.yahoo.restapi.Path; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.restapi.StringResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PaymentInstrument; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Invoice; +import com.yahoo.vespa.hosted.controller.api.integration.billing.InstrumentOwner; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.yolean.Exceptions; + +import javax.ws.rs.BadRequestException; +import javax.ws.rs.ForbiddenException; +import javax.ws.rs.NotFoundException; +import java.io.IOException; +import java.math.BigDecimal; +import java.security.Principal; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; + +/** + * @author andreer + * @author olaa + */ +public class BillingApiHandler extends LoggingRequestHandler { + + private static final String OPTIONAL_PREFIX = "/api"; + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + + private final BillingController billingController; + private final ApplicationController applicationController; + + public BillingApiHandler(Executor executor, + AccessLog accessLog, + Controller controller) { + super(executor, accessLog); + this.billingController = controller.serviceRegistry().billingController(); + this.applicationController = controller.applications(); + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + String userId = userIdOrThrow(request); + switch (request.getMethod()) { + case GET: + return handleGET(request, path, userId); + case PATCH: + return handlePATCH(request, path, userId); + case DELETE: + return handleDELETE(path, userId); + case POST: + return handlePOST(path, request, userId); + default: + return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + } + } catch (NotFoundException e) { + return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); + } catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } catch (Exception e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + // Don't expose internal billing details in error message to user + return ErrorResponse.internalServerError("Internal problem while handling billing API request"); + } + } + + private HttpResponse handleGET(HttpRequest request, Path path, String userId) { + if (path.matches("/billing/v1/tenant/{tenant}/token")) return getToken(path.get("tenant"), userId); + if (path.matches("/billing/v1/tenant/{tenant}/instrument")) return getInstruments(path.get("tenant"), userId); + if (path.matches("/billing/v1/tenant/{tenant}/billing")) return getBilling(path.get("tenant"), request.getProperty("until")); + if (path.matches("/billing/v1/tenant/{tenant}/plan")) return getPlan(path.get("tenant")); + if (path.matches("/billing/v1/billing")) return getBillingAllTenants(request.getProperty("until")); + if (path.matches("/billing/v1/invoice/tenant/{tenant}/line-item")) return getLineItems(path.get("tenant")); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse handlePATCH(HttpRequest request, Path path, String userId) { + if (path.matches("/billing/v1/tenant/{tenant}/instrument")) return patchActiveInstrument(request, path.get("tenant"), userId); + if (path.matches("/billing/v1/tenant/{tenant}/plan")) return patchPlan(request, path.get("tenant")); + return ErrorResponse.notFoundError("Nothing at " + path); + + } + + private HttpResponse handleDELETE(Path path, String userId) { + if (path.matches("/billing/v1/tenant/{tenant}/instrument/{instrument}")) return deleteInstrument(path.get("tenant"), userId, path.get("instrument")); + if (path.matches("/billing/v1/invoice/line-item/{line-item-id}")) return deleteLineItem(path.get("line-item-id")); + return ErrorResponse.notFoundError("Nothing at " + path); + + } + + private HttpResponse handlePOST(Path path, HttpRequest request, String userId) { + if (path.matches("/billing/v1/invoice")) return createInvoice(request, userId); + if (path.matches("/billing/v1/invoice/{invoice-id}/status")) return setInvoiceStatus(request, path.get("invoice-id")); + if (path.matches("/billing/v1/invoice/tenant/{tenant}/line-item")) return addLineItem(request, path.get("tenant")); + return ErrorResponse.notFoundError("Nothing at " + path); + + } + + private HttpResponse getPlan(String tenant) { + var plan = billingController.getPlan(TenantName.from(tenant)); + var slime = new Slime(); + var root = slime.setObject(); + root.setString("tenant", tenant); + root.setString("plan", plan.value()); + return new SlimeJsonResponse(slime); + } + + private HttpResponse patchPlan(HttpRequest request, String tenant) { + var tenantName = TenantName.from(tenant); + var slime = inspectorOrThrow(request); + var planId = PlanId.from(slime.field("plan").asString()); + var hasApplications = applicationController.asList(tenantName).size() > 0; + + if (billingController.setPlan(tenantName, planId, hasApplications)) { + return new StringResponse("Plan: " + planId.value()); + } else { + return ErrorResponse.forbidden("Invalid plan change with active deployments"); + } + + } + + private HttpResponse getBillingAllTenants(String until) { + try { + var untilDate = untilParameter(until); + var uncommittedInvoices = billingController.createUncommittedInvoices(untilDate); + + var slime = new Slime(); + var root = slime.setObject(); + root.setString("until", untilDate.format(DateTimeFormatter.ISO_DATE)); + var tenants = root.setArray("tenants"); + + uncommittedInvoices.forEach((tenant, invoice) -> { + var tc = tenants.addObject(); + tc.setString("tenant", tenant.value()); + getPlanForTenant(tc, tenant); + renderCurrentUsage(tc.setObject("current"), invoice); + renderAdditionalItems(tc.setObject("additional").setArray("items"), billingController.getUnusedLineItems(tenant)); + + billingController.getDefaultInstrument(tenant).ifPresent(card -> + renderInstrument(tc.setObject("payment"), card) + ); + }); + + return new SlimeJsonResponse(slime); + } catch (DateTimeParseException e) { + return ErrorResponse.badRequest("Could not parse date: " + until); + } + } + + private HttpResponse addLineItem(HttpRequest request, String tenant) { + Inspector inspector = inspectorOrThrow(request); + billingController.addLineItem( + TenantName.from(tenant), + getInspectorFieldOrThrow(inspector, "description"), + new BigDecimal(getInspectorFieldOrThrow(inspector, "amount")), + userIdOrThrow(request)); + return new MessageResponse("Added line item for tenant " + tenant); + } + + private HttpResponse setInvoiceStatus(HttpRequest request, String invoiceId) { + Inspector inspector = inspectorOrThrow(request); + String status = getInspectorFieldOrThrow(inspector, "status"); + billingController.updateInvoiceStatus(Invoice.Id.of(invoiceId), userIdOrThrow(request), status); + return new MessageResponse("Updated status of invoice " + invoiceId); + } + + private HttpResponse createInvoice(HttpRequest request, String userId) { + Inspector inspector = inspectorOrThrow(request); + TenantName tenantName = TenantName.from(getInspectorFieldOrThrow(inspector, "tenant")); + + LocalDate startDate = LocalDate.parse(getInspectorFieldOrThrow(inspector, "startTime")); + LocalDate endDate = LocalDate.parse(getInspectorFieldOrThrow(inspector, "endTime")); + ZonedDateTime startTime = startDate.atStartOfDay(ZoneId.of("UTC")); + ZonedDateTime endTime = endDate.atStartOfDay(ZoneId.of("UTC")); + + var invoiceId = billingController.createInvoiceForPeriod(tenantName, startTime, endTime, userId); + + return new MessageResponse("Created invoice with ID " + invoiceId.value()); + } + + private HttpResponse getInstruments(String tenant, String userId) { + var instrumentListResponse = billingController.listInstruments(TenantName.from(tenant), userId); + return new JacksonJsonResponse<>(200, instrumentListResponse); + } + + private HttpResponse getToken(String tenant, String userId) { + return new StringResponse(billingController.createClientToken(tenant, userId)); + } + + private HttpResponse getBilling(String tenant, String until) { + try { + var untilDate = untilParameter(until); + var tenantId = TenantName.from(tenant); + var slimeResponse = new Slime(); + var root = slimeResponse.setObject(); + + root.setString("until", untilDate.format(DateTimeFormatter.ISO_DATE)); + + getPlanForTenant(root, tenantId); + renderCurrentUsage(root.setObject("current"), getCurrentUsageForTenant(tenantId, untilDate)); + renderAdditionalItems(root.setObject("additional").setArray("items"), billingController.getUnusedLineItems(tenantId)); + renderInvoices(root.setArray("bills"), getInvoicesForTenant(tenantId)); + + billingController.getDefaultInstrument(tenantId).ifPresent( card -> + renderInstrument(root.setObject("payment"), card) + ); + + return new SlimeJsonResponse(slimeResponse); + } catch (DateTimeParseException e) { + return ErrorResponse.badRequest("Could not parse date: " + until); + } + } + + private HttpResponse getLineItems(String tenant) { + var slimeResponse = new Slime(); + var root = slimeResponse.setObject(); + var lineItems = root.setArray("lineItems"); + + billingController.getUnusedLineItems(TenantName.from(tenant)) + .forEach(lineItem -> { + var itemCursor = lineItems.addObject(); + renderLineItemToCursor(itemCursor, lineItem); + }); + + return new SlimeJsonResponse(slimeResponse); + } + + private void getPlanForTenant(Cursor cursor, TenantName tenant) { + cursor.setString("plan", billingController.getPlan(tenant).value()); + } + + private void renderInstrument(Cursor cursor, PaymentInstrument instrument) { + cursor.setString("type", instrument.getType()); + cursor.setString("brand", instrument.getBrand()); + cursor.setString("endingWith", instrument.getEndingWith()); + } + + private void renderCurrentUsage(Cursor cursor, Invoice currentUsage) { + cursor.setString("amount", currentUsage.sum().toPlainString()); + cursor.setString("status", "accrued"); + cursor.setString("from", currentUsage.getStartTime().format(DATE_TIME_FORMATTER)); + var itemsCursor = cursor.setArray("items"); + currentUsage.lineItems().forEach(lineItem -> { + var itemCursor = itemsCursor.addObject(); + renderLineItemToCursor(itemCursor, lineItem); + }); + } + + private void renderAdditionalItems(Cursor cursor, List<Invoice.LineItem> items) { + items.forEach(item -> { + renderLineItemToCursor(cursor.addObject(), item); + }); + } + + private Invoice getCurrentUsageForTenant(TenantName tenant, LocalDate until) { + return billingController.createUncommittedInvoice(tenant, until); + } + + private List<Invoice> getInvoicesForTenant(TenantName tenant) { + return billingController.getInvoices(tenant); + } + + private void renderInvoices(Cursor cursor, List<Invoice> invoices) { + invoices.forEach(invoice -> { + var invoiceCursor = cursor.addObject(); + renderInvoiceToCursor(invoiceCursor, invoice); + }); + } + + private void renderInvoiceToCursor(Cursor invoiceCursor, Invoice invoice) { + invoiceCursor.setString("id", invoice.id().value()); + invoiceCursor.setString("from", invoice.getStartTime().format(DATE_TIME_FORMATTER)); + invoiceCursor.setString("to", invoice.getEndTime().format(DATE_TIME_FORMATTER)); + + invoiceCursor.setString("amount", invoice.sum().toString()); + invoiceCursor.setString("status", invoice.status()); + var statusCursor = invoiceCursor.setArray("statusHistory"); + renderStatusHistory(statusCursor, invoice.statusHistory()); + + + var lineItemsCursor = invoiceCursor.setArray("items"); + invoice.lineItems().forEach(lineItem -> { + var itemCursor = lineItemsCursor.addObject(); + renderLineItemToCursor(itemCursor, lineItem); + }); + } + + private void renderStatusHistory(Cursor cursor, Invoice.StatusHistory statusHistory) { + statusHistory.getHistory() + .entrySet() + .stream() + .forEach(entry -> { + var c = cursor.addObject(); + c.setString("at", entry.getKey().format(DATE_TIME_FORMATTER)); + c.setString("status", entry.getValue()); + }); + } + + private void renderLineItemToCursor(Cursor cursor, Invoice.LineItem lineItem) { + cursor.setString("id", lineItem.id()); + cursor.setString("description", lineItem.description()); + cursor.setString("amount", lineItem.amount().toString()); + lineItem.applicationId().ifPresent(appId -> { + cursor.setString("application", appId.application().value()); + }); + } + + private HttpResponse deleteInstrument(String tenant, String userId, String instrument) { + if (billingController.deleteInstrument(TenantName.from(tenant), userId, instrument)) { + return new StringResponse("OK"); + } else { + return ErrorResponse.forbidden("Cannot delete payment instrument you don't own"); + } + } + + private HttpResponse deleteLineItem(String lineItemId) { + billingController.deleteLineItem(lineItemId); + return new MessageResponse("Succesfully deleted line item " + lineItemId); + } + + private HttpResponse patchActiveInstrument(HttpRequest request, String tenant, String userId) { + var inspector = inspectorOrThrow(request); + String instrumentId = getInspectorFieldOrThrow(inspector, "active"); + InstrumentOwner paymentInstrument = new InstrumentOwner(TenantName.from(tenant), userId, instrumentId, true); + boolean success = billingController.setActivePaymentInstrument(paymentInstrument); + return success ? new StringResponse("OK") : ErrorResponse.internalServerError("Failed to patch active instrument"); + } + + private Inspector inspectorOrThrow(HttpRequest request) { + try { + return SlimeUtils.jsonToSlime(request.getData().readAllBytes()).get(); + } catch (IOException e) { + throw new BadRequestException("Failed to parse request body"); + } + } + + private static String userIdOrThrow(HttpRequest request) { + return Optional.ofNullable(request.getJDiscRequest().getUserPrincipal()) + .map(Principal::getName) + .orElseThrow(() -> new ForbiddenException("Must be authenticated to use this API")); + } + + private static String getInspectorFieldOrThrow(Inspector inspector, String field) { + if (!inspector.field(field).valid()) + throw new BadRequestException("Field " + field + " cannot be null"); + return inspector.field(field).asString(); + } + + private DeploymentId getDeploymentIdOrNull(Inspector inspector) { + if (inspector.field("applicationId").valid() != inspector.field("zoneId").valid() ) { + throw new BadRequestException("Either both application id and zone id should be set, or neither."); + } + if (inspector.field("applicationId").valid()) { + return new DeploymentId( + ApplicationId.fromSerializedForm(inspector.field("applicationId").asString()), + com.yahoo.config.provision.zone.ZoneId.from(inspector.field("zoneId").asString()) + ); + } + return null; + } + + private LocalDate untilParameter(String until) { + if (until == null || until.isEmpty() || until.isBlank()) + return LocalDate.now().plusDays(1); + return LocalDate.parse(until); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index a429c444e0b..e080b7babce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.application.api.DeploymentSpec; @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; +import com.yahoo.vespa.hosted.controller.api.integration.dns.LatencyAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; @@ -125,7 +126,7 @@ public class RoutingPolicies { for (var policy : routeEntry.getValue()) { if (policy.dnsZone().isEmpty()) continue; if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(RoutingMethod.exclusive)) continue; - var target = new AliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone()); + var target = new LatencyAliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone()); var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); // Remove target zone if global routing status is set out at: // - zone level (ZoneRoutingPolicy) @@ -361,7 +362,7 @@ public class RoutingPolicies { public void removeRecords(Record.Type type, RecordName name) {} @Override - public void createAlias(RecordName name, Set<AliasTarget> target) {} + public void createAlias(RecordName name, Set<AliasTarget> targets) {} @Override public void createCname(RecordName name, RecordData data) {} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index bd0143ef879..13cf992cd52 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; @@ -36,13 +36,13 @@ public class CloudAccessControl implements AccessControl { private final UserManagement userManagement; private final BooleanFlag enablePublicSignup; - private final PlanController planController; + private final BillingController planController; @Inject public CloudAccessControl(UserManagement userManagement, FlagSource flagSource, ServiceRegistry serviceRegistry) { this.userManagement = userManagement; this.enablePublicSignup = Flags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource); - planController = serviceRegistry.planController(); + planController = serviceRegistry.billingController(); } @Override @@ -109,7 +109,7 @@ public class CloudAccessControl implements AccessControl { } private boolean isTrial(TenantName tenant) { - return planController.getPlan(tenant).id().equals("trial"); + return planController.getPlan(tenant).value().equals("trial"); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index dfca2d69a5e..9e6eb9ca2e1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.google.common.collect.Sets; @@ -11,6 +11,7 @@ import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; @@ -24,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.dns.LatencyAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; @@ -623,6 +625,8 @@ public class ControllerTest { // Create application var context = tester.newDeploymentContext(); ZoneId zone = ZoneId.from("dev", "us-east-1"); + tester.controllerTester().zoneRegistry() + .setRoutingMethod(ZoneApiMock.from(zone), RoutingMethod.shared, RoutingMethod.sharedLayer4); // Deploy tester.controller().applications().deploy(context.instanceId(), zone, Optional.of(applicationPackage), DeployOptions.none()); @@ -631,6 +635,14 @@ public class ControllerTest { assertTrue("No job status added", context.instanceJobs().isEmpty()); assertEquals("DeploymentSpec is not persisted", DeploymentSpec.empty, context.application().deploymentSpec()); + + // Verify zone supports shared layer 4 and shared routing methods + Set<RoutingMethod> routingMethods = tester.controller().routing().endpointsOf(context.deploymentIdIn(zone)) + .asList() + .stream() + .map(Endpoint::routingMethod) + .collect(Collectors.toSet()); + assertEquals(routingMethods, Set.of(RoutingMethod.shared, RoutingMethod.sharedLayer4)); } @Test @@ -813,7 +825,8 @@ public class ControllerTest { // The 'east' global endpoint, pointing to zone 2 with exclusive routing new Record(Record.Type.ALIAS, RecordName.from("east.application.tenant.global.vespa.oath.cloud"), - RecordData.from("lb-0--tenant:application:default--prod.us-east-3/dns-zone-1/prod.us-east-3")), + new LatencyAliasTarget(HostName.from("lb-0--tenant:application:default--prod.us-east-3"), + "dns-zone-1", ZoneId.from("prod.us-east-3")).pack()), // The 'default' global endpoint, pointing to both zones with shared routing, via rotation new Record(Record.Type.CNAME, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java index 30ed9b5432c..c187552b0b3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java @@ -1,9 +1,9 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; +import com.yahoo.vespa.hosted.controller.api.integration.dns.LatencyAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; @@ -27,19 +27,17 @@ public class NameServiceQueueTest { var nameService = new MemoryNameService(); var r1 = new Record(Record.Type.CNAME, RecordName.from("cname.vespa.oath.cloud"), RecordData.from("example.com")); var r2 = new Record(Record.Type.TXT, RecordName.from("txt.example.com"), RecordData.from("text")); - var r3 = List.of(new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), RecordData.from("alias1/dns-zone-01/prod.us-north-1")), - new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), RecordData.from("alias2/dns-zone-02/prod.us-north-2"))); + var r3 = List.of(new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), + new LatencyAliasTarget(HostName.from("alias1"), + "dns-zone-01", + ZoneId.from("prod", "us-north-1")).pack()), + new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), + new LatencyAliasTarget(HostName.from("alias2"), + "dns-zone-02", + ZoneId.from("prod", "us-north-2")).pack())); var req1 = new CreateRecord(r1); var req2 = new CreateRecords(List.of(r2)); - var req3 = new CreateRecords(List.of(new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), - new AliasTarget(HostName.from("alias1"), - "dns-zone-01", - ZoneId.from("prod", "us-north-1")).asData()), - new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), - new AliasTarget(HostName.from("alias2"), - "dns-zone-02", - ZoneId.from("prod", "us-north-2")).asData())) - ); + var req3 = new CreateRecords(r3); var req4 = new RemoveRecords(r3.get(0).type(), r3.get(0).name()); var req5 = new RemoveRecords(r2.type(), r2.data()); var req6 = new RemoveRecords(Record.Type.CNAME, r1.data()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index b7e7c9814e3..1b21f7db7c4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -12,14 +12,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.aws.NoopApplicationRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; -import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; @@ -60,7 +60,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final MockRunDataStore mockRunDataStore = new MockRunDataStore(); private final MockResourceTagger mockResourceTagger = new MockResourceTagger(); private final ApplicationRoleService applicationRoleService = new NoopApplicationRoleService(); - private final PlanController planController = (tenantName) -> null; + private final BillingController billingController = new MockBillingController(); public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); @@ -187,6 +187,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return systemMonitor; } + @Override + public BillingController billingController() { + return billingController; + } + public ConfigServerMock configServerMock() { return configServerMock; } @@ -203,9 +208,4 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return endpointCertificateMock; } - @Override - public PlanController planController() { - return planController; - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java index e852710c604..a2c995eaaf0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializerTest.java @@ -1,9 +1,9 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; +import com.yahoo.vespa.hosted.controller.api.integration.dns.LatencyAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; @@ -32,13 +32,13 @@ public class NameServiceQueueSerializerTest { new CreateRecord(record1), new CreateRecords(List.of(record2)), new CreateRecords(List.of(new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), - new AliasTarget(HostName.from("alias1"), - "dns-zone-01", - ZoneId.from("prod", "us-north-1")).asData()), + new LatencyAliasTarget(HostName.from("alias1"), + "dns-zone-01", + ZoneId.from("prod", "us-north-1")).pack()), new Record(Record.Type.ALIAS, RecordName.from("alias.example.com"), - new AliasTarget(HostName.from("alias2"), - "dns-zone-02", - ZoneId.from("prod", "us-north-2")).asData())) + new LatencyAliasTarget(HostName.from("alias2"), + "dns-zone-02", + ZoneId.from("prod", "us-north-2")).pack())) ), new RemoveRecords(record1.type(), record1.name()), new RemoveRecords(record2.type(), record2.data()) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java new file mode 100644 index 00000000000..19cfa95c682 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java @@ -0,0 +1,214 @@ +package com.yahoo.vespa.hosted.controller.restapi.billing; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Invoice; +import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import static com.yahoo.application.container.handler.Request.Method.*; +import static org.junit.Assert.*; + +/** + * @author olaa + */ +public class BillingApiHandlerTest extends ControllerContainerCloudTest { + + private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/"; + private static final TenantName tenant = TenantName.from("tenant1"); + private static final TenantName tenant2 = TenantName.from("tenant2"); + private static final Set<Role> tenantRole = Set.of(Role.administrator(tenant)); + private static final Set<Role> financeAdmin = Set.of(Role.hostedAccountant()); + private MockBillingController billingController; + + private ContainerTester tester; + + @Before + public void setup() { + tester = new ContainerTester(container, responseFiles); + billingController = (MockBillingController) tester.serviceRegistry().billingController(); + } + + @Override + protected SystemName system() { + return SystemName.PublicCd; + } + + @Override + protected String variablePartXml() { + return " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControlRequests'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControl'/>\n" + + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.billing.BillingApiHandler'>\n" + + " <binding>http://*/billing/v1/*</binding>\n" + + " </handler>\n" + + + " <http>\n" + + " <server id='default' port='8080' />\n" + + " <filtering>\n" + + " <request-chain id='default'>\n" + + " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" + + " <binding>http://*/*</binding>\n" + + " </request-chain>\n" + + " </filtering>\n" + + " </http>\n"; + } + + @Test + public void setting_and_deleting_instrument() { + assertTrue(billingController.getDefaultInstrument(tenant).isEmpty()); + + var instrumentRequest = request("/billing/v1/tenant/tenant1/instrument", PATCH) + .data("{\"active\": \"id-1\"}") + .roles(tenantRole); + + tester.assertResponse(instrumentRequest,"OK"); + assertEquals("id-1", billingController.getDefaultInstrument(tenant).get().getId()); + + var deleteInstrumentRequest = request("/billing/v1/tenant/tenant1/instrument/id-1", DELETE) + .roles(tenantRole); + + tester.assertResponse(deleteInstrumentRequest,"OK"); + assertTrue(billingController.getDefaultInstrument(tenant).isEmpty()); + } + + @Test + public void response_list_bills() { + var invoice = createInvoice(); + + billingController.addInvoice(tenant, invoice, true); + billingController.addInvoice(tenant, invoice, false); + billingController.setPlan(tenant, PlanId.from("some-plan"), true); + + var request = request("/billing/v1/tenant/tenant1/billing?until=2020-05-28").roles(tenantRole); + tester.assertResponse(request, new File("tenant-billing-view")); + + } + + @Test + public void test_invoice_creation() { + var invoices = billingController.getInvoices(tenant); + assertEquals(0, invoices.size()); + + String requestBody = "{\"tenant\":\"tenant1\", \"startTime\":\"2020-04-20\", \"endTime\":\"2020-05-20\"}"; + var request = request("/billing/v1/invoice", POST) + .data(requestBody) + .roles(tenantRole); + + tester.assertResponse(request, accessDenied, 403); + request.roles(financeAdmin); + tester.assertResponse(request, new File("invoice-creation-response")); + + invoices = billingController.getInvoices(tenant); + assertEquals(1, invoices.size()); + Invoice invoice = invoices.get(0); + assertEquals(invoice.getStartTime().toString(), "2020-04-20T00:00Z[UTC]"); + assertEquals(invoice.getEndTime().toString(), "2020-05-20T00:00Z[UTC]"); + } + + @Test + public void adding_and_listing_line_item() { + + var requestBody = "{" + + "\"description\":\"some description\"," + + "\"amount\":\"123.45\" " + + "}"; + + var request = request("/billing/v1/invoice/tenant/tenant1/line-item", POST) + .data(requestBody) + .roles(financeAdmin); + + tester.assertResponse(request, "{\"message\":\"Added line item for tenant tenant1\"}"); + + var lineItems = billingController.getUnusedLineItems(tenant); + Assert.assertEquals(1, lineItems.size()); + Invoice.LineItem lineItem = lineItems.get(0); + assertEquals("some description", lineItem.description()); + assertEquals(new BigDecimal("123.45"), lineItem.amount()); + + request = request("/billing/v1/invoice/tenant/tenant1/line-item") + .roles(financeAdmin); + + tester.assertResponse(request, new File("line-item-list")); + } + + @Test + public void adding_new_status() { + billingController.addInvoice(tenant, createInvoice(), true); + + var requestBody = "{\"status\":\"DONE\"}"; + var request = request("/billing/v1/invoice/id-1/status", POST) + .data(requestBody) + .roles(financeAdmin); + tester.assertResponse(request, "{\"message\":\"Updated status of invoice id-1\"}"); + + var invoice = billingController.getInvoices(tenant).get(0); + assertEquals("DONE", invoice.status()); + } + + @Test + public void list_all_uninvoiced_items() { + var invoice = createInvoice(); + billingController.setPlan(tenant, PlanId.from("some-plan"), true); + billingController.setPlan(tenant2, PlanId.from("some-plan"), true); + billingController.addInvoice(tenant, invoice, false); + billingController.addLineItem(tenant, "support", new BigDecimal("42"), "Smith"); + billingController.addInvoice(tenant2, invoice, false); + + + var request = request("/billing/v1/billing?until=2020-05-28").roles(financeAdmin); + + tester.assertResponse(request, new File("billing-all-tenants")); + } + + @Test + public void setting_plans() { + var planRequest = request("/billing/v1/tenant/tenant1/plan", PATCH) + .data("{\"plan\": \"new-plan\"}") + .roles(tenantRole); + tester.assertResponse(planRequest, "Plan: new-plan"); + assertEquals("new-plan", billingController.getPlan(tenant).value()); + } + + private Invoice createInvoice() { + var start = LocalDate.of(2020, 5, 23).atStartOfDay(ZoneId.systemDefault()); + var end = start.plusDays(5); + var statusHistory = new Invoice.StatusHistory(new TreeMap<>(Map.of(start, "OPEN"))); + return new Invoice( + Invoice.Id.of("id-1"), + statusHistory, + List.of(createLineItem(start)), + start, + end + ); + } + + + private Invoice.LineItem createLineItem(ZonedDateTime addedAt) { + return new Invoice.LineItem( + "some-id", + "description", + new BigDecimal("123.00"), + "plan", + "Smith", + addedAt + ); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants new file mode 100644 index 00000000000..c5bf0c88c2c --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants @@ -0,0 +1,48 @@ +{ + "until":"2020-05-28", + "tenants":[ + { + "tenant":"tenant2", + "plan":"some-plan", + "current":{ + "amount":"123.00", + "status":"accrued", + "from":"2020-05-23", + "items":[ + { + "id":"some-id", + "description":"description", + "amount":"123.00" + } + ] + }, + "additional":{"items":[]} + }, + { + "tenant":"tenant1", + "plan":"some-plan", + "current":{ + "amount":"123.00", + "status":"accrued", + "from":"2020-05-23", + "items":[ + { + "id":"some-id", + "description":"description", + "amount":"123.00" + } + ] + }, + "additional": + { + "items":[ + { + "id":"line-item-id", + "description":"support", + "amount":"42.00" + } + ] + } + } + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response new file mode 100644 index 00000000000..0a92229025b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response @@ -0,0 +1 @@ +{"message":"Created invoice with ID id-123"}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list new file mode 100644 index 00000000000..cd5aec2f8f4 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list @@ -0,0 +1,9 @@ +{ + "lineItems":[ + { + "id":"line-item-id", + "description":"some description", + "amount":"123.45" + } + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view new file mode 100644 index 00000000000..8bc39771b31 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view @@ -0,0 +1,38 @@ +{ + "until":"2020-05-28", + "plan":"some-plan", + "current":{ + "amount":"123.00", + "status":"accrued", + "from":"2020-05-23", + "items":[ + { + "id":"some-id", + "description":"description", + "amount":"123.00" + } + ] + }, + "additional":{"items":[]}, + "bills":[ + { + "id":"id-1", + "from":"2020-05-23", + "to":"2020-05-28","amount":"123.00", + "status":"OPEN", + "statusHistory":[ + { + "at":"2020-05-23", + "status":"OPEN" + } + ], + "items":[ + { + "id":"some-id", + "description":"description", + "amount":"123.00" + } + ] + } + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 760d80d230c..3b2fe95d0f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; import com.google.common.collect.Sets; @@ -247,7 +247,7 @@ public class RoutingPoliciesTest { var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud"; assertEquals(endpoint + " points to c0 in all regions", - List.of("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), + List.of("latency/lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), tester.aliasDataOf(endpoint)); assertTrue("No rotations assigned", context.application().instances().values().stream() .map(Instance::rotations) @@ -699,7 +699,7 @@ public class RoutingPoliciesTest { .map(Endpoint::dnsName) .orElse("<none>"); var zoneTargets = Arrays.stream(zone) - .map(z -> "lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + + .map(z -> "latency/lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + z.value() + "/dns-zone-1/" + z.value()) .collect(Collectors.toSet()); assertEquals("Global endpoint " + endpoint + " points to expected zones", zoneTargets, diff --git a/dist/vespa.spec b/dist/vespa.spec index 1e1658d3d55..9353acf5c68 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -441,6 +441,7 @@ fi %{_prefix}/lib/jars/application-model-jar-with-dependencies.jar %{_prefix}/lib/jars/application-preprocessor-jar-with-dependencies.jar %{_prefix}/lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar +%{_prefix}/lib/jars/cloud-tenant-cd-jar-with-dependencies.jar %{_prefix}/lib/jars/clustercontroller-apps-jar-with-dependencies.jar %{_prefix}/lib/jars/clustercontroller-apputil-jar-with-dependencies.jar %{_prefix}/lib/jars/clustercontroller-core-jar-with-dependencies.jar @@ -460,7 +461,9 @@ fi %{_prefix}/lib/jars/searchlib.jar %{_prefix}/lib/jars/searchlib-jar-with-dependencies.jar %{_prefix}/lib/jars/service-monitor-jar-with-dependencies.jar +%{_prefix}/lib/jars/tenant-cd-api-jar-with-dependencies.jar %{_prefix}/lib/jars/vespa_feed_perf-jar-with-dependencies.jar +%{_prefix}/lib/jars/vespa-osgi-testrunner-jar-with-dependencies.jar %{_prefix}/lib/jars/vespa-testrunner-components.jar %{_prefix}/lib/jars/vespa-testrunner-components-jar-with-dependencies.jar %{_prefix}/lib/jars/zookeeper-command-line-client-jar-with-dependencies.jar diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanTree.java b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java index 05e5ff41cf1..9df45ab6c2e 100644 --- a/document/src/main/java/com/yahoo/document/annotation/SpanTree.java +++ b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java @@ -25,7 +25,7 @@ import java.util.Map; * A SpanTree holds a root node of a tree of SpanNodes, and a List of Annotations pointing to these nodes * or each other. It also has a name. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @see com.yahoo.document.annotation.SpanNode * @see com.yahoo.document.annotation.Annotation */ @@ -39,8 +39,7 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl /** * WARNING! Only to be used by deserializers! Creates an empty SpanTree instance. */ - public SpanTree() { - } + public SpanTree() { } /** * Creates a new SpanTree with a given root node. @@ -227,18 +226,12 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl root.setParent(this); } - /** - * Returns the name of this span tree. - * @return the name of this span tree. - */ + /** Returns the name of this span tree. */ public String getName() { return name; } - /** - * Returns the root node of this span tree. - * @return the root node of this span tree. - */ + /** Returns the root node of this span tree. */ public SpanNode getRoot() { return root; } diff --git a/document/src/main/java/com/yahoo/document/select/simple/SelectionParser.java b/document/src/main/java/com/yahoo/document/select/simple/SelectionParser.java index 0614731265b..c250d3bede1 100644 --- a/document/src/main/java/com/yahoo/document/select/simple/SelectionParser.java +++ b/document/src/main/java/com/yahoo/document/select/simple/SelectionParser.java @@ -8,8 +8,11 @@ import com.yahoo.document.select.rule.ExpressionNode; * @author baldersheim */ public class SelectionParser extends Parser { + private ExpressionNode node; + public ExpressionNode getNode() { return node; } + public boolean parse(CharSequence s) { boolean retval = false; IdSpecParser id = new IdSpecParser(); @@ -40,4 +43,5 @@ public class SelectionParser extends Parser { return retval; } + } diff --git a/document/src/tests/CMakeLists.txt b/document/src/tests/CMakeLists.txt index 6458adccfd8..e203fc4b464 100644 --- a/document/src/tests/CMakeLists.txt +++ b/document/src/tests/CMakeLists.txt @@ -1,5 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) + # Runner for unit tests written in gtest. # NOTE: All new test classes should be added here. vespa_add_executable(document_gtest_runner_app TEST @@ -32,7 +34,7 @@ vespa_add_executable(document_gtest_runner_app TEST weightedsetfieldvaluetest.cpp DEPENDS document - gtest + GTest::GTest AFTER document_documentconfig ) diff --git a/document/src/vespa/document/select/CMakeLists.txt b/document/src/vespa/document/select/CMakeLists.txt index f210e8abdd7..e0adfe2b2d9 100644 --- a/document/src/vespa/document/select/CMakeLists.txt +++ b/document/src/vespa/document/select/CMakeLists.txt @@ -13,6 +13,9 @@ FLEX_TARGET(DocSelLexer grammar/lexer.ll ADD_FLEX_BISON_DEPENDENCY(DocSelLexer DocSelParser) include_directories(${CMAKE_CURRENT_BINARY_DIR}) +vespa_add_source_target(bisongen_document_select DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/parser.cxx ${CMAKE_CURRENT_BINARY_DIR}/parser.hxx) +vespa_add_source_target(flexgen_document_select DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lexer.cxx ${CMAKE_CURRENT_BINARY_DIR}/lexer.hxx) + vespa_add_library(document_select OBJECT SOURCES bodyfielddetector.cpp diff --git a/documentapi/OWNERS b/documentapi/OWNERS index e030acdbc5b..8e9a0415de5 100644 --- a/documentapi/OWNERS +++ b/documentapi/OWNERS @@ -1 +1,2 @@ -freva +vekterli +baldersheim diff --git a/documentapi/src/tests/loadtypes/CMakeLists.txt b/documentapi/src/tests/loadtypes/CMakeLists.txt index 495c05c0e43..3e3e64310f8 100644 --- a/documentapi/src/tests/loadtypes/CMakeLists.txt +++ b/documentapi/src/tests/loadtypes/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(documentapi_loadtype_test_app TEST SOURCES loadtypetest.cpp DEPENDS documentapi vdstestlib - gtest + GTest::GTest ) vespa_add_test(NAME documentapi_loadtype_test_app COMMAND documentapi_loadtype_test_app) diff --git a/eval/src/tests/eval/inline_operation/CMakeLists.txt b/eval/src/tests/eval/inline_operation/CMakeLists.txt index 04cdbca3abf..050e9287c21 100644 --- a/eval/src/tests/eval/inline_operation/CMakeLists.txt +++ b/eval/src/tests/eval/inline_operation/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(eval_inline_operation_test_app TEST SOURCES inline_operation_test.cpp DEPENDS vespaeval - gtest + GTest::GTest ) vespa_add_test(NAME eval_inline_operation_test_app COMMAND eval_inline_operation_test_app) diff --git a/eval/src/tests/eval/multiply_add/CMakeLists.txt b/eval/src/tests/eval/multiply_add/CMakeLists.txt index c50aa4f50a2..687c9b3b576 100644 --- a/eval/src/tests/eval/multiply_add/CMakeLists.txt +++ b/eval/src/tests/eval/multiply_add/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(eval_multiply_add_test_app TEST SOURCES multiply_add_test.cpp DEPENDS vespaeval - gtest + GTest::GTest ) vespa_add_test(NAME eval_multiply_add_test_app COMMAND eval_multiply_add_test_app) diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp index 055c7e582bf..54bcd0694e2 100644 --- a/eval/src/tests/eval/value_type/value_type_test.cpp +++ b/eval/src/tests/eval/value_type/value_type_test.cpp @@ -266,6 +266,20 @@ TEST("require that dimension names can be obtained") { EXPECT_EQUAL(type("tensor<float>(y[10],x[30],z{})").dimension_names(), str_list({"x", "y", "z"})); } +TEST("require that nontrivial dimensions can be obtained") { + auto my_check = [](const auto &list) + { + ASSERT_EQUAL(list.size(), 2u); + EXPECT_EQUAL(list[0].name, "x"); + EXPECT_EQUAL(list[0].size, 10u); + EXPECT_EQUAL(list[1].name, "y"); + EXPECT_TRUE(list[1].is_mapped()); + }; + EXPECT_TRUE(type("double").nontrivial_dimensions().empty()); + TEST_DO(my_check(type("tensor(x[10],y{})").nontrivial_dimensions())); + TEST_DO(my_check(type("tensor(a[1],b[1],x[10],y{},z[1])").nontrivial_dimensions())); +} + TEST("require that dimension index can be obtained") { EXPECT_EQUAL(type("error").dimension_index("x"), ValueType::Dimension::npos); EXPECT_EQUAL(type("double").dimension_index("x"), ValueType::Dimension::npos); diff --git a/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp b/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp index f9c563c9bf8..31dae9e7cf8 100644 --- a/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp +++ b/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp @@ -26,6 +26,7 @@ const TensorEngine &prod_engine = DefaultTensorEngine::ref(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() .add_dense({{"A", 2}, {"B", 1}, {"C", 3}, {"a", 2}, {"d", 3}}) // inner/inner + .add_dense({{"A", 2}, {"B", 1}, {"C", 3}, {"D", 1}, {"a", 2}, {"c", 1}, {"d", 3}, {"e", 1}}) // inner/inner, extra dims .add_dense({{"B", 1}, {"C", 3}, {"a", 2}, {"d", 3}}) // inner/inner, missing A .add_dense({{"A", 1}, {"a", 2}, {"d", 3}}) // inner/inner, single mat .add_dense({{"A", 2}, {"D", 3}, {"a", 2}, {"b", 1}, {"c", 3}}) // inner/inner, inverted @@ -109,6 +110,11 @@ TEST("require that multi matmul must have inner nesting of matmul dimensions") { TEST_DO(verify_not_optimized("reduce(B5D3a2b1c3*A2D3a2b1c3,sum,D)")); } +TEST("require that multi matmul ignores trivial dimensions") { + TEST_DO(verify_optimized("reduce(A2B1C3D1a2c1d3e1*A2B1C3b5d3,sum,d)", 2, 3, 5, 6, true, true)); + TEST_DO(verify_optimized("reduce(A2B1C3b5d3*A2B1C3D1a2c1d3e1,sum,d)", 2, 3, 5, 6, true, true)); +} + TEST("require that multi matmul function can be debug dumped") { EvalFixture fixture(prod_engine, "reduce(A2B1C3a2d3*A2B1C3b5d3,sum,d)", param_repo, true); auto info = fixture.find_all<DenseMultiMatMulFunction>(); diff --git a/eval/src/tests/tensor/dense_pow_as_map_optimizer/CMakeLists.txt b/eval/src/tests/tensor/dense_pow_as_map_optimizer/CMakeLists.txt index cf5103f8b4f..60cb72a8ed7 100644 --- a/eval/src/tests/tensor/dense_pow_as_map_optimizer/CMakeLists.txt +++ b/eval/src/tests/tensor/dense_pow_as_map_optimizer/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(eval_dense_pow_as_map_optimizer_test_app TEST SOURCES dense_pow_as_map_optimizer_test.cpp DEPENDS vespaeval - gtest + GTest::GTest ) vespa_add_test(NAME eval_dense_pow_as_map_optimizer_test_app COMMAND eval_dense_pow_as_map_optimizer_test_app) diff --git a/eval/src/tests/tensor/dense_simple_expand_function/CMakeLists.txt b/eval/src/tests/tensor/dense_simple_expand_function/CMakeLists.txt index 4e7409a6139..42be6f4c613 100644 --- a/eval/src/tests/tensor/dense_simple_expand_function/CMakeLists.txt +++ b/eval/src/tests/tensor/dense_simple_expand_function/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(eval_dense_simple_expand_function_test_app TEST SOURCES dense_simple_expand_function_test.cpp DEPENDS vespaeval - gtest + GTest::GTest ) vespa_add_test(NAME eval_dense_simple_expand_function_test_app COMMAND eval_dense_simple_expand_function_test_app) diff --git a/eval/src/tests/tensor/dense_simple_map_function/CMakeLists.txt b/eval/src/tests/tensor/dense_simple_map_function/CMakeLists.txt index 766986bd628..daa68665d2e 100644 --- a/eval/src/tests/tensor/dense_simple_map_function/CMakeLists.txt +++ b/eval/src/tests/tensor/dense_simple_map_function/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(eval_dense_simple_map_function_test_app TEST SOURCES dense_simple_map_function_test.cpp DEPENDS vespaeval - gtest + GTest::GTest ) vespa_add_test(NAME eval_dense_simple_map_function_test_app COMMAND eval_dense_simple_map_function_test_app) diff --git a/eval/src/tests/tensor/index_lookup_table/CMakeLists.txt b/eval/src/tests/tensor/index_lookup_table/CMakeLists.txt index f2dc5d31045..3669c663896 100644 --- a/eval/src/tests/tensor/index_lookup_table/CMakeLists.txt +++ b/eval/src/tests/tensor/index_lookup_table/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(eval_index_lookup_table_test_app TEST SOURCES index_lookup_table_test.cpp DEPENDS vespaeval - gtest + GTest::GTest ) vespa_add_test(NAME eval_index_lookup_table_test_app COMMAND eval_index_lookup_table_test_app) diff --git a/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt b/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt index e511b23c5ec..3dcd14a2189 100644 --- a/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt +++ b/eval/src/tests/tensor/tensor_add_operation/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(eval_tensor_add_operation_test_app TEST SOURCES tensor_add_operation_test.cpp DEPENDS vespaeval - gtest + GTest::GTest ) vespa_add_test(NAME eval_tensor_add_operation_test_app COMMAND eval_tensor_add_operation_test_app) diff --git a/eval/src/tests/tensor/tensor_modify_operation/CMakeLists.txt b/eval/src/tests/tensor/tensor_modify_operation/CMakeLists.txt index 481a0aac7fb..9673c6d0c80 100644 --- a/eval/src/tests/tensor/tensor_modify_operation/CMakeLists.txt +++ b/eval/src/tests/tensor/tensor_modify_operation/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(eval_tensor_modify_operation_test_app TEST SOURCES tensor_modify_operation_test.cpp DEPENDS vespaeval - gtest + GTest::GTest ) vespa_add_test(NAME eval_tensor_modify_operation_test_app COMMAND eval_tensor_modify_operation_test_app) diff --git a/eval/src/tests/tensor/tensor_remove_operation/CMakeLists.txt b/eval/src/tests/tensor/tensor_remove_operation/CMakeLists.txt index ffc71563107..0d0c9b17473 100644 --- a/eval/src/tests/tensor/tensor_remove_operation/CMakeLists.txt +++ b/eval/src/tests/tensor/tensor_remove_operation/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(eval_tensor_remove_operation_test_app TEST SOURCES tensor_remove_operation_test.cpp DEPENDS vespaeval - gtest + GTest::GTest ) vespa_add_test(NAME eval_tensor_remove_operation_test_app COMMAND eval_tensor_remove_operation_test_app) diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index 36c7c49c8b9..2287e8599eb 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -185,6 +185,17 @@ ValueType::dense_subspace_size() const return size; } +std::vector<ValueType::Dimension> +ValueType::nontrivial_dimensions() const { + std::vector<ValueType::Dimension> result; + for (const auto &dim: dimensions()) { + if (!dim.is_trivial()) { + result.push_back(dim); + } + } + return result; +} + size_t ValueType::dimension_index(const vespalib::string &name) const { return my_dimension_index(_dimensions, name); diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index a8ae9c44bb0..a3dbb901eb0 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -33,6 +33,7 @@ public: bool operator!=(const Dimension &rhs) const { return !(*this == rhs); } bool is_mapped() const { return (size == npos); } bool is_indexed() const { return (size != npos); } + bool is_trivial() const { return (size == 1); } }; private: @@ -61,6 +62,7 @@ public: bool is_dense() const; size_t dense_subspace_size() const; const std::vector<Dimension> &dimensions() const { return _dimensions; } + std::vector<Dimension> nontrivial_dimensions() const; size_t dimension_index(const vespalib::string &name) const; std::vector<vespalib::string> dimension_names() const; bool operator==(const ValueType &rhs) const { diff --git a/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp index 73942f7f044..23e54e7fc02 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_multi_matmul_function.cpp @@ -20,6 +20,9 @@ using eval::Aggr; using namespace eval::tensor_function; using namespace eval::operation; +using Dim = ValueType::Dimension; +using DimList = std::vector<Dim>; + namespace { void my_cblas_double_multi_matmul_op(InterpretedFunction::State &state, uint64_t param) { @@ -77,45 +80,37 @@ InterpretedFunction::op_function my_select(CellType cell_type) { struct CommonDim { bool valid; bool inner; - CommonDim(const ValueType &type, const vespalib::string &dim) + CommonDim(const DimList &list, const vespalib::string &dim) : valid(true), inner(false) { - size_t size = type.dimensions().size(); - if (type.dimensions()[size - 1].name == dim) { + if (list[list.size() - 1].name == dim) { inner = true; - } else if (type.dimensions()[size - 2].name != dim) { + } else if (list[list.size() - 2].name != dim) { valid = false; } } - const ValueType::Dimension &get(const ValueType &type) const { - size_t size = type.dimensions().size(); - return type.dimensions()[size - (inner ? 1 : 2)]; - } - const ValueType::Dimension &get(const TensorFunction &expr) const { - return get(expr.result_type()); - } - const ValueType::Dimension &inv(const ValueType &type) const { - size_t size = type.dimensions().size(); - return type.dimensions()[size - (inner ? 2 : 1)]; + const Dim &get(const DimList &dims) const { + return dims[dims.size() - (inner ? 1 : 2)]; } - const ValueType::Dimension &inv(const TensorFunction &expr) const { - return inv(expr.result_type()); + const Dim &inv(const DimList &dims) const { + return dims[dims.size() - (inner ? 2 : 1)]; } }; -// Currently, non-matmul dimensions are required to be identical. This -// restriction is added to reduce complexity and might be removed in -// the future if/when a relevant use-case arises. +// Currently, non-matmul dimensions are required to be identical +// (after trivial dimensions are ignored). This restriction is added +// to reduce complexity and might be removed in the future if/when a +// relevant use-case arises. struct DimPrefix { bool valid; size_t size; - DimPrefix(const ValueType &a, const ValueType &b) + DimPrefix(const DimList &a, const DimList &b) : valid(true), size(1) { - if (a.dimensions().size() == b.dimensions().size()) { - for (size_t i = 0; i < (a.dimensions().size() - 2); ++i) { - if (a.dimensions()[i] == b.dimensions()[i]) { - size *= a.dimensions()[i].size; + if (a.size() == b.size()) { + for (size_t i = 0; i < (a.size() - 2); ++i) { + if (a[i] == b[i]) { + size *= a[i].size; } else { valid = false; } @@ -126,20 +121,22 @@ struct DimPrefix { } }; -bool check_input_type(const ValueType &type) { +bool check_input_type(const ValueType &type, const DimList &relevant) { return (type.is_dense() && - (type.dimensions().size() >= 2) && + (relevant.size() >= 2) && ((type.cell_type() == CellType::FLOAT) || (type.cell_type() == CellType::DOUBLE))); } bool is_multi_matmul(const ValueType &a, const ValueType &b, const vespalib::string &reduce_dim) { - if (check_input_type(a) && check_input_type(b) && (a.cell_type() == b.cell_type())) { - CommonDim cd_a(a, reduce_dim); - CommonDim cd_b(b, reduce_dim); - DimPrefix prefix(a, b); + auto dims_a = a.nontrivial_dimensions(); + auto dims_b = b.nontrivial_dimensions(); + if (check_input_type(a, dims_a) && check_input_type(b, dims_b) && (a.cell_type() == b.cell_type())) { + CommonDim cd_a(dims_a, reduce_dim); + CommonDim cd_b(dims_b, reduce_dim); + DimPrefix prefix(dims_a, dims_b); return (cd_a.valid && cd_b.valid && prefix.valid && - (b.dimension_index(cd_a.inv(a).name) == ValueType::Dimension::npos) && - (a.dimension_index(cd_b.inv(b).name) == ValueType::Dimension::npos)); + (b.dimension_index(cd_a.inv(dims_a).name) == Dim::npos) && + (a.dimension_index(cd_b.inv(dims_b).name) == Dim::npos)); } return false; } @@ -147,13 +144,15 @@ bool is_multi_matmul(const ValueType &a, const ValueType &b, const vespalib::str const TensorFunction &create_multi_matmul(const TensorFunction &a, const TensorFunction &b, const vespalib::string &reduce_dim, const ValueType &result_type, Stash &stash) { - CommonDim cd_a(a.result_type(), reduce_dim); - CommonDim cd_b(b.result_type(), reduce_dim); - DimPrefix prefix(a.result_type(), b.result_type()); - size_t a_size = cd_a.inv(a).size; - size_t b_size = cd_b.inv(b).size; - size_t common_size = cd_a.get(a).size; - bool a_is_lhs = (cd_a.inv(a).name < cd_b.inv(b).name); + auto dims_a = a.result_type().nontrivial_dimensions(); + auto dims_b = b.result_type().nontrivial_dimensions(); + CommonDim cd_a(dims_a, reduce_dim); + CommonDim cd_b(dims_b, reduce_dim); + DimPrefix prefix(dims_a, dims_b); + size_t a_size = cd_a.inv(dims_a).size; + size_t b_size = cd_b.inv(dims_b).size; + size_t common_size = cd_a.get(dims_a).size; + bool a_is_lhs = (cd_a.inv(dims_a).name < cd_b.inv(dims_b).name); if (a_is_lhs) { return stash.create<DenseMultiMatMulFunction>(result_type, a, b, a_size, common_size, b_size, prefix.size, cd_a.inner, cd_b.inner); } else { diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp index d45e0d936a9..e4e3ffc27d6 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp @@ -71,16 +71,9 @@ using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool>; //----------------------------------------------------------------------------- -std::vector<ValueType::Dimension> strip_trivial(const std::vector<ValueType::Dimension> &dim_list) { - std::vector<ValueType::Dimension> result; - std::copy_if(dim_list.begin(), dim_list.end(), std::back_inserter(result), - [](const auto &dim){ return (dim.size != 1); }); - return result; -} - std::optional<Inner> detect_simple_expand(const TensorFunction &lhs, const TensorFunction &rhs) { - std::vector<ValueType::Dimension> a = strip_trivial(lhs.result_type().dimensions()); - std::vector<ValueType::Dimension> b = strip_trivial(rhs.result_type().dimensions()); + std::vector<ValueType::Dimension> a = lhs.result_type().nontrivial_dimensions(); + std::vector<ValueType::Dimension> b = rhs.result_type().nontrivial_dimensions(); if (a.empty() || b.empty()) { return std::nullopt; } else if (a.back().name < b.front().name) { diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp index 5f8fbcac9bb..c407ef6cdff 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp @@ -129,16 +129,9 @@ Primary select_primary(const TensorFunction &lhs, const TensorFunction &rhs, Val } } -std::vector<ValueType::Dimension> strip_trivial(const std::vector<ValueType::Dimension> &dim_list) { - std::vector<ValueType::Dimension> result; - std::copy_if(dim_list.begin(), dim_list.end(), std::back_inserter(result), - [](const auto &dim){ return (dim.size != 1); }); - return result; -} - std::optional<Overlap> detect_overlap(const TensorFunction &primary, const TensorFunction &secondary) { - std::vector<ValueType::Dimension> a = strip_trivial(primary.result_type().dimensions()); - std::vector<ValueType::Dimension> b = strip_trivial(secondary.result_type().dimensions()); + std::vector<ValueType::Dimension> a = primary.result_type().nontrivial_dimensions(); + std::vector<ValueType::Dimension> b = secondary.result_type().nontrivial_dimensions(); if (b.size() > a.size()) { return std::nullopt; } else if (b == a) { diff --git a/eval/src/vespa/eval/tensor/dense/typed_cells.h b/eval/src/vespa/eval/tensor/dense/typed_cells.h index 0f22c85735e..d1d2baa535e 100644 --- a/eval/src/vespa/eval/tensor/dense/typed_cells.h +++ b/eval/src/vespa/eval/tensor/dense/typed_cells.h @@ -48,15 +48,6 @@ struct TypedCells { }; template <typename TGT, typename... Args> -decltype(auto) dispatch_0(CellType ct, Args &&...args) { - switch (ct) { - case CellType::DOUBLE: return TGT::template call<double>(std::forward<Args>(args)...); - case CellType::FLOAT: return TGT::template call<float>(std::forward<Args>(args)...); - } - abort(); -} - -template <typename TGT, typename... Args> decltype(auto) dispatch_1(const TypedCells &a, Args &&...args) { switch (a.type) { case CellType::DOUBLE: return TGT::call(a.unsafe_typify<double>(), std::forward<Args>(args)...); @@ -74,22 +65,4 @@ decltype(auto) dispatch_2(A1 &&a, const TypedCells &b, Args &&...args) { abort(); } -template <typename T, typename... Args> -decltype(auto) select_1(CellType a_type) { - switch(a_type) { - case CellType::DOUBLE: return T::template get_fun<double, Args...>(); - case CellType::FLOAT: return T::template get_fun<float, Args...>(); - } - abort(); -} - -template <typename T> -decltype(auto) select_2(CellType a_type, CellType b_type) { - switch(b_type) { - case CellType::DOUBLE: return select_1<T, double>(a_type); - case CellType::FLOAT: return select_1<T, float>(a_type); - } - abort(); -} - } // namespace diff --git a/fbench/src/test/authority/CMakeLists.txt b/fbench/src/test/authority/CMakeLists.txt index 00f804f43f6..09c1152383e 100644 --- a/fbench/src/test/authority/CMakeLists.txt +++ b/fbench/src/test/authority/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(fbench_authority_test_app TEST SOURCES authority_test.cpp DEPENDS fbench_util vespalib - gtest + GTest::GTest ) vespa_add_test(NAME fbench_authority_test_app COMMAND fbench_authority_test_app) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index b96560276d2..64d020f4d46 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -275,13 +275,6 @@ public class Flags { CONSOLE_USER_EMAIL ); - public static final UnboundBooleanFlag CONFIGSERVER_PROVISION_LB = defineFeatureFlag( - "configserver-provision-lb", true, - "Provision load balancer for config server cluster", - "Takes effect when zone-config-servers application is redeployed", - ZONE_ID - ); - public static final UnboundBooleanFlag CONTROLLER_PROVISION_LB = defineFeatureFlag( "controller-provision-lb", false, "Provision load balancer for controller cluster", @@ -289,6 +282,19 @@ public class Flags { ZONE_ID ); + public static final UnboundIntFlag TENANT_NODE_QUOTA = defineIntFlag( + "tenant-node-quota", 5, + "The number of nodes a tenant is allowed to request", + "Takes effect on next deployment", + APPLICATION_ID + ); + + public static final UnboundBooleanFlag ONLY_PUBLIC_ACCESS = defineFeatureFlag( + "enable-public-only", false, + "Only access public hosts from container", + "Takes effect on next tick" + ); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/functions.cmake b/functions.cmake index 43dd2d30853..162cb6852b0 100644 --- a/functions.cmake +++ b/functions.cmake @@ -101,6 +101,18 @@ function(vespa_add_target_system_dependency TARGET PACKAGE_NAME) endif() endfunction() +function(vespa_add_source_target TARGET) + cmake_parse_arguments( + ARG + "" + "" + "DEPENDS" + ${ARGN} + ) + add_custom_target(${TARGET} DEPENDS ${ARG_DEPENDS}) + __add_source_target_to_module(${TARGET}) +endfunction() + function(vespa_generate_config TARGET RELATIVE_CONFIG_DEF_PATH) # Generate config-<name>.cpp and config-<name>.h from <name>.def # Destination directory is always where generate_config is called (or, in the case of out-of-source builds, in the build-tree parallel) @@ -134,6 +146,7 @@ function(vespa_generate_config TARGET RELATIVE_CONFIG_DEF_PATH) OUTPUT ${CONFIG_H_PATH} ${CONFIG_CPP_PATH} COMMAND java -Dconfig.spec=${CONFIG_DEF_PATH} -Dconfig.dest=${CONFIG_DEST_PARENT_DIR} -Dconfig.lang=cpp -Dconfig.subdir=${CONFIG_DEST_DIRNAME} -Dconfig.dumpTree=false -Xms64m -Xmx64m -jar ${PROJECT_SOURCE_DIR}/configgen/target/configgen.jar WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. + COMMENT "Generating cpp config for ${CONFIG_NAME} in ${CMAKE_CURRENT_SOURCE_DIR}" MAIN_DEPENDENCY ${CONFIG_DEF_PATH} ) @@ -148,6 +161,7 @@ function(vespa_generate_config TARGET RELATIVE_CONFIG_DEF_PATH) # Needed to be able to do a #include <config-<name>.h> for this target # This is used within some unit tests target_include_directories(${TARGET} PRIVATE ${CONFIG_DEST_DIR}) + vespa_add_source_target("configgen_${TARGET}_${CONFIG_NAME}" DEPENDS ${CONFIG_H_PATH} ${CONFIG_CPP_PATH}) endfunction() function(vespa_add_library TARGET) @@ -497,6 +511,10 @@ function(__add_test_target_to_module TARGET) set_property(GLOBAL APPEND PROPERTY MODULE_${MODULE_NAME}_TEST_TARGETS ${TARGET}) endfunction() +function(__add_source_target_to_module TARGET) + set_property(GLOBAL APPEND PROPERTY MODULE_${MODULE_NAME}_SOURCE_TARGETS ${TARGET}) +endfunction() + macro(__handle_test_targets) # If this is a test executable, add it to the test target for this module # If building of unit tests is not specified, exclude this target from the all target diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java index 14b5af53b5a..31633cdc88b 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java @@ -27,24 +27,30 @@ public final class ExactExpression extends Expression { @Override protected void doExecute(ExecutionContext ctx) { StringFieldValue input = (StringFieldValue)ctx.getValue(); - if (input.getString().isEmpty()) { - return; - } + if (input.getString().isEmpty()) return; + StringFieldValue output = input.clone(); ctx.setValue(output); String prev = output.getString(); String next = toLowerCase(prev); - SpanList root = new SpanList(); - SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS, root); + SpanTree tree = output.getSpanTree(SpanTrees.LINGUISTICS); + SpanList root; + if (tree == null) { + root = new SpanList(); + tree = new SpanTree(SpanTrees.LINGUISTICS, root); + output.setSpanTree(tree); + } + else { + root = (SpanList)tree.getRoot(); + } SpanNode node = new Span(0, prev.length()); tree.annotate(node, new Annotation(AnnotationTypes.TERM, next.equals(prev) ? null : new StringFieldValue(next))); tree.annotate(node, new Annotation(AnnotationTypes.TOKEN_TYPE, new IntegerFieldValue(TokenType.ALPHABETIC.getValue()))); root.add(node); - output.setSpanTree(tree); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java index e43744420f8..91f46381def 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java @@ -24,6 +24,7 @@ public final class FlattenExpression extends Expression { public FlattenExpression() { super(DataType.STRING); } + @Override protected void doExecute(ExecutionContext ctx) { StringFieldValue input = (StringFieldValue)ctx.getValue(); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java index d91338e3d3f..adf3e4ecaaa 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java @@ -65,10 +65,10 @@ public final class NGramExpression extends Expression { // annotate gram as a word term String gramString = gram.extractFrom(input.getString()); - typedSpan(gram.getStart(), gram.getLength(), TokenType.ALPHABETIC, spanList). + typedSpan(gram.getStart(), gram.getCodePointCount(), TokenType.ALPHABETIC, spanList). annotate(LinguisticsAnnotator.lowerCaseTermAnnotation(gramString, gramString)); - lastPosition = gram.getStart() + gram.getLength(); + lastPosition = gram.getStart() + gram.getCodePointCount(); } // handle punctuation at the end if (lastPosition < input.toString().length()) { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java index 25194a5502a..e0292bbe026 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java @@ -123,13 +123,11 @@ public class Request extends AbstractResource { /** * Returns the Uniform Resource Identifier used by the {@link Container} to resolve the appropriate {@link - * RequestHandler} for this Request + * RequestHandler} for this Request. * * @see #setUri(URI) */ - public URI getUri() { - return uri; - } + public URI getUri() { return uri; } /** * Sets the Uniform Resource Identifier used by the {@link Container} to resolve the appropriate {@link diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseHandler.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseHandler.java index 9451191a31c..57d9402de5f 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseHandler.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseHandler.java @@ -20,11 +20,11 @@ import com.yahoo.jdisc.service.ClientProvider; public interface ResponseHandler { /** - * <p>This method will process the given {@link Response} and return a {@link ContentChannel} into which the caller - * can write the Response's content.</p> + * This method will process the given {@link Response} and return a {@link ContentChannel} into which the caller + * can write the Response's content. * - * @param response The Response to handle. - * @return The ContentChannel to write the Response content to. Notice that the ContentChannel holds a Container + * @param response the Response to handle + * @return the ContentChannel to write the Response content to. Notice that the ContentChannel holds a Container * reference, so failure to close this will prevent the Container from ever shutting down. */ ContentChannel handleResponse(Response response); diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index 68b68dca068..aca8b673cca 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -703,6 +703,7 @@ "public com.yahoo.jdisc.http.ServerConfig$Builder maxWorkerThreads(int)", "public com.yahoo.jdisc.http.ServerConfig$Builder stopTimeout(double)", "public com.yahoo.jdisc.http.ServerConfig$Builder jmx(com.yahoo.jdisc.http.ServerConfig$Jmx$Builder)", + "public com.yahoo.jdisc.http.ServerConfig$Builder metric(com.yahoo.jdisc.http.ServerConfig$Metric$Builder)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", "public final java.lang.String getDefMd5()", "public final java.lang.String getDefName()", @@ -711,7 +712,8 @@ ], "fields": [ "public java.util.List filter", - "public com.yahoo.jdisc.http.ServerConfig$Jmx$Builder jmx" + "public com.yahoo.jdisc.http.ServerConfig$Jmx$Builder jmx", + "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder metric" ] }, "com.yahoo.jdisc.http.ServerConfig$Filter$Builder": { @@ -776,6 +778,39 @@ ], "fields": [] }, + "com.yahoo.jdisc.http.ServerConfig$Metric$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.jdisc.http.ServerConfig$Metric)", + "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder monitoringHandlerPaths(java.lang.String)", + "public com.yahoo.jdisc.http.ServerConfig$Metric$Builder monitoringHandlerPaths(java.util.Collection)", + "public com.yahoo.jdisc.http.ServerConfig$Metric build()" + ], + "fields": [ + "public java.util.List monitoringHandlerPaths" + ] + }, + "com.yahoo.jdisc.http.ServerConfig$Metric": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.jdisc.http.ServerConfig$Metric$Builder)", + "public java.util.List monitoringHandlerPaths()", + "public java.lang.String monitoringHandlerPaths(int)" + ], + "fields": [] + }, "com.yahoo.jdisc.http.ServerConfig$Producer": { "superClass": "java.lang.Object", "interfaces": [ @@ -813,7 +848,8 @@ "public com.yahoo.jdisc.http.ServerConfig$Filter filter(int)", "public int maxWorkerThreads()", "public double stopTimeout()", - "public com.yahoo.jdisc.http.ServerConfig$Jmx jmx()" + "public com.yahoo.jdisc.http.ServerConfig$Jmx jmx()", + "public com.yahoo.jdisc.http.ServerConfig$Metric metric()" ], "fields": [ "public static final java.lang.String CONFIG_DEF_MD5", diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java index 4239d2120cf..731e737d683 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java @@ -34,6 +34,7 @@ import java.util.concurrent.atomic.LongAdder; */ public class HttpResponseStatisticsCollector extends HandlerWrapper implements Graceful { private final AtomicReference<FutureCallback> shutdown = new AtomicReference<>(); + private final List<String> monitoringHandlerPaths; public static enum HttpMethod { GET, PATCH, POST, PUT, DELETE, OPTIONS, HEAD, OTHER @@ -43,20 +44,28 @@ public class HttpResponseStatisticsCollector extends HandlerWrapper implements G HTTP, HTTPS, OTHER } + public enum RequestType { + READ, WRITE, MONITORING + } + private static final String[] HTTP_RESPONSE_GROUPS = { Metrics.RESPONSES_1XX, Metrics.RESPONSES_2XX, Metrics.RESPONSES_3XX, Metrics.RESPONSES_4XX, Metrics.RESPONSES_5XX, Metrics.RESPONSES_401, Metrics.RESPONSES_403}; private final AtomicLong inFlight = new AtomicLong(); - private final LongAdder statistics[][][]; + private final LongAdder statistics[][][][]; - public HttpResponseStatisticsCollector() { + public HttpResponseStatisticsCollector(List<String> monitoringHandlerPaths) { super(); - statistics = new LongAdder[HttpScheme.values().length][HttpMethod.values().length][]; + this.monitoringHandlerPaths = monitoringHandlerPaths; + statistics = new LongAdder[HttpScheme.values().length][HttpMethod.values().length][][]; for (int scheme = 0; scheme < HttpScheme.values().length; ++scheme) { for (int method = 0; method < HttpMethod.values().length; method++) { - statistics[scheme][method] = new LongAdder[HTTP_RESPONSE_GROUPS.length]; + statistics[scheme][method] = new LongAdder[HTTP_RESPONSE_GROUPS.length][]; for (int group = 0; group < HTTP_RESPONSE_GROUPS.length; group++) { - statistics[scheme][method][group] = new LongAdder(); + statistics[scheme][method][group] = new LongAdder[RequestType.values().length]; + for (int requestType = 0 ; requestType < RequestType.values().length; requestType++) { + statistics[scheme][method][group][requestType] = new LongAdder(); + } } } } @@ -117,9 +126,11 @@ public class HttpResponseStatisticsCollector extends HandlerWrapper implements G if (group >= 0) { HttpScheme scheme = getScheme(request); HttpMethod method = getMethod(request); - statistics[scheme.ordinal()][method.ordinal()][group].increment(); + RequestType requestType = getRequestType(request); + + statistics[scheme.ordinal()][method.ordinal()][group][requestType.ordinal()].increment(); if (group == 5 || group == 6) { // if 401/403, also increment 4xx - statistics[scheme.ordinal()][method.ordinal()][3].increment(); + statistics[scheme.ordinal()][method.ordinal()][3][requestType.ordinal()].increment(); } } @@ -184,6 +195,18 @@ public class HttpResponseStatisticsCollector extends HandlerWrapper implements G } } + private RequestType getRequestType(Request request) { + String path = request.getRequestURI(); + for (String monitoringHandlerPath : monitoringHandlerPaths) { + if (path.startsWith(monitoringHandlerPath)) return RequestType.MONITORING; + } + if ("GET".equals(request.getMethod())) { + return RequestType.READ; + } else { + return RequestType.WRITE; + } + } + public List<StatisticsEntry> takeStatistics() { var ret = new ArrayList<StatisticsEntry>(); for (HttpScheme scheme : HttpScheme.values()) { @@ -191,9 +214,11 @@ public class HttpResponseStatisticsCollector extends HandlerWrapper implements G for (HttpMethod method : HttpMethod.values()) { int methodIndex = method.ordinal(); for (int group = 0; group < HTTP_RESPONSE_GROUPS.length; group++) { - long value = statistics[schemeIndex][methodIndex][group].sumThenReset(); - if (value > 0) { - ret.add(new StatisticsEntry(scheme.name().toLowerCase(), method.name(), HTTP_RESPONSE_GROUPS[group], value)); + for (RequestType type : RequestType.values()) { + long value = statistics[schemeIndex][methodIndex][group][type.ordinal()].sumThenReset(); + if (value > 0) { + ret.add(new StatisticsEntry(scheme.name().toLowerCase(), method.name(), HTTP_RESPONSE_GROUPS[group], type.name().toLowerCase(), value)); + } } } } @@ -239,13 +264,15 @@ public class HttpResponseStatisticsCollector extends HandlerWrapper implements G public final String scheme; public final String method; public final String name; + public final String requestType; public final long value; - public StatisticsEntry(String scheme, String method, String name, long value) { + public StatisticsEntry(String scheme, String method, String name, String requestType, long value) { this.scheme = scheme; this.method = method; this.name = name; + this.requestType = requestType; this.value = value; } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index 04db58f6d07..a33278c7e02 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -70,6 +70,7 @@ public class JettyHttpServer extends AbstractServerProvider { String PORT_DIMENSION = "serverPort"; String METHOD_DIMENSION = "httpMethod"; String SCHEME_DIMENSION = "scheme"; + String REQUEST_TYPE_DIMENSION = "requestType"; String NUM_OPEN_CONNECTIONS = "serverNumOpenConnections"; String NUM_CONNECTIONS_OPEN_MAX = "serverConnectionsOpenMax"; @@ -255,7 +256,7 @@ public class JettyHttpServer extends AbstractServerProvider { GzipHandler gzipHandler = newGzipHandler(serverConfig); gzipHandler.setHandler(authEnforcer); - HttpResponseStatisticsCollector statisticsCollector = new HttpResponseStatisticsCollector(); + HttpResponseStatisticsCollector statisticsCollector = new HttpResponseStatisticsCollector(serverConfig.metric().monitoringHandlerPaths()); statisticsCollector.setHandler(gzipHandler); StatisticsHandler statisticsHandler = newStatisticsHandler(); @@ -380,6 +381,7 @@ public class JettyHttpServer extends AbstractServerProvider { Map<String, Object> dimensions = new HashMap<>(); dimensions.put(Metrics.METHOD_DIMENSION, metricEntry.method); dimensions.put(Metrics.SCHEME_DIMENSION, metricEntry.scheme); + dimensions.put(Metrics.REQUEST_TYPE_DIMENSION, metricEntry.requestType); metric.add(metricEntry.name, metricEntry.value, metric.createContext(dimensions)); } } diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def index 33f82963243..e968d17df85 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def @@ -38,3 +38,6 @@ jmx.enabled bool default = false # Listen port for the JMX server. jmx.listenPort int default = 1099 + +# Paths that should be reported with monitoring dimensions where applicable +metric.monitoringHandlerPaths[] string
\ No newline at end of file diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java index 7176a0a1ff6..e06905a0212 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java @@ -35,7 +35,8 @@ import static org.hamcrest.Matchers.equalTo; */ public class HttpResponseStatisticsCollectorTest { private Connector connector; - private HttpResponseStatisticsCollector collector = new HttpResponseStatisticsCollector(); + private List<String> monitoringPaths = List.of("/status.html"); + private HttpResponseStatisticsCollector collector = new HttpResponseStatisticsCollector(monitoringPaths); private int httpResponseCode = 500; @Test @@ -98,6 +99,24 @@ public class HttpResponseStatisticsCollectorTest { assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_2XX, 1L); } + @Test + public void statistics_include_request_type_dimension() throws Exception { + testRequest("http", 200, "GET", "/search"); + testRequest("http", 200, "POST", "/feed"); + testRequest("http", 200, "GET", "/status.html?foo=bar"); + + var stats = collector.takeStatistics(); + assertStatisticsEntryWithRequestTypePresent(stats, "http", "GET", Metrics.RESPONSES_2XX, "monitoring", 1L); + assertStatisticsEntryWithRequestTypePresent(stats, "http", "GET", Metrics.RESPONSES_2XX, "read", 1L); + assertStatisticsEntryWithRequestTypePresent(stats, "http", "POST", Metrics.RESPONSES_2XX, "write", 1L); + + testRequest("http", 200, "GET"); + + stats = collector.takeStatistics(); + assertStatisticsEntryPresent(stats, "http", "GET", Metrics.RESPONSES_2XX, 1L); + + } + @Before public void initializeCollector() throws Exception { Server server = new Server(); @@ -124,8 +143,11 @@ public class HttpResponseStatisticsCollectorTest { } private Request testRequest(String scheme, int responseCode, String httpMethod) throws Exception { + return testRequest(scheme, responseCode, httpMethod, "foo/bar"); + } + private Request testRequest(String scheme, int responseCode, String httpMethod, String path) throws Exception { HttpChannel channel = new HttpChannel(connector, new HttpConfiguration(), null, new DummyTransport()); - MetaData.Request metaData = new MetaData.Request(httpMethod, new HttpURI(scheme + "://foo/bar"), HttpVersion.HTTP_1_1, new HttpFields()); + MetaData.Request metaData = new MetaData.Request(httpMethod, new HttpURI(scheme + "://" + path), HttpVersion.HTTP_1_1, new HttpFields()); Request req = channel.getRequest(); req.setMetaData(metaData); @@ -143,6 +165,15 @@ public class HttpResponseStatisticsCollectorTest { assertThat(value, equalTo(expectedValue)); } + private static void assertStatisticsEntryWithRequestTypePresent(List<StatisticsEntry> result, String scheme, String method, String name, String requestType, long expectedValue) { + long value = result.stream() + .filter(entry -> entry.method.equals(method) && entry.scheme.equals(scheme) && entry.name.equals(name) && entry.requestType.equals(requestType)) + .mapToLong(entry -> entry.value) + .reduce(Long::sum) + .orElseThrow(() -> new AssertionError(String.format("Not matching entry in result (scheme=%s, method=%s, name=%s, type=%s)", scheme, method, name, requestType))); + assertThat(value, equalTo(expectedValue)); + } + private final class DummyTransport implements HttpTransport { @Override public void send(Response info, boolean head, ByteBuffer content, boolean lastContent, Callback callback) { diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index d56e56e23b5..58b838d7332 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -337,8 +337,9 @@ "methods": [ "public void <init>(int, int)", "public int getStart()", - "public int getLength()", + "public int getCodePointCount()", "public java.lang.String extractFrom(java.lang.String)", + "public java.lang.String extractFrom(com.yahoo.language.process.GramSplitter$UnicodeString)", "public boolean equals(java.lang.Object)", "public int hashCode()" ], diff --git a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java index aa7ae59edf9..8a255dd5370 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java +++ b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java @@ -49,12 +49,12 @@ public class GramSplitter { private final CharacterClasses characterClasses; /** Text to split */ - private final String input; + private final UnicodeString input; - /** Gram size */ + /** Gram size in code points */ private final int n; - /** Current index */ + /** Current position in the string */ private int i = 0; /** Whether the last thing that happened was being on a separator (including the start of the string) */ @@ -64,7 +64,7 @@ public class GramSplitter { private Gram nextGram = null; public GramSplitterIterator(String input, int n, CharacterClasses characterClasses) { - this.input = input; + this.input = new UnicodeString(input); this.n = n; this.characterClasses = characterClasses; } @@ -90,38 +90,40 @@ public class GramSplitter { private Gram findNext() { // Skip to next word character while (i < input.length() && !characterClasses.isLetterOrDigit(input.codePointAt(i))) { - i++; + i = input.next(i); isFirstAfterSeparator = true; } if (i >= input.length()) return null; - String gram = input.substring(i, Math.min(i + n, input.length())); - int nonWordChar = indexOfNonWordChar(gram); + UnicodeString gram = input.substring(i, n); + int nonWordChar = indexOfNonWordCodepoint(gram); if (nonWordChar == 0) throw new RuntimeException("Programming error"); if (nonWordChar > 0) - gram = gram.substring(0, nonWordChar); + gram = new UnicodeString(gram.toString().substring(0, nonWordChar)); - if (gram.length() == n) { // normal case: got a full length gram - i++; + if (gram.codePointCount() == n) { // normal case: got a full length gram + Gram g = new Gram(i, gram.codePointCount()); + i = input.next(i); isFirstAfterSeparator = false; - return new Gram(i - 1, gram.length()); + return g; } else { // gram is too short due either to a non-word separator or end of string if (isFirstAfterSeparator) { // make a gram anyway - i++; + Gram g = new Gram(i, gram.codePointCount()); + i = input.next(i); isFirstAfterSeparator = false; - return new Gram(i - 1, gram.length()); + return g; } else { // skip to next - i += gram.length() + 1; + i = input.skip(gram.codePointCount() + 1, i); isFirstAfterSeparator = true; return findNext(); } } } - private int indexOfNonWordChar(String s) { - for (int i = 0; i < s.length(); i++) { + private int indexOfNonWordCodepoint(UnicodeString s) { + for (int i = 0; i < s.length(); i = s.next(i)) { if ( ! characterClasses.isLetterOrDigit(s.codePointAt(i))) return i; } @@ -151,24 +153,29 @@ public class GramSplitter { */ public static final class Gram { - private int start, length; + private int start, codePointCount; - public Gram(int start, int length) { + public Gram(int start, int codePointCount) { this.start = start; - this.length = length; + this.codePointCount = codePointCount; } public int getStart() { return start; } - public int getLength() { - return length; + public int getCodePointCount() { + return codePointCount; } /** Returns this gram as a string from the input string */ public String extractFrom(String input) { - return input.substring(start, start + length); + return extractFrom(new UnicodeString(input)); + } + + /** Returns this gram as a string from the input string */ + public String extractFrom(UnicodeString input) { + return input.substring(start, codePointCount).toString(); } @Override @@ -177,7 +184,7 @@ public class GramSplitter { if ( ! (o instanceof Gram)) return false; Gram gram = (Gram)o; - if (length != gram.length) return false; + if (codePointCount != gram.codePointCount) return false; if (start != gram.start) return false; return true; } @@ -185,10 +192,64 @@ public class GramSplitter { @Override public int hashCode() { int result = start; - result = 31 * result + length; + result = 31 * result + codePointCount; return result; } } + /** + * A string wrapper with some convenience methods for dealing with UTF-16 surrogate pairs + * (a crime against humanity for which we'll be negatively impacted for at least the next million years). + */ + private static class UnicodeString { + + private final String s; + + public UnicodeString(String s) { + this.s = s; + } + + /** Substring in code point space */ + public UnicodeString substring(int start, int codePoints) { + int offset = s.offsetByCodePoints(start, Math.min(codePoints, s.codePointCount(start, s.length()))); + if (offset < 0) + return new UnicodeString(s.substring(start)); + else + return new UnicodeString(s.substring(start, offset)); + } + + /** Returns the position count code points after start (which may be past the end of the string) */ + public int skip(int codePointCount, int start) { + int index = start; + for (int i = 0; i < codePointCount; i++) { + index = next(index); + if (index > s.length()) break; + } + return index; + } + + /** Returns the index of the next code point after start (which may be past the end of the string) */ + public int next(int index) { + int next = index + 1; + if (next < s.length() && Character.isLowSurrogate(s.charAt(next))) + next++; + return next; + } + + /** Returns the number of positions (not code points) in this */ + public int length() { return s.length(); } + + /** Returns the number of code points in this */ + public int codePointCount() { return s.codePointCount(0, s.length()); } + + public int codePointAt(int index) { + return s.codePointAt(index); + } + + @Override + public String toString() { return s; } + + } + } diff --git a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java index d862280550c..8fa23626193 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java @@ -4,9 +4,9 @@ package com.yahoo.language.process; import com.yahoo.language.simple.SimpleLinguistics; import org.junit.Test; +import java.util.Arrays; import java.util.Iterator; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; /** @@ -113,6 +113,30 @@ public class GramSplitterTestCase { "\u7345\u9069\u5e02]"); } + @Test + public void testSurrogatePairs() { + // A surrogate pair representing a code point in the "letter" class + String s = "\uD800\uDC00"; + + assertGramSplits(s, 1, s); + assertGramSplits(s, 2, s); + assertGramSplits(s + s, 1, s, s); + assertGramSplits(s + s, 2, s + s); + assertGramSplits(s + s, 3, s + s); + assertGramSplits(s + " " + s + s + " " + s, 1, s, s, s, s); + assertGramSplits(s + " " + s + s + " " + s, 2, s, s + s, s); + assertGramSplits(s + " " + s + s + " " + s, 3, s, s + s, s); + assertGramSplits(" " + s + " " + s + s + " " + s + " ", 1, s, s, s, s); + assertGramSplits(" " + s + " " + s + s + " " + s + " ", 2, s, s + s, s); + assertGramSplits(" " + s + " " + s + s + " " + s + " ", 3, s, s + s, s); + assertGramSplits(" " + s + " " + s + s + " " + s + " ", 1, s, s, s, s); + assertGramSplits(" " + s + " " + s + s + " " + s + " ", 2, s, s + s, s); + assertGramSplits(" " + s + " " + s + s + " " + s + " ", 3, s, s + s, s); + assertGramSplits(s + " " + s + " " + s, 4, s, s, s); + assertGramSplits(s + s + s + s, 3, s + s + s, s + s + s); + assertGramSplits(s + s + s + s + " " + s, 3, s + s + s, s + s + s, s); + } + @Test(expected = IllegalArgumentException.class) public void testInvalidSplitSize() { gramSplitter.split("en", 0); @@ -128,23 +152,27 @@ public class GramSplitterTestCase { String text = "en gul bille sang"; Iterator<GramSplitter.Gram> grams = gramSplitter.split(text, 3); - assertThat(grams.next().extractFrom(text), is("en")); + assertEquals("en", grams.next().extractFrom(text)); assertTrue(grams.hasNext()); assertTrue(grams.hasNext()); - assertThat(grams.next().extractFrom(text), is("gul")); - assertThat(grams.next().extractFrom(text), is("bil")); - assertThat(grams.next().extractFrom(text), is("ill")); - assertThat(grams.next().extractFrom(text), is("lle")); + assertEquals("gul", grams.next().extractFrom(text)); + assertEquals("bil", grams.next().extractFrom(text)); + assertEquals("ill", grams.next().extractFrom(text)); + assertEquals("lle", grams.next().extractFrom(text)); assertTrue(grams.hasNext()); assertTrue(grams.hasNext()); - assertThat(grams.next().extractFrom(text), is("san")); - assertThat(grams.next().extractFrom(text), is("ang")); + assertEquals("san", grams.next().extractFrom(text)); + assertEquals("ang", grams.next().extractFrom(text)); assertFalse(grams.hasNext()); assertFalse(grams.hasNext()); } + private void assertGramSplits(String input, int gramSize, String ... expected) { + assertEquals(Arrays.asList(expected), gramSplitter.split(input, gramSize).toExtractedList()); + } + private void assertGramSplit(String input, int gramSize, String expected) { - assertThat(gramSplitter.split(input, gramSize).toExtractedList().toString(), is(expected)); + assertEquals(expected, gramSplitter.split(input, gramSize).toExtractedList().toString()); } } diff --git a/logd/src/logd/CMakeLists.txt b/logd/src/logd/CMakeLists.txt index 865d810fe0e..92f70ef9102 100644 --- a/logd/src/logd/CMakeLists.txt +++ b/logd/src/logd/CMakeLists.txt @@ -6,6 +6,8 @@ find_package(Protobuf REQUIRED) protobuf_generate_cpp(logd_PROTOBUF_SRCS logd_PROTOBUF_HDRS ../../../logserver/src/protobuf/log_protocol.proto) +vespa_add_source_target(protobufgen_logd DEPENDS ${logd_PROTOBUF_SRCS} ${logd_PROTOBUF_HDRS}) + # protoc-generated files emit compiler warnings that we normally treat as errors. if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") set_source_files_properties(${logd_PROTOBUF_SRCS} PROPERTIES COMPILE_FLAGS "-Wno-array-bounds -Wno-suggest-override") diff --git a/logd/src/tests/empty_forwarder/CMakeLists.txt b/logd/src/tests/empty_forwarder/CMakeLists.txt index 4c4ab8efa40..978a425a738 100644 --- a/logd/src/tests/empty_forwarder/CMakeLists.txt +++ b/logd/src/tests/empty_forwarder/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(logd_empty_forwarder_test_app TEST SOURCES empty_forwarder_test.cpp DEPENDS logd - gtest + GTest::GTest ) vespa_add_test(NAME logd_empty_forwarder_test_app COMMAND logd_empty_forwarder_test_app) diff --git a/logd/src/tests/proto_converter/CMakeLists.txt b/logd/src/tests/proto_converter/CMakeLists.txt index 5ca048ecd4e..ba4c7c2b26d 100644 --- a/logd/src/tests/proto_converter/CMakeLists.txt +++ b/logd/src/tests/proto_converter/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(logd_proto_converter_test_app TEST SOURCES proto_converter_test.cpp DEPENDS logd - gtest + GTest::GTest ) vespa_add_test(NAME logd_proto_converter_test_app COMMAND logd_proto_converter_test_app) diff --git a/logd/src/tests/rpc_forwarder/CMakeLists.txt b/logd/src/tests/rpc_forwarder/CMakeLists.txt index 66a30777b41..430647c9b82 100644 --- a/logd/src/tests/rpc_forwarder/CMakeLists.txt +++ b/logd/src/tests/rpc_forwarder/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(logd_rpc_forwarder_test_app TEST SOURCES rpc_forwarder_test.cpp DEPENDS logd - gtest + GTest::GTest ) vespa_add_test(NAME logd_rpc_forwarder_test_app COMMAND logd_rpc_forwarder_test_app) diff --git a/logd/src/tests/watcher/CMakeLists.txt b/logd/src/tests/watcher/CMakeLists.txt index 0bf0a574fa9..42fddae48b3 100644 --- a/logd/src/tests/watcher/CMakeLists.txt +++ b/logd/src/tests/watcher/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(logd_watcher_test_app TEST SOURCES watcher_test.cpp DEPENDS logd - gtest + GTest::GTest ) vespa_add_test(NAME logd_watcher_test_app COMMAND logd_watcher_test_app) diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java b/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java index 6d1ec7586b3..39fc9aa8314 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java @@ -59,8 +59,8 @@ public class Sequencer implements MessageHandler, ReplyHandler { * queued for later sending due to sequencing restrictions. This method also sets the sequence id as message * context. * - * @param msg The message to filter. - * @return True if the message was consumed. + * @param msg the message to filter + * @return true if the message was consumed */ private boolean filter(Message msg) { long seqId = msg.getSequenceId(); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java index c3231daab5f..6b1376452fa 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java @@ -3,7 +3,12 @@ package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.core.MetricsConsumers; +import ai.vespa.metricsproxy.http.TextResponse; import ai.vespa.metricsproxy.metric.model.ConsumerId; +import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; +import ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil; +import ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil; import com.google.inject.Inject; import com.yahoo.container.handler.metrics.ErrorResponse; import com.yahoo.container.handler.metrics.HttpHandlerBase; @@ -16,9 +21,13 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.Executor; import java.util.logging.Level; +import java.util.stream.Collectors; import static ai.vespa.metricsproxy.http.ValuesFetcher.getConsumerOrDefault; import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericApplicationModel; +import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericJsonModel; +import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toMetricsPackets; +import static ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil.toPrometheusModel; import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; import static com.yahoo.jdisc.Response.Status.OK; @@ -29,8 +38,11 @@ import static com.yahoo.jdisc.Response.Status.OK; */ public class ApplicationMetricsHandler extends HttpHandlerBase { - public static final String V1_PATH = "/applicationmetrics/v1"; - public static final String VALUES_PATH = V1_PATH + "/values"; + public static final String METRICS_V1_PATH = "/applicationmetrics/v1"; + public static final String METRICS_VALUES_PATH = METRICS_V1_PATH + "/values"; + + public static final String PROMETHEUS_V1_PATH = "/applicationprometheus/v1"; + public static final String PROMETHEUS_VALUES_PATH = PROMETHEUS_V1_PATH + "/values"; private final ApplicationMetricsRetriever metricsRetriever; private final MetricsConsumers metricsConsumers; @@ -46,8 +58,12 @@ public class ApplicationMetricsHandler extends HttpHandlerBase { @Override public Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) { - if (apiPath.matches(V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH))); - if (apiPath.matches(VALUES_PATH)) return Optional.of(applicationMetricsResponse(consumer)); + if (apiPath.matches(METRICS_V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(METRICS_VALUES_PATH))); + if (apiPath.matches(METRICS_VALUES_PATH)) return Optional.of(applicationMetricsResponse(consumer)); + + if (apiPath.matches(PROMETHEUS_V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(PROMETHEUS_VALUES_PATH))); + if (apiPath.matches(PROMETHEUS_VALUES_PATH)) return Optional.of(applicationPrometheusResponse(consumer)); + return Optional.empty(); } @@ -64,4 +80,17 @@ public class ApplicationMetricsHandler extends HttpHandlerBase { } } + private TextResponse applicationPrometheusResponse(String requestedConsumer) { + ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers); + var metricsByNode = metricsRetriever.getMetrics(consumer); + + + List<GenericJsonModel> genericNodes = toGenericApplicationModel(metricsByNode).nodes; + List<MetricsPacket> metricsForAllNodes = genericNodes.stream() + .flatMap(element -> toMetricsPackets(element).stream() + .map(MetricsPacket.Builder::build)) + .collect(Collectors.toList()); + return new TextResponse(200, toPrometheusModel(metricsForAllNodes).serialize()); + } + } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java index 155bbf094a1..46c4af4c1f4 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java @@ -25,8 +25,8 @@ import java.util.concurrent.Executors; import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID; -import static ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler.V1_PATH; -import static ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler.VALUES_PATH; +import static ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler.METRICS_V1_PATH; +import static ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler.METRICS_VALUES_PATH; import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; @@ -48,8 +48,8 @@ public class ApplicationMetricsHandlerTest { private static final String HOST = "localhost"; private static final String URI_BASE = "http://" + HOST; - private static final String APP_METRICS_V1_URI = URI_BASE + V1_PATH; - private static final String APP_METRICS_VALUES_URI = URI_BASE + VALUES_PATH; + private static final String APP_METRICS_V1_URI = URI_BASE + METRICS_V1_PATH; + private static final String APP_METRICS_VALUES_URI = URI_BASE + METRICS_VALUES_PATH; private static final String TEST_FILE = "generic-sample.json"; private static final String RESPONSE = getFileContents(TEST_FILE); diff --git a/metrics/src/tests/CMakeLists.txt b/metrics/src/tests/CMakeLists.txt index c7ca296e7b5..435be3fd1dd 100644 --- a/metrics/src/tests/CMakeLists.txt +++ b/metrics/src/tests/CMakeLists.txt @@ -1,5 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) + # Runner for unit tests written in gtest. vespa_add_executable(metrics_gtest_runner_app TEST SOURCES @@ -17,7 +19,7 @@ vespa_add_executable(metrics_gtest_runner_app TEST DEPENDS metrics vdstestlib - gtest + GTest::GTest ) vespa_add_test( diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java index 91ff5d9cdd8..d67f064916b 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java @@ -87,19 +87,29 @@ public class Gather extends IntermediateOperation { addSliceDimension(dataSliceDimensions, dataType.dimensions().get(i).name(), i); } - List<Slice.DimensionValue<Reference>> indicesSliceDimensions = new ArrayList<>(); - for (int i = 0; i < indicesType.rank(); ++i) { - addSliceDimension(indicesSliceDimensions, indicesType.dimensions().get(i).name(), axis + i); + if (indicesType.rank() == 0 && indices.isConstant()) { + double constantValue = indices.getConstantValue().get().asDouble(); + ExpressionNode indexExpression = new ConstantNode(new DoubleValue(constantValue)); + if (constantValue < 0) { + ExpressionNode axisSize = new ConstantNode(new DoubleValue(dataType.dimensions().get(axis).size().get())); + indexExpression = new EmbracedNode(new ArithmeticNode(indexExpression, ArithmeticOperator.PLUS, axisSize)); + } + addSliceDimension(dataSliceDimensions, dataType.dimensions().get(axis).name(), indexExpression); + } else { + List<Slice.DimensionValue<Reference>> indicesSliceDimensions = new ArrayList<>(); + for (int i = 0; i < indicesType.rank(); ++i) { + addSliceDimension(indicesSliceDimensions, indicesType.dimensions().get(i).name(), axis + i); + } + ExpressionNode sliceExpression = createSliceExpression(indicesSliceDimensions, indicesFunctionName); + ExpressionNode indexExpression = createIndexExpression(dataType, sliceExpression); + addSliceDimension(dataSliceDimensions, dataType.dimensions().get(axis).name(), indexExpression); } - ExpressionNode sliceExpression = createSliceExpression(indicesSliceDimensions, indicesFunctionName); - ExpressionNode indexExpression = createIndexExpression(dataType, sliceExpression); - addSliceDimension(dataSliceDimensions, dataType.dimensions().get(axis).name(), indexExpression); for (int i = axis + 1; i < dataType.rank(); ++i) { addSliceDimension(dataSliceDimensions, dataType.dimensions().get(i).name(), i + indicesType.rank() - 1); } - sliceExpression = createSliceExpression(dataSliceDimensions, dataFunctionName); + ExpressionNode sliceExpression = createSliceExpression(dataSliceDimensions, dataFunctionName); return Generate.bound(type.type(), wrapScalar(sliceExpression)); } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Range.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Range.java index 6df686cf910..18018a1a73a 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Range.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Range.java @@ -51,7 +51,7 @@ public class Range extends IntermediateOperation { delta = getConstantInput(2, "delta"); elements = (long) Math.ceil((limit - start) / delta); - OrderedTensorType type = new OrderedTensorType.Builder() + OrderedTensorType type = new OrderedTensorType.Builder(inputs.get(0).type().get().type().valueType()) .add(TensorType.Dimension.indexed(vespaName(), elements)) .build(); return type; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java index 121cb244715..52900e35fe2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.file; +import com.yahoo.lang.MutableInteger; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import java.io.IOException; @@ -110,17 +111,19 @@ public class FileFinder { * @return true iff anything was matched and deleted */ public boolean deleteRecursively(TaskContext context) { + final int maxNumberOfDeletedPathsToLog = 20; + MutableInteger numDeleted = new MutableInteger(0); List<Path> deletedPaths = new ArrayList<>(); try { forEach(attributes -> { if (attributes.unixPath().deleteRecursively()) { - deletedPaths.add(attributes.path()); + if (numDeleted.next() <= maxNumberOfDeletedPathsToLog) deletedPaths.add(attributes.path()); } }); } finally { - if (deletedPaths.size() > 20) { - context.log(logger, "Deleted " + deletedPaths.size() + " paths under " + basePath); + if (numDeleted.get() > maxNumberOfDeletedPathsToLog) { + context.log(logger, "Deleted " + numDeleted.get() + " paths under " + basePath); } else if (deletedPaths.size() > 0) { List<Path> paths = deletedPaths.stream() .map(basePath::relativize) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 44bfed90106..0a32970e056 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -110,7 +110,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { /** * Provision the nodes necessary to satisfy given capacity. * - * @return Excess hosts that can safely be deprovisioned, if any. + * @return excess hosts that can safely be deprovisioned, if any */ private List<Node> provision(List<NodeResources> capacity, NodeList nodes) { List<Node> existingHosts = availableHostsOf(nodes); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index e918c1a815a..1e978682fa9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.HostLivenessTracker; @@ -199,6 +200,9 @@ public class NodeFailer extends NodeRepositoryMaintainer { clearDownRecord(node, lock); } } + catch (UncheckedTimeoutException e) { + // Ignore - node may be locked on this round due to deployment + } }); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 5c990f0a3f3..39f784a6aac 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -19,12 +19,8 @@ public class CapacityPolicies { private final Zone zone; - /* Deployments must match 1-to-1 the advertised resources of a physical host */ - private final boolean isUsingAdvertisedResources; - public CapacityPolicies(NodeRepository nodeRepository) { this.zone = nodeRepository.zone(); - this.isUsingAdvertisedResources = zone.getCloud().dynamicProvisioning(); } public int decideSize(int requested, Capacity capacity, ClusterSpec cluster, ApplicationId application) { @@ -64,14 +60,14 @@ public class CapacityPolicies { // Use small logserver in dev system return new NodeResources(0.1, 1, 10, 0.3); } - return isUsingAdvertisedResources ? - new NodeResources(0.5, 4, 50, 0.3) : - new NodeResources(0.5, 2, 50, 0.3); + return zone.getCloud().allowHostSharing() ? + new NodeResources(0.5, 2, 50, 0.3) : + new NodeResources(0.5, 4, 50, 0.3); } - return isUsingAdvertisedResources ? - new NodeResources(2.0, 8, 50, 0.3) : - new NodeResources(1.5, 8, 50, 0.3); + return zone.getCloud().allowHostSharing() ? + new NodeResources(1.5, 8, 50, 0.3) : + new NodeResources(2.0, 8, 50, 0.3); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index bb25e8371a2..e710d70f20f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -50,14 +50,12 @@ public class LoadBalancerProvisioner { private final NodeRepository nodeRepository; private final CuratorDatabaseClient db; private final LoadBalancerService service; - private final BooleanFlag provisionConfigServerLoadBalancer; private final BooleanFlag provisionControllerLoadBalancer; public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service, FlagSource flagSource) { this.nodeRepository = nodeRepository; this.db = nodeRepository.database(); this.service = service; - this.provisionConfigServerLoadBalancer = Flags.CONFIGSERVER_PROVISION_LB.bindTo(flagSource); this.provisionControllerLoadBalancer = Flags.CONTROLLER_PROVISION_LB.bindTo(flagSource); // Read and write all load balancers to make sure they are stored in the latest version of the serialization format for (var id : db.readLoadBalancerIds()) { @@ -149,11 +147,10 @@ public class LoadBalancerProvisioner { db.writeLoadBalancers(deactivatedLoadBalancers, transaction); } - // TODO(mpolden): Inline when feature flags are removed + // TODO(mpolden): Inline when feature flag is removed private boolean canForwardTo(NodeType type, ClusterSpec cluster) { boolean canForwardTo = service.canForwardTo(type, cluster.type()); if (canForwardTo) { - if (type == NodeType.config) return provisionConfigServerLoadBalancer.value(); if (type == NodeType.controller) return provisionControllerLoadBalancer.value(); } return canForwardTo; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index faae52c72ec..a41cab14fe8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -16,7 +16,10 @@ import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -55,6 +58,7 @@ public class NodeRepositoryProvisioner implements Provisioner { private final Activator activator; private final Optional<LoadBalancerProvisioner> loadBalancerProvisioner; private final NodeResourceLimits nodeResourceLimits; + private final IntFlag tenantNodeQuota; @Inject public NodeRepositoryProvisioner(NodeRepository nodeRepository, Zone zone, @@ -72,6 +76,7 @@ public class NodeRepositoryProvisioner implements Provisioner { flagSource, loadBalancerProvisioner); this.activator = new Activator(nodeRepository, loadBalancerProvisioner); + this.tenantNodeQuota = Flags.TENANT_NODE_QUOTA.bindTo(flagSource); } @@ -184,7 +189,8 @@ public class NodeRepositoryProvisioner implements Provisioner { if (application.tenant().value().hashCode() == 3857) return requestedNodes <= 60; if (application.tenant().value().hashCode() == -1271827001) return requestedNodes <= 75; - return requestedNodes <= 5; + + return requestedNodes <= tenantNodeQuota.with(FetchVector.Dimension.APPLICATION_ID, application.tenant().value()).value(); } private List<HostSpec> asSortedHosts(List<Node> nodes, NodeResources requestedResources) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index b4a1a4afe9b..29a121b8a21 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -217,7 +217,6 @@ public class LoadBalancerProvisionerTest { @Test public void provision_load_balancer_config_server_cluster() { - flagSource.withBooleanFlag(Flags.CONFIGSERVER_PROVISION_LB.id(), true); ApplicationId configServerApp = ApplicationId.from("hosted-vespa", "zone-config-servers", "default"); Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(configServerApp).asList(); var cluster = ClusterSpec.Id.from("zone-config-servers"); diff --git a/persistence/CMakeLists.txt b/persistence/CMakeLists.txt index e9fd742af27..3883929265c 100644 --- a/persistence/CMakeLists.txt +++ b/persistence/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_define_module( DEPENDS fastos @@ -20,7 +21,7 @@ vespa_define_module( src/vespa/persistence/spi TEST_DEPENDS - gtest + GTest::GTest TESTS src/tests diff --git a/persistence/src/vespa/persistence/CMakeLists.txt b/persistence/src/vespa/persistence/CMakeLists.txt index 77f3f865b6f..456c3f8f87f 100644 --- a/persistence/src/vespa/persistence/CMakeLists.txt +++ b/persistence/src/vespa/persistence/CMakeLists.txt @@ -11,5 +11,5 @@ vespa_add_library(persistence_persistence_conformancetest $<TARGET_OBJECTS:persistence_conformancetest_lib> DEPENDS persistence - gtest + GTest::GTest ) diff --git a/searchcommon/src/tests/schema/CMakeLists.txt b/searchcommon/src/tests/schema/CMakeLists.txt index aafe015d9a1..76c8d3f8483 100644 --- a/searchcommon/src/tests/schema/CMakeLists.txt +++ b/searchcommon/src/tests/schema/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcommon_schema_test_app TEST SOURCES schema_test.cpp DEPENDS searchcommon - gtest + GTest::GTest ) vespa_add_test(NAME searchcommon_schema_test_app NO_VALGRIND COMMAND searchcommon_schema_test_app) diff --git a/searchcommon/src/vespa/searchcommon/attribute/distance_metric.h b/searchcommon/src/vespa/searchcommon/attribute/distance_metric.h index 9309a0a86dc..aa4ff22cdf3 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/distance_metric.h +++ b/searchcommon/src/vespa/searchcommon/attribute/distance_metric.h @@ -4,6 +4,6 @@ namespace search::attribute { -enum class DistanceMetric { Euclidean, Angular, GeoDegrees }; +enum class DistanceMetric { Euclidean, Angular, GeoDegrees, InnerProduct }; } diff --git a/searchcore/src/tests/proton/attribute/CMakeLists.txt b/searchcore/src/tests/proton/attribute/CMakeLists.txt index c23d97c6e88..95f55c28f18 100644 --- a/searchcore/src/tests/proton/attribute/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcore_attribute_test_app TEST SOURCES attribute_test.cpp @@ -8,7 +9,7 @@ vespa_add_executable(searchcore_attribute_test_app TEST searchcore_flushengine searchcore_pcommon searchlib_test - gtest + GTest::GTest ) vespa_add_test(NAME searchcore_attribute_test_app COMMAND searchcore_attribute_test_app) diff --git a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/CMakeLists.txt index de1ab5851f4..54c028fbfe8 100644 --- a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcore_attribute_aspect_delayer_test_app TEST SOURCES attribute_aspect_delayer_test.cpp DEPENDS searchcore_attribute searchcore_pcommon - gtest + GTest::GTest ) vespa_add_test(NAME searchcore_attribute_aspect_delayer_test_app COMMAND searchcore_attribute_aspect_delayer_test_app) diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 407b14c5f7d..abcf35051fc 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include <vespa/config-attributes.h> #include <vespa/fastos/file.h> +#include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_manager_initializer.h> @@ -9,24 +11,22 @@ #include <vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h> -#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> +#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchcore/proton/initializer/initializer_task.h> #include <vespa/searchcore/proton/initializer/task_runner.h> #include <vespa/searchcore/proton/server/executor_thread_service.h> #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchcore/proton/test/attribute_vectors.h> +#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchlib/attribute/attributevector.hpp> -#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/attribute/imported_attribute_vector_factory.h> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/searchlib/attribute/reference_attribute.h> #include <vespa/searchlib/attribute/singlenumericattribute.hpp> -#include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/searchlib/common/indexmetainfo.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/predicate/predicate_index.h> @@ -34,6 +34,8 @@ #include <vespa/searchlib/test/directory_handler.h> #include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/foreground_thread_executor.h> +#include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> @@ -53,6 +55,7 @@ using proton::test::AttributeUtils; using proton::test::createInt32Attribute; using proton::test::Int32Attribute; using vespalib::ForegroundTaskExecutor; +using vespalib::ForegroundThreadExecutor; using search::TuneFileAttributes; using search::attribute::BasicType; using search::attribute::IAttributeContext; @@ -152,15 +155,21 @@ struct BaseFixture DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; + ForegroundThreadExecutor _shared; HwInfo _hwInfo; BaseFixture(); ~BaseFixture(); + proton::AttributeManager::SP make_manager() { + return std::make_shared<proton::AttributeManager>(test_dir, "test.subdb", TuneFileAttributes(), + _fileHeaderContext, _attributeFieldWriter, _shared, _hwInfo); + } }; BaseFixture::BaseFixture() : _dirHandler(test_dir), _fileHeaderContext(), _attributeFieldWriter(), + _shared(), _hwInfo() { } @@ -185,8 +194,7 @@ struct AttributeManagerFixture }; AttributeManagerFixture::AttributeManagerFixture(BaseFixture &bf) - : _msp(std::make_shared<proton::AttributeManager>(test_dir, "test.subdb", TuneFileAttributes(), - bf._fileHeaderContext, bf._attributeFieldWriter, bf._hwInfo)), + : _msp(bf.make_manager()), _m(*_msp), _builder() {} @@ -503,11 +511,7 @@ TEST_F("require that new attributes after reconfig are initialized", Fixture) TEST_F("require that removed attributes cannot resurrect", BaseFixture) { - proton::AttributeManager::SP am1( - new proton::AttributeManager(test_dir, "test.subdb", - TuneFileAttributes(), - f._fileHeaderContext, - f._attributeFieldWriter, f._hwInfo)); + auto am1 = f.make_manager(); { AttributeVector::SP a1 = am1->addAttribute({"a1", INT32_SINGLE}, 0); fillAttribute(a1, 2, 10, 15); @@ -801,9 +805,7 @@ TEST_F("require that attribute vector of wrong type is dropped", BaseFixture) predicateParams2.setArity(4); predicate2.setPredicateParams(predicateParams2); - auto am1(std::make_shared<proton::AttributeManager> - (test_dir, "test.subdb", TuneFileAttributes(), - f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo)); + auto am1 = f.make_manager(); am1->addAttribute({"a1", INT32_SINGLE}, 1); am1->addAttribute({"a2", INT32_SINGLE}, 2); am1->addAttribute({"a3", generic_tensor}, 3); @@ -840,17 +842,13 @@ void assertShrinkTargetSerial(proton::AttributeManager &mgr, const vespalib::str TEST_F("require that we can guess flushed serial number for shrink flushtarget", BaseFixture) { - auto am1(std::make_shared<proton::AttributeManager> - (test_dir, "test.subdb", TuneFileAttributes(), - f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo)); + auto am1 = f.make_manager(); am1->addAttribute({"a1", INT32_SINGLE}, 1); am1->addAttribute({"a2", INT32_SINGLE}, 2); TEST_DO(assertShrinkTargetSerial(*am1, "a1", 0)); TEST_DO(assertShrinkTargetSerial(*am1, "a2", 1)); am1->flushAll(10); - am1 = std::make_shared<proton::AttributeManager> - (test_dir, "test.subdb", TuneFileAttributes(), - f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo); + am1 = f.make_manager(); am1->addAttribute({"a1", INT32_SINGLE}, 1); am1->addAttribute({"a2", INT32_SINGLE}, 2); TEST_DO(assertShrinkTargetSerial(*am1, "a1", 10)); @@ -859,9 +857,7 @@ TEST_F("require that we can guess flushed serial number for shrink flushtarget", TEST_F("require that shrink flushtarget is handed over to new attribute manager", BaseFixture) { - auto am1(std::make_shared<proton::AttributeManager> - (test_dir, "test.subdb", TuneFileAttributes(), - f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo)); + auto am1 = f.make_manager(); am1->addAttribute({"a1", INT32_SINGLE}, 4); AttrSpecList newSpec; newSpec.push_back(AttributeSpec("a1", INT32_SINGLE)); diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp index 788aa4fb1ee..f81644c3f55 100644 --- a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp @@ -1,22 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("attribute_populator_test"); -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/repo/configbuilder.h> #include <vespa/document/fieldvalue/intfieldvalue.h> +#include <vespa/document/repo/configbuilder.h> #include <vespa/searchcore/proton/attribute/attribute_populator.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/test/test.h> -#include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/foreground_thread_executor.h> +#include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_populator_test"); + using document::config_builder::DocumenttypesConfigBuilderHelper; using document::config_builder::Struct; using vespalib::ForegroundTaskExecutor; +using vespalib::ForegroundThreadExecutor; using namespace document; using namespace proton; using namespace search; @@ -62,6 +65,7 @@ struct Fixture DirectoryHandler _testDir; DummyFileHeaderContext _fileHeader; ForegroundTaskExecutor _attributeFieldWriter; + ForegroundThreadExecutor _shared; HwInfo _hwInfo; AttributeManager::SP _mgr; std::unique_ptr<AttributePopulator> _pop; @@ -70,10 +74,10 @@ struct Fixture : _testDir(TEST_DIR), _fileHeader(), _attributeFieldWriter(), + _shared(), _hwInfo(), - _mgr(new AttributeManager(TEST_DIR, "test.subdb", - TuneFileAttributes(), - _fileHeader, _attributeFieldWriter, _hwInfo)), + _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), + _fileHeader, _attributeFieldWriter, _shared, _hwInfo)), _pop(), _ctx() { diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index dabab649497..258ee5f32d8 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -42,6 +42,7 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/foreground_thread_executor.h> #include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/vespalib/util/sequencedtaskexecutorobserver.h> @@ -78,6 +79,7 @@ using search::tensor::TensorAttribute; using search::test::DirectoryHandler; using std::string; using vespalib::ForegroundTaskExecutor; +using vespalib::ForegroundThreadExecutor; using vespalib::SequencedTaskExecutorObserver; using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; @@ -121,12 +123,12 @@ fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64 const std::shared_ptr<IDestructorCallback> emptyCallback; - class AttributeWriterTest : public ::testing::Test { public: DirectoryHandler _dirHandler; std::unique_ptr<ForegroundTaskExecutor> _attributeFieldWriterReal; std::unique_ptr<SequencedTaskExecutorObserver> _attributeFieldWriter; + ForegroundThreadExecutor _shared; std::shared_ptr<MockAttributeManager> _mgr; std::unique_ptr<AttributeWriter> _aw; @@ -134,6 +136,7 @@ public: : _dirHandler(test_dir), _attributeFieldWriterReal(), _attributeFieldWriter(), + _shared(), _mgr(), _aw() { @@ -146,6 +149,7 @@ public: _attributeFieldWriter = std::make_unique<SequencedTaskExecutorObserver>(*_attributeFieldWriterReal); _mgr = std::make_shared<MockAttributeManager>(); _mgr->set_writer(*_attributeFieldWriter); + _mgr->set_shared_executor(_shared); allocAttributeWriter(); } void allocAttributeWriter() { @@ -573,6 +577,7 @@ public: DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; + ForegroundThreadExecutor _shared; HwInfo _hwInfo; proton::AttributeManager::SP _baseMgr; FilterAttributeManager _filterMgr; @@ -581,11 +586,13 @@ public: : _dirHandler(test_dir), _fileHeaderContext(), _attributeFieldWriter(), + _shared(), _hwInfo(), _baseMgr(new proton::AttributeManager(test_dir, "test.subdb", TuneFileAttributes(), _fileHeaderContext, _attributeFieldWriter, + _shared, _hwInfo)), _filterMgr(ACCEPTED_ATTRIBUTES, _baseMgr) { @@ -768,7 +775,7 @@ TEST_F(AttributeWriterTest, spreads_write_over_2_write_contexts) TEST_F(AttributeWriterTest, spreads_write_over_3_write_contexts) { setup(8); - putAttributes(*this, {0, 1, 2}); + putAttributes(*this, {0, 1, 3}); } struct MockPrepareResult : public PrepareResult { @@ -798,7 +805,7 @@ public: return std::make_unique<MockPrepareResult>(docid, tensor); } - virtual void complete_set_tensor(DocId docid, const Tensor& tensor, std::unique_ptr<PrepareResult> prepare_result) override { + void complete_set_tensor(DocId docid, const Tensor& tensor, std::unique_ptr<PrepareResult> prepare_result) override { ++complete_set_tensor_cnt; assert(prepare_result); auto* mock_result = dynamic_cast<MockPrepareResult*>(prepare_result.get()); @@ -887,6 +894,11 @@ public: startAttributeField("a1"). addTensor(std::unique_ptr<vespalib::tensor::Tensor>()).endField().endDocument(); } + void expect_shared_executor_tasks(size_t exp_accepted_tasks) { + auto stats = _shared.getStats(); + EXPECT_EQ(exp_accepted_tasks, stats.acceptedTasks); + EXPECT_EQ(0, stats.rejectedTasks); + } }; TEST_F(TwoPhasePutTest, handles_put_in_two_phases_when_specified_for_tensor_attribute) @@ -895,16 +907,13 @@ TEST_F(TwoPhasePutTest, handles_put_in_two_phases_when_specified_for_tensor_attr put(1, *doc, 1); expect_tensor_attr_calls(1, 1); - assertExecuteHistory({1, 0}); + expect_shared_executor_tasks(1); + assertExecuteHistory({0}); put(2, *doc, 2); expect_tensor_attr_calls(2, 2); - assertExecuteHistory({1, 0, 0, 0}); - - put(3, *doc, 3); - expect_tensor_attr_calls(3, 3); - // Note that the prepare step is executed round-robin between the 2 threads. - assertExecuteHistory({1, 0, 0, 0, 1, 0}); + expect_shared_executor_tasks(2); + assertExecuteHistory({0, 0}); } TEST_F(TwoPhasePutTest, put_is_ignored_when_serial_number_is_older_or_equal_to_attribute) @@ -913,7 +922,8 @@ TEST_F(TwoPhasePutTest, put_is_ignored_when_serial_number_is_older_or_equal_to_a attr->commit(7, 7); put(7, *doc, 1); expect_tensor_attr_calls(0, 0); - assertExecuteHistory({1, 0}); + expect_shared_executor_tasks(1); + assertExecuteHistory({0}); } TEST_F(TwoPhasePutTest, document_is_cleared_if_field_is_not_set) @@ -921,7 +931,8 @@ TEST_F(TwoPhasePutTest, document_is_cleared_if_field_is_not_set) auto doc = make_no_field_doc(); put(1, *doc, 1); expect_tensor_attr_calls(0, 0, 1); - assertExecuteHistory({1, 0}); + expect_shared_executor_tasks(1); + assertExecuteHistory({0}); } TEST_F(TwoPhasePutTest, document_is_cleared_if_tensor_in_field_is_not_set) @@ -929,7 +940,8 @@ TEST_F(TwoPhasePutTest, document_is_cleared_if_tensor_in_field_is_not_set) auto doc = make_no_tensor_doc(); put(1, *doc, 1); expect_tensor_attr_calls(0, 0, 1); - assertExecuteHistory({1, 0}); + expect_shared_executor_tasks(1); + assertExecuteHistory({0}); } diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt index 2cb6bc65df3..a48d7f19abe 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcore_attribute_usage_sampler_functor_test_app TEST SOURCES attribute_usage_sampler_functor_test.cpp DEPENDS searchcore_attribute searchcore_pcommon - gtest + GTest::GTest ) vespa_add_test(NAME searchcore_attribute_usage_sampler_functor_test_app COMMAND searchcore_attribute_usage_sampler_functor_test_app) diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp index 4604129248b..adf2f4a7d7d 100644 --- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp @@ -8,13 +8,14 @@ #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/integerbase.h> -#include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/searchlib/common/indexmetainfo.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/datastore/datastorebase.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/foreground_thread_executor.h> +#include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> @@ -246,6 +247,7 @@ struct BaseFixture test::DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; + ForegroundThreadExecutor _shared; HwInfo _hwInfo; BaseFixture(); BaseFixture(const HwInfo &hwInfo); @@ -256,12 +258,14 @@ BaseFixture::BaseFixture() : _dirHandler(test_dir), _fileHeaderContext(), _attributeFieldWriter(), + _shared(), _hwInfo() { } BaseFixture::BaseFixture(const HwInfo &hwInfo) : _dirHandler(test_dir), _fileHeaderContext(), _attributeFieldWriter(), + _shared(), _hwInfo(hwInfo) {} BaseFixture::~BaseFixture() = default; @@ -289,7 +293,7 @@ struct AttributeManagerFixture AttributeManagerFixture::AttributeManagerFixture(BaseFixture &bf) : _msp(std::make_shared<AttributeManager>(test_dir, "test.subdb", TuneFileAttributes(), - bf._fileHeaderContext, bf._attributeFieldWriter, bf._hwInfo)), + bf._fileHeaderContext, bf._attributeFieldWriter, bf._shared, bf._hwInfo)), _m(*_msp) {} AttributeManagerFixture::~AttributeManagerFixture() = default; diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp index 01452f1361e..094ae1a1400 100644 --- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp @@ -1,16 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/attribute/attribute_manager_explorer.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/common/hw_info.h> -#include <vespa/searchcore/proton/test/attribute_vectors.h> #include <vespa/searchcore/proton/test/attribute_utils.h> -#include <vespa/vespalib/util/foregroundtaskexecutor.h> +#include <vespa/searchcore/proton/test/attribute_vectors.h> +#include <vespa/searchlib/attribute/singlenumericattribute.hpp> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/searchlib/attribute/singlenumericattribute.hpp> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/foreground_thread_executor.h> +#include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/log/log.h> LOG_SETUP("attributes_state_explorer_test"); @@ -19,6 +20,7 @@ using namespace proton; using namespace proton::test; using search::AttributeVector; using vespalib::ForegroundTaskExecutor; +using vespalib::ForegroundThreadExecutor; using search::TuneFileAttributes; using search::index::DummyFileHeaderContext; using search::test::DirectoryHandler; @@ -30,6 +32,7 @@ struct Fixture DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; + ForegroundThreadExecutor _shared; HwInfo _hwInfo; AttributeManager::SP _mgr; AttributeManagerExplorer _explorer; @@ -37,10 +40,12 @@ struct Fixture : _dirHandler(TEST_DIR), _fileHeaderContext(), _attributeFieldWriter(), + _shared(), _hwInfo(), _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), _fileHeaderContext, _attributeFieldWriter, + _shared, _hwInfo)), _explorer(_mgr) { diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp index 6f91b1a6f6f..19af6a699c4 100644 --- a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp +++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp @@ -38,7 +38,7 @@ TEST_F("require that attribute write thread is blocked while guard is held", Fix { ReadGuard::UP guard = f.accessor.takeGuard(); Gate gate; - f.writer->execute(f.writer->getExecutorId(f.attribute->getNamePrefix()), [&gate]() { gate.countDown(); }); + f.writer->execute(f.writer->getExecutorIdFromName(f.attribute->getNamePrefix()), [&gate]() { gate.countDown(); }); bool reachedZero = gate.await(100); EXPECT_FALSE(reachedZero); EXPECT_EQUAL(1u, gate.getCount()); diff --git a/searchcore/src/tests/proton/common/operation_rate_tracker/CMakeLists.txt b/searchcore/src/tests/proton/common/operation_rate_tracker/CMakeLists.txt index f5e6a791124..3c5152935d7 100644 --- a/searchcore/src/tests/proton/common/operation_rate_tracker/CMakeLists.txt +++ b/searchcore/src/tests/proton/common/operation_rate_tracker/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcore_operation_rate_tracker_test_app TEST SOURCES operation_rate_tracker_test.cpp DEPENDS searchcore_pcommon - gtest + GTest::GTest ) vespa_add_test(NAME searchcore_operation_rate_tracker_test_app COMMAND searchcore_operation_rate_tracker_test_app) diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index b6d6d2437d8..7d27c3b21f4 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -791,7 +791,7 @@ Test::requireThatAttributesAreUsed() search::AttributeVector *bjAttr = attributeManager->getWritableAttribute("bj"); auto bjTensorAttr = dynamic_cast<search::tensor::TensorAttribute *>(bjAttr); - attributeFieldWriter.execute(attributeFieldWriter.getExecutorId(bjAttr->getNamePrefix()), + attributeFieldWriter.execute(attributeFieldWriter.getExecutorIdFromName(bjAttr->getNamePrefix()), [&]() { bjTensorAttr->setTensor(3, *make_tensor(TensorSpec("tensor(x{},y{})") .add({{"x", "a"}, {"y", "b"}}, 4))); diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 89930a6b1a3..15f7d5798ea 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -192,7 +192,7 @@ Fixture::initViewSet(ViewSet &views) views._reconfigurer, views._writeService, _summaryExecutor, TuneFileIndexManager(), TuneFileAttributes(), views._fileHeaderContext); auto attrMgr = make_shared<AttributeManager>(BASE_DIR, "test.subdb", TuneFileAttributes(), views._fileHeaderContext, - views._writeService.attributeFieldWriter(),views._hwInfo); + views._writeService.attributeFieldWriter(), views._writeService.shared(), views._hwInfo); auto summaryMgr = make_shared<SummaryManager> (_summaryExecutor, search::LogDocumentStore::Config(), search::GrowStrategy(), BASE_DIR, views._docTypeName, TuneFileSummary(), views._fileHeaderContext,views._noTlSyncer, search::IBucketizer::SP()); @@ -273,7 +273,7 @@ struct MyFastAccessFeedView _writeService, *_lidReuseDelayer, _commitTimeTracker); StoreOnlyFeedView::PersistentParams params(1, 1, DocTypeName(DOC_TYPE), 0, SubDbType::NOTREADY); auto mgr = make_shared<AttributeManager>(BASE_DIR, "test.subdb", TuneFileAttributes(), _fileHeaderContext, - _writeService.attributeFieldWriter(), _hwInfo); + _writeService.attributeFieldWriter(), _writeService.shared(), _hwInfo); IAttributeWriter::SP writer(new AttributeWriter(mgr)); FastAccessFeedView::Context fastUpdateCtx(writer, _docIdLimit); _feedView.set(FastAccessFeedView::SP(new FastAccessFeedView(storeOnlyCtx, params, fastUpdateCtx)));; diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt index c4e69ace22c..3e008e3f5d0 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcore_lid_space_compaction_test_app TEST SOURCES lid_space_compaction_test.cpp @@ -10,6 +11,6 @@ vespa_add_executable(searchcore_lid_space_compaction_test_app TEST searchcore_documentmetastore searchcore_bucketdb searchcore_pcommon - gtest + GTest::GTest ) vespa_add_test(NAME searchcore_lid_space_compaction_test_app COMMAND searchcore_lid_space_compaction_test_app) diff --git a/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt b/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt index f31d82240e0..c35d736acd1 100644 --- a/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentmetastore/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcore_documentmetastore_test_app TEST SOURCES documentmetastore_test.cpp @@ -9,7 +10,7 @@ vespa_add_executable(searchcore_documentmetastore_test_app TEST searchcore_attribute searchcore_feedoperation searchcore_fconfig - gtest + GTest::GTest ) vespa_add_test(NAME searchcore_documentmetastore_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/documentmetastore_test.sh DEPENDS searchcore_documentmetastore_test_app) diff --git a/searchcore/src/tests/proton/index/CMakeLists.txt b/searchcore/src/tests/proton/index/CMakeLists.txt index 1ffad6cdbf8..b967bf304c3 100644 --- a/searchcore/src/tests/proton/index/CMakeLists.txt +++ b/searchcore/src/tests/proton/index/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcore_indexmanager_test_app TEST SOURCES indexmanager_test.cpp @@ -7,7 +8,7 @@ vespa_add_executable(searchcore_indexmanager_test_app TEST searchcore_index searchcore_flushengine searchcore_pcommon - gtest + GTest::GTest ) vespa_add_executable(searchcore_fusionrunner_test_app TEST SOURCES @@ -28,7 +29,7 @@ vespa_add_executable(searchcore_indexcollection_test_app TEST indexcollection_test.cpp DEPENDS searchcore_index - gtest + GTest::GTest ) vespa_add_test(NAME searchcore_index_test COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/index_test.sh DEPENDS searchcore_indexmanager_test_app searchcore_fusionrunner_test_app searchcore_diskindexcleaner_test_app searchcore_indexcollection_test_app) diff --git a/searchcore/src/tests/proton/matching/handle_recorder/CMakeLists.txt b/searchcore/src/tests/proton/matching/handle_recorder/CMakeLists.txt index a56d22ab154..3f4fd071e5c 100644 --- a/searchcore/src/tests/proton/matching/handle_recorder/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/handle_recorder/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcore_matching_handle_recorder_test_app TEST SOURCES handle_recorder_test.cpp DEPENDS searchcore_matching - gtest + GTest::GTest ) vespa_add_test(NAME searchcore_matching_handle_recorder_test_app COMMAND searchcore_matching_handle_recorder_test_app) diff --git a/searchcore/src/tests/proton/matching/request_context/CMakeLists.txt b/searchcore/src/tests/proton/matching/request_context/CMakeLists.txt index 54696112302..c044a9d4869 100644 --- a/searchcore/src/tests/proton/matching/request_context/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/request_context/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcore_matching_request_context_test_app TEST SOURCES request_context_test.cpp DEPENDS searchcore_matching - gtest + GTest::GTest ) vespa_add_test(NAME searchcore_matching_request_context_test_app COMMAND searchcore_matching_request_context_test_app) diff --git a/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/CMakeLists.txt b/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/CMakeLists.txt index 6fce7151664..764208c0eba 100644 --- a/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcore_unpacking_iterators_optimizer_test_app TEST SOURCES unpacking_iterators_optimizer_test.cpp DEPENDS searchcore_matching - gtest + GTest::GTest ) vespa_add_test(NAME searchcore_unpacking_iterators_optimizer_test_app COMMAND searchcore_unpacking_iterators_optimizer_test_app) diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt index 048ed1938e5..f96a31b7765 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchcore_attribute_reprocessing_initializer_test_app TEST SOURCES attribute_reprocessing_initializer_test.cpp @@ -6,6 +7,6 @@ vespa_add_executable(searchcore_attribute_reprocessing_initializer_test_app TEST searchcore_reprocessing searchcore_attribute searchcore_pcommon - gtest + GTest::GTest ) vespa_add_test(NAME searchcore_attribute_reprocessing_initializer_test_app COMMAND searchcore_attribute_reprocessing_initializer_test_app) diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp index 898825016b3..21bb011901e 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp @@ -11,11 +11,12 @@ #include <vespa/searchcore/proton/reprocessing/i_reprocessing_handler.h> #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/util/foreground_thread_executor.h> +#include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/log/log.h> LOG_SETUP("attribute_reprocessing_initializer_test"); @@ -30,6 +31,7 @@ using search::attribute::Config; using search::index::schema::DataType; using search::test::DirectoryHandler; using vespalib::ForegroundTaskExecutor; +using vespalib::ForegroundThreadExecutor; const vespalib::string TEST_DIR = "test_output"; const SerialNum INIT_SERIAL_NUM = 10; @@ -54,6 +56,7 @@ struct MyConfig { DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; + ForegroundThreadExecutor _shared; HwInfo _hwInfo; AttributeManager::SP _mgr; search::index::Schema _schema; @@ -87,10 +90,10 @@ struct MyConfig MyConfig::MyConfig() : _fileHeaderContext(), _attributeFieldWriter(), + _shared(), _hwInfo(), _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), - _fileHeaderContext, - _attributeFieldWriter, _hwInfo)), + _fileHeaderContext, _attributeFieldWriter, _shared, _hwInfo)), _schema() {} MyConfig::~MyConfig() = default; @@ -132,6 +135,7 @@ public: DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; + ForegroundThreadExecutor _shared; HwInfo _hwInfo; AttributeManager::SP _mgr; MyConfig _oldCfg; @@ -143,10 +147,10 @@ public: : _dirHandler(TEST_DIR), _fileHeaderContext(), _attributeFieldWriter(), + _shared(), _hwInfo(), - _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), - _fileHeaderContext, - _attributeFieldWriter, _hwInfo)), + _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), _fileHeaderContext, + _attributeFieldWriter, _shared, _hwInfo)), _oldCfg(), _newCfg(), _inspector(_oldCfg, _newCfg), diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp index 91e935fe7c0..9ede0bda577 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp @@ -18,8 +18,8 @@ class PopulateDoneContext : public IDestructorCallback { std::shared_ptr<document::Document> _doc; public: - PopulateDoneContext(const std::shared_ptr<document::Document> &doc) - : _doc(doc) + PopulateDoneContext(std::shared_ptr<document::Document> doc) + : _doc(std::move(doc)) { } ~PopulateDoneContext() override = default; @@ -40,6 +40,7 @@ AttributePopulator::getNames() const std::vector<search::AttributeGuard> attrs; _writer.getAttributeManager()->getAttributeList(attrs); std::vector<vespalib::string> names; + names.reserve(attrs.size()); for (const search::AttributeGuard &attr : attrs) { names.push_back(_subDbName + ".attribute." + attr->getName()); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 9b54ae816e0..a49b27caf36 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -12,9 +12,10 @@ #include <vespa/searchcore/proton/common/attribute_updater.h> #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/imported_attribute_vector.h> -#include <vespa/searchlib/tensor/prepare_result.h> #include <vespa/searchlib/common/idestructorcallback.h> +#include <vespa/searchlib/tensor/prepare_result.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/threadexecutor.h> #include <future> #include <vespa/log/log.h> @@ -281,7 +282,7 @@ public: FieldContext::FieldContext(ISequencedTaskExecutor &writer, AttributeVector *attr) : _name(attr->getName()), - _executorId(writer.getExecutorId(attr->getNamePrefix())), + _executorId(writer.getExecutorIdFromName(attr->getNamePrefix())), _attr(attr), _use_two_phase_put(use_two_phase_put_for_attribute(*attr)) { @@ -607,8 +608,7 @@ AttributeWriter::internalPut(SerialNum serialNum, const Document &doc, DocumentI assert(wc.getFields().size() == 1); auto prepare_task = std::make_unique<PreparePutTask>(serialNum, lid, wc.getFields()[0], extractor); auto complete_task = std::make_unique<CompletePutTask>(*prepare_task, immediateCommit, onWriteDone); - // We use the local docid to create an executor id to round-robin between the threads. - _attributeFieldWriter.executeTask(_attributeFieldWriter.getExecutorId(lid), std::move(prepare_task)); + _shared_executor.execute(std::move(prepare_task)); _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(complete_task)); } else { if (allAttributes || wc.hasStructFieldAttribute()) { @@ -633,6 +633,7 @@ AttributeWriter::internalRemove(SerialNum serialNum, DocumentIdT lid, bool immed AttributeWriter::AttributeWriter(proton::IAttributeManager::SP mgr) : _mgr(std::move(mgr)), _attributeFieldWriter(_mgr->getAttributeFieldWriter()), + _shared_executor(_mgr->get_shared_executor()), _writeContexts(), _dataType(nullptr), _hasStructFieldAttribute(false), @@ -645,7 +646,7 @@ AttributeWriter::AttributeWriter(proton::IAttributeManager::SP mgr) void AttributeWriter::setupAttriuteMapping() { for (auto attr : getWritableAttributes()) { vespalib::stringref name = attr->getName(); - _attrMap[name] = AttrWithId(attr, _attributeFieldWriter.getExecutorId(attr->getNamePrefix())); + _attrMap[name] = AttrWithId(attr, _attributeFieldWriter.getExecutorIdFromName(attr->getNamePrefix())); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index 726379220e3..eaf8abe4872 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -26,6 +26,7 @@ private: using FieldValue = document::FieldValue; const IAttributeManager::SP _mgr; vespalib::ISequencedTaskExecutor &_attributeFieldWriter; + vespalib::ThreadExecutor& _shared_executor; using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId; public: /** diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index 319ae2dcad1..504f6841daf 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -63,7 +63,9 @@ std::shared_ptr<ShrinkLidSpaceFlushTarget> allocShrinker(const AttributeVector:: using Type = IFlushTarget::Type; using Component = IFlushTarget::Component; - auto shrinkwrap = std::make_shared<ThreadedCompactableLidSpace>(attr, attributeFieldWriter, attributeFieldWriter.getExecutorId(attr->getNamePrefix())); + auto shrinkwrap = std::make_shared<ThreadedCompactableLidSpace>(attr, attributeFieldWriter, + attributeFieldWriter.getExecutorIdFromName( + attr->getNamePrefix())); const vespalib::string &name = attr->getName(); auto dir = diskLayout.createAttributeDir(name); search::SerialNum shrinkSerialNum = estimateShrinkSerialNum(*attr); @@ -223,6 +225,7 @@ AttributeManager::AttributeManager(const vespalib::string &baseDir, const TuneFileAttributes &tuneFileAttributes, const FileHeaderContext &fileHeaderContext, vespalib::ISequencedTaskExecutor &attributeFieldWriter, + vespalib::ThreadExecutor& shared_executor, const HwInfo &hwInfo) : proton::IAttributeManager(), _attributes(), @@ -235,17 +238,18 @@ AttributeManager::AttributeManager(const vespalib::string &baseDir, _factory(std::make_shared<AttributeFactory>()), _interlock(std::make_shared<search::attribute::Interlock>()), _attributeFieldWriter(attributeFieldWriter), + _shared_executor(shared_executor), _hwInfo(hwInfo), _importedAttributes() { } - AttributeManager::AttributeManager(const vespalib::string &baseDir, const vespalib::string &documentSubDbName, const search::TuneFileAttributes &tuneFileAttributes, const search::common::FileHeaderContext &fileHeaderContext, vespalib::ISequencedTaskExecutor &attributeFieldWriter, + vespalib::ThreadExecutor& shared_executor, const IAttributeFactory::SP &factory, const HwInfo &hwInfo) : proton::IAttributeManager(), @@ -259,6 +263,7 @@ AttributeManager::AttributeManager(const vespalib::string &baseDir, _factory(factory), _interlock(std::make_shared<search::attribute::Interlock>()), _attributeFieldWriter(attributeFieldWriter), + _shared_executor(shared_executor), _hwInfo(hwInfo), _importedAttributes() { @@ -278,6 +283,7 @@ AttributeManager::AttributeManager(const AttributeManager &currMgr, _factory(currMgr._factory), _interlock(currMgr._interlock), _attributeFieldWriter(currMgr._attributeFieldWriter), + _shared_executor(currMgr._shared_executor), _hwInfo(currMgr._hwInfo), _importedAttributes() { @@ -537,6 +543,11 @@ AttributeManager::getAttributeFieldWriter() const return _attributeFieldWriter; } +vespalib::ThreadExecutor& +AttributeManager::get_shared_executor() const +{ + return _shared_executor; +} AttributeVector * AttributeManager::getWritableAttribute(const vespalib::string &name) const @@ -548,14 +559,12 @@ AttributeManager::getWritableAttribute(const vespalib::string &name) const return itr->second.getAttribute().get(); } - const std::vector<AttributeVector *> & AttributeManager::getWritableAttributes() const { return _writableAttributes; } - void AttributeManager::asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor> func) const { @@ -564,7 +573,7 @@ AttributeManager::asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor> continue; } AttributeVector::SP attrsp = attr.second.getAttribute(); - _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorId(attrsp->getNamePrefix()), + _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorIdFromName(attrsp->getNamePrefix()), [attrsp, func]() { (*func)(*attrsp); }); } } @@ -577,7 +586,7 @@ AttributeManager::asyncForAttribute(const vespalib::string &name, std::unique_pt } AttributeVector::SP attrsp = itr->second.getAttribute(); vespalib::string attrName = attrsp->getNamePrefix(); - _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorId(attrName), + _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorIdFromName(attrName), [attr=std::move(attrsp), func=std::move(func)]() { (*func)(*attr); }); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index 350b986add4..10c017c84d3 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -17,6 +17,8 @@ namespace search::common { class FileHeaderContext; } namespace searchcorespi { class IFlushTarget; } +namespace vespalib { class ThreadExecutor; } + namespace proton { class AttributeDiskLayout; @@ -78,6 +80,7 @@ private: IAttributeFactory::SP _factory; std::shared_ptr<search::attribute::Interlock> _interlock; vespalib::ISequencedTaskExecutor &_attributeFieldWriter; + vespalib::ThreadExecutor& _shared_executor; HwInfo _hwInfo; std::unique_ptr<ImportedAttributesRepo> _importedAttributes; @@ -104,6 +107,7 @@ public: const search::TuneFileAttributes &tuneFileAttributes, const search::common::FileHeaderContext & fileHeaderContext, vespalib::ISequencedTaskExecutor &attributeFieldWriter, + vespalib::ThreadExecutor& shared_executor, const HwInfo &hwInfo); AttributeManager(const vespalib::string &baseDir, @@ -111,6 +115,7 @@ public: const search::TuneFileAttributes &tuneFileAttributes, const search::common::FileHeaderContext & fileHeaderContext, vespalib::ISequencedTaskExecutor &attributeFieldWriter, + vespalib::ThreadExecutor& shared_executor, const IAttributeFactory::SP &factory, const HwInfo &hwInfo); @@ -166,6 +171,8 @@ public: vespalib::ISequencedTaskExecutor &getAttributeFieldWriter() const override; + vespalib::ThreadExecutor& get_shared_executor() const override; + search::AttributeVector *getWritableAttribute(const vespalib::string &name) const override; const std::vector<search::AttributeVector *> &getWritableAttributes() const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp index 9543897407d..16f06bdde93 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp @@ -51,7 +51,7 @@ ExclusiveAttributeReadAccessor::takeGuard() { GateSP entranceGate = std::make_shared<Gate>(); GateSP exitGate = std::make_shared<Gate>(); - _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorId(_attribute->getNamePrefix()), + _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorIdFromName(_attribute->getNamePrefix()), [this, entranceGate, exitGate]() { attributeWriteBlockingTask(_attribute, entranceGate, exitGate); }); entranceGate->await(); return std::make_unique<Guard>(*_attribute, exitGate); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index 0fd9849443a..3b1269b031c 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -27,8 +27,7 @@ public: _type(type) { } - - ~FlushTargetFilter() { } + ~FlushTargetFilter(); bool match(const IFlushTarget::SP &flushTarget) const { const vespalib::string &targetName = flushTarget->getName(); @@ -45,6 +44,8 @@ public: } }; +FlushTargetFilter::~FlushTargetFilter() = default; + FlushTargetFilter syncFilter(FLUSH_TARGET_NAME_PREFIX, IFlushTarget::Type::SYNC); FlushTargetFilter shrinkFilter(SHRINK_TARGET_NAME_PREFIX, IFlushTarget::Type::GC); @@ -57,9 +58,9 @@ FilterAttributeManager::acceptAttribute(const vespalib::string &name) const } FilterAttributeManager::FilterAttributeManager(const AttributeSet &acceptedAttributes, - const IAttributeManager::SP &mgr) + IAttributeManager::SP mgr) : _acceptedAttributes(acceptedAttributes), - _mgr(mgr) + _mgr(std::move(mgr)) { // Assume that list of attributes in mgr doesn't change for (const auto attr : _mgr->getWritableAttributes()) { @@ -160,13 +161,17 @@ FilterAttributeManager::getFlushedSerialNum(const vespalib::string &name) const return 0; } - vespalib::ISequencedTaskExecutor & FilterAttributeManager::getAttributeFieldWriter() const { return _mgr->getAttributeFieldWriter(); } +vespalib::ThreadExecutor& +FilterAttributeManager::get_shared_executor() const +{ + return _mgr->get_shared_executor(); +} search::AttributeVector * FilterAttributeManager::getWritableAttribute(const vespalib::string &name) const @@ -195,7 +200,7 @@ FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IConstAttributeFun search::AttributeVector::SP attrsp = guard.getSP(); // Name must be extracted in document db master thread or attribute // writer thread - attributeFieldWriter.execute(attributeFieldWriter.getExecutorId(attrsp->getNamePrefix()), + attributeFieldWriter.execute(attributeFieldWriter.getExecutorIdFromName(attrsp->getNamePrefix()), [attrsp, func]() { (*func)(*attrsp); }); } } @@ -206,7 +211,7 @@ FilterAttributeManager::asyncForAttribute(const vespalib::string &name, std::uni if (!attr) { return; } vespalib::ISequencedTaskExecutor &attributeFieldWriter = getAttributeFieldWriter(); vespalib::string attrName = (*attr)->getNamePrefix(); - attributeFieldWriter.execute(attributeFieldWriter.getExecutorId(attrName), + attributeFieldWriter.execute(attributeFieldWriter.getExecutorIdFromName(attrName), [attr=std::move(attr), func=std::move(func)]() mutable { (*func)(**attr); }); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h index 3755946037d..0b478a34144 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h @@ -28,7 +28,7 @@ private: public: FilterAttributeManager(const AttributeSet &acceptedAttributes, - const IAttributeManager::SP &mgr); + IAttributeManager::SP mgr); ~FilterAttributeManager() override; // Implements search::IAttributeManager @@ -47,6 +47,7 @@ public: void pruneRemovedFields(search::SerialNum serialNum) override; const IAttributeFactory::SP &getFactory() const override; vespalib::ISequencedTaskExecutor & getAttributeFieldWriter() const override; + vespalib::ThreadExecutor& get_shared_executor() const override; search::AttributeVector * getWritableAttribute(const vespalib::string &name) const override; const std::vector<search::AttributeVector *> & getWritableAttributes() const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index 2b5f4b028dc..4edb64b861a 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -230,7 +230,7 @@ FlushableAttribute::initFlush(SerialNum currentSerial) // Called by document db executor std::promise<IFlushTarget::Task::UP> promise; std::future<IFlushTarget::Task::UP> future = promise.get_future(); - _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorId(_attr->getNamePrefix()), + _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorIdFromName(_attr->getNamePrefix()), [&]() { promise.set_value(internalInitFlush(currentSerial)); }); return future.get(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h index 25d0a508438..fa5c52617fd 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h @@ -14,7 +14,10 @@ namespace search { class IDestructorCallback;} namespace search::attribute { class IAttributeFunctor; } -namespace vespalib { class ISequencedTaskExecutor; } +namespace vespalib { + class ISequencedTaskExecutor; + class ThreadExecutor; +} namespace proton { @@ -74,6 +77,8 @@ struct IAttributeManager : public search::IAttributeManager virtual vespalib::ISequencedTaskExecutor &getAttributeFieldWriter() const = 0; + virtual vespalib::ThreadExecutor& get_shared_executor() const = 0; + /* * Get pointer to named writable attribute. If attribute isn't * found or is an extra attribute then nullptr is returned. diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp index 7d5eabe7de8..57e284cc61b 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp @@ -11,7 +11,7 @@ GidToLidChangeListener::GidToLidChangeListener(vespalib::ISequencedTaskExecutor const vespalib::string &name, const vespalib::string &docTypeName) : _attributeFieldWriter(attributeFieldWriter), - _executorId(_attributeFieldWriter.getExecutorId(attr->getNamePrefix())), + _executorId(_attributeFieldWriter.getExecutorIdFromName(attr->getNamePrefix())), _attr(std::move(attr)), _refCount(refCount), _name(name), diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 744b5060ca5..063561347b4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -69,6 +69,7 @@ FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &co configSnapshot.getTuneFileDocumentDBSP()->_attr, _fileHeaderContext, _writeService.attributeFieldWriter(), + _writeService.shared(), attrFactory, _hwInfo); return std::make_shared<AttributeManagerInitializer>(configSerialNum, @@ -186,11 +187,8 @@ FastAccessDocSubDB::createReprocessingTask(IReprocessingInitializer &initializer { uint32_t docIdLimit = _metaStoreCtx->get().getCommittedDocIdLimit(); assert(docIdLimit > 0); - return IReprocessingTask::UP(new ReprocessDocumentsTask(initializer, - getSummaryManager(), - docTypeRepo, - getSubDbName(), - docIdLimit)); + return std::make_unique<ReprocessDocumentsTask>(initializer, getSummaryManager(), docTypeRepo, + getSubDbName(), docIdLimit); } FastAccessDocSubDB::FastAccessDocSubDB(const Config &cfg, const Context &ctx) @@ -234,8 +232,8 @@ FastAccessDocSubDB::initViews(const DocumentDBConfig &configSnapshot, { // Called by executor thread (void) sessionManager; - _iSearchView.set(ISearchHandler::SP(new EmptySearchView)); - IAttributeWriter::SP writer(new AttributeWriter(getAndResetInitAttributeManager())); + _iSearchView.set(std::make_shared<EmptySearchView>()); + auto writer = std::make_shared<AttributeWriter>(getAndResetInitAttributeManager()); { std::lock_guard<std::mutex> guard(_configMutex); initFeedView(writer, configSnapshot); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 962ee65c10d..1e579739d85 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -102,8 +102,8 @@ diskMemUsageSamplerConfig(const ProtonConfig &proton, const HwInfo &hwInfo) } size_t -deriveCompactionCompressionThreads(const ProtonConfig &proton, - const HwInfo::Cpu &cpuInfo) { +derive_shared_threads(const ProtonConfig &proton, + const HwInfo::Cpu &cpuInfo) { size_t scaledCores = (size_t)std::ceil(cpuInfo.cores() * proton.feeding.concurrency); // We need at least 1 guaranteed free worker in order to ensure progress so #documentsdbs + 1 should suffice, @@ -301,7 +301,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::string fileConfigId; _warmupExecutor = std::make_unique<vespalib::ThreadStackExecutor>(4, 128*1024, index_warmup_executor); - const size_t sharedThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu()); + const size_t sharedThreads = derive_shared_threads(protonConfig, hwInfo.cpu()); _sharedExecutor = std::make_shared<vespalib::BlockingThreadStackExecutor>(sharedThreads, 128*1024, sharedThreads*16, proton_shared_executor); _compile_cache_executor_binding = vespalib::eval::CompileCache::bind(_sharedExecutor); InitializeThreads initializeThreads; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index e169f51ef9d..46d5a3282be 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -75,12 +75,8 @@ reconfigureMatchView(const Matchers::SP &matchers, const IAttributeManager::SP &attrMgr) { SearchView::SP curr = _searchView.get(); - MatchView::SP matchView(new MatchView(matchers, - indexSearchable, - attrMgr, - curr->getSessionManager(), - curr->getDocumentMetaStore(), - curr->getDocIdLimit())); + auto matchView = std::make_shared<MatchView>(matchers, indexSearchable, attrMgr, curr->getSessionManager(), + curr->getDocumentMetaStore(), curr->getDocIdLimit()); reconfigureSearchView(matchView); } @@ -126,7 +122,7 @@ Matchers::UP SearchableDocSubDBConfigurer::createMatchers(const Schema::SP &schema, const RankProfilesConfig &cfg) { - Matchers::UP newMatchers(new Matchers(_clock, _queryLimiter, _constantValueRepo)); + auto newMatchers = std::make_unique<Matchers>(_clock, _queryLimiter, _constantValueRepo); for (const auto &profile : cfg.rankprofile) { vespalib::string name = profile.name; search::fef::Properties properties; @@ -222,11 +218,11 @@ SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, attrMgr = newAttrMgr; shouldMatchViewChange = true; - IAttributeWriter::SP newAttrWriter(new AttributeWriter(newAttrMgr)); + auto newAttrWriter = std::make_shared<AttributeWriter>(newAttrMgr); attrWriter = newAttrWriter; shouldFeedViewChange = true; - initializer.reset(createAttributeReprocessingInitializer(newConfig, newAttrMgr, - oldConfig, oldAttrMgr, _subDbName, attrSpec.getCurrentSerialNum()).release()); + initializer = createAttributeReprocessingInitializer(newConfig, newAttrMgr, oldConfig, oldAttrMgr, + _subDbName, attrSpec.getCurrentSerialNum()); } else if (params.shouldAttributeWriterChange()) { attrWriter = std::make_shared<AttributeWriter>(attrMgr); shouldFeedViewChange = true; diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp index 22eb64acdee..cf9f27a94af 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp @@ -28,13 +28,17 @@ ThreadingServiceConfig::ThreadingServiceConfig(uint32_t indexingThreads_, namespace { uint32_t -calculateIndexingThreads(uint32_t cfgIndexingThreads, double concurrency, const HwInfo::Cpu &cpuInfo) +calculateIndexingThreads(const ProtonConfig::Indexing & indexing, double concurrency, const HwInfo::Cpu &cpuInfo) { - // We are capping at 12 threads to reduce cost of waking up threads - // to achieve a better throughput. - // TODO: Fix this in a simpler/better way. - double scaledCores = std::min(12.0, cpuInfo.cores() * concurrency); - uint32_t indexingThreads = std::max((uint32_t)std::ceil(scaledCores / 3), cfgIndexingThreads); + double scaledCores = cpuInfo.cores() * concurrency; + if (indexing.optimize != ProtonConfig::Indexing::Optimize::ADAPTIVE) { + // We are capping at 12 threads to reduce cost of waking up threads + // to achieve a better throughput. + // TODO: Fix this in a simpler/better way. + scaledCores = std::min(12.0, scaledCores); + } + + uint32_t indexingThreads = std::max((int32_t)std::ceil(scaledCores / 3), indexing.threads); return std::max(indexingThreads, 1u); } @@ -54,7 +58,7 @@ selectOptimization(ProtonConfig::Indexing::Optimize optimize) { ThreadingServiceConfig ThreadingServiceConfig::make(const ProtonConfig &cfg, double concurrency, const HwInfo::Cpu &cpuInfo) { - uint32_t indexingThreads = calculateIndexingThreads(cfg.indexing.threads, concurrency, cpuInfo); + uint32_t indexingThreads = calculateIndexingThreads(cfg.indexing, concurrency, cpuInfo); return ThreadingServiceConfig(indexingThreads, cfg.indexing.tasklimit, (cfg.indexing.semiunboundtasklimit / indexingThreads), selectOptimization(cfg.indexing.optimize), diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index 8e5d3018532..3c301e66057 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -14,13 +14,15 @@ private: std::vector<search::AttributeVector*> _writables; std::unique_ptr<ImportedAttributesRepo> _importedAttributes; vespalib::ISequencedTaskExecutor* _writer; + vespalib::ThreadExecutor* _shared; public: MockAttributeManager() : _mock(), _writables(), _importedAttributes(), - _writer() + _writer(), + _shared() {} search::AttributeVector::SP addAttribute(const vespalib::string &name, const search::AttributeVector::SP &attr) { @@ -28,11 +30,12 @@ public: _writables.push_back(attr.get()); return attr; } - void set_writer(vespalib::ISequencedTaskExecutor& writer) { _writer = &writer; } - + void set_shared_executor(vespalib::ThreadExecutor& shared) { + _shared = &shared; + } search::AttributeGuard::UP getAttribute(const vespalib::string &name) const override { return _mock.getAttribute(name); } @@ -69,6 +72,10 @@ public: assert(_writer != nullptr); return *_writer; } + vespalib::ThreadExecutor& get_shared_executor() const override { + assert(_shared != nullptr); + return *_shared; + } search::AttributeVector *getWritableAttribute(const vespalib::string &name) const override { auto attr = getAttribute(name); if (attr) { diff --git a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h index 2ace9a8ac6b..18c376a4c2b 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h +++ b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h @@ -8,26 +8,36 @@ namespace vespalib { class ISequencedTaskExecutor; } namespace searchcorespi::index { /** - * Interface for the thread model used for write tasks. + * Interface for the thread model used for write tasks for a single document database. * * We have multiple write threads: * - * 1. The master write thread used for the majority of write tasks. + * 1. The "master" write thread used for the majority of write tasks. * - * 2. The index write thread used for doing changes to the memory + * 2. The "index" write thread used for doing changes to the memory * index, either directly (for data not bound to a field) or via * index field inverter executor or index field writer executor. * - * 3. The index field inverter executor is used to populate field + * 3. The "summary" thread is used for doing changes to the document store. + * + * 4. The "index field inverter" executor is used to populate field * inverters with data from document fields. Scheduled tasks for * the same field are executed in sequence. * - * 4. The index field writer executor is used to sort data in field + * 5. The "index field writer" executor is used to sort data in field * inverters before pushing the data to the memory field indexes. * Scheduled tasks for the same field are executed in sequence. * - * The master write thread is always the one giving tasks to the index - * write thread. + * 6. The "attribute field writer" executor is used to write data to attribute vectors. + * Each attribute is always handled by the same thread, + * and scheduled tasks for the same attribute are executed in sequence. + * + * The master write thread is always the one giving tasks to the other write threads above. + * + * In addition this interface exposes the "shared" executor that is used by all document databases. + * This is among others used for compressing / de-compressing documents in the document store, + * merging files as part of disk index fusion, and running the prepare step when doing two-phase + * puts against a tensor attribute with a HNSW index. * * The index write thread extracts fields from documents and gives * task to the index field inverter executor and the index field diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index c5dd468e4fd..1db9018bd21 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -78,6 +78,7 @@ vespa_define_module( src/tests/attribute/changevector src/tests/attribute/compaction src/tests/attribute/document_weight_iterator + src/tests/attribute/document_weight_or_filter_search src/tests/attribute/enum_attribute_compaction src/tests/attribute/enum_comparator src/tests/attribute/enumeratedsave @@ -186,7 +187,6 @@ vespa_define_module( src/tests/query src/tests/queryeval src/tests/queryeval/blueprint - src/tests/queryeval/booleanmatchiteratorwrapper src/tests/queryeval/dot_product src/tests/queryeval/equiv src/tests/queryeval/fake_searchable @@ -205,6 +205,7 @@ vespa_define_module( src/tests/queryeval/weak_and_heap src/tests/queryeval/weak_and_scorers src/tests/queryeval/weighted_set_term + src/tests/queryeval/wrappers src/tests/rankingexpression/feature_name_extractor src/tests/rankingexpression/intrinsic_blueprint_adapter src/tests/ranksetup diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index c22d906e2b2..0768999c52f 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1614,6 +1614,7 @@ "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", + "public java.util.Optional asScalarFunction()", "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", "public java.lang.String toString()", "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java index befe2179dc1..86541343edb 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java @@ -25,6 +25,7 @@ import java.util.Deque; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -157,6 +158,11 @@ public class TensorFunctionNode extends CompositeNode { } @Override + public Optional<TensorFunction<Reference>> asTensorFunction() { + return Optional.of(new ExpressionTensorFunction(expression)); + } + + @Override public String toString() { return toString(ExpressionToStringContext.empty); } @@ -230,6 +236,11 @@ public class TensorFunctionNode extends CompositeNode { } @Override + public Optional<ScalarFunction<Reference>> asScalarFunction() { + return Optional.of(new ExpressionScalarFunction(expression)); + } + + @Override public Tensor evaluate(EvaluationContext<Reference> context) { return expression.evaluate((Context)context).asTensor(); } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java index a41fb02f784..bfaff0712ee 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java @@ -29,6 +29,8 @@ public class ConstantDereferencerTestCase { assertEquals("1.0 + 2.0 + 3.5", c.transform(new RankingExpression("a + b + c"), context).toString()); assertEquals("myFunction(1.0,2.0)", c.transform(new RankingExpression("myFunction(a, b)"), context).toString()); + assertEquals("tensor(x[2],y[3])((x + y == 1.0))", c.transform(new RankingExpression("tensor(x[2],y[3])(x+y==a)"), context).toString()); + } } diff --git a/searchlib/src/tests/attribute/attribute_header/CMakeLists.txt b/searchlib/src/tests/attribute/attribute_header/CMakeLists.txt index e72c0c6a528..2327cf5116f 100644 --- a/searchlib/src/tests/attribute/attribute_header/CMakeLists.txt +++ b/searchlib/src/tests/attribute/attribute_header/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_attribute_header_test_app TEST SOURCES attribute_header_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_attribute_header_test_app COMMAND searchlib_attribute_header_test_app) diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index b94186626c2..1191a7aa2e2 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -289,6 +289,12 @@ AttributeManagerTest::testConfigConvert() auto out = ConfigConverter::convert(a); EXPECT_TRUE(out.distance_metric() == DistanceMetric::GeoDegrees); } + { // distance metric (explicit) + CACA a; + a.distancemetric = AttributesConfig::Attribute::Distancemetric::INNERPRODUCT; + auto out = ConfigConverter::convert(a); + EXPECT_TRUE(out.distance_metric() == DistanceMetric::InnerProduct); + } { // hnsw index params (enabled) auto dm_in = AttributesConfig::Attribute::Distancemetric::ANGULAR; auto dm_out = DistanceMetric::Angular; @@ -306,6 +312,7 @@ AttributeManagerTest::testConfigConvert() EXPECT_TRUE(params.distance_metric() == dm_out); EXPECT_TRUE(params.multi_threaded_indexing()); } + { // hnsw index params (disabled) CACA a; a.index.hnsw.enabled = false; diff --git a/searchlib/src/tests/attribute/document_weight_or_filter_search/CMakeLists.txt b/searchlib/src/tests/attribute/document_weight_or_filter_search/CMakeLists.txt new file mode 100644 index 00000000000..8f25f6b66b8 --- /dev/null +++ b/searchlib/src/tests/attribute/document_weight_or_filter_search/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) +vespa_add_executable(searchlib_document_weight_or_filter_search_test_app TEST + SOURCES + document_weight_or_filter_search_test.cpp + DEPENDS + searchlib + searchlib_test + GTest::GTest +) +vespa_add_test(NAME searchlib_document_weight_or_filter_search_test_app COMMAND searchlib_document_weight_or_filter_search_test_app) diff --git a/searchlib/src/tests/attribute/document_weight_or_filter_search/document_weight_or_filter_search_test.cpp b/searchlib/src/tests/attribute/document_weight_or_filter_search/document_weight_or_filter_search_test.cpp new file mode 100644 index 00000000000..22f928ddf45 --- /dev/null +++ b/searchlib/src/tests/attribute/document_weight_or_filter_search/document_weight_or_filter_search_test.cpp @@ -0,0 +1,261 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/searchlib/attribute/i_document_weight_attribute.h> +#include <vespa/searchlib/attribute/document_weight_or_filter_search.h> +#include <vespa/searchlib/queryeval/searchiterator.h> +#include <vespa/searchlib/common/bitvector.h> +#define ENABLE_GTEST_MIGRATION +#include <vespa/searchlib/test/searchiteratorverifier.h> + +using PostingList = search::attribute::PostingListTraits<int32_t>::PostingStoreBase; +using Iterator = search::attribute::PostingListTraits<int32_t>::const_iterator; +using KeyData = PostingList::KeyDataType; +using search::BitVector; +using search::attribute::DocumentWeightOrFilterSearch; +using search::queryeval::SearchIterator; +using vespalib::datastore::EntryRef; + +class DocumentWeightOrFilterSearchTest : public ::testing::Test { + PostingList _postings; + vespalib::GenerationHandler _gens; + std::vector<EntryRef> _trees; + uint32_t _range_start; + uint32_t _range_end; +public: + DocumentWeightOrFilterSearchTest(); + ~DocumentWeightOrFilterSearchTest(); + void inc_generation(); + size_t num_trees() const { return _trees.size(); } + Iterator get_tree(size_t idx) const { + if (idx < _trees.size()) { + return _postings.beginFrozen(_trees[idx]); + } else { + return Iterator(); + } + } + void ensure_tree(size_t idx) { + if (idx <= _trees.size()) { + _trees.resize(idx + 1); + } + } + void add_tree(size_t idx, std::vector<uint32_t> keys) { + ensure_tree(idx); + std::vector<KeyData> adds; + std::vector<uint32_t> removes; + adds.reserve(keys.size()); + for (auto& key : keys) { + adds.emplace_back(KeyData(key, 1)); + } + _postings.apply(_trees[idx], &*adds.begin(), &*adds.end(), &*removes.begin(), &*removes.end()); + } + + void clear_tree(size_t idx) { + if (idx < _trees.size()) { + _postings.clear(_trees[idx]); + _trees[idx] = EntryRef(); + } + } + + std::unique_ptr<SearchIterator> make_iterator() const { + std::vector<Iterator> iterators; + for (size_t i = 0; i < num_trees(); ++i) { + iterators.emplace_back(get_tree(i)); + } + auto result = DocumentWeightOrFilterSearch::create(std::move(iterators)); + result->initRange(_range_start, _range_end); + return result; + }; + + std::vector<uint32_t> eval_daat(SearchIterator &iterator) { + std::vector<uint32_t> result; + uint32_t doc_id = _range_start; + while (doc_id < _range_end) { + if (iterator.seek(doc_id)) { + result.emplace_back(doc_id); + ++doc_id; + } else { + doc_id = std::max(doc_id + 1, iterator.getDocId()); + } + } + return result; + } + + std::vector<uint32_t> frombv(const BitVector &bv) { + std::vector<uint32_t> result; + uint32_t doc_id = _range_start; + doc_id = bv.getNextTrueBit(doc_id); + while (doc_id < _range_end) { + result.emplace_back(doc_id); + ++doc_id; + doc_id = bv.getNextTrueBit(doc_id); + } + return result; + } + + std::unique_ptr<BitVector> tobv(std::vector<uint32_t> values) { + auto bv = BitVector::create(_range_start, _range_end); + for (auto value : values) { + bv->setBit(value); + } + bv->invalidateCachedCount(); + return bv; + } + + void expect_result(std::vector<uint32_t> exp, std::vector<uint32_t> act) + { + EXPECT_EQ(exp, act); + } + + void make_sample_data() { + add_tree(0, { 10, 11 }); + add_tree(1, { 14, 17, 20 }); + add_tree(2, { 3 }); + add_tree(3, { 17 }); + } + + uint32_t get_range_start() const { return _range_start; } + void set_range(uint32_t start, uint32_t end) { + _range_start = start; + _range_end = end; + } +}; + +DocumentWeightOrFilterSearchTest::DocumentWeightOrFilterSearchTest() + : _postings(true), + _gens(), + _range_start(1), + _range_end(10000) +{ +} + +DocumentWeightOrFilterSearchTest::~DocumentWeightOrFilterSearchTest() +{ + for (auto& tree : _trees) { + _postings.clear(tree); + } + _postings.clearBuilder(); + _postings.clearHoldLists(); + inc_generation(); +} + +void +DocumentWeightOrFilterSearchTest::inc_generation() +{ + _postings.freeze(); + _postings.transferHoldLists(_gens.getCurrentGeneration()); + _gens.incGeneration(); + _gens.updateFirstUsedGeneration(); + _postings.trimHoldLists(_gens.getFirstUsedGeneration()); +} + +TEST_F(DocumentWeightOrFilterSearchTest, daat_or) +{ + make_sample_data(); + expect_result(eval_daat(*make_iterator()), { 3, 10, 11, 14, 17, 20 }); +} + +TEST_F(DocumentWeightOrFilterSearchTest, taat_get_hits) +{ + make_sample_data(); + expect_result(frombv(*make_iterator()->get_hits(get_range_start())), { 3, 10, 11, 14, 17, 20 }); +} + +TEST_F(DocumentWeightOrFilterSearchTest, taat_or_hits_into) +{ + make_sample_data(); + auto bv = tobv({13, 14}); + make_iterator()->or_hits_into(*bv, get_range_start()); + expect_result(frombv(*bv), { 3, 10, 11, 13, 14, 17, 20 }); +} + +TEST_F(DocumentWeightOrFilterSearchTest, taat_and_hits_into) +{ + make_sample_data(); + auto bv = tobv({13, 14}); + make_iterator()->and_hits_into(*bv, get_range_start()); + expect_result(frombv(*bv), { 14 }); +} + +TEST_F(DocumentWeightOrFilterSearchTest, daat_or_ranged) +{ + make_sample_data(); + set_range(4, 15); + expect_result(eval_daat(*make_iterator()), {10, 11, 14 }); +} + +TEST_F(DocumentWeightOrFilterSearchTest, taat_get_hits_ranged) +{ + make_sample_data(); + set_range(4, 15); + expect_result(frombv(*make_iterator()->get_hits(get_range_start())), { 10, 11, 14 }); +} + +TEST_F(DocumentWeightOrFilterSearchTest, taat_or_hits_into_ranged) +{ + make_sample_data(); + set_range(4, 15); + auto bv = tobv({13, 14}); + make_iterator()->or_hits_into(*bv, get_range_start()); + expect_result(frombv(*bv), { 10, 11, 13, 14 }); +} + +TEST_F(DocumentWeightOrFilterSearchTest, taat_and_hits_into_ranged) +{ + make_sample_data(); + set_range(4, 15); + auto bv = tobv({13, 14}); + make_iterator()->and_hits_into(*bv, get_range_start()); + expect_result(frombv(*bv), { 14 }); +} + +namespace { + +class Verifier : public search::test::SearchIteratorVerifier { + DocumentWeightOrFilterSearchTest &_test; +public: + Verifier(DocumentWeightOrFilterSearchTest &test, int num_trees) + : _test(test) + { + std::vector<std::vector<uint32_t>> trees(num_trees); + uint32_t tree_id = 0; + for (const auto doc_id : getExpectedDocIds()) { + trees[tree_id++ % trees.size()].emplace_back(doc_id); + } + tree_id = 0; + for (const auto &tree : trees) { + _test.add_tree(tree_id++, tree); + } + _test.inc_generation(); + } + ~Verifier() { + for (uint32_t tree_id = 0; tree_id < _test.num_trees(); ++tree_id) { + _test.clear_tree(tree_id); + } + _test.inc_generation(); + } + std::unique_ptr<SearchIterator> create(bool) const override { + return _test.make_iterator(); + } + +}; + +TEST_F(DocumentWeightOrFilterSearchTest, iterator_conformance) +{ + { + Verifier verifier(*this, 1); + verifier.verify(); + } + { + Verifier verifier(*this, 2); + verifier.verify(); + } + { + Verifier verifier(*this, 3); + verifier.verify(); + } +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/attribute/enumstore/CMakeLists.txt b/searchlib/src/tests/attribute/enumstore/CMakeLists.txt index e8f6068cebe..b9a361d05fc 100644 --- a/searchlib/src/tests/attribute/enumstore/CMakeLists.txt +++ b/searchlib/src/tests/attribute/enumstore/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_enumstore_test_app TEST SOURCES enumstore_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_enumstore_test_app COMMAND searchlib_enumstore_test_app) diff --git a/searchlib/src/tests/attribute/multi_value_mapping/CMakeLists.txt b/searchlib/src/tests/attribute/multi_value_mapping/CMakeLists.txt index 6b1f5f3d656..53856bf7b88 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/CMakeLists.txt +++ b/searchlib/src/tests/attribute/multi_value_mapping/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_multi_value_mapping_test_app TEST SOURCES multi_value_mapping_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_multi_value_mapping_test_app COMMAND searchlib_multi_value_mapping_test_app) diff --git a/searchlib/src/tests/attribute/reference_attribute/CMakeLists.txt b/searchlib/src/tests/attribute/reference_attribute/CMakeLists.txt index 6638bf886b7..9473f9f7fbf 100644 --- a/searchlib/src/tests/attribute/reference_attribute/CMakeLists.txt +++ b/searchlib/src/tests/attribute/reference_attribute/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_reference_attribute_test_app TEST SOURCES reference_attribute_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_reference_attribute_test_app COMMAND searchlib_reference_attribute_test_app) diff --git a/searchlib/src/tests/attribute/save_target/CMakeLists.txt b/searchlib/src/tests/attribute/save_target/CMakeLists.txt index e127f66579e..dac326207da 100644 --- a/searchlib/src/tests/attribute/save_target/CMakeLists.txt +++ b/searchlib/src/tests/attribute/save_target/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_attribute_save_target_test_app TEST SOURCES attribute_save_target_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_attribute_save_target_test_app COMMAND searchlib_attribute_save_target_test_app) diff --git a/searchlib/src/tests/attribute/searchable/CMakeLists.txt b/searchlib/src/tests/attribute/searchable/CMakeLists.txt index e754253c34a..80c23e299f4 100644 --- a/searchlib/src/tests/attribute/searchable/CMakeLists.txt +++ b/searchlib/src/tests/attribute/searchable/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_attribute_searchable_adapter_test_app TEST SOURCES attribute_searchable_adapter_test.cpp @@ -19,6 +20,6 @@ vespa_add_executable(searchlib_attribute_blueprint_test_app TEST attributeblueprint_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_attribute_blueprint_test_app COMMAND searchlib_attribute_blueprint_test_app) diff --git a/searchlib/src/tests/attribute/searchcontextelementiterator/CMakeLists.txt b/searchlib/src/tests/attribute/searchcontextelementiterator/CMakeLists.txt index 1a27b446d30..6810d145124 100644 --- a/searchlib/src/tests/attribute/searchcontextelementiterator/CMakeLists.txt +++ b/searchlib/src/tests/attribute/searchcontextelementiterator/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_attribute_searchcontextelementiterator_test_app TEST SOURCES searchcontextelementiterator_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_attribute_searchcontextelementiterator_test_app COMMAND searchlib_attribute_searchcontextelementiterator_test_app) diff --git a/searchlib/src/tests/common/matching_elements/CMakeLists.txt b/searchlib/src/tests/common/matching_elements/CMakeLists.txt index cd1d3560c15..ef7aa316dfd 100644 --- a/searchlib/src/tests/common/matching_elements/CMakeLists.txt +++ b/searchlib/src/tests/common/matching_elements/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_common_matching_elements_test_app TEST SOURCES matching_elements_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_common_matching_elements_test_app COMMAND searchlib_common_matching_elements_test_app) diff --git a/searchlib/src/tests/common/matching_elements_fields/CMakeLists.txt b/searchlib/src/tests/common/matching_elements_fields/CMakeLists.txt index 3d6f338315a..f5a61787328 100644 --- a/searchlib/src/tests/common/matching_elements_fields/CMakeLists.txt +++ b/searchlib/src/tests/common/matching_elements_fields/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_common_matching_elements_fields_test_app TEST SOURCES matching_elements_fields_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_common_matching_elements_fields_test_app COMMAND searchlib_common_matching_elements_fields_test_app) diff --git a/searchlib/src/tests/engine/proto_converter/CMakeLists.txt b/searchlib/src/tests/engine/proto_converter/CMakeLists.txt index 718e8120c2c..82e8e1b8be7 100644 --- a/searchlib/src/tests/engine/proto_converter/CMakeLists.txt +++ b/searchlib/src/tests/engine/proto_converter/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_engine_proto_converter_test_app TEST SOURCES proto_converter_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_engine_proto_converter_test_app COMMAND searchlib_engine_proto_converter_test_app) diff --git a/searchlib/src/tests/engine/proto_rpc_adapter/CMakeLists.txt b/searchlib/src/tests/engine/proto_rpc_adapter/CMakeLists.txt index 3fa638f16e5..7d7259c40d8 100644 --- a/searchlib/src/tests/engine/proto_rpc_adapter/CMakeLists.txt +++ b/searchlib/src/tests/engine/proto_rpc_adapter/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_engine_proto_rpc_adapter_test_app TEST SOURCES proto_rpc_adapter_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_engine_proto_rpc_adapter_test_app COMMAND searchlib_engine_proto_rpc_adapter_test_app) diff --git a/searchlib/src/tests/features/bm25/CMakeLists.txt b/searchlib/src/tests/features/bm25/CMakeLists.txt index 3f9b92684f8..35ad629169f 100644 --- a/searchlib/src/tests/features/bm25/CMakeLists.txt +++ b/searchlib/src/tests/features/bm25/CMakeLists.txt @@ -1,11 +1,12 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_features_bm25_test_app TEST SOURCES bm25_test.cpp DEPENDS searchlib searchlib_test - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_features_bm25_test_app COMMAND searchlib_features_bm25_test_app) diff --git a/searchlib/src/tests/index/field_length_calculator/CMakeLists.txt b/searchlib/src/tests/index/field_length_calculator/CMakeLists.txt index df09d0abaa7..e35927662e6 100644 --- a/searchlib/src/tests/index/field_length_calculator/CMakeLists.txt +++ b/searchlib/src/tests/index/field_length_calculator/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_field_length_calculator_test_app TEST SOURCES field_length_calculator_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_field_length_calculator_test_app COMMAND searchlib_field_length_calculator_test_app) diff --git a/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt index 754ff796690..71e13db15fa 100644 --- a/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_compact_words_store_test_app TEST SOURCES compact_words_store_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_compact_words_store_test_app COMMAND searchlib_compact_words_store_test_app) diff --git a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt index be1a193cd3c..770118a65b4 100644 --- a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_feature_store_test_app TEST SOURCES feature_store_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_feature_store_test_app COMMAND searchlib_feature_store_test_app) vespa_add_executable(searchlib_word_store_test_app TEST @@ -12,6 +13,6 @@ vespa_add_executable(searchlib_word_store_test_app TEST word_store_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_word_store_test_app COMMAND searchlib_word_store_test_app) diff --git a/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt index ecf33ee48fd..0831e87252f 100644 --- a/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_document_inverter_test_app TEST SOURCES document_inverter_test.cpp DEPENDS searchlib_test searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_document_inverter_test_app COMMAND searchlib_document_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt index a09d6baf1a5..2568249c2df 100644 --- a/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt @@ -1,11 +1,12 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_field_index_test_app TEST SOURCES field_index_test.cpp DEPENDS searchlib searchlib_test - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_field_index_test_app COMMAND searchlib_field_index_test_app) diff --git a/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt index f18b4ba29cd..bc3a62c4f26 100644 --- a/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_field_index_remover_test_app TEST SOURCES field_index_remover_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_field_index_remover_test_app COMMAND searchlib_field_index_remover_test_app) diff --git a/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt index 6fefada6570..b670e0ccf57 100644 --- a/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_field_inverter_test_app TEST SOURCES field_inverter_test.cpp DEPENDS searchlib_test searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_field_inverter_test_app COMMAND searchlib_field_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt index 0dd42eacf30..611aa6e58c6 100644 --- a/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_memory_index_test_app TEST SOURCES memory_index_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_memory_index_test_app COMMAND searchlib_memory_index_test_app) diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt index db9418b7190..0378d209d5e 100644 --- a/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_url_field_inverter_test_app TEST SOURCES url_field_inverter_test.cpp DEPENDS searchlib_test searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_url_field_inverter_test_app COMMAND searchlib_url_field_inverter_test_app) diff --git a/searchlib/src/tests/postinglistbm/CMakeLists.txt b/searchlib/src/tests/postinglistbm/CMakeLists.txt index 6e90f44726a..004cff76efd 100644 --- a/searchlib/src/tests/postinglistbm/CMakeLists.txt +++ b/searchlib/src/tests/postinglistbm/CMakeLists.txt @@ -1,11 +1,12 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_posting_list_test_app TEST SOURCES posting_list_test.cpp DEPENDS searchlib_test searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_posting_list_test_app NO_VALGRIND COMMAND searchlib_posting_list_test_app) diff --git a/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/.cvsignore b/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/.cvsignore deleted file mode 100644 index 9e6565f9d16..00000000000 --- a/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/.cvsignore +++ /dev/null @@ -1,3 +0,0 @@ -.depend -Makefile -booleanmatchiteratorwrapper_test diff --git a/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/.gitignore b/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/.gitignore deleted file mode 100644 index b568b87514a..00000000000 --- a/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -booleanmatchiteratorwrapper_test -searchlib_booleanmatchiteratorwrapper_test_app diff --git a/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/CMakeLists.txt b/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/CMakeLists.txt deleted file mode 100644 index d97fe5b12d7..00000000000 --- a/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_booleanmatchiteratorwrapper_test_app TEST - SOURCES - booleanmatchiteratorwrapper_test.cpp - DEPENDS - searchlib - searchlib_test -) -vespa_add_test(NAME searchlib_booleanmatchiteratorwrapper_test_app COMMAND searchlib_booleanmatchiteratorwrapper_test_app) diff --git a/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/booleanmatchiteratorwrapper_test.cpp b/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/booleanmatchiteratorwrapper_test.cpp deleted file mode 100644 index 7d4551a5c7d..00000000000 --- a/searchlib/src/tests/queryeval/booleanmatchiteratorwrapper/booleanmatchiteratorwrapper_test.cpp +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> -#include <vespa/searchlib/fef/termfieldmatchdata.h> -#include <vespa/searchlib/common/bitvectoriterator.h> -#include <vespa/searchlib/test/searchiteratorverifier.h> - -using namespace search::fef; -using namespace search::queryeval; -using search::BitVector; -using search::BitVectorIterator; - -struct DummyItr : public SearchIterator { - static uint32_t seekCnt; - static uint32_t unpackCnt; - static uint32_t dtorCnt; - static uint32_t _unpackedDocId; - TermFieldMatchData *match; - - DummyItr(TermFieldMatchData *m) { - match = m; - } - - ~DummyItr() { - ++dtorCnt; - } - - void doSeek(uint32_t docid) override { - ++seekCnt; - if (docid <= 10) { - setDocId(10); - } else if (docid <= 20) { - setDocId(20); - } else { - setAtEnd(); - } - } - - void doUnpack(uint32_t docid) override { - ++unpackCnt; - if (match != 0) { - _unpackedDocId = docid; - } - } -}; -uint32_t DummyItr::seekCnt = 0; -uint32_t DummyItr::unpackCnt = 0; -uint32_t DummyItr::dtorCnt = 0; -uint32_t DummyItr::_unpackedDocId = 0; - - -TEST("mostly everything") { - EXPECT_EQUAL(DummyItr::seekCnt, 0u); - EXPECT_EQUAL(DummyItr::unpackCnt, 0u); - EXPECT_EQUAL(DummyItr::dtorCnt, 0u); - { // without wrapper - TermFieldMatchData match; - DummyItr::_unpackedDocId = 0; - SearchIterator::UP search(new DummyItr(&match)); - search->initFullRange(); - EXPECT_EQUAL(DummyItr::_unpackedDocId, 0u); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_EQUAL(search->getDocId(), 10u); - EXPECT_TRUE(search->seek(10)); - search->unpack(10); - EXPECT_EQUAL(DummyItr::_unpackedDocId, 10u); - EXPECT_TRUE(!search->seek(15)); - EXPECT_EQUAL(search->getDocId(), 20u); - EXPECT_TRUE(search->seek(20)); - search->unpack(20); - EXPECT_EQUAL(DummyItr::_unpackedDocId, 20u); - EXPECT_TRUE(!search->seek(25)); - EXPECT_TRUE(search->isAtEnd()); - } - EXPECT_EQUAL(DummyItr::seekCnt, 3u); - EXPECT_EQUAL(DummyItr::unpackCnt, 2u); - EXPECT_EQUAL(DummyItr::dtorCnt, 1u); - { // with wrapper - TermFieldMatchData match; - TermFieldMatchDataArray tfmda; - tfmda.add(&match); - DummyItr::_unpackedDocId = 0; - SearchIterator::UP search(new BooleanMatchIteratorWrapper(SearchIterator::UP(new DummyItr(&match)), tfmda)); - search->initFullRange(); - EXPECT_EQUAL(DummyItr::_unpackedDocId, 0u); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_EQUAL(search->getDocId(), 10u); - EXPECT_TRUE(search->seek(10)); - search->unpack(10); - EXPECT_EQUAL(DummyItr::_unpackedDocId, 0u); - EXPECT_TRUE(!search->seek(15)); - EXPECT_EQUAL(search->getDocId(), 20u); - EXPECT_TRUE(search->seek(20)); - search->unpack(20); - EXPECT_EQUAL(DummyItr::_unpackedDocId, 0u); - EXPECT_TRUE(!search->seek(25)); - EXPECT_TRUE(search->isAtEnd()); - } - EXPECT_EQUAL(DummyItr::seekCnt, 6u); - EXPECT_EQUAL(DummyItr::unpackCnt, 2u); - EXPECT_EQUAL(DummyItr::dtorCnt, 2u); - { // with wrapper, without match data - SearchIterator::UP search(new BooleanMatchIteratorWrapper(SearchIterator::UP(new DummyItr(0)), TermFieldMatchDataArray())); - search->initFullRange(); - EXPECT_TRUE(!search->seek(1u)); - EXPECT_EQUAL(search->getDocId(), 10u); - EXPECT_TRUE(search->seek(10)); - search->unpack(10); - EXPECT_TRUE(!search->seek(15)); - EXPECT_EQUAL(search->getDocId(), 20u); - EXPECT_TRUE(search->seek(20)); - search->unpack(20); - EXPECT_TRUE(!search->seek(25)); - EXPECT_TRUE(search->isAtEnd()); - } - EXPECT_EQUAL(DummyItr::seekCnt, 9u); - EXPECT_EQUAL(DummyItr::unpackCnt, 2u); - EXPECT_EQUAL(DummyItr::dtorCnt, 3u); -} - -class Verifier : public search::test::SearchIteratorVerifier { -public: - ~Verifier(); - SearchIterator::UP create(bool strict) const override { - return std::make_unique<BooleanMatchIteratorWrapper>(createIterator(getExpectedDocIds(), strict), _tfmda);; - } -private: - mutable TermFieldMatchDataArray _tfmda; -}; - -Verifier::~Verifier() {} - -TEST("Test that boolean wrapper iterators adheres to SearchIterator requirements") { - Verifier searchIteratorVerifier; - searchIteratorVerifier.verify(); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/queryeval/fake_searchable/CMakeLists.txt b/searchlib/src/tests/queryeval/fake_searchable/CMakeLists.txt index c98306aa179..d21524d158a 100644 --- a/searchlib/src/tests/queryeval/fake_searchable/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/fake_searchable/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_fake_searchable_test_app TEST SOURCES fake_searchable_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_fake_searchable_test_app COMMAND searchlib_fake_searchable_test_app) diff --git a/searchlib/src/tests/queryeval/wrappers/CMakeLists.txt b/searchlib/src/tests/queryeval/wrappers/CMakeLists.txt new file mode 100644 index 00000000000..d1d566b1cab --- /dev/null +++ b/searchlib/src/tests/queryeval/wrappers/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(searchlib_wrappers_test_app TEST + SOURCES + wrappers_test.cpp + DEPENDS + searchlib + searchlib_test + gtest +) +vespa_add_test(NAME searchlib_wrappers_test_app COMMAND searchlib_wrappers_test_app) diff --git a/searchlib/src/tests/queryeval/wrappers/wrappers_test.cpp b/searchlib/src/tests/queryeval/wrappers/wrappers_test.cpp new file mode 100644 index 00000000000..62bff93ab0e --- /dev/null +++ b/searchlib/src/tests/queryeval/wrappers/wrappers_test.cpp @@ -0,0 +1,197 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/queryeval/filter_wrapper.h> +#include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/vespalib/gtest/gtest.h> + +#define ENABLE_GTEST_MIGRATION +#include <vespa/searchlib/test/searchiteratorverifier.h> + +using namespace search::fef; +using namespace search::queryeval; + +struct ObservedData { + uint32_t seekCnt; + uint32_t unpackCnt; + uint32_t dtorCnt; + uint32_t unpackedDocId; +}; + +class WrapperTest : public ::testing::Test { +public: + class DummyItr : public SearchIterator { + private: + ObservedData &_data; + TermFieldMatchData *_match; + public: + DummyItr(ObservedData &data, TermFieldMatchData *m) : _data(data), _match(m) {} + ~DummyItr() { + ++_data.dtorCnt; + } + void doSeek(uint32_t docid) override { + ++_data.seekCnt; + if (docid <= 10) { + setDocId(10); + } else if (docid <= 20) { + setDocId(20); + } else { + setAtEnd(); + } + } + void doUnpack(uint32_t docid) override { + ++_data.unpackCnt; + if (_match != 0) { + _data.unpackedDocId = docid; + } + } + }; + WrapperTest() : _data{0,0,0,0} {} +protected: + ObservedData _data; + + void verify_unwrapped() { + EXPECT_EQ(_data.seekCnt, 0u); + EXPECT_EQ(_data.unpackCnt, 0u); + EXPECT_EQ(_data.dtorCnt, 0u); + + // without wrapper + TermFieldMatchData match; + _data.unpackedDocId = 0; + auto search = std::make_unique<DummyItr>(_data, &match); + search->initFullRange(); + EXPECT_EQ(_data.unpackedDocId, 0u); + EXPECT_TRUE(!search->seek(1u)); + EXPECT_EQ(search->getDocId(), 10u); + EXPECT_TRUE(search->seek(10)); + search->unpack(10); + EXPECT_EQ(_data.unpackedDocId, 10u); + EXPECT_TRUE(!search->seek(15)); + EXPECT_EQ(search->getDocId(), 20u); + EXPECT_TRUE(search->seek(20)); + search->unpack(20); + EXPECT_EQ(_data.unpackedDocId, 20u); + EXPECT_TRUE(!search->seek(25)); + EXPECT_TRUE(search->isAtEnd()); + + search.reset(nullptr); + EXPECT_EQ(_data.seekCnt, 3u); + EXPECT_EQ(_data.unpackCnt, 2u); + EXPECT_EQ(_data.dtorCnt, 1u); + } +}; + +TEST_F(WrapperTest, filter_wrapper) +{ + verify_unwrapped(); + + // with FilterWrapper + TermFieldMatchData match; + TermFieldMatchDataArray tfmda; + tfmda.add(&match); + _data.unpackedDocId = 0; + auto search = std::make_unique<FilterWrapper>(1); +search->wrap(std::make_unique<DummyItr>(_data, search->tfmda()[0])); + search->initFullRange(); + EXPECT_EQ(_data.unpackedDocId, 0u); + EXPECT_TRUE(!search->seek(1u)); + EXPECT_EQ(search->getDocId(), 10u); + EXPECT_TRUE(search->seek(10)); + search->unpack(10); + EXPECT_EQ(_data.unpackedDocId, 0u); + EXPECT_TRUE(!search->seek(15)); + EXPECT_EQ(search->getDocId(), 20u); + EXPECT_TRUE(search->seek(20)); + search->unpack(20); + EXPECT_EQ(_data.unpackedDocId, 0u); + EXPECT_TRUE(!search->seek(25)); + EXPECT_TRUE(search->isAtEnd()); + + search.reset(nullptr); + EXPECT_EQ(_data.seekCnt, 6u); + EXPECT_EQ(_data.unpackCnt, 2u); + EXPECT_EQ(_data.dtorCnt, 2u); +} + +TEST_F(WrapperTest, boolean_match_iterator_wrapper) +{ + verify_unwrapped(); + { // with wrapper + TermFieldMatchData match; + TermFieldMatchDataArray tfmda; + tfmda.add(&match); + _data.unpackedDocId = 0; + auto to_wrap = std::make_unique<DummyItr>(_data, &match); + auto search = std::make_unique<BooleanMatchIteratorWrapper>(std::move(to_wrap), tfmda); + search->initFullRange(); + EXPECT_EQ(_data.unpackedDocId, 0u); + EXPECT_TRUE(!search->seek(1u)); + EXPECT_EQ(search->getDocId(), 10u); + EXPECT_TRUE(search->seek(10)); + search->unpack(10); + EXPECT_EQ(_data.unpackedDocId, 0u); + EXPECT_TRUE(!search->seek(15)); + EXPECT_EQ(search->getDocId(), 20u); + EXPECT_TRUE(search->seek(20)); + search->unpack(20); + EXPECT_EQ(_data.unpackedDocId, 0u); + EXPECT_TRUE(!search->seek(25)); + EXPECT_TRUE(search->isAtEnd()); + } + EXPECT_EQ(_data.seekCnt, 6u); + EXPECT_EQ(_data.unpackCnt, 2u); + EXPECT_EQ(_data.dtorCnt, 2u); + { // with wrapper, without match data + + auto to_wrap = std::make_unique<DummyItr>(_data, nullptr); + auto search = std::make_unique<BooleanMatchIteratorWrapper>(std::move(to_wrap), TermFieldMatchDataArray()); + search->initFullRange(); + EXPECT_TRUE(!search->seek(1u)); + EXPECT_EQ(search->getDocId(), 10u); + EXPECT_TRUE(search->seek(10)); + search->unpack(10); + EXPECT_TRUE(!search->seek(15)); + EXPECT_EQ(search->getDocId(), 20u); + EXPECT_TRUE(search->seek(20)); + search->unpack(20); + EXPECT_TRUE(!search->seek(25)); + EXPECT_TRUE(search->isAtEnd()); + } + EXPECT_EQ(_data.seekCnt, 9u); + EXPECT_EQ(_data.unpackCnt, 2u); + EXPECT_EQ(_data.dtorCnt, 3u); +} + +class FilterWrapperVerifier : public search::test::SearchIteratorVerifier { +public: + ~FilterWrapperVerifier() {} + SearchIterator::UP create(bool strict) const override { + auto search = std::make_unique<FilterWrapper>(1); + search->wrap(createIterator(getExpectedDocIds(), strict)); + return search; + } +}; + +TEST(FilterWrapperTest, adheres_to_search_iterator_requirements) +{ + FilterWrapperVerifier verifier; + verifier.verify(); +} + +class BooleanMatchIteratorWrapperVerifier : public search::test::SearchIteratorVerifier { +public: + SearchIterator::UP create(bool strict) const override { + return std::make_unique<BooleanMatchIteratorWrapper>(createIterator(getExpectedDocIds(), strict), _tfmda); + } + ~BooleanMatchIteratorWrapperVerifier() {} +private: + mutable TermFieldMatchDataArray _tfmda; +}; + +TEST(BooleanMatchIteratorWrapperWrapperTest, adheres_to_search_iterator_requirements) +{ + BooleanMatchIteratorWrapperVerifier verifier; + verifier.verify(); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/tensor/distance_functions/CMakeLists.txt b/searchlib/src/tests/tensor/distance_functions/CMakeLists.txt index 9356658f90e..6f7c71d4d54 100644 --- a/searchlib/src/tests/tensor/distance_functions/CMakeLists.txt +++ b/searchlib/src/tests/tensor/distance_functions/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_distance_functions_test_app TEST SOURCES distance_functions_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_distance_functions_test_app COMMAND searchlib_distance_functions_test_app) diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index 59532919347..082d04f104b 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -31,12 +31,11 @@ void verify_geo_miles(const DistanceFunction *dist_fun, } -TEST(DistanceFunctionsTest, gives_expected_score) +TEST(DistanceFunctionsTest, euclidean_gives_expected_score) { auto ct = vespalib::eval::ValueType::CellType::DOUBLE; auto euclid = make_distance_function(DistanceMetric::Euclidean, ct); - auto angular = make_distance_function(DistanceMetric::Angular, ct); std::vector<double> p0{0.0, 0.0, 0.0}; std::vector<double> p1{1.0, 0.0, 0.0}; @@ -44,33 +43,103 @@ TEST(DistanceFunctionsTest, gives_expected_score) std::vector<double> p3{0.0, 0.0, 1.0}; std::vector<double> p4{0.5, 0.5, 0.707107}; std::vector<double> p5{0.0,-1.0, 0.0}; + std::vector<double> p6{1.0, 2.0, 2.0}; double n4 = euclid->calc(t(p0), t(p4)); - EXPECT_GT(n4, 0.99999); - EXPECT_LT(n4, 1.00001); + EXPECT_FLOAT_EQ(n4, 1.0); double d12 = euclid->calc(t(p1), t(p2)); EXPECT_EQ(d12, 2.0); + EXPECT_DOUBLE_EQ(euclid->to_rawscore(d12), 1.0/(1.0 + sqrt(2.0))); +} + +TEST(DistanceFunctionsTest, angular_gives_expected_score) +{ + auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + auto angular = make_distance_function(DistanceMetric::Angular, ct); + + std::vector<double> p0{0.0, 0.0, 0.0}; + std::vector<double> p1{1.0, 0.0, 0.0}; + std::vector<double> p2{0.0, 1.0, 0.0}; + std::vector<double> p3{0.0, 0.0, 1.0}; + std::vector<double> p4{0.5, 0.5, 0.707107}; + std::vector<double> p5{0.0,-1.0, 0.0}; + std::vector<double> p6{1.0, 2.0, 2.0}; + + constexpr double pi = 3.14159265358979323846; double a12 = angular->calc(t(p1), t(p2)); double a13 = angular->calc(t(p1), t(p3)); double a23 = angular->calc(t(p2), t(p3)); - EXPECT_EQ(a12, 1.0); - EXPECT_EQ(a13, 1.0); - EXPECT_EQ(a23, 1.0); + EXPECT_DOUBLE_EQ(a12, 1.0); + EXPECT_DOUBLE_EQ(a13, 1.0); + EXPECT_DOUBLE_EQ(a23, 1.0); + EXPECT_FLOAT_EQ(angular->to_rawscore(a12), 1.0/(1.0 + pi/2)); + double a14 = angular->calc(t(p1), t(p4)); double a24 = angular->calc(t(p2), t(p4)); - EXPECT_EQ(a14, 0.5); - EXPECT_EQ(a24, 0.5); + EXPECT_FLOAT_EQ(a14, 0.5); + EXPECT_FLOAT_EQ(a24, 0.5); + EXPECT_FLOAT_EQ(angular->to_rawscore(a14), 1.0/(1.0 + pi/3)); + double a34 = angular->calc(t(p3), t(p4)); - EXPECT_GT(a34, 0.999999 - 0.707107); - EXPECT_LT(a34, 1.000001 - 0.707107); + EXPECT_FLOAT_EQ(a34, (1.0 - 0.707107)); + EXPECT_FLOAT_EQ(angular->to_rawscore(a34), 1.0/(1.0 + pi/4)); double a25 = angular->calc(t(p2), t(p5)); - EXPECT_EQ(a25, 2.0); + EXPECT_DOUBLE_EQ(a25, 2.0); + EXPECT_FLOAT_EQ(angular->to_rawscore(a25), 1.0/(1.0 + pi)); double a44 = angular->calc(t(p4), t(p4)); EXPECT_GE(a44, 0.0); EXPECT_LT(a44, 0.000001); + EXPECT_FLOAT_EQ(angular->to_rawscore(a44), 1.0); + + double a66 = angular->calc(t(p6), t(p6)); + EXPECT_GE(a66, 0.0); + EXPECT_LT(a66, 0.000001); + EXPECT_FLOAT_EQ(angular->to_rawscore(a66), 1.0); + + double a16 = angular->calc(t(p1), t(p6)); + double a26 = angular->calc(t(p2), t(p6)); + double a36 = angular->calc(t(p3), t(p6)); + EXPECT_FLOAT_EQ(a16, 1.0 - (1.0/3.0)); + EXPECT_FLOAT_EQ(a26, 1.0 - (2.0/3.0)); + EXPECT_FLOAT_EQ(a36, 1.0 - (2.0/3.0)); +} + +TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) +{ + auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + + auto innerproduct = make_distance_function(DistanceMetric::InnerProduct, ct); + + std::vector<double> p0{0.0, 0.0, 0.0}; + std::vector<double> p1{1.0, 0.0, 0.0}; + std::vector<double> p2{0.0, 1.0, 0.0}; + std::vector<double> p3{0.0, 0.0, 1.0}; + std::vector<double> p4{0.5, 0.5, 0.707107}; + std::vector<double> p5{0.0,-1.0, 0.0}; + std::vector<double> p6{1.0, 2.0, 2.0}; + + double i12 = innerproduct->calc(t(p1), t(p2)); + double i13 = innerproduct->calc(t(p1), t(p3)); + double i23 = innerproduct->calc(t(p2), t(p3)); + EXPECT_DOUBLE_EQ(i12, 1.0); + EXPECT_DOUBLE_EQ(i13, 1.0); + EXPECT_DOUBLE_EQ(i23, 1.0); + double i14 = innerproduct->calc(t(p1), t(p4)); + double i24 = innerproduct->calc(t(p2), t(p4)); + EXPECT_DOUBLE_EQ(i14, 0.5); + EXPECT_DOUBLE_EQ(i24, 0.5); + double i34 = innerproduct->calc(t(p3), t(p4)); + EXPECT_FLOAT_EQ(i34, 1.0 - 0.707107); + + double i25 = innerproduct->calc(t(p2), t(p5)); + EXPECT_DOUBLE_EQ(i25, 2.0); + + double i44 = innerproduct->calc(t(p4), t(p4)); + EXPECT_GE(i44, 0.0); + EXPECT_LT(i44, 0.000001); } TEST(GeoDegreesTest, gives_expected_score) diff --git a/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt index b6a87502fdf..0fd2f9a205a 100644 --- a/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt +++ b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_hnsw_index_test_app TEST SOURCES hnsw_index_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_hnsw_index_test_app COMMAND searchlib_hnsw_index_test_app) @@ -13,5 +14,5 @@ vespa_add_executable(mt_stress_hnsw_app TEST stress_hnsw_mt.cpp DEPENDS searchlib - gtest + GTest::GTest ) diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 7dc0efc106d..cd989c03b4e 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -82,7 +82,7 @@ public: level_generator = generator.get(); index = std::make_unique<HnswIndex>(vectors, std::make_unique<FloatSqEuclideanDistance>(), std::move(generator), - HnswIndex::Config(5, 2, 10, heuristic_select_neighbors)); + HnswIndex::Config(5, 2, 10, 0, heuristic_select_neighbors)); } void add_document(uint32_t docid, uint32_t max_level = 0) { level_generator->level = max_level; diff --git a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp index 4dec9550f6f..1e10d94bb18 100644 --- a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp @@ -234,7 +234,7 @@ public: uint32_t m = 16; index = std::make_unique<HnswIndex>(vectors, std::make_unique<FloatSqEuclideanDistance>(), std::make_unique<InvLogLevelGenerator>(m), - HnswIndex::Config(2*m, m, 200, true)); + HnswIndex::Config(2*m, m, 200, 10, true)); } size_t get_rnd(size_t size) { return rng.nextUniform() * size; diff --git a/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt index 90202e222a7..0dbd80c68d0 100644 --- a/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt +++ b/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_hnsw_save_load_test_app TEST SOURCES hnsw_save_load_test.cpp DEPENDS searchlib - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_hnsw_save_load_test_app COMMAND searchlib_hnsw_save_load_test_app) diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index 6ab70e11fc0..bf56b476cc9 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -34,6 +34,7 @@ vespa_add_library(searchlib_attribute OBJECT defines.cpp diversity.cpp dociditerator.cpp + document_weight_or_filter_search.cpp searchcontextelementiterator.cpp enumattribute.cpp enumattributesaver.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 84bed6bb5e9..c4467df5a1c 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -6,6 +6,7 @@ #include "iterator_pack.h" #include "predicate_attribute.h" #include "attribute_blueprint_params.h" +#include "document_weight_or_filter_search.h" #include <vespa/eval/eval/value.h> #include <vespa/eval/tensor/dense/dense_tensor_view.h> #include <vespa/searchlib/common/location.h> @@ -342,8 +343,22 @@ public: } return SearchType::create(*tfmda[0], _weights, std::move(iterators)); } + + std::unique_ptr<SearchIterator> createFilterSearch(bool strict, FilterConstraint constraint) const override; }; +template <typename SearchType> +std::unique_ptr<SearchIterator> +DirectWeightedSetBlueprint<SearchType>::createFilterSearch(bool, FilterConstraint) const +{ + std::vector<DocumentWeightIterator> iterators; + iterators.reserve(_terms.size()); + for (const IDocumentWeightAttribute::LookupResult &r : _terms) { + _attr.create(r.posting_idx, iterators); + } + return attribute::DocumentWeightOrFilterSearch::create(std::move(iterators)); +} + //----------------------------------------------------------------------------- class DirectWandBlueprint : public queryeval::ComplexLeafBlueprint diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp index e97b0364af8..acf0d3d2fd6 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp @@ -27,6 +27,7 @@ const vespalib::string hnsw_distance_metric = "hnsw.distance_metric"; const vespalib::string euclidean = "euclidean"; const vespalib::string angular = "angular"; const vespalib::string geodegrees = "geodegrees"; +const vespalib::string innerproduct = "innerproduct"; const vespalib::string doc_id_limit_tag = "docIdLimit"; const vespalib::string enumerated_tag = "enumerated"; const vespalib::string unique_value_count_tag = "uniqueValueCount"; @@ -97,6 +98,7 @@ to_string(DistanceMetric metric) case DistanceMetric::Euclidean: return euclidean; case DistanceMetric::Angular: return angular; case DistanceMetric::GeoDegrees: return geodegrees; + case DistanceMetric::InnerProduct: return innerproduct; } throw vespalib::IllegalArgumentException("Unknown distance metric " + std::to_string(static_cast<int>(metric))); } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp index 121cb736471..0959476158f 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp @@ -9,6 +9,8 @@ #include <vespa/vespalib/objects/visit.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/searchlib/queryeval/filter_wrapper.h> +#include <vespa/searchlib/queryeval/orsearch.h> namespace search { @@ -182,6 +184,21 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr } } +queryeval::SearchIterator::UP +AttributeWeightedSetBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + (void) constraint; + std::vector<std::unique_ptr<queryeval::SearchIterator>> children; + children.reserve(_contexts.size()); + for (auto& context : _contexts) { + auto wrapper = std::make_unique<search::queryeval::FilterWrapper>(1); + wrapper->wrap(context->createIterator(wrapper->tfmda()[0], strict)); + children.emplace_back(std::move(wrapper)); + } + search::queryeval::UnpackInfo unpack_info; + return search::queryeval::OrSearch::create(std::move(children), strict, unpack_info); +} + void AttributeWeightedSetBlueprint::fetchPostings(const queryeval::ExecuteInfo &execInfo) { diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h index 6af3405c91d..7ab11aace81 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h @@ -29,6 +29,7 @@ public: ~AttributeWeightedSetBlueprint(); void addToken(std::unique_ptr<ISearchContext> context, int32_t weight); queryeval::SearchIterator::UP createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool strict) const override; + queryeval::SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; void fetchPostings(const queryeval::ExecuteInfo &execInfo) override; void visitMembers(vespalib::ObjectVisitor &visitor) const override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index f435f79bf65..5a8b32ec01b 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -85,6 +85,9 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg) case CfgDm::GEODEGREES: dm = DistanceMetric::GeoDegrees; break; + case CfgDm::INNERPRODUCT: + dm = DistanceMetric::InnerProduct; + break; } retval.set_distance_metric(dm); if (cfg.index.hnsw.enabled) { diff --git a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp new file mode 100644 index 00000000000..d9134417d27 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp @@ -0,0 +1,80 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_weight_or_filter_search.h" +#include "iterator_pack.h" +#include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchlib/queryeval/emptysearch.h> + +namespace search::attribute { + +class DocumentWeightOrFilterSearchImpl : public DocumentWeightOrFilterSearch +{ + AttributeIteratorPack _children; +public: + DocumentWeightOrFilterSearchImpl(AttributeIteratorPack&& children); + ~DocumentWeightOrFilterSearchImpl(); + + void doSeek(uint32_t docId) override; + + void doUnpack(uint32_t) override; + + void initRange(uint32_t begin, uint32_t end) override { + SearchIterator::initRange(begin, end); + _children.initRange(begin, end); + } + + void or_hits_into(BitVector &result, uint32_t begin_id) override { + return _children.or_hits_into(result, begin_id); + } + + void and_hits_into(BitVector &result, uint32_t begin_id) override { + return result.andWith(*get_hits(begin_id)); + } + + std::unique_ptr<BitVector> get_hits(uint32_t begin_id) override { + return _children.get_hits(begin_id, getEndId()); + } + + Trinary is_strict() const override { return Trinary::True; } +}; + +DocumentWeightOrFilterSearchImpl::DocumentWeightOrFilterSearchImpl(AttributeIteratorPack&& children) + : DocumentWeightOrFilterSearch(), + _children(std::move(children)) +{ +} + +DocumentWeightOrFilterSearchImpl::~DocumentWeightOrFilterSearchImpl() = default; + +void +DocumentWeightOrFilterSearchImpl::doSeek(uint32_t docId) +{ + if (_children.get_docid(0) < docId) { + _children.seek(0, docId); + } + uint32_t min_doc_id = _children.get_docid(0); + for (uint16_t i = 1; i < _children.size(); ++i) { + if (_children.get_docid(i) < docId) { + _children.seek(i, docId); + } + min_doc_id = std::min(min_doc_id, _children.get_docid(i)); + } + setDocId(min_doc_id); +} + +void +DocumentWeightOrFilterSearchImpl::doUnpack(uint32_t) +{ +} + +std::unique_ptr<search::queryeval::SearchIterator> +DocumentWeightOrFilterSearch::create(std::vector<DocumentWeightIterator>&& children) +{ + if (children.empty()) { + return std::make_unique<queryeval::EmptySearch>(); + } else { + return std::make_unique<DocumentWeightOrFilterSearchImpl>(AttributeIteratorPack(std::move(children))); + } +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h new file mode 100644 index 00000000000..d9721871b81 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h @@ -0,0 +1,23 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "i_document_weight_attribute.h" +#include <vespa/searchlib/queryeval/searchiterator.h> + +namespace search::attribute { + +/** + * Filter iterator on top of document weight iterators with OR semantics used during + * calculation of global filter for weighted set terms, wand terms and dot product terms. + */ +class DocumentWeightOrFilterSearch : public search::queryeval::SearchIterator +{ +protected: + DocumentWeightOrFilterSearch() + : search::queryeval::SearchIterator() + { + } +public: + static std::unique_ptr<search::queryeval::SearchIterator> create(std::vector<DocumentWeightIterator>&& children); +}; + +} diff --git a/searchlib/src/vespa/searchlib/engine/CMakeLists.txt b/searchlib/src/vespa/searchlib/engine/CMakeLists.txt index 21a5b232ae0..082af18d32b 100644 --- a/searchlib/src/vespa/searchlib/engine/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/engine/CMakeLists.txt @@ -3,6 +3,8 @@ find_package(Protobuf REQUIRED) protobuf_generate_cpp(searchlib_engine_PROTOBUF_SRCS searchlib_engine_PROTOBUF_HDRS ../../../../src/protobuf/search_protocol.proto) +vespa_add_source_target(protobufgen_searchlib_engine DEPENDS ${searchlib_engine_PROTOBUF_SRCS} ${searchlib_engine_PROTOBUF_HDRS}) + # protoc-generated files emit compiler warnings that we normally treat as errors. if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") set_source_files_properties(${searchlib_engine_PROTOBUF_SRCS} diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 1a7a0c5eba2..0b4b5b29a8c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -6,6 +6,8 @@ #include "emptysearch.h" #include "full_search.h" #include "field_spec.hpp" +#include "andsearch.h" +#include "orsearch.h" #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/objects/visit.hpp> #include <vespa/vespalib/objects/objectdumper.h> @@ -132,6 +134,38 @@ Blueprint::createFilterSearch(bool /*strict*/, FilterConstraint constraint) cons } } +namespace { + +template <typename Op> +std::unique_ptr<SearchIterator> +create_op_filter(const std::vector<Blueprint *>& children, bool strict, Blueprint::FilterConstraint constraint) +{ + MultiSearch::Children sub_searches; + sub_searches.reserve(children.size()); + for (size_t i = 0; i < children.size(); ++i) { + bool child_strict = strict && (std::is_same_v<Op,AndSearch> ? (i == 0) : true); + auto search = children[i]->createFilterSearch(child_strict, constraint); + sub_searches.push_back(std::move(search)); + } + UnpackInfo unpack_info; + auto search = Op::create(std::move(sub_searches), strict, unpack_info); + return search; +} + +} + +std::unique_ptr<SearchIterator> +Blueprint::create_and_filter(const std::vector<Blueprint *>& children, bool strict, Blueprint::FilterConstraint constraint) +{ + return create_op_filter<AndSearch>(children, strict, constraint); +} + +std::unique_ptr<SearchIterator> +Blueprint::create_or_filter(const std::vector<Blueprint *>& children, bool strict, Blueprint::FilterConstraint constraint) +{ + return create_op_filter<OrSearch>(children, strict, constraint); +} + vespalib::string Blueprint::asString() const { diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index ef15736073e..1ee58f8b97d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -208,6 +208,9 @@ public: virtual SearchIteratorUP createSearch(fef::MatchData &md, bool strict) const = 0; virtual SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const; + static std::unique_ptr<SearchIterator> create_and_filter(const std::vector<Blueprint *>& children, bool strict, FilterConstraint constraint); + static std::unique_ptr<SearchIterator> create_or_filter(const std::vector<Blueprint *>& children, bool strict, FilterConstraint constraint); + // for debug dumping vespalib::string asString() const; vespalib::slime::Cursor & asSlime(const vespalib::slime::Inserter & cursor) const; @@ -274,6 +277,8 @@ protected: bool should_do_termwise_eval(const UnpackInfo &unpack, double match_limit) const; + const Children& get_children() const { return _children; } + public: typedef std::vector<size_t> IndexList; IntermediateBlueprint(); diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index 3d3a703cd7b..aa65342c114 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -84,18 +84,11 @@ need_normal_features_for_children(const IntermediateBlueprint &blueprint, fef::M /** utility for operators that degrade to AND when creating filter */ SearchIterator::UP createAndFilter(const IntermediateBlueprint &self, + const std::vector<Blueprint *>& children, bool strict, Blueprint::FilterConstraint constraint) { - MultiSearch::Children sub_searches; - sub_searches.reserve(self.childCnt()); - for (size_t i = 0; i < self.childCnt(); ++i) { - bool child_strict = strict && (i == 0); - auto search = self.getChild(i).createFilterSearch(child_strict, constraint); - sub_searches.push_back(std::move(search)); - } - UnpackInfo unpack_info; - auto search = AndSearch::create(std::move(sub_searches), strict, unpack_info); - search->estimate(self.getState().estimate().estHits); + auto search = Blueprint::create_and_filter(children, strict, constraint); + static_cast<AndSearch &>(*search).estimate(self.getState().estimate().estHits); return search; } @@ -292,7 +285,7 @@ AndBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, SearchIterator::UP AndBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const { - return createAndFilter(*this, strict, constraint); + return createAndFilter(*this, get_children(), strict, constraint); } double @@ -492,7 +485,7 @@ SearchIterator::UP NearBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const { if (constraint == Blueprint::FilterConstraint::UPPER_BOUND) { - return createAndFilter(*this, strict, constraint); + return createAndFilter(*this, get_children(), strict, constraint); } else { return std::make_unique<EmptySearch>(); } @@ -552,7 +545,7 @@ SearchIterator::UP ONearBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const { if (constraint == Blueprint::FilterConstraint::UPPER_BOUND) { - return createAndFilter(*this, strict, constraint); + return createAndFilter(*this, get_children(), strict, constraint); } else { return std::make_unique<EmptySearch>(); } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index 2cf6e7e29c4..877ed6b5094 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -141,7 +141,7 @@ NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchData { assert(tfmda.size() == 1); fef::TermFieldMatchData &tfmd = *tfmda[0]; // always search in only one field - if (strict && ! _found_hits.empty()) { + if (! _found_hits.empty()) { return NnsIndexIterator::create(tfmd, _found_hits, _dist_fun); } const vespalib::tensor::DenseTensorView &qT = *_query_tensor; diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp index cea35d976f0..3cfa928da87 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp @@ -2,6 +2,7 @@ #include "weighted_set_term_blueprint.h" #include "weighted_set_term_search.h" +#include "orsearch.h" #include <vespa/vespalib/objects/visit.hpp> namespace search::queryeval { @@ -56,6 +57,12 @@ WeightedSetTermBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &t return SearchIterator::UP(WeightedSetTermSearch::create(children, *tfmda[0], _weights, std::move(md))); } +SearchIterator::UP +WeightedSetTermBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + return create_or_filter(_terms, strict, constraint); +} + void WeightedSetTermBlueprint::fetchPostings(const ExecuteInfo &execInfo) { diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h index 8ae42607a9d..864e3e7fc7f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h @@ -34,6 +34,7 @@ public: void addTerm(Blueprint::UP term, int32_t weight); SearchIteratorUP createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool strict) const override; + SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override; void visitMembers(vespalib::ObjectVisitor &visitor) const override; private: diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 0f106f693f8..35615b255c0 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -6,6 +6,7 @@ vespa_add_library(searchlib_tensor OBJECT dense_tensor_attribute_saver.cpp dense_tensor_store.cpp distance_function_factory.cpp + distance_functions.cpp generic_tensor_attribute.cpp generic_tensor_attribute_saver.cpp generic_tensor_store.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp index 067280e9a23..0bb6f339455 100644 --- a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp @@ -36,6 +36,7 @@ DefaultNearestNeighborIndexFactory::make(const DocVectorAccess& vectors, HnswIndex::Config cfg(m * 2, m, params.neighbors_to_explore_at_insert(), + 10000, true); return std::make_unique<HnswIndex>(vectors, make_distance_function(params.distance_metric(), cell_type), diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp index 6b24a062727..b76994d6092 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp @@ -33,6 +33,13 @@ make_distance_function(DistanceMetric variant, ValueType::CellType cell_type) return std::make_unique<GeoDegreesDistance<double>>(); } break; + case DistanceMetric::InnerProduct: + if (cell_type == ValueType::CellType::FLOAT) { + return std::make_unique<InnerProductDistance<float>>(); + } else { + return std::make_unique<InnerProductDistance<double>>(); + } + break; } // not reached: return DistanceFunction::UP(); diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.cpp b/searchlib/src/vespa/searchlib/tensor/distance_functions.cpp new file mode 100644 index 00000000000..9017628d42c --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.cpp @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "distance_functions.h" + +namespace search::tensor { + +template class SquaredEuclideanDistance<float>; +template class SquaredEuclideanDistance<double>; + +template class AngularDistance<float>; +template class AngularDistance<double>; + +template class InnerProductDistance<float>; +template class InnerProductDistance<double>; + +template class GeoDegreesDistance<float>; +template class GeoDegreesDistance<double>; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h index d37495e85da..7e75920619f 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h @@ -50,11 +50,8 @@ public: const vespalib::hwaccelrated::IAccelrated & _computer; }; -template class SquaredEuclideanDistance<float>; -template class SquaredEuclideanDistance<double>; - /** - * Calculates angular distance between vectors with assumed norm 1. + * Calculates angular distance between vectors */ template <typename FloatType> class AngularDistance : public DistanceFunction { @@ -67,6 +64,51 @@ public: auto rhs_vector = rhs.typify<FloatType>(); size_t sz = lhs_vector.size(); assert(sz == rhs_vector.size()); + auto a = &lhs_vector[0]; + auto b = &rhs_vector[0]; + double a_norm_sq = _computer.dotProduct(a, a, sz); + double b_norm_sq = _computer.dotProduct(b, b, sz); + double squared_norms = a_norm_sq * b_norm_sq; + double dot_product = _computer.dotProduct(a, b, sz); + double div = (squared_norms > 0) ? sqrt(squared_norms) : 1.0; + double cosine_similarity = dot_product / div; + double distance = 1.0 - cosine_similarity; // in range [0,2] + return distance; + } + double to_rawscore(double distance) const override { + double cosine_similarity = 1.0 - distance; + // should be in in range [-1,1] but roundoff may cause problems: + cosine_similarity = std::min(1.0, cosine_similarity); + cosine_similarity = std::max(-1.0, cosine_similarity); + double angle_distance = acos(cosine_similarity); // in range [0,pi] + double score = 1.0 / (1.0 + angle_distance); + return score; + } + double calc_with_limit(const vespalib::tensor::TypedCells& lhs, + const vespalib::tensor::TypedCells& rhs, + double /*limit*/) const override + { + return calc(lhs, rhs); + } + + const vespalib::hwaccelrated::IAccelrated & _computer; +}; + +/** + * Calculates inner-product "distance" between vectors with assumed norm 1. + * Should give same ordering as Angular distance, but is less expensive. + */ +template <typename FloatType> +class InnerProductDistance : public DistanceFunction { +public: + InnerProductDistance() + : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()) + {} + double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override { + auto lhs_vector = lhs.typify<FloatType>(); + auto rhs_vector = rhs.typify<FloatType>(); + size_t sz = lhs_vector.size(); + assert(sz == rhs_vector.size()); double score = 1.0 - _computer.dotProduct(&lhs_vector[0], &rhs_vector[0], sz); return std::max(0.0, score); } @@ -84,9 +126,6 @@ public: const vespalib::hwaccelrated::IAccelrated & _computer; }; -template class AngularDistance<float>; -template class AngularDistance<double>; - /** * Calculates great-circle distance between Latitude/Longitude pairs, * measured in degrees. Output distance is measured in meters. @@ -139,7 +178,4 @@ public: }; -template class GeoDegreesDistance<float>; -template class GeoDegreesDistance<double>; - } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 36d970dfd01..2a17378f58a 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -227,11 +227,13 @@ HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, NearestPriQ candidates; uint32_t doc_id_limit = _graph.node_refs.size(); if (filter) { - assert(filter->size() >= doc_id_limit); + doc_id_limit = std::min(filter->size(), doc_id_limit); } auto visited = _visited_set_pool.get(doc_id_limit); for (const auto &entry : best_neighbors.peek()) { - assert(entry.docid < doc_id_limit); + if (entry.docid >= doc_id_limit) { + continue; + } candidates.push(entry); visited.mark(entry.docid); if (filter && !filter->testBit(entry.docid)) { @@ -287,16 +289,17 @@ HnswIndex::~HnswIndex() = default; void HnswIndex::add_document(uint32_t docid) { - PreparedAddDoc op = internal_prepare_add(docid, get_vector(docid)); + vespalib::GenerationHandler::Guard no_guard_needed; + PreparedAddDoc op = internal_prepare_add(docid, get_vector(docid), no_guard_needed); internal_complete_add(docid, op); } HnswIndex::PreparedAddDoc -HnswIndex::internal_prepare_add(uint32_t docid, TypedCells input_vector) const +HnswIndex::internal_prepare_add(uint32_t docid, TypedCells input_vector, vespalib::GenerationHandler::Guard read_guard) const { // TODO: Add capping on num_levels int level = _level_generator->max_level(); - PreparedAddDoc op(docid, level); + PreparedAddDoc op(docid, level, std::move(read_guard)); auto entry = _graph.get_entry_node(); if (entry.docid == 0) { // graph has no entry point @@ -371,8 +374,13 @@ HnswIndex::prepare_add_document(uint32_t docid, TypedCells vector, vespalib::GenerationHandler::Guard read_guard) const { - PreparedAddDoc op = internal_prepare_add(docid, vector); - (void) read_guard; // must keep guard until this point + uint32_t max_nodes = _graph.node_refs.size(); + if (max_nodes < _cfg.min_size_before_two_phase()) { + // the first documents added will do all work in write thread + // to ensure they are linked together: + return std::unique_ptr<PrepareResult>(); + } + PreparedAddDoc op = internal_prepare_add(docid, vector, std::move(read_guard)); return std::make_unique<PreparedAddDoc>(std::move(op)); } @@ -383,7 +391,11 @@ HnswIndex::complete_add_document(uint32_t docid, std::unique_ptr<PrepareResult> if (prepared && (prepared->docid == docid)) { internal_complete_add(docid, *prepared); } else { - LOG(warning, "complete_add_document called with invalid prepare_result"); + // we expect this for the first documents added, so no warning for them + if (_graph.node_refs.size() > 1.25 * _cfg.min_size_before_two_phase()) { + LOG(warning, "complete_add_document(%u) called with invalid prepare_result %s/%u", + docid, (prepared ? "valid ptr" : "nullptr"), (prepared ? prepared->docid : 0u)); + } // fallback to normal add add_document(docid); } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index ab3eced8fdc..f9adef0b86c 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -39,21 +39,25 @@ public: uint32_t _max_links_at_level_0; uint32_t _max_links_on_inserts; uint32_t _neighbors_to_explore_at_construction; + uint32_t _min_size_before_two_phase; bool _heuristic_select_neighbors; public: Config(uint32_t max_links_at_level_0_in, uint32_t max_links_on_inserts_in, uint32_t neighbors_to_explore_at_construction_in, + uint32_t min_size_before_two_phase_in, bool heuristic_select_neighbors_in) : _max_links_at_level_0(max_links_at_level_0_in), _max_links_on_inserts(max_links_on_inserts_in), _neighbors_to_explore_at_construction(neighbors_to_explore_at_construction_in), + _min_size_before_two_phase(min_size_before_two_phase_in), _heuristic_select_neighbors(heuristic_select_neighbors_in) {} uint32_t max_links_at_level_0() const { return _max_links_at_level_0; } uint32_t max_links_on_inserts() const { return _max_links_on_inserts; } uint32_t neighbors_to_explore_at_construction() const { return _neighbors_to_explore_at_construction; } + uint32_t min_size_before_two_phase() const { return _min_size_before_two_phase; } bool heuristic_select_neighbors() const { return _heuristic_select_neighbors; } }; @@ -122,17 +126,22 @@ protected: const BitVector *filter, uint32_t explore_k) const; struct PreparedAddDoc : public PrepareResult { + using ReadGuard = vespalib::GenerationHandler::Guard; uint32_t docid; int32_t max_level; + ReadGuard read_guard; using Links = std::vector<std::pair<uint32_t, HnswGraph::NodeRef>>; std::vector<Links> connections; - PreparedAddDoc(uint32_t docid_in, int32_t max_level_in) - : docid(docid_in), max_level(max_level_in), connections(max_level+1) + PreparedAddDoc(uint32_t docid_in, int32_t max_level_in, ReadGuard read_guard_in) + : docid(docid_in), max_level(max_level_in), + read_guard(std::move(read_guard_in)), + connections(max_level+1) {} ~PreparedAddDoc() = default; PreparedAddDoc(PreparedAddDoc&& other) = default; }; - PreparedAddDoc internal_prepare_add(uint32_t docid, TypedCells input_vector) const; + PreparedAddDoc internal_prepare_add(uint32_t docid, TypedCells input_vector, + vespalib::GenerationHandler::Guard read_guard) const; LinkArray filter_valid_docids(uint32_t level, const PreparedAddDoc::Links &neighbors, uint32_t me); void internal_complete_add(uint32_t docid, PreparedAddDoc &op); public: diff --git a/searchlib/src/vespa/searchlib/test/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/CMakeLists.txt index 41084148c87..4d722e687ef 100644 --- a/searchlib/src/vespa/searchlib/test/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(searchlib_test SOURCES document_weight_attribute_helper.cpp @@ -12,7 +13,17 @@ vespa_add_library(searchlib_test statestring.cpp $<TARGET_OBJECTS:searchlib_test_fakedata> $<TARGET_OBJECTS:searchlib_searchlib_test_diskindex> + $<TARGET_OBJECTS:searchlib_test_gtest_migration> DEPENDS searchlib searchlib_searchlib_test_memoryindex + GTest::GTest ) + +vespa_add_library(searchlib_test_gtest_migration OBJECT + SOURCES + initrange.cpp + searchiteratorverifier.cpp +) + +target_compile_definitions(searchlib_test_gtest_migration PRIVATE ENABLE_GTEST_MIGRATION) diff --git a/searchlib/src/vespa/searchlib/test/initrange.cpp b/searchlib/src/vespa/searchlib/test/initrange.cpp index 2292a8e775e..1e23f7c5b8c 100644 --- a/searchlib/src/vespa/searchlib/test/initrange.cpp +++ b/searchlib/src/vespa/searchlib/test/initrange.cpp @@ -1,6 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "initrange.h" +#ifdef ENABLE_GTEST_MIGRATION +#include <vespa/vespalib/gtest/gtest.h> +#define ASSERT_EQUAL ASSERT_EQ +#define EXPECT_EQUAL EXPECT_EQ +#else #include <vespa/vespalib/testkit/test_kit.h> +#endif #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/queryeval/truesearch.h> #include <algorithm> diff --git a/searchlib/src/vespa/searchlib/test/initrange.h b/searchlib/src/vespa/searchlib/test/initrange.h index a143dfdb119..4fbb851779a 100644 --- a/searchlib/src/vespa/searchlib/test/initrange.h +++ b/searchlib/src/vespa/searchlib/test/initrange.h @@ -7,6 +7,10 @@ namespace search::test { +#ifdef ENABLE_GTEST_MIGRATION +#define InitRangeVerifier InitRangeVerifierForGTest +#endif + class InitRangeVerifier { public: typedef queryeval::SearchIterator SearchIterator; diff --git a/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp b/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp index ec53d6d9d00..276f8fbc08d 100644 --- a/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp +++ b/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp @@ -2,7 +2,14 @@ #include "searchiteratorverifier.h" #include "initrange.h" +#ifdef ENABLE_GTEST_MIGRATION +#include <vespa/vespalib/gtest/gtest.h> +#define TEST_DO(x) x +#define EXPECT_EQUAL EXPECT_EQ +#define ASSERT_EQUAL ASSERT_EQ +#else #include <vespa/vespalib/testkit/test_kit.h> +#endif #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/queryeval/truesearch.h> #include <vespa/searchlib/queryeval/termwise_search.h> diff --git a/searchlib/src/vespa/searchlib/test/searchiteratorverifier.h b/searchlib/src/vespa/searchlib/test/searchiteratorverifier.h index 3d35731dab1..afeb46f0c16 100644 --- a/searchlib/src/vespa/searchlib/test/searchiteratorverifier.h +++ b/searchlib/src/vespa/searchlib/test/searchiteratorverifier.h @@ -7,6 +7,10 @@ namespace search::test { +#ifdef ENABLE_GTEST_MIGRATION +#define SearchIteratorVerifier SearchIteratorVerifierForGTest +#endif + class SearchIteratorVerifier { public: typedef queryeval::SearchIterator SearchIterator; diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp index 10f3f6089e3..2ca49105610 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp +++ b/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp @@ -63,6 +63,8 @@ public: } }; +vespalib::stringref ZERO("0"); + TEST_F("testExecute", Fixture) { std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); EXPECT_EQUAL(0, tv->_val); @@ -97,7 +99,7 @@ TEST_F("require that task with different component ids are not serialized", Fixt std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); EXPECT_EQUAL(0, tv->_val); f._threads.execute(0, [&]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute(2, [&]() { tv->modify(14, 42); }); + f._threads.execute(1, [&]() { tv->modify(14, 42); }); tv->wait(2); if (tv->_fail != 1) { continue; @@ -118,8 +120,8 @@ TEST_F("require that task with same string component id are serialized", Fixture std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); EXPECT_EQUAL(0, tv->_val); auto test2 = [&]() { tv->modify(14, 42); }; - f._threads.execute(f._threads.getExecutorId("0"), [&]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute(f._threads.getExecutorId("0"), test2); + f._threads.execute(f._threads.getExecutorIdFromName(ZERO), [&]() { usleep(2000); tv->modify(0, 14); }); + f._threads.execute(f._threads.getExecutorIdFromName(ZERO), test2); tv->wait(2); EXPECT_EQUAL(0, tv->_fail); EXPECT_EQUAL(42, tv->_val); @@ -136,8 +138,8 @@ int detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int t for (tryCnt = 0; tryCnt < tryLimit; ++tryCnt) { std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); EXPECT_EQUAL(0, tv->_val); - f._threads.execute(f._threads.getExecutorId("0"), [&]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute(f._threads.getExecutorId(altComponentId), [&]() { tv->modify(14, 42); }); + f._threads.execute(f._threads.getExecutorIdFromName(ZERO), [&]() { usleep(2000); tv->modify(0, 14); }); + f._threads.execute(f._threads.getExecutorIdFromName(altComponentId), [&]() { tv->modify(14, 42); }); tv->wait(2); if (tv->_fail != 1) { continue; @@ -156,10 +158,10 @@ vespalib::string makeAltComponentId(Fixture &f) { int tryCnt = 0; char altComponentId[20]; - ISequencedTaskExecutor::ExecutorId executorId0 = f._threads.getExecutorId("0"); + ISequencedTaskExecutor::ExecutorId executorId0 = f._threads.getExecutorIdFromName(ZERO); for (tryCnt = 1; tryCnt < 100; ++tryCnt) { sprintf(altComponentId, "%d", tryCnt); - if (f._threads.getExecutorId(altComponentId) == executorId0) { + if (f._threads.getExecutorIdFromName(altComponentId) == executorId0) { break; } } @@ -236,13 +238,9 @@ TEST("require that you get correct number of executors") { TEST("require that you distribute well") { AdaptiveSequencedExecutor seven(7, 1, 0, 10); EXPECT_EQUAL(7u, seven.getNumExecutors()); - EXPECT_EQUAL(97u, seven.getComponentHashSize()); - EXPECT_EQUAL(0u, seven.getComponentEffectiveHashSize()); for (uint32_t id=0; id < 1000; id++) { - EXPECT_EQUAL((id%97)%7, seven.getExecutorId(id).getId()); + EXPECT_EQUAL(id%7, seven.getExecutorId(id).getId()); } - EXPECT_EQUAL(97u, seven.getComponentHashSize()); - EXPECT_EQUAL(97u, seven.getComponentEffectiveHashSize()); } } diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index 70d0f1c743d..df94e70f9d6 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp @@ -65,6 +65,8 @@ public: } }; +vespalib::stringref ZERO("0"); + TEST_F("testExecute", Fixture) { std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); EXPECT_EQUAL(0, tv->_val); @@ -120,8 +122,8 @@ TEST_F("require that task with same string component id are serialized", Fixture std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); EXPECT_EQUAL(0, tv->_val); auto test2 = [=]() { tv->modify(14, 42); }; - f._threads->execute(f._threads->getExecutorId("0"), [=]() { usleep(2000); tv->modify(0, 14); }); - f._threads->execute(f._threads->getExecutorId("0"), test2); + f._threads->execute(f._threads->getExecutorIdFromName(ZERO), [=]() { usleep(2000); tv->modify(0, 14); }); + f._threads->execute(f._threads->getExecutorIdFromName(ZERO), test2); tv->wait(2); EXPECT_EQUAL(0, tv->_fail); EXPECT_EQUAL(42, tv->_val); @@ -138,8 +140,8 @@ int detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int t for (tryCnt = 0; tryCnt < tryLimit; ++tryCnt) { std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); EXPECT_EQUAL(0, tv->_val); - f._threads->execute(f._threads->getExecutorId("0"), [=]() { usleep(2000); tv->modify(0, 14); }); - f._threads->execute(f._threads->getExecutorId(altComponentId), [=]() { tv->modify(14, 42); }); + f._threads->execute(f._threads->getExecutorIdFromName(ZERO), [=]() { usleep(2000); tv->modify(0, 14); }); + f._threads->execute(f._threads->getExecutorIdFromName(altComponentId), [=]() { tv->modify(14, 42); }); tv->wait(2); if (tv->_fail != 1) { continue; @@ -158,10 +160,10 @@ vespalib::string makeAltComponentId(Fixture &f) { int tryCnt = 0; char altComponentId[20]; - ISequencedTaskExecutor::ExecutorId executorId0 = f._threads->getExecutorId("0"); + ISequencedTaskExecutor::ExecutorId executorId0 = f._threads->getExecutorIdFromName(ZERO); for (tryCnt = 1; tryCnt < 100; ++tryCnt) { sprintf(altComponentId, "%d", tryCnt); - if (f._threads->getExecutorId(altComponentId) == executorId0) { + if (f._threads->getExecutorIdFromName(altComponentId) == executorId0) { break; } } @@ -237,14 +239,15 @@ TEST("require that you get correct number of executors") { TEST("require that you distribute well") { auto seven = SequencedTaskExecutor::create(7); + const SequencedTaskExecutor & seq = dynamic_cast<const SequencedTaskExecutor &>(*seven); EXPECT_EQUAL(7u, seven->getNumExecutors()); - EXPECT_EQUAL(97u, seven->getComponentHashSize()); - EXPECT_EQUAL(0u, seven->getComponentEffectiveHashSize()); + EXPECT_EQUAL(97u, seq.getComponentHashSize()); + EXPECT_EQUAL(0u, seq.getComponentEffectiveHashSize()); for (uint32_t id=0; id < 1000; id++) { EXPECT_EQUAL((id%97)%7, seven->getExecutorId(id).getId()); } - EXPECT_EQUAL(97u, seven->getComponentHashSize()); - EXPECT_EQUAL(97u, seven->getComponentEffectiveHashSize()); + EXPECT_EQUAL(97u, seq.getComponentHashSize()); + EXPECT_EQUAL(97u, seq.getComponentEffectiveHashSize()); } TEST("Test creation of different types") { diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp index 50bc3b020a8..3e87749c794 100644 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp @@ -256,6 +256,11 @@ AdaptiveSequencedExecutor::~AdaptiveSequencedExecutor() assert(_worker_stack.empty()); } +ISequencedTaskExecutor::ExecutorId +AdaptiveSequencedExecutor::getExecutorId(uint64_t component) const { + return ExecutorId(component % _strands.size()); +} + void AdaptiveSequencedExecutor::executeTask(ExecutorId id, Task::UP task) { diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h index bc3457a72ef..a4d3ac97758 100644 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h +++ b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h @@ -117,6 +117,7 @@ public: AdaptiveSequencedExecutor(size_t num_strands, size_t num_threads, size_t max_waiting, size_t max_pending); ~AdaptiveSequencedExecutor() override; + ExecutorId getExecutorId(uint64_t component) const override; void executeTask(ExecutorId id, Task::UP task) override; void sync() override; void setTaskLimit(uint32_t task_limit) override; diff --git a/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h b/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h new file mode 100644 index 00000000000..575552971fa --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/threadexecutor.h> +#include <atomic> + +namespace vespalib { + +/** + * Implementation of the ThreadExecutor interface that runs all tasks in the foreground by the calling thread. + */ +class ForegroundThreadExecutor : public vespalib::ThreadExecutor { +private: + std::atomic<size_t> _accepted; + +public: + ForegroundThreadExecutor() : _accepted(0) { } + Task::UP execute(Task::UP task) override { + task->run(); + ++_accepted; + return Task::UP(); + } + size_t getNumThreads() const override { return 0; } + Stats getStats() override { + return ExecutorStats(ExecutorStats::QueueSizeT(), _accepted.load(std::memory_order_relaxed), 0); + } + virtual void setTaskLimit(uint32_t taskLimit) override { (void) taskLimit; } +}; + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp index b45ada1c58c..f295d2b30c1 100644 --- a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp @@ -39,4 +39,9 @@ vespalib::ExecutorStats ForegroundTaskExecutor::getStats() { return vespalib::ExecutorStats(vespalib::ExecutorStats::QueueSizeT(0) , _accepted.load(std::memory_order_relaxed), 0); } +ISequencedTaskExecutor::ExecutorId +ForegroundTaskExecutor::getExecutorId(uint64_t componentId) const { + return ExecutorId(componentId%getNumExecutors()); +} + } // namespace search diff --git a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h index d9a348ed012..f7b3ff8eab0 100644 --- a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h @@ -21,11 +21,10 @@ public: ForegroundTaskExecutor(uint32_t threads); ~ForegroundTaskExecutor() override; + ExecutorId getExecutorId(uint64_t componentId) const override; void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override; void sync() override; - void setTaskLimit(uint32_t taskLimit) override; - vespalib::ExecutorStats getStats() override; private: std::atomic<uint64_t> _accepted; diff --git a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp index d05702cc85b..af3ce5fe64f 100644 --- a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp @@ -2,45 +2,20 @@ #include "isequencedtaskexecutor.h" #include <vespa/vespalib/stllike/hash_fun.h> -#include <vespa/vespalib/stllike/hashtable.h> -#include <cassert> namespace vespalib { -namespace { - constexpr uint8_t MAGIC = 255; -} - ISequencedTaskExecutor::ISequencedTaskExecutor(uint32_t numExecutors) - : _component2Id(vespalib::hashtable_base::getModuloStl(numExecutors*8), MAGIC), - _mutex(), - _numExecutors(numExecutors), - _nextId(0) + : _numExecutors(numExecutors) { - assert(numExecutors < 256); } ISequencedTaskExecutor::~ISequencedTaskExecutor() = default; ISequencedTaskExecutor::ExecutorId -ISequencedTaskExecutor::getExecutorId(vespalib::stringref componentId) const { +ISequencedTaskExecutor::getExecutorIdFromName(vespalib::stringref componentId) const { vespalib::hash<vespalib::stringref> hashfun; return getExecutorId(hashfun(componentId)); } -ISequencedTaskExecutor::ExecutorId -ISequencedTaskExecutor::getExecutorId(uint64_t componentId) const { - uint32_t shrunkId = componentId % _component2Id.size(); - uint8_t executorId = _component2Id[shrunkId]; - if (executorId == MAGIC) { - std::lock_guard guard(_mutex); - if (_component2Id[shrunkId] == MAGIC) { - _component2Id[shrunkId] = _nextId % getNumExecutors(); - _nextId++; - } - executorId = _component2Id[shrunkId]; - } - return ExecutorId(executorId); -} - } diff --git a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h index cd2a6c6f0d8..d457de26f54 100644 --- a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h @@ -37,10 +37,10 @@ public: * @param componentId component id * @return executor id */ - ExecutorId getExecutorId(uint64_t componentId) const; + virtual ExecutorId getExecutorId(uint64_t componentId) const = 0; uint32_t getNumExecutors() const { return _numExecutors; } - ExecutorId getExecutorId(vespalib::stringref componentId) const; + ExecutorId getExecutorIdFromName(vespalib::stringref componentId) const; /** * Schedule a task to run after all previously scheduled tasks with @@ -98,16 +98,9 @@ public: void execute(ExecutorId id, FunctionType &&function) { executeTask(id, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); } - /** - * For testing only - */ - uint32_t getComponentHashSize() const { return _component2Id.size(); } - uint32_t getComponentEffectiveHashSize() const { return _nextId; } + private: - mutable std::vector<uint8_t> _component2Id; - mutable std::mutex _mutex; uint32_t _numExecutors; - mutable uint32_t _nextId; }; } diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp index a0c2f0ac237..963264a62e7 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp @@ -4,12 +4,15 @@ #include "adaptive_sequenced_executor.h" #include "singleexecutor.h" #include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/stllike/hashtable.h> +#include <cassert> namespace vespalib { namespace { constexpr uint32_t stackSize = 128 * 1024; +constexpr uint8_t MAGIC = 255; } @@ -18,7 +21,8 @@ std::unique_ptr<ISequencedTaskExecutor> SequencedTaskExecutor::create(uint32_t threads, uint32_t taskLimit, OptimizeFor optimize, uint32_t kindOfWatermark, duration reactionTime) { if (optimize == OptimizeFor::ADAPTIVE) { - return std::make_unique<AdaptiveSequencedExecutor>(threads, threads, kindOfWatermark, taskLimit); + size_t num_strands = std::min(taskLimit, threads*32); + return std::make_unique<AdaptiveSequencedExecutor>(num_strands, threads, kindOfWatermark, taskLimit); } else { auto executors = std::make_unique<std::vector<std::unique_ptr<SyncableThreadExecutor>>>(); executors->reserve(threads); @@ -41,8 +45,12 @@ SequencedTaskExecutor::~SequencedTaskExecutor() SequencedTaskExecutor::SequencedTaskExecutor(std::unique_ptr<std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>>> executors) : ISequencedTaskExecutor(executors->size()), - _executors(std::move(executors)) + _executors(std::move(executors)), + _component2Id(vespalib::hashtable_base::getModuloStl(getNumExecutors()*8), MAGIC), + _mutex(), + _nextId(0) { + assert(getNumExecutors() < 256); } void @@ -86,4 +94,19 @@ SequencedTaskExecutor::getStats() return accumulatedStats; } +ISequencedTaskExecutor::ExecutorId +SequencedTaskExecutor::getExecutorId(uint64_t componentId) const { + uint32_t shrunkId = componentId % _component2Id.size(); + uint8_t executorId = _component2Id[shrunkId]; + if (executorId == MAGIC) { + std::lock_guard guard(_mutex); + if (_component2Id[shrunkId] == MAGIC) { + _component2Id[shrunkId] = _nextId % getNumExecutors(); + _nextId++; + } + executorId = _component2Id[shrunkId]; + } + return ExecutorId(executorId); +} + } // namespace search diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h index b3dd400478a..c37bd2eecf4 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h @@ -14,11 +14,8 @@ class SyncableThreadExecutor; */ class SequencedTaskExecutor final : public ISequencedTaskExecutor { - using Stats = vespalib::ExecutorStats; - std::unique_ptr<std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>>> _executors; - - SequencedTaskExecutor(std::unique_ptr<std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>>> executor); public: + using Stats = vespalib::ExecutorStats; using ISequencedTaskExecutor::getExecutorId; using OptimizeFor = vespalib::Executor::OptimizeFor; @@ -26,6 +23,7 @@ public: void setTaskLimit(uint32_t taskLimit) override; void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override; + ExecutorId getExecutorId(uint64_t componentId) const override; void sync() override; Stats getStats() override; @@ -35,6 +33,19 @@ public: */ static std::unique_ptr<ISequencedTaskExecutor> create(uint32_t threads, uint32_t taskLimit = 1000, OptimizeFor optimize = OptimizeFor::LATENCY, uint32_t kindOfWatermark = 0, duration reactionTime = 10ms); + /** + * For testing only + */ + uint32_t getComponentHashSize() const { return _component2Id.size(); } + uint32_t getComponentEffectiveHashSize() const { return _nextId; } +private: + SequencedTaskExecutor(std::unique_ptr<std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>>> executor); + + std::unique_ptr<std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>>> _executors; + mutable std::vector<uint8_t> _component2Id; + mutable std::mutex _mutex; + mutable uint32_t _nextId; + }; } // namespace search diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp index 3d9ed4e21f4..d6a89117d68 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp @@ -49,4 +49,9 @@ vespalib::ExecutorStats SequencedTaskExecutorObserver::getStats() { return _executor.getStats(); } +ISequencedTaskExecutor::ExecutorId +SequencedTaskExecutorObserver::getExecutorId(uint64_t componentId) const { + return _executor.getExecutorId(componentId); +} + } // namespace search diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h index 9307a7ddb37..6bcdf08ae5c 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h @@ -23,16 +23,15 @@ public: SequencedTaskExecutorObserver(ISequencedTaskExecutor &executor); ~SequencedTaskExecutorObserver() override; + ExecutorId getExecutorId(uint64_t componentId) const override; void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override; void sync() override; + void setTaskLimit(uint32_t taskLimit) override; + vespalib::ExecutorStats getStats() override; uint32_t getExecuteCnt() const { return _executeCnt; } uint32_t getSyncCnt() const { return _syncCnt; } std::vector<uint32_t> getExecuteHistory(); - - void setTaskLimit(uint32_t taskLimit) override; - - vespalib::ExecutorStats getStats() override; }; } // namespace search diff --git a/storage/src/tests/bucketdb/CMakeLists.txt b/storage/src/tests/bucketdb/CMakeLists.txt index 2468e587aff..5dc269e54f9 100644 --- a/storage/src/tests/bucketdb/CMakeLists.txt +++ b/storage/src/tests/bucketdb/CMakeLists.txt @@ -1,5 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(storage_bucketdb_gtest_runner_app TEST SOURCES bucketinfotest.cpp @@ -12,7 +13,7 @@ vespa_add_executable(storage_bucketdb_gtest_runner_app TEST DEPENDS storage storage_testcommon - gtest + GTest::GTest ) vespa_add_test( diff --git a/storage/src/tests/common/CMakeLists.txt b/storage/src/tests/common/CMakeLists.txt index 1922d13ca61..5d5b04d8095 100644 --- a/storage/src/tests/common/CMakeLists.txt +++ b/storage/src/tests/common/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(storage_testcommon TEST SOURCES dummystoragelink.cpp @@ -19,7 +20,7 @@ vespa_add_executable(storage_common_gtest_runner_app TEST DEPENDS storage_testcommon storage - gtest + GTest::GTest ) vespa_add_test( diff --git a/storage/src/tests/common/hostreporter/CMakeLists.txt b/storage/src/tests/common/hostreporter/CMakeLists.txt index 2fcb159cb08..4328b8156e6 100644 --- a/storage/src/tests/common/hostreporter/CMakeLists.txt +++ b/storage/src/tests/common/hostreporter/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(storage_testhostreporter TEST SOURCES util.cpp @@ -14,7 +15,7 @@ vespa_add_executable(storage_hostreporter_gtest_runner_app TEST DEPENDS storage storage_testhostreporter - gtest + GTest::GTest ) vespa_add_test( diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt index 3148540d86d..1403021a9c3 100644 --- a/storage/src/tests/distributor/CMakeLists.txt +++ b/storage/src/tests/distributor/CMakeLists.txt @@ -1,5 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(storage_distributor_gtest_runner_app TEST SOURCES blockingoperationstartertest.cpp @@ -49,7 +50,7 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST storage_testcommon storage_testhostreporter storage_distributor - gtest + GTest::GTest ) vespa_add_test( diff --git a/storage/src/tests/frameworkimpl/status/CMakeLists.txt b/storage/src/tests/frameworkimpl/status/CMakeLists.txt index 1b49b1bac45..cf2ee5fd51b 100644 --- a/storage/src/tests/frameworkimpl/status/CMakeLists.txt +++ b/storage/src/tests/frameworkimpl/status/CMakeLists.txt @@ -1,5 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(storage_status_gtest_runner_app TEST SOURCES gtest_runner.cpp @@ -8,7 +9,7 @@ vespa_add_executable(storage_status_gtest_runner_app TEST DEPENDS storage storage_testcommon - gtest + GTest::GTest ) vespa_add_test( diff --git a/storage/src/tests/persistence/CMakeLists.txt b/storage/src/tests/persistence/CMakeLists.txt index e8095109806..ae96748ff9d 100644 --- a/storage/src/tests/persistence/CMakeLists.txt +++ b/storage/src/tests/persistence/CMakeLists.txt @@ -1,5 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(storage_persistence_gtest_runner_app TEST SOURCES bucketownershipnotifiertest.cpp @@ -15,7 +16,7 @@ vespa_add_executable(storage_persistence_gtest_runner_app TEST DEPENDS storage storage_testpersistence_common - gtest + GTest::GTest ) vespa_add_test( diff --git a/storage/src/tests/persistence/common/CMakeLists.txt b/storage/src/tests/persistence/common/CMakeLists.txt index 53ec3fd7c0c..29b826c0c9b 100644 --- a/storage/src/tests/persistence/common/CMakeLists.txt +++ b/storage/src/tests/persistence/common/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(storage_testpersistence_common TEST SOURCES filestortestfixture.cpp persistenceproviderwrapper.cpp DEPENDS - gtest + GTest::GTest persistence storage_testcommon ) diff --git a/storage/src/tests/persistence/filestorage/CMakeLists.txt b/storage/src/tests/persistence/filestorage/CMakeLists.txt index 5209bcce73d..8cabfb865cd 100644 --- a/storage/src/tests/persistence/filestorage/CMakeLists.txt +++ b/storage/src/tests/persistence/filestorage/CMakeLists.txt @@ -1,5 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(storage_filestorage_gtest_runner_app TEST SOURCES deactivatebucketstest.cpp @@ -16,7 +17,7 @@ vespa_add_executable(storage_filestorage_gtest_runner_app TEST storage storageapi storage_testpersistence_common - gtest + GTest::GTest ) vespa_add_test( diff --git a/storage/src/tests/storageserver/CMakeLists.txt b/storage/src/tests/storageserver/CMakeLists.txt index 1d759a534f6..95ce08265ad 100644 --- a/storage/src/tests/storageserver/CMakeLists.txt +++ b/storage/src/tests/storageserver/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(storage_teststorageserver TEST SOURCES testvisitormessagesession.cpp @@ -25,7 +26,7 @@ vespa_add_executable(storage_storageserver_gtest_runner_app TEST storage_storageserver storage_testcommon storage_teststorageserver - gtest + GTest::GTest ) vespa_add_test( diff --git a/storage/src/tests/visiting/CMakeLists.txt b/storage/src/tests/visiting/CMakeLists.txt index c1b19960cea..8bfc28e7eb9 100644 --- a/storage/src/tests/visiting/CMakeLists.txt +++ b/storage/src/tests/visiting/CMakeLists.txt @@ -1,5 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(storage_visiting_gtest_runner_app TEST SOURCES commandqueuetest.cpp @@ -10,7 +11,7 @@ vespa_add_executable(storage_visiting_gtest_runner_app TEST DEPENDS storage storage_teststorageserver - gtest + GTest::GTest ) vespa_add_test( diff --git a/storage/src/vespa/storage/distributor/activecopy.cpp b/storage/src/vespa/storage/distributor/activecopy.cpp index 4174ddb726f..5654e986882 100644 --- a/storage/src/vespa/storage/distributor/activecopy.cpp +++ b/storage/src/vespa/storage/distributor/activecopy.cpp @@ -86,22 +86,29 @@ namespace { } }; - void buildValidNodeIndexList(BucketDatabase::Entry& e, std::vector<uint16_t>& result) { + std::vector<uint16_t> + buildValidNodeIndexList(BucketDatabase::Entry& e) { + std::vector<uint16_t> result; + result.reserve(e->getNodeCount()); for (uint32_t i=0, n=e->getNodeCount(); i < n; ++i) { const BucketCopy& cp = e->getNodeRef(i); if (!cp.valid()) continue; result.push_back(cp.getNode()); } + return result; } - void buildNodeList(BucketDatabase::Entry& e, - const std::vector<uint16_t>& nodeIndexes, - const std::vector<uint16_t>& idealState, - std::vector<ActiveCopy>& result) + std::vector<ActiveCopy> + buildNodeList(BucketDatabase::Entry& e, + const std::vector<uint16_t>& nodeIndexes, + const std::vector<uint16_t>& idealState) { + std::vector<ActiveCopy> result; + result.reserve(nodeIndexes.size()); for (uint16_t nodeIndex : nodeIndexes) { result.emplace_back(nodeIndex, e, idealState); } + return result; } } @@ -118,26 +125,25 @@ ActiveCopy::calculate(const std::vector<uint16_t>& idealState, BucketDatabase::Entry& e) { DEBUG(std::cerr << "Ideal state is " << idealState << "\n"); - std::vector<uint16_t> validNodesWithCopy; - buildValidNodeIndexList(e, validNodesWithCopy); + std::vector<uint16_t> validNodesWithCopy = buildValidNodeIndexList(e); if (validNodesWithCopy.empty()) { return ActiveList(); } typedef std::vector<uint16_t> IndexList; std::vector<IndexList> groups; if (distribution.activePerGroup()) { - groups = distribution.splitNodesIntoLeafGroups(validNodesWithCopy); + groups = distribution.splitNodesIntoLeafGroups(std::move(validNodesWithCopy)); } else { - groups.push_back(validNodesWithCopy); + groups.push_back(std::move(validNodesWithCopy)); } std::vector<ActiveCopy> result; + result.reserve(groups.size()); for (uint32_t i=0; i<groups.size(); ++i) { - std::vector<ActiveCopy> entries; - buildNodeList(e, groups[i], idealState, entries); + std::vector<ActiveCopy> entries = buildNodeList(e, groups[i], idealState); DEBUG(std::cerr << "Finding active for group " << entries << "\n"); auto best = std::min_element(entries.begin(), entries.end(), ActiveStateOrder()); DEBUG(std::cerr << "Best copy " << *best << "\n"); - result.push_back(ActiveCopy(*best)); + result.emplace_back(*best); } return ActiveList(std::move(result)); } @@ -165,8 +171,8 @@ ActiveList::print(std::ostream& out, bool verbose, bool ActiveList::contains(uint16_t node) const { - for (uint32_t i=0; i<_v.size(); ++i) { - if (node == _v[i]._nodeIndex) return true; + for (const auto & candadate : _v) { + if (node == candadate._nodeIndex) return true; } return false; } diff --git a/storageapi/src/tests/CMakeLists.txt b/storageapi/src/tests/CMakeLists.txt index 8b820adb467..f07b9d0a2bc 100644 --- a/storageapi/src/tests/CMakeLists.txt +++ b/storageapi/src/tests/CMakeLists.txt @@ -1,5 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(storageapi_gtest_runner_app TEST SOURCES gtest_runner.cpp @@ -8,7 +9,7 @@ vespa_add_executable(storageapi_gtest_runner_app TEST storageapi_testmbusprot storageapi_testmessageapi storageapi - gtest + GTest::GTest ) vespa_add_test( diff --git a/storageapi/src/tests/buckets/CMakeLists.txt b/storageapi/src/tests/buckets/CMakeLists.txt index 42310be56f0..ea2cc109986 100644 --- a/storageapi/src/tests/buckets/CMakeLists.txt +++ b/storageapi/src/tests/buckets/CMakeLists.txt @@ -1,8 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(storageapi_testbuckets SOURCES bucketinfotest.cpp DEPENDS storageapi - gtest + GTest::GTest ) diff --git a/storageapi/src/tests/mbusprot/CMakeLists.txt b/storageapi/src/tests/mbusprot/CMakeLists.txt index 2801c9a91dd..84b8ff0f1b0 100644 --- a/storageapi/src/tests/mbusprot/CMakeLists.txt +++ b/storageapi/src/tests/mbusprot/CMakeLists.txt @@ -1,8 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(storageapi_testmbusprot SOURCES storageprotocoltest.cpp DEPENDS storageapi - gtest + GTest::GTest ) diff --git a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt index 113a4372068..1b1e224f034 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt +++ b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt @@ -7,6 +7,8 @@ PROTOBUF_GENERATE_CPP(storageapi_PROTOBUF_SRCS storageapi_PROTOBUF_HDRS protobuf/visiting.proto protobuf/maintenance.proto) +vespa_add_source_target(protobufgen_storageapi_mbusprot DEPENDS ${storageapi_PROTOBUF_SRCS} ${storageapi_PROTOBUF_HDRS}) + # protoc-generated files emit compiler warnings that we normally treat as errors. # Instead of rolling our own compiler plugin we'll pragmatically disable the noise. if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") diff --git a/storageframework/src/tests/CMakeLists.txt b/storageframework/src/tests/CMakeLists.txt index d658605a911..1cf40b8a2ef 100644 --- a/storageframework/src/tests/CMakeLists.txt +++ b/storageframework/src/tests/CMakeLists.txt @@ -1,5 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) + # Runner for unit tests written in gtest. # NOTE: All new test classes should be added here. vespa_add_executable(storageframework_gtest_runner_app TEST @@ -8,7 +10,7 @@ vespa_add_executable(storageframework_gtest_runner_app TEST DEPENDS storageframework_testclock storageframework_testthread - gtest + GTest::GTest ) vespa_add_test( diff --git a/storageframework/src/tests/clock/CMakeLists.txt b/storageframework/src/tests/clock/CMakeLists.txt index f887de61239..62bb6eeedc1 100644 --- a/storageframework/src/tests/clock/CMakeLists.txt +++ b/storageframework/src/tests/clock/CMakeLists.txt @@ -1,8 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(storageframework_testclock SOURCES timetest.cpp DEPENDS storageframework - gtest + GTest::GTest ) diff --git a/storageframework/src/tests/thread/CMakeLists.txt b/storageframework/src/tests/thread/CMakeLists.txt index 3e6db8c9b57..810b0c59912 100644 --- a/storageframework/src/tests/thread/CMakeLists.txt +++ b/storageframework/src/tests/thread/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(storageframework_testthread SOURCES tickingthreadtest.cpp taskthreadtest.cpp DEPENDS storageframework - gtest + GTest::GTest ) diff --git a/storageserver/src/tests/CMakeLists.txt b/storageserver/src/tests/CMakeLists.txt index 9c475543b81..2981d452d5f 100644 --- a/storageserver/src/tests/CMakeLists.txt +++ b/storageserver/src/tests/CMakeLists.txt @@ -1,5 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(storageserver_gtest_runner_app TEST SOURCES storageservertest.cpp @@ -8,7 +9,7 @@ vespa_add_executable(storageserver_gtest_runner_app TEST DEPENDS storageserver_storageapp vdstestlib - gtest + GTest::GTest ) vespa_add_test( diff --git a/tenant-cd-api/CMakeLists.txt b/tenant-cd-api/CMakeLists.txt new file mode 100644 index 00000000000..971aa974aa2 --- /dev/null +++ b/tenant-cd-api/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(tenant-cd-api) diff --git a/tenant-cd-api/pom.xml b/tenant-cd-api/pom.xml index b19d42d094f..23e5f3ec3f4 100644 --- a/tenant-cd-api/pom.xml +++ b/tenant-cd-api/pom.xml @@ -57,9 +57,7 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> <configuration> - <attachBundleArtifact>true</attachBundleArtifact> - <bundleClassifierName>deploy</bundleClassifierName> - <useCommonAssemblyIds>false</useCommonAssemblyIds> + <useCommonAssemblyIds>true</useCommonAssemblyIds> </configuration> </plugin> <plugin> diff --git a/vdslib/src/tests/CMakeLists.txt b/vdslib/src/tests/CMakeLists.txt index f552808f97c..458f3bda01d 100644 --- a/vdslib/src/tests/CMakeLists.txt +++ b/vdslib/src/tests/CMakeLists.txt @@ -1,5 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) + # Runner for unit tests written in gtest. # NOTE: All new test classes should be added here. vespa_add_executable(vdslib_gtest_runner_app TEST @@ -11,7 +13,7 @@ vespa_add_executable(vdslib_gtest_runner_app TEST vdslib_testdistribution vdslib_teststate vdslib_testthread - gtest + GTest::GTest ) vespa_add_test( diff --git a/vdslib/src/tests/bucketdistribution/CMakeLists.txt b/vdslib/src/tests/bucketdistribution/CMakeLists.txt index ecb29e9b12d..986a7ff26ff 100644 --- a/vdslib/src/tests/bucketdistribution/CMakeLists.txt +++ b/vdslib/src/tests/bucketdistribution/CMakeLists.txt @@ -1,8 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(vdslib_bucketdistributiontest SOURCES bucketdistributiontest.cpp DEPENDS vdslib - gtest + GTest::GTest ) diff --git a/vdslib/src/tests/container/CMakeLists.txt b/vdslib/src/tests/container/CMakeLists.txt index 9f7b2e33efa..47ffbcf7b82 100644 --- a/vdslib/src/tests/container/CMakeLists.txt +++ b/vdslib/src/tests/container/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(vdslib_containertest SOURCES parameterstest.cpp @@ -6,5 +7,5 @@ vespa_add_library(vdslib_containertest documentsummarytest.cpp DEPENDS vdslib - gtest + GTest::GTest ) diff --git a/vdslib/src/tests/distribution/CMakeLists.txt b/vdslib/src/tests/distribution/CMakeLists.txt index cdcfcfce5f9..61563c4c448 100644 --- a/vdslib/src/tests/distribution/CMakeLists.txt +++ b/vdslib/src/tests/distribution/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(vdslib_testdistribution SOURCES distributiontest.cpp @@ -6,5 +7,5 @@ vespa_add_library(vdslib_testdistribution idealnodecalculatorimpltest.cpp DEPENDS vdslib - gtest + GTest::GTest ) diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index f0b48faebef..5a337433bdb 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -13,10 +13,14 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/lexical_cast.h> #include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/vespalib/util/benchmark_timer.h> +#include <gmock/gmock.h> #include <chrono> #include <thread> #include <fstream> +using namespace ::testing; + namespace storage::lib { template <typename T> @@ -1143,4 +1147,106 @@ TEST(DistributionTest, test_hierarchical_distribute_less_than_redundancy) } } +TEST(DistributionTest, wildcard_top_level_distribution_gives_expected_node_results) { + std::string raw_config = R"(redundancy 2 +initial_redundancy 2 +ensure_primary_persisted true +ready_copies 2 +active_per_leaf_group false +distributor_auto_ownership_transfer_on_whole_group_down true +group[0].index "invalid" +group[0].name "invalid" +group[0].capacity 5 +group[0].partitions "*" +group[1].index "0" +group[1].name "switch0" +group[1].capacity 3 +group[1].partitions "" +group[1].nodes[0].index 0 +group[1].nodes[0].retired false +group[1].nodes[1].index 1 +group[1].nodes[1].retired false +group[1].nodes[2].index 2 +group[1].nodes[2].retired false +group[2].index "1" +group[2].name "switch1" +group[2].capacity 2 +group[2].partitions "" +group[2].nodes[0].index 3 +group[2].nodes[0].retired false +group[2].nodes[1].index 4 +group[2].nodes[1].retired false +disk_distribution "MODULO_BID" +)"; + Distribution distr(raw_config); + ClusterState state("version:1 distributor:5 storage:5"); + + auto nodes_of = [&](uint32_t bucket){ + std::vector<uint16_t> actual; + distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, bucket), actual); + return actual; + }; + + EXPECT_THAT(nodes_of(1), ElementsAre(0, 2)); + EXPECT_THAT(nodes_of(2), ElementsAre(2, 0)); + EXPECT_THAT(nodes_of(3), ElementsAre(4, 3)); + EXPECT_THAT(nodes_of(4), ElementsAre(3, 4)); + EXPECT_THAT(nodes_of(5), ElementsAre(4, 3)); + EXPECT_THAT(nodes_of(6), ElementsAre(1, 0)); + EXPECT_THAT(nodes_of(7), ElementsAre(2, 0)); +} + +namespace { + +std::string generate_config_with_n_1node_groups(int n_groups) { + std::ostringstream config_os; + std::ostringstream partition_os; + for (int i = 0; i < n_groups - 1; ++i) { + partition_os << "1|"; + } + partition_os << '*'; + config_os << "redundancy " << n_groups << "\n" + << "initial_redundancy " << n_groups << "\n" + << "ensure_primary_persisted true\n" + << "ready_copies " << n_groups << "\n" + << "active_per_leaf_group true\n" + << "distributor_auto_ownership_transfer_on_whole_group_down true\n" + << "group[0].index \"invalid\"\n" + << "group[0].name \"invalid\"\n" + << "group[0].capacity " << n_groups << "\n" + << "group[0].partitions \"" << partition_os.str() << "\"\n"; + + for (int i = 0; i < n_groups; ++i) { + int g = i + 1; + config_os << "group[" << g << "].index \"" << i << "\"\n" + << "group[" << g << "].name \"group" << g << "\"\n" + << "group[" << g << "].capacity 1\n" + << "group[" << g << "].partitions \"\"\n" + << "group[" << g << "].nodes[0].index \"" << i << "\"\n" + << "group[" << g << "].nodes[0].retired false\n"; + } + return config_os.str(); +} + +std::string generate_state_with_n_nodes_up(int n_nodes) { + std::ostringstream state_os; + state_os << "version:1 bits:8 distributor:" << n_nodes << " storage:" << n_nodes; + return state_os.str(); +} + +} + +TEST(DistributionTest, DISABLED_benchmark_ideal_state_for_many_groups) { + const int n_groups = 150; + Distribution distr(generate_config_with_n_1node_groups(n_groups)); + ClusterState state(generate_state_with_n_nodes_up(n_groups)); + + std::vector<uint16_t> actual; + uint32_t bucket = 0; + auto min_time = vespalib::BenchmarkTimer::benchmark([&]{ + distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, (bucket++ & 0xffffU)), actual); + }, 5.0); + fprintf(stderr, "%.10f seconds\n", min_time); +} + } diff --git a/vdslib/src/tests/state/CMakeLists.txt b/vdslib/src/tests/state/CMakeLists.txt index 0e057d12226..d28ff47798a 100644 --- a/vdslib/src/tests/state/CMakeLists.txt +++ b/vdslib/src/tests/state/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(vdslib_teststate SOURCES cluster_state_bundle_test.cpp @@ -7,5 +8,5 @@ vespa_add_library(vdslib_teststate nodestatetest.cpp DEPENDS vdslib - gtest + GTest::GTest ) diff --git a/vdslib/src/tests/thread/CMakeLists.txt b/vdslib/src/tests/thread/CMakeLists.txt index 4d1e753a8f6..df2e8bf43f9 100644 --- a/vdslib/src/tests/thread/CMakeLists.txt +++ b/vdslib/src/tests/thread/CMakeLists.txt @@ -1,8 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_library(vdslib_testthread SOURCES taskschedulertest.cpp DEPENDS vdslib - gtest + GTest::GTest ) diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.cpp b/vdslib/src/vespa/vdslib/distribution/distribution.cpp index 52d523071e6..4c74923c58b 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/distribution.cpp @@ -39,6 +39,7 @@ VESPA_IMPLEMENT_EXCEPTION(TooFewBucketBitsInUseException, vespalib::Exception); Distribution::Distribution() : _distributionBitMasks(getDistributionBitMasks()), _nodeGraph(), + _node2Group(), _redundancy(), _initialRedundancy(0), _ensurePrimaryPersisted(true), @@ -55,6 +56,7 @@ Distribution::Distribution() Distribution::Distribution(const Distribution& d) : _distributionBitMasks(getDistributionBitMasks()), _nodeGraph(), + _node2Group(), _redundancy(), _initialRedundancy(0), _ensurePrimaryPersisted(true), @@ -69,7 +71,7 @@ Distribution::Distribution(const Distribution& d) Distribution::ConfigWrapper::ConfigWrapper(std::unique_ptr<DistributionConfig> cfg) : _cfg(std::move(cfg)) { } -Distribution::ConfigWrapper::~ConfigWrapper() { } +Distribution::ConfigWrapper::~ConfigWrapper() = default; Distribution::Distribution(const ConfigWrapper & config) : Distribution(config.get()) @@ -78,6 +80,7 @@ Distribution::Distribution(const ConfigWrapper & config) : Distribution::Distribution(const vespa::config::content::StorDistributionConfig & config) : _distributionBitMasks(getDistributionBitMasks()), _nodeGraph(), + _node2Group(), _redundancy(), _initialRedundancy(0), _ensurePrimaryPersisted(true), @@ -93,6 +96,7 @@ Distribution::Distribution(const vespa::config::content::StorDistributionConfig Distribution::Distribution(const vespalib::string& serialized) : _distributionBitMasks(getDistributionBitMasks()), _nodeGraph(), + _node2Group(), _redundancy(), _initialRedundancy(0), _ensurePrimaryPersisted(true), @@ -113,7 +117,7 @@ Distribution::operator=(const Distribution& d) return *this; } -Distribution::~Distribution() { } +Distribution::~Distribution() = default; namespace { using ConfigDiskDistribution = vespa::config::content::StorDistributionConfig::DiskDistribution; @@ -142,34 +146,35 @@ Distribution::configure(const vespa::config::content::StorDistributionConfig& co { typedef vespa::config::content::StorDistributionConfig::Group ConfigGroup; std::unique_ptr<Group> nodeGraph; + std::vector<const Group *> node2Group; for (uint32_t i=0, n=config.group.size(); i<n; ++i) { const ConfigGroup& cg(config.group[i]); std::vector<uint16_t> path; - if (nodeGraph.get() != nullptr) { + if (nodeGraph) { path = DistributionConfigUtil::getGroupPath(cg.index); } bool isLeafGroup = (cg.nodes.size() > 0); - std::unique_ptr<Group> group; uint16_t index = (path.empty() ? 0 : path.back()); - if (isLeafGroup) { - group.reset(new Group(index, cg.name)); - } else { - group.reset(new Group( - index, cg.name, - Group::Distribution(cg.partitions), config.redundancy)); - } + std::unique_ptr<Group> group = (isLeafGroup) + ? std::make_unique<Group>(index, cg.name) + : std::make_unique<Group>(index, cg.name, Group::Distribution(cg.partitions), config.redundancy); group->setCapacity(cg.capacity); if (isLeafGroup) { std::vector<uint16_t> nodes(cg.nodes.size()); for (uint32_t j=0, m=nodes.size(); j<m; ++j) { - nodes[j] = cg.nodes[j].index; + uint16_t nodeIndex = cg.nodes[j].index; + nodes[j] = nodeIndex; + if (node2Group.size() <= nodeIndex) { + node2Group.resize(nodeIndex + 1); + } + node2Group[nodeIndex] = group.get(); } group->setNodes(nodes); } if (path.empty()) { nodeGraph = std::move(group); } else { - assert(nodeGraph.get() != nullptr); + assert(nodeGraph); Group* parent = nodeGraph.get(); for (uint32_t j=0; j<path.size() - 1; ++j) { parent = parent->getSubGroups()[path[j]]; @@ -177,7 +182,7 @@ Distribution::configure(const vespa::config::content::StorDistributionConfig& co parent->addSubGroup(std::move(group)); } } - if (nodeGraph.get() == nullptr) { + if ( ! nodeGraph) { throw vespalib::IllegalStateException( "Got config that didn't seem to specify even a root group. Must " "have a root group at minimum:\n" @@ -185,6 +190,7 @@ Distribution::configure(const vespa::config::content::StorDistributionConfig& co } nodeGraph->calculateDistributionHashValues(); _nodeGraph = std::move(nodeGraph); + _node2Group = std::move(node2Group); _redundancy = config.redundancy; _initialRedundancy = config.initialRedundancy; _ensurePrimaryPersisted = config.ensurePrimaryPersisted; @@ -345,6 +351,7 @@ namespace { const Group* _group; double _score; + ScoredGroup() : _group(nullptr), _score(0) {} ScoredGroup(const Group* group, double score) : _group(group), _score(score) {} @@ -424,40 +431,36 @@ Distribution::getIdealGroups(const document::BucketId& bucket, std::vector<ResultGroup>& results) const { if (parent.isLeafGroup()) { - results.push_back(ResultGroup(parent, redundancy)); + results.emplace_back(parent, redundancy); return; } - const Group::Distribution& redundancyArray( - parent.getDistribution(redundancy)); - std::vector<ScoredGroup> tmpResults(redundancyArray.size(), - ScoredGroup(0, 0)); - uint32_t seed(getGroupSeed(bucket, clusterState, parent)); + const Group::Distribution& redundancyArray = parent.getDistribution(redundancy); + uint32_t seed = getGroupSeed(bucket, clusterState, parent); RandomGen random(seed); uint32_t currentIndex = 0; - const std::map<uint16_t, Group*>& subGroups(parent.getSubGroups()); - for (std::map<uint16_t, Group*>::const_iterator it = subGroups.begin(); - it != subGroups.end(); ++it) - { - while (it->first < currentIndex++) random.nextDouble(); - double score = random.nextDouble(); - if (it->second->getCapacity() != 1) { - // Capacity shouldn't possibly be 0. - // Verified in Group::setCapacity() - score = std::pow(score, 1.0 / it->second->getCapacity().getValue()); + const auto& subGroups = parent.getSubGroups(); + std::vector<ScoredGroup> tmpResults; + tmpResults.reserve(subGroups.size()); + for (const auto& g : subGroups) { + while (g.first < currentIndex++) { + random.nextDouble(); } - if (score > tmpResults.back()._score) { - tmpResults.push_back(ScoredGroup(it->second, score)); - std::sort(tmpResults.begin(), tmpResults.end()); - tmpResults.pop_back(); + double score = random.nextDouble(); + if (g.second->getCapacity() != 1) { + // Capacity shouldn't possibly be 0. + // Verified in Group::setCapacity() + score = std::pow(score, 1.0 / g.second->getCapacity().getValue()); } + tmpResults.emplace_back(g.second, score); } - while (tmpResults.back()._group == nullptr) { - tmpResults.pop_back(); + std::sort(tmpResults.begin(), tmpResults.end()); + if (tmpResults.size() > redundancyArray.size()) { + tmpResults.resize(redundancyArray.size()); } for (uint32_t i=0, n=tmpResults.size(); i<n; ++i) { ScoredGroup& group(tmpResults[i]); - // This should never happen. Config should verify that each group - // has enough groups beneath them. + // This should never happen. Config should verify that each group + // has enough groups beneath them. assert(group._group != nullptr); getIdealGroups(bucket, clusterState, *group._group, redundancyArray[i], results); @@ -669,20 +672,19 @@ Distribution::splitNodesIntoLeafGroups(IndexList nodeList) const { std::vector<IndexList> result; std::map<uint16_t, IndexList> nodes; - for (uint32_t i=0, n=nodeList.size(); i<n; ++i) { - const Group* group(_nodeGraph->getGroupForNode(nodeList[i])); + for (auto node : nodeList) { + const Group* group(_node2Group[node]); if (group == nullptr) { LOGBP(warning, "Node %u is not assigned to a group. " - "Should not happen?", nodeList[i]); + "Should not happen?", node); } else { assert(group->isLeafGroup()); - nodes[group->getIndex()].push_back(nodeList[i]); + nodes[group->getIndex()].push_back(node); } } - for (std::map<uint16_t, IndexList>::const_iterator it(nodes.begin()); - it != nodes.end(); ++it) - { - result.push_back(it->second); + result.reserve(nodes.size()); + for (auto & node : nodes) { + result.emplace_back(std::move(node.second)); } return result; } diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.h b/vdslib/src/vespa/vdslib/distribution/distribution.h index 6da60e084bb..9ee505be7cd 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.h +++ b/vdslib/src/vespa/vdslib/distribution/distribution.h @@ -33,8 +33,9 @@ public: enum DiskDistribution { MODULO, MODULO_INDEX, MODULO_KNUTH, MODULO_BID }; private: - std::vector<uint32_t> _distributionBitMasks; - std::unique_ptr<Group> _nodeGraph; + std::vector<uint32_t> _distributionBitMasks; + std::unique_ptr<Group> _nodeGraph; + std::vector<const Group *> _node2Group; uint16_t _redundancy; uint16_t _initialRedundancy; uint16_t _readyCopies; diff --git a/vdslib/src/vespa/vdslib/distribution/group.cpp b/vdslib/src/vespa/vdslib/distribution/group.cpp index 91e27715911..45f8f0f8aea 100644 --- a/vdslib/src/vespa/vdslib/distribution/group.cpp +++ b/vdslib/src/vespa/vdslib/distribution/group.cpp @@ -39,11 +39,9 @@ Group::Group(uint16_t index, vespalib::stringref name, Group::~Group() { - for (std::map<uint16_t, Group*>::iterator it = _subGroups.begin(); - it != _subGroups.end(); ++it) - { - delete it->second; - it->second = 0; + for (auto & subGroup : _subGroups) { + delete subGroup.second; + subGroup.second = nullptr; } } @@ -75,8 +73,8 @@ Group::print(std::ostream& out, bool verbose, } if (_distributionSpec.size() == 0) { out << ", nodes( "; - for (uint32_t i = 0; i < _nodes.size(); i++) { - out << _nodes[i] << " "; + for (auto node : _nodes) { + out << node << " "; } out << ")"; } @@ -88,10 +86,9 @@ Group::print(std::ostream& out, bool verbose, out << ") {"; if (_subGroups.size()>0) { - for (std::map<uint16_t, Group*>::const_iterator it = _subGroups.begin(); - it != _subGroups.end(); ++it) { + for (const auto & subGroup : _subGroups) { out << "\n" << indent << " "; - it->second->print(out, verbose, indent + " "); + subGroup.second->print(out, verbose, indent + " "); } } @@ -106,12 +103,11 @@ Group::addSubGroup(Group::UP group) "Cannot add sub groups to a group without a valid distribution", VESPA_STRLOC); } - if (!group.get()) { + if (!group) { throw vespalib::IllegalArgumentException( "Cannot add null group.", VESPA_STRLOC); } - std::map<uint16_t, Group*>::const_iterator it( - _subGroups.find(group->getIndex())); + auto it =_subGroups.find(group->getIndex()); if (it != _subGroups.end()) { throw vespalib::IllegalArgumentException( "Another subgroup with same index is already added.", @@ -148,32 +144,28 @@ Group::setNodes(const std::vector<uint16_t>& nodes) const Group* Group::getGroupForNode(uint16_t nodeIdx) const { - for (uint32_t i = 0; i < _nodes.size(); ++i) { - if (_nodes[i] == nodeIdx) { + for (auto node : _nodes) { + if (node == nodeIdx) { return this; } } - for (std::map<uint16_t, Group*>::const_iterator iter = _subGroups.begin(); - iter != _subGroups.end(); - ++iter) { - const Group* g = iter->second->getGroupForNode(nodeIdx); - if (g != NULL) { + for (const auto & subGroup : _subGroups) { + const Group* g = subGroup.second->getGroupForNode(nodeIdx); + if (g != nullptr) { return g; } } - return NULL; + return nullptr; } void Group::calculateDistributionHashValues(uint32_t parentHash) { _distributionHash = parentHash ^ (1664525L * _index + 1013904223L); - for (std::map<uint16_t, Group*>::iterator it = _subGroups.begin(); - it != _subGroups.end(); ++it) - { - it->second->calculateDistributionHashValues(_distributionHash); + for (const auto & subGroup : _subGroups) { + subGroup.second->calculateDistributionHashValues(_distributionHash); } } diff --git a/vdslib/src/vespa/vdslib/distribution/group.h b/vdslib/src/vespa/vdslib/distribution/group.h index 238a56d0295..d6af005130d 100644 --- a/vdslib/src/vespa/vdslib/distribution/group.h +++ b/vdslib/src/vespa/vdslib/distribution/group.h @@ -19,8 +19,7 @@ namespace vespalib { class asciistream; } -namespace storage { -namespace lib { +namespace storage::lib { class IdealGroup; class SystemState; @@ -101,6 +100,4 @@ public: vespalib::string getDistributionConfigHash() const; }; -} // lib -} // storage - +} diff --git a/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp b/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp index 99974225e9e..e52ad78c2ba 100644 --- a/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp @@ -1,14 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vdslib/distribution/redundancygroupdistribution.h> - -#include <algorithm> -#include <boost/lexical_cast.hpp> +#include "redundancygroupdistribution.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/text/stringtokenizer.h> +#include <boost/lexical_cast.hpp> +#include <algorithm> -namespace storage { -namespace lib { +namespace storage::lib { namespace { void verifyLegal(vespalib::StringTokenizer& st, @@ -144,6 +142,4 @@ RedundancyGroupDistribution::divideSpecifiedCopies( return redundancy; } -} // lib -} // storage - +} diff --git a/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.h b/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.h index 33f895cadf0..73013b6ce81 100644 --- a/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.h +++ b/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.h @@ -11,8 +11,7 @@ #include <vector> #include <vespa/vespalib/stllike/string.h> -namespace storage { -namespace lib { +namespace storage::lib { class RedundancyGroupDistribution : public document::Printable { std::vector<uint16_t> _values; @@ -47,5 +46,4 @@ private: uint16_t redundancy, const std::vector<uint16_t>& maxValues); }; -} // lib -} // storage +} diff --git a/vespa-osgi-testrunner/CMakeLists.txt b/vespa-osgi-testrunner/CMakeLists.txt new file mode 100644 index 00000000000..58aba186710 --- /dev/null +++ b/vespa-osgi-testrunner/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(vespa-osgi-testrunner) diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml index 5ed9f75c9eb..62ea578f14f 100644 --- a/vespa-osgi-testrunner/pom.xml +++ b/vespa-osgi-testrunner/pom.xml @@ -79,9 +79,7 @@ <version>${project.version}</version> <extensions>true</extensions> <configuration> - <attachBundleArtifact>true</attachBundleArtifact> - <bundleClassifierName>deploy</bundleClassifierName> - <useCommonAssemblyIds>false</useCommonAssemblyIds> + <useCommonAssemblyIds>true</useCommonAssemblyIds> </configuration> </plugin> <plugin> diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 154b6871392..7e7e376a8df 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1904,6 +1904,7 @@ ], "methods": [ "public abstract java.lang.Double apply(com.yahoo.tensor.evaluation.EvaluationContext)", + "public java.util.Optional asTensorFunction()", "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", "public bridge synthetic java.lang.Object apply(java.lang.Object)" ], @@ -2609,6 +2610,7 @@ "public abstract com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public final com.yahoo.tensor.Tensor evaluate()", "public abstract java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public java.util.Optional asScalarFunction()", "public java.lang.String toString()" ], "fields": [] diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java index fa3d70a4ddf..1a12c7a6370 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java @@ -72,13 +72,23 @@ public class Generate<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAM } @Override - public List<TensorFunction<NAMETYPE>> arguments() { return Collections.emptyList(); } + public List<TensorFunction<NAMETYPE>> arguments() { + return boundGenerator != null && boundGenerator.asTensorFunction().isPresent() + ? List.of(boundGenerator.asTensorFunction().get()) + : List.of(); + } @Override public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { - if ( arguments.size() != 0) - throw new IllegalArgumentException("Generate must have 0 arguments, got " + arguments.size()); - return this; + if ( arguments.size() > 1) + throw new IllegalArgumentException("Generate must have 0 or 1 arguments, got " + arguments.size()); + if (arguments.isEmpty()) return this; + + if (arguments.get(0).asScalarFunction().isEmpty()) + throw new IllegalArgumentException("The argument to generate must be convertible to a tensor function, " + + "but got " + arguments.get(0)); + + return new Generate<>(type, null, arguments.get(0).asScalarFunction().get()); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunction.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunction.java index ec579a90e4f..f8ab9dfa636 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunction.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunction.java @@ -4,6 +4,7 @@ package com.yahoo.tensor.functions; import com.yahoo.tensor.evaluation.EvaluationContext; import com.yahoo.tensor.evaluation.Name; +import java.util.Optional; import java.util.function.Function; /** @@ -16,6 +17,9 @@ public interface ScalarFunction<NAMETYPE extends Name> extends Function<Evaluati @Override Double apply(EvaluationContext<NAMETYPE> context); + /** Returns this as a tensor function, or empty if it cannot be represented as a tensor function */ + default Optional<TensorFunction<NAMETYPE>> asTensorFunction() { return Optional.empty(); } + default String toString(ToStringContext context) { return toString(); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java index b4c5dedbf4e..5c0d0a99441 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java @@ -9,6 +9,7 @@ import com.yahoo.tensor.evaluation.Name; import com.yahoo.tensor.evaluation.TypeContext; import java.util.List; +import java.util.Optional; /** * A representation of a tensor function which is able to be translated to a set of primitive @@ -61,6 +62,9 @@ public abstract class TensorFunction<NAMETYPE extends Name> { */ public abstract String toString(ToStringContext context); + /** Returns this as a scalar function, or empty if it cannot be represented as a scalar function */ + public Optional<ScalarFunction<NAMETYPE>> asScalarFunction() { return Optional.empty(); } + @Override public String toString() { return toString(ToStringContext.empty()); } diff --git a/vespalib/src/tests/alloc/alloc_test.cpp b/vespalib/src/tests/alloc/alloc_test.cpp index d46d2374dfc..4dbeba62ee1 100644 --- a/vespalib/src/tests/alloc/alloc_test.cpp +++ b/vespalib/src/tests/alloc/alloc_test.cpp @@ -8,6 +8,14 @@ using namespace vespalib; using namespace vespalib::alloc; +#ifndef __SANITIZE_ADDRESS__ +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define __SANITIZE_ADDRESS__ +#endif +#endif +#endif + template <typename T> void testSwap(T & a, T & b) @@ -192,6 +200,13 @@ TEST("auto alloced mmap alloc can not be extended if no room") { TEST_DO(verifyNoExtensionWhenNoRoom(buf, reserved, SZ)); } +/* + * The two following tests are disabled when address sanitizer is + * enabled since extra instrumentation code might trigger extra mmap + * or munmap calls, breaking some of the assumptions in the disabled + * tests. + */ +#ifndef __SANITIZE_ADDRESS__ TEST("mmap alloc can be extended if room") { Alloc dummy = Alloc::allocMMap(100); Alloc reserved = Alloc::allocMMap(100); @@ -209,6 +224,7 @@ TEST("mmap alloc can not be extended if no room") { TEST_DO(verifyNoExtensionWhenNoRoom(buf, reserved, 4096)); } #endif +#endif TEST("heap alloc can not be shrinked") { Alloc buf = Alloc::allocHeap(101); diff --git a/vespalib/src/tests/crypto/CMakeLists.txt b/vespalib/src/tests/crypto/CMakeLists.txt index b930b5715b5..d0661461dd4 100644 --- a/vespalib/src/tests/crypto/CMakeLists.txt +++ b/vespalib/src/tests/crypto/CMakeLists.txt @@ -1,10 +1,11 @@ # Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalib_crypto_crypto_test_app TEST SOURCES crypto_test.cpp DEPENDS vespalib - gtest + GTest::GTest ) vespa_add_test(NAME vespalib_crypto_crypto_test_app COMMAND vespalib_crypto_crypto_test_app) diff --git a/vespalib/src/tests/datastore/datastore/CMakeLists.txt b/vespalib/src/tests/datastore/datastore/CMakeLists.txt index eb3e0a4576a..631c1812a7f 100644 --- a/vespalib/src/tests/datastore/datastore/CMakeLists.txt +++ b/vespalib/src/tests/datastore/datastore/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalib_datastore_test_app TEST SOURCES datastore_test.cpp DEPENDS vespalib - gtest + GTest::GTest ) vespa_add_test(NAME vespalib_datastore_test_app COMMAND vespalib_datastore_test_app) diff --git a/vespalib/src/tests/datastore/unique_store/CMakeLists.txt b/vespalib/src/tests/datastore/unique_store/CMakeLists.txt index d72e8c10ad5..29b34e93247 100644 --- a/vespalib/src/tests/datastore/unique_store/CMakeLists.txt +++ b/vespalib/src/tests/datastore/unique_store/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalib_unique_store_test_app TEST SOURCES unique_store_test.cpp DEPENDS vespalib - gtest + GTest::GTest ) vespa_add_test(NAME vespalib_unique_store_test_app COMMAND vespalib_unique_store_test_app) diff --git a/vespalib/src/tests/datastore/unique_store_dictionary/CMakeLists.txt b/vespalib/src/tests/datastore/unique_store_dictionary/CMakeLists.txt index b1478dea22c..2208902b5bb 100644 --- a/vespalib/src/tests/datastore/unique_store_dictionary/CMakeLists.txt +++ b/vespalib/src/tests/datastore/unique_store_dictionary/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalib_unique_store_dictionary_test_app TEST SOURCES unique_store_dictionary_test.cpp DEPENDS vespalib - gtest + GTest::GTest ) vespa_add_test(NAME vespalib_unique_store_dictionary_test_app COMMAND vespalib_unique_store_dictionary_test_app) diff --git a/vespalib/src/tests/datastore/unique_store_string_allocator/CMakeLists.txt b/vespalib/src/tests/datastore/unique_store_string_allocator/CMakeLists.txt index c2a9999d545..9984877ab21 100644 --- a/vespalib/src/tests/datastore/unique_store_string_allocator/CMakeLists.txt +++ b/vespalib/src/tests/datastore/unique_store_string_allocator/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalib_unique_store_string_allocator_test_app TEST SOURCES unique_store_string_allocator_test.cpp DEPENDS vespalib - gtest + GTest::GTest ) vespa_add_test(NAME vespalib_unique_store_string_allocator_test_app COMMAND vespalib_unique_store_string_allocator_test_app) diff --git a/vespalib/src/tests/overload/CMakeLists.txt b/vespalib/src/tests/overload/CMakeLists.txt index 67aa6230225..a03f6fbdc8d 100644 --- a/vespalib/src/tests/overload/CMakeLists.txt +++ b/vespalib/src/tests/overload/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalib_overload_test_app TEST SOURCES overload_test.cpp DEPENDS vespalib - gtest + GTest::GTest ) vespa_add_test(NAME vespalib_overload_test_app COMMAND vespalib_overload_test_app) diff --git a/vespalib/src/tests/stllike/CMakeLists.txt b/vespalib/src/tests/stllike/CMakeLists.txt index ebf7de9c747..41e0b9e8507 100644 --- a/vespalib/src/tests/stllike/CMakeLists.txt +++ b/vespalib/src/tests/stllike/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalib_hash_test_app TEST SOURCES hash_test.cpp @@ -52,6 +53,6 @@ vespa_add_executable(vespalib_replace_variable_test_app TEST replace_variable_test.cpp DEPENDS vespalib - gtest + GTest::GTest ) vespa_add_test(NAME vespalib_replace_variable_test_app COMMAND vespalib_replace_variable_test_app) diff --git a/vespalib/src/tests/time/CMakeLists.txt b/vespalib/src/tests/time/CMakeLists.txt index e43bd9097e5..ba639f2392e 100644 --- a/vespalib/src/tests/time/CMakeLists.txt +++ b/vespalib/src/tests/time/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalib_time_box_test_app TEST SOURCES time_box_test.cpp @@ -11,6 +12,6 @@ vespa_add_executable(vespalib_time_test_app TEST time_test.cpp DEPENDS vespalib - gtest + GTest::GTest ) vespa_add_test(NAME vespalib_time_test_app COMMAND vespalib_time_test_app) diff --git a/vespalib/src/tests/typify/CMakeLists.txt b/vespalib/src/tests/typify/CMakeLists.txt index 29e95af1988..c8e53d6baca 100644 --- a/vespalib/src/tests/typify/CMakeLists.txt +++ b/vespalib/src/tests/typify/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalib_typify_test_app TEST SOURCES typify_test.cpp DEPENDS vespalib - gtest + GTest::GTest ) vespa_add_test(NAME vespalib_typify_test_app COMMAND vespalib_typify_test_app) diff --git a/vespalib/src/tests/util/reusable_set/CMakeLists.txt b/vespalib/src/tests/util/reusable_set/CMakeLists.txt index 9c46b5ba61e..edbbc2ff11f 100644 --- a/vespalib/src/tests/util/reusable_set/CMakeLists.txt +++ b/vespalib/src/tests/util/reusable_set/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalib_reusable_set_test_app TEST SOURCES reusable_set_test.cpp DEPENDS vespalib - gtest + GTest::GTest ) vespa_add_test(NAME vespalib_reusable_set_test_app COMMAND vespalib_reusable_set_test_app) diff --git a/vespalib/src/tests/visit_ranges/CMakeLists.txt b/vespalib/src/tests/visit_ranges/CMakeLists.txt index de94b2ebb1e..3c51d7c1e34 100644 --- a/vespalib/src/tests/visit_ranges/CMakeLists.txt +++ b/vespalib/src/tests/visit_ranges/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalib_visit_ranges_test_app TEST SOURCES visit_ranges_test.cpp DEPENDS vespalib - gtest + GTest::GTest ) vespa_add_test(NAME vespalib_visit_ranges_test_app COMMAND vespalib_visit_ranges_test_app) diff --git a/vespalog/src/test/log_message/CMakeLists.txt b/vespalog/src/test/log_message/CMakeLists.txt index 51c888f67f3..a42e7f24bf3 100644 --- a/vespalog/src/test/log_message/CMakeLists.txt +++ b/vespalog/src/test/log_message/CMakeLists.txt @@ -1,9 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(vespalog_log_message_test_app TEST SOURCES log_message_test.cpp DEPENDS vespalog - gtest + GTest::GTest ) vespa_add_test(NAME vespalog_log_message_test_app COMMAND vespalog_log_message_test_app) |