diff options
422 files changed, 8794 insertions, 6803 deletions
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a32300043a3..52c655eebfe 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -37,3 +37,12 @@ jobs: - name: Run notebook tests run: | nbdev_test_nbs + - name: Build and publish + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.test_pypi_password }} + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + python setup.py sdist bdist_wheel + twine upload --repository testpypi dist/* diff --git a/CMakeLists.txt b/CMakeLists.txt index 044af8ff499..873e4c5fad7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,6 +85,7 @@ add_subdirectory(filedistribution) add_subdirectory(flags) add_subdirectory(fnet) add_subdirectory(fsa) +add_subdirectory(hosted-zone-api) add_subdirectory(jdisc_core) add_subdirectory(jdisc-security-filters) add_subdirectory(jdisc_http_service) diff --git a/cloud-tenant-base/OWNERS b/cloud-tenant-base/OWNERS new file mode 100644 index 00000000000..ff9741f2060 --- /dev/null +++ b/cloud-tenant-base/OWNERS @@ -0,0 +1,2 @@ +mortent +bjorncs
\ No newline at end of file diff --git a/cloud-tenant-base/README b/cloud-tenant-base/README new file mode 100644 index 00000000000..3863ef33b12 --- /dev/null +++ b/cloud-tenant-base/README @@ -0,0 +1 @@ +Parent pom for Vespa Cloud applications diff --git a/cloud-tenant-base/pom.xml b/cloud-tenant-base/pom.xml new file mode 100644 index 00000000000..8ffa6be4f9c --- /dev/null +++ b/cloud-tenant-base/pom.xml @@ -0,0 +1,383 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <artifactId>cloud-tenant-base</artifactId> + <name>Vespa Cloud tenant base</name> + <version>7-SNAPSHOT</version> + <description>Parent POM for all Vespa Cloud applications.</description> + <url>https://github.com/vespa-engine</url> + <packaging>pom</packaging> + + <parent> + <artifactId>hosted-tenant-base</artifactId> + <groupId>com.yahoo.vespa</groupId> + <version>7-SNAPSHOT</version> + <relativePath>../hosted-tenant-base/pom.xml</relativePath> + </parent> + + <licenses> + <license> + <name>The Apache License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + </license> + </licenses> + <developers> + <developer> + <name>Vespa</name> + <url>https://github.com/vespa-engine</url> + </developer> + </developers> + <scm> + <connection>scm:git:git@github.com:vespa-engine/vespa.git</connection> + <developerConnection>scm:git:git@github.com:vespa-engine/vespa.git</developerConnection> + <url>git@github.com:vespa-engine/vespa.git</url> + </scm> + + <properties> + <endpoint>https://api.vespa-external.aws.oath.cloud:4443</endpoint> + </properties> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dependency-versions</artifactId> + <version>${vespaversion}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <version>${junit.version}</version> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <version>${junit.version}</version> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container</artifactId> + <version>${vespaversion}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${vespaversion}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>cloud-tenant-cd</artifactId> + <version>${test-framework.version}</version> + <scope>runtime</scope> + </dependency> + </dependencies> + + <profiles> + <profile> + <!-- Build *-fat-test.jar file that includes all non-test classes and resources + that are part of the class path during test and and test.jar that includes + all test classes and resources, and put it inside a zip: + 1. application classes and resources + 2. test classes and resources + 3. classes and resources in all dependencies of both (1) and (2) + 4. copy the fat-test-jar and test-jar to application-test/artifacts directory + 5. zip application-test --> + <id>fat-test-application</id> + <build> + <plugins> + <plugin> + <!-- dependencies, see (3) above --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <!-- JAR-like dependencies --> + <id>unpack-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <configuration> + <includeTypes>jar,test-jar</includeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + <!-- WARNING(2018-06-27): bcpkix-jdk15on-1.58.jar and + bcprov-jdk15on-1.58.jar are pulled in via + container-dev and both contains the same set of + bouncycastle signature files in META-INF: + BC1024KE.DSA, BC1024KE.SF, BC2048KE.DSA, and + BC2048KE.SF. By merging any of these two with any + other JAR file like we're doing here, the signatures + are wrong. Worse, what we're doing is WRONG but not + yet fatal. + + The symptom of this happening is that the tester fails + to load the SystemTest class(!?), and subsequently + tries to run all test-like files in the fat test JAR. + + The solution is to exclude such files. This happens + automatically with maven-assembly-plugin. --> + <excludes>META-INF/*.SF,META-INF/*.DSA</excludes> + </configuration> + </execution> + <execution> + <!-- non-JAR-like dependencies --> + <id>non-jar-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <excludeTypes>jar,test-jar</excludeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>copy-resources</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>target/fat-test-classes</outputDirectory> + <resources> + <!-- application classes and resources, see 1. above --> + <resource> + <directory>target/classes</directory> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>fat-test-jar</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <classesDirectory>target/fat-test-classes</classesDirectory> + <classifier>fat-test</classifier> + </configuration> + </execution> + <execution> + <id>test-jar</id> + <phase>package</phase> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>attach-artifact</id> + <phase>package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <tasks> + <!-- copy fat test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-fat-test.jar" + todir="target/application-test/artifacts/" /> + + <!-- copy slim test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-tests.jar" + todir="target/application-test/artifacts/" /> + + <!-- zip application-test, see 5. above --> + <zip destfile="target/application-test.zip" + basedir="target/application-test/" /> + </tasks> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + + <profile> <!-- Alias vespaversion with a more descriptive vespa.compile.version --> + <id>set-vespa-compile-version</id> + <activation> + <property> + <name>vespa.compile.version</name> + </property> + </activation> + <properties> + <vespaversion>${vespa.compile.version}</vespaversion> + </properties> + </profile> + + <profile> <!-- Alias vespaVersion with a more descriptive vespa.runtime.version --> + <id>set-vespa-runtime-version</id> + <activation> + <property> + <name>vespa.runtime.version</name> + </property> + </activation> + <properties> + <vespaVersion>${vespa.runtime.version}</vespaVersion> + </properties> + </profile> + </profiles> + + <build> + <finalName>${project.artifactId}</finalName> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <groups>${test.categories}</groups> + <redirectTestOutputToFile>false</redirectTestOutputToFile> + <trimStackTrace>false</trimStackTrace> + <systemPropertyVariables> + <application>${application}</application> + <tenant>${tenant}</tenant> + <instance>${instance}</instance> + <environment>${environment}</environment> + <region>${region}</region> + <endpoint>${endpoint}</endpoint> + <apiKeyFile>${apiKeyFile}</apiKeyFile> + <apiCertificateFile>${apiCertificateFile}</apiCertificateFile> + <dataPlaneKeyFile>${dataPlaneKeyFile}</dataPlaneKeyFile> + <dataPlaneCertificateFile>${dataPlaneCertificateFile}</dataPlaneCertificateFile> + </systemPropertyVariables> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + </configuration> + </plugin> + </plugins> + </pluginManagement> + + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>3.0.0-M2</version> + <executions> + <execution> + <id>enforce-java</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireJavaVersion> + <version>[11, )</version> + </requireJavaVersion> + <requireMavenVersion> + <version>[3.5, )</version> + </requireMavenVersion> + </rules> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>${maven-compiler-plugin.version}</version> + <configuration> + <source>${target_jdk_version}</source> + <target>${target_jdk_version}</target> + <showWarnings>true</showWarnings> + <showDeprecation>true</showDeprecation> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-maven-plugin</artifactId> + <version>${vespaversion}</version> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-application-maven-plugin</artifactId> + <version>${vespaversion}</version> + <executions> + <execution> + <goals> + <goal>packageApplication</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${vespaversion}</version> + <extensions>true</extensions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/cloud-tenant-cd/pom.xml b/cloud-tenant-cd/pom.xml new file mode 100644 index 00000000000..ba4b7d02020 --- /dev/null +++ b/cloud-tenant-cd/pom.xml @@ -0,0 +1,86 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <artifactId>cloud-tenant-cd</artifactId> + <name>Vespa Cloud tenant CD implementation</name> + <description>Test library implementation for Vespa Cloud applications.</description> + <url>https://github.com/vespa-engine</url> + <packaging>container-plugin</packaging> + + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent</relativePath> + </parent> + + <dependencies> + <!-- provided scope --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-cd-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-zone-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <!-- compile scope --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-auth</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-api</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <attachBundleArtifact>true</attachBundleArtifact> + <bundleClassifierName>deploy</bundleClassifierName> + <useCommonAssemblyIds>false</useCommonAssemblyIds> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> +</project>
\ No newline at end of file diff --git a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java new file mode 100644 index 00000000000..75aaaec78ba --- /dev/null +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.hosted.cd; + +import ai.vespa.cloud.Zone; +import ai.vespa.hosted.api.ControllerHttpClient; +import ai.vespa.hosted.api.Properties; +import ai.vespa.hosted.api.TestConfig; +import ai.vespa.hosted.cd.http.HttpDeployment; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.zone.ZoneId; + +import java.nio.file.Files; +import java.nio.file.Paths; + +/** + * @author mortent + */ +public class VespaTestRuntime implements TestRuntime { + private final TestConfig config; + private final Deployment deploymentToTest; + + public VespaTestRuntime() { + String configPath = System.getProperty("vespa.test.config"); + TestConfig config = configPath != null ? fromFile(configPath) : fromController(); + this.config = config; + this.deploymentToTest = new HttpDeployment(config.deployments().get(config.zone()), new ai.vespa.hosted.auth.EndpointAuthenticator(config.system())); + } + + @Override + public Zone zone() { + return new Zone( + ai.vespa.cloud.Environment.valueOf(config.zone().environment().name()), + config.zone().region().value()); } + + /** Returns the deployment this is testing. */ + @Override + public Deployment deploymentToTest() { return deploymentToTest; } + + private static TestConfig fromFile(String path) { + try { + return TestConfig.fromJson(Files.readAllBytes(Paths.get(path))); + } + catch (Exception e) { + throw new IllegalArgumentException("Failed reading config from '" + path + "'!", e); + } + } + + private static TestConfig fromController() { + ControllerHttpClient controller = new ai.vespa.hosted.auth.ApiAuthenticator().controller(); + ApplicationId id = Properties.application(); + Environment environment = Properties.environment().orElse(Environment.dev); + ZoneId zone = Properties.region().map(region -> ZoneId.from(environment, region)) + .orElseGet(() -> controller.defaultZone(environment)); + return controller.testConfig(id, zone); + } +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java index e17589c325d..80d5416ab34 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.http; import ai.vespa.hosted.api.EndpointAuthenticator; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java index 74eb765dd0b..a803fc3e0e2 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.http; import ai.vespa.hosted.api.EndpointAuthenticator; diff --git a/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime b/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime new file mode 100644 index 00000000000..695fe363e4e --- /dev/null +++ b/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime @@ -0,0 +1,2 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +ai.vespa.hosted.cd.VespaTestRuntime
\ No newline at end of file diff --git a/config-lib/abi-spec.json b/config-lib/abi-spec.json index 0a6a673e39e..fa352d8f6bd 100644 --- a/config-lib/abi-spec.json +++ b/config-lib/abi-spec.json @@ -178,8 +178,8 @@ "methods": [ "public void <init>(java.lang.String)", "public java.lang.String value()", - "public int hashCode()", "public boolean equals(java.lang.Object)", + "public int hashCode()", "public java.lang.String toString()", "public static java.util.List toValues(java.util.Collection)", "public static java.util.Map toValueMap(java.util.Map)", diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index ad91ad6e1bc..6c0c2c1260f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -87,7 +87,13 @@ public interface ModelContext { return 0.9999; } - String docprocLoadBalancerType(); + // TODO Remove when 7.238 is last + default String docprocLoadBalancerType() { + return "adaptive"; + } + + /// Default setting for the gc-options attribute if not specified explicit by application + String jvmGCOptions(); boolean useDistributorBtreeDb(); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index b00eb900221..2232e57a06f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -42,7 +42,7 @@ public class TestProperties implements ModelContext.Properties { private double defaultTermwiseLimit = 1.0; private double threadPoolSizeFactor = 0.0; private double queueSizeFactor = 0.0; - private String docprocLoadBalancerType = null; + private String jvmGCOptions = null; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private AthenzDomain athenzDomain; private ApplicationRoles applicationRoles; @@ -56,13 +56,13 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean hostedVespa() { return hostedVespa; } @Override public Zone zone() { return zone; } @Override public Set<ContainerEndpoint> endpoints() { return endpoints; } + @Override public String jvmGCOptions() { return jvmGCOptions; } @Override public boolean isBootstrap() { return false; } @Override public boolean isFirstTimeDeployment() { return false; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } - @Override public String docprocLoadBalancerType() { return docprocLoadBalancerType; } @Override public double threadPoolSizeFactor() { return threadPoolSizeFactor; } @@ -74,8 +74,8 @@ public class TestProperties implements ModelContext.Properties { @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); } @Override public Optional<ApplicationRoles> applicationRoles() { return Optional.ofNullable(applicationRoles); } - public TestProperties setDocprocLoadBalancerType(String type) { - docprocLoadBalancerType = type; + public TestProperties setJvmGCOptions(String gcOptions) { + jvmGCOptions = gcOptions; return this; } public TestProperties setDefaultTermwiseLimit(double limit) { diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index 41a30c4553d..df9f72b2182 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -66,7 +66,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private final Map<Integer, NewDocumentType> inherits = new LinkedHashMap<>(); private final AnnotationTypeRegistry annotations = new AnnotationTypeRegistry(); private final StructDataType header; - private final StructDataType body; private final Set<FieldSet> fieldSets = new LinkedHashSet<>(); private final Set<Name> documentReferences; // Imported fields are virtual and therefore exist outside of the SD's document field definition @@ -84,7 +83,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp this( name, new StructDataType(name.getName() + ".header"), - new StructDataType(name.getName() + ".body"), new FieldSets(), documentReferences, importedFieldNames); @@ -96,14 +94,12 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp public NewDocumentType(Name name, StructDataType header, - StructDataType body, FieldSets fs, Set<Name> documentReferences, Set<String> importedFieldNames) { super(name.getName()); this.name = name; this.header = header; - this.body = body; if (fs != null) { this.fieldSets.addAll(fs.userFieldSets().values()); for (FieldSet f : fs.builtInFieldSets().values()) { @@ -122,7 +118,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } public DataType getHeader() { return header; } - public DataType getBody() { return body; } public Collection<NewDocumentType> getInherited() { return inherits.values(); } public NewDocumentType getInherited(Name inherited) { return inherits.get(inherited.getId()); } public NewDocumentType removeInherited(Name inherited) { return inherits.remove(inherited.getId()); } @@ -144,23 +139,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp return ret; } - /** - * Data type of the body fields of this and all inherited document types - * @return merged {@link StructDataType} - */ - public StructDataType allBody() { - StructDataType ret = new StructDataType(body.getName()); - for (Field f : body.getFields()) { - ret.addField(f); - } - for (NewDocumentType inherited : getInherited()) { - for (Field f : ((StructDataType) inherited.getBody()).getFields()) { - ret.addField(f); - } - } - return ret; - } - @Override public Class getValueClass() { return Document.class; @@ -219,9 +197,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp public Field getField(String name) { Field field = header.getField(name); if (field == null) { - field = body.getField(name); - } - if (field == null) { for (NewDocumentType inheritedType : inherits.values()) { field = inheritedType.getField(name); if (field != null) { @@ -240,9 +215,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp public Field getField(int id) { Field field = header.getField(id); if (field == null) { - field = body.getField(id); - } - if (field == null) { for (NewDocumentType inheritedType : inherits.values()) { field = inheritedType.getField(id); if (field != null) { @@ -261,14 +233,12 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } collection.addAll(header.getFields()); - collection.addAll(body.getFields()); return Collections.unmodifiableCollection(collection); } public Collection<Field> getFields() { Collection<Field> collection = new LinkedList<>(); collection.addAll(header.getFields()); - collection.addAll(body.getFields()); return Collections.unmodifiableCollection(collection); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index d3a78321106..fed35382b21 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -338,7 +338,6 @@ public class DocumentModelBuilder { Map<StructDataType, String> structInheritance = new HashMap<>(); NewDocumentType dt = new NewDocumentType(new NewDocumentType.Name(sdoc.getName()), sdoc.getDocumentType().contentStruct(), - sdoc.getDocumentType().getBodyType(), sdoc.getFieldSets(), convertDocumentReferencesToNames(sdoc.getDocumentReferences()), convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); @@ -391,7 +390,6 @@ public class DocumentModelBuilder { } } handleStruct(dt, sdoc.getDocumentType().contentStruct()); - handleStruct(dt, sdoc.getDocumentType().getBodyType()); extractDataTypesFromFields(dt, sdoc.fieldSet()); return dt; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index b3f98cc6f26..c44bf44f4fe 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -155,7 +155,6 @@ public class SDDocumentType implements Cloneable, Serializable { public SDDocumentType(String name, Search search) { docType = new DocumentType(name); docType.contentStruct().setCompressionConfig(new CompressionConfig()); - docType.getBodyType().setCompressionConfig(new CompressionConfig()); validateId(search); inherit(VESPA_DOCUMENT); } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index e0d015cc8b2..9b516767c9b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -94,8 +94,7 @@ public class DocumentManager { builder.documenttype(doc); doc. name(dt.getName()). - headerstruct(dt.contentStruct().getId()). - bodystruct(dt.getBodyType().getId()); + headerstruct(dt.contentStruct().getId()); for (DocumentType inherited : dt.getInheritedTypes()) { doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); } @@ -105,8 +104,7 @@ public class DocumentManager { builder.documenttype(doc); doc. name(dt.getName()). - headerstruct(dt.getHeader().getId()). - bodystruct(dt.getBody().getId()); + headerstruct(dt.getHeader().getId()); for (NewDocumentType inherited : dt.getInherited()) { doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index e6bf826dccc..4f075264608 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -42,8 +42,7 @@ public class DocumentTypes { db. id(documentType.getId()). name(documentType.getName()). - headerstruct(documentType.getHeader().getId()). - bodystruct(documentType.getBody().getId()); + headerstruct(documentType.getHeader().getId()); Set<Integer> built = new HashSet<>(); for (NewDocumentType inherited : documentType.getInherited()) { db.inherits(new DocumenttypesConfig.Documenttype.Inherits.Builder().id(inherited.getId())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index bb5ba71700c..c8908495c0a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -87,7 +87,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.DeprecatedSecretStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); - addSimpleComponent("ai.vespa.cloud.SystemInfo"); + addSimpleComponent("com.yahoo.container.jdisc.SystemInfoProvider"); addMetricsV2Handler(); addTestrunnerComponentsIfTester(deployState); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 82fbfe87de3..5127616ad5e 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -18,7 +18,6 @@ import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.core.document.ContainerDocumentConfig; -import com.yahoo.container.handler.ThreadPoolProvider; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; @@ -162,7 +161,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> private final double threadPoolSizeFactor; private final double queueSizeFactor; - private final String docprocLoadbalancerType; public ContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { @@ -172,7 +170,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); this.threadPoolSizeFactor = deployState.getProperties().threadPoolSizeFactor(); this.queueSizeFactor = deployState.getProperties().queueSizeFactor(); - this.docprocLoadbalancerType = deployState.getProperties().docprocLoadBalancerType(); componentGroup = new ComponentGroup<>(this, "component"); @@ -202,10 +199,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> return queueSizeFactor; } - public String getDocprocLoadbalancerType() { - return docprocLoadbalancerType; - } - public void setZone(Zone zone) { this.zone = zone; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 559a4b8b668..b83632a58a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -513,17 +513,16 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return (gcAlgorithm.matcher(jvmargs).find() ||cmsArgs.matcher(jvmargs).find()); } - private static String buildJvmGCOptions(Zone zone, String jvmGCOPtions, boolean isHostedVespa) { - if (jvmGCOPtions != null) { - return jvmGCOPtions; - } else if ((zone.system() == SystemName.dev) || isHostedVespa) { - return null; - } else { - return ContainerCluster.G1GC; - } + private static String buildJvmGCOptions(DeployState deployState, String jvmGCOPtions) { + String options = (jvmGCOPtions != null) + ? jvmGCOPtions + : deployState.getProperties().jvmGCOptions(); + return (options == null ||options.isEmpty()) + ? (deployState.isHosted() ? ContainerCluster.CMS : ContainerCluster.G1GC) + : options; } private static String getJvmOptions(ApplicationContainerCluster cluster, Element nodesElement, DeployLogger deployLogger) { - String jvmOptions = ""; + String jvmOptions; if (nodesElement.hasAttribute(VespaDomBuilder.JVM_OPTIONS)) { jvmOptions = nodesElement.getAttribute(VespaDomBuilder.JVM_OPTIONS); if (nodesElement.hasAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME)) { @@ -541,15 +540,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return jvmOptions; } + private static String extractAttribute(Element element, String attrName) { + return element.hasAttribute(attrName) ? element.getAttribute(attrName) : null; + } + void extractJvmFromLegacyNodesTag(List<ApplicationContainer> nodes, ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { applyNodesTagJvmArgs(nodes, getJvmOptions(cluster, nodesElement, context.getDeployLogger())); if (!cluster.getJvmGCOptions().isPresent()) { - String jvmGCOptions = nodesElement.hasAttribute(VespaDomBuilder.JVM_GC_OPTIONS) - ? nodesElement.getAttribute(VespaDomBuilder.JVM_GC_OPTIONS) - : null; - cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState().zone(), jvmGCOptions, context.getDeployState().isHosted())); + String jvmGCOptions = extractAttribute(nodesElement, VespaDomBuilder.JVM_GC_OPTIONS); + cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState(), jvmGCOptions)); } applyMemoryPercentage(cluster, nodesElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); @@ -559,10 +560,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element jvmElement, ConfigModelContext context) { applyNodesTagJvmArgs(nodes, jvmElement.getAttribute(VespaDomBuilder.OPTIONS)); applyMemoryPercentage(cluster, jvmElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); - String jvmGCOptions = jvmElement.hasAttribute(VespaDomBuilder.GC_OPTIONS) - ? jvmElement.getAttribute(VespaDomBuilder.GC_OPTIONS) - : null; - cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState().zone(), jvmGCOptions, context.getDeployState().isHosted())); + String jvmGCOptions = extractAttribute(jvmElement, VespaDomBuilder.GC_OPTIONS); + cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState(), jvmGCOptions)); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index 3b9b8778e31..c2a85790f89 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -150,23 +150,20 @@ public final class DocumentProtocol implements Protocol, Documentrouteselectorpo String policy = policy(docproc); for (DocprocChain chain : docproc.getChains().allChains().allComponents()) { - addChainHop(table, cluster.getConfigId(), policy, chain, cluster.getDocprocLoadbalancerType()); + addChainHop(table, cluster.getConfigId(), policy, chain); } } } } - private static void addChainHop(RoutingTableSpec table, String configId, String policy, DocprocChain chain, String docprocLoadBalancerType) { + private static void addChainHop(RoutingTableSpec table, String configId, String policy, DocprocChain chain) { final StringBuilder selector = new StringBuilder(); if (policy != null) { selector.append(configId).append("/").append(policy).append("/").append(chain.getSessionName()); } else { selector.append("[LoadBalancer:cluster=").append(configId) - .append(";session=").append(chain.getSessionName()); - if ((docprocLoadBalancerType != null) && ! docprocLoadBalancerType.isEmpty()) { - selector.append(";type=").append(docprocLoadBalancerType); - } - selector.append("]"); + .append(";session=").append(chain.getSessionName()) + .append("]"); } table.addHop(new HopSpec(chain.getServiceName(), selector.toString())); } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 5586b6a24e7..75ef3a831e8 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -586,9 +586,6 @@ void compression(SDDocumentType document, String name) : if (name == null || name.equals("header")) { document.getDocumentType().contentStruct().setCompressionConfig(cfg); } - if (name == null || name.equals("body")) { - document.getDocumentType().getBodyType().setCompressionConfig(cfg); - } } } diff --git a/config-model/src/test/configmodel/types/documentmanager.cfg b/config-model/src/test/configmodel/types/documentmanager.cfg index 631886181d7..f59dbeeb3ca 100644 --- a/config-model/src/test/configmodel/types/documentmanager.cfg +++ b/config-model/src/test/configmodel/types/documentmanager.cfg @@ -214,44 +214,37 @@ datatype[26].structtype[0].field[26].detailedtype "" datatype[26].structtype[0].field[27].name "other" datatype[26].structtype[0].field[27].datatype 4 datatype[26].structtype[0].field[27].detailedtype "" -datatype[27].id 348447225 -datatype[27].structtype[0].name "types.body" -datatype[27].structtype[0].version 0 -datatype[27].structtype[0].compresstype NONE -datatype[27].structtype[0].compresslevel 0 -datatype[27].structtype[0].compressthreshold 95 -datatype[27].structtype[0].compressminsize 800 -datatype[28].id -853072901 -datatype[28].documenttype[0].name "types" -datatype[28].documenttype[0].version 0 -datatype[28].documenttype[0].inherits[0].name "document" -datatype[28].documenttype[0].inherits[0].version 0 -datatype[28].documenttype[0].headerstruct 1328581348 -datatype[28].documenttype[0].bodystruct 348447225 -datatype[28].documenttype[0].fieldsets{[document]}.fields[0] "Folders" -datatype[28].documenttype[0].fieldsets{[document]}.fields[1] "abyte" -datatype[28].documenttype[0].fieldsets{[document]}.fields[2] "album0" -datatype[28].documenttype[0].fieldsets{[document]}.fields[3] "album1" -datatype[28].documenttype[0].fieldsets{[document]}.fields[4] "along" -datatype[28].documenttype[0].fieldsets{[document]}.fields[5] "arrarr" -datatype[28].documenttype[0].fieldsets{[document]}.fields[6] "arrayfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[7] "arraymapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[8] "complexarray" -datatype[28].documenttype[0].fieldsets{[document]}.fields[9] "doublemapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[10] "floatmapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[11] "intmapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[12] "juletre" -datatype[28].documenttype[0].fieldsets{[document]}.fields[13] "longmapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[14] "maparr" -datatype[28].documenttype[0].fieldsets{[document]}.fields[15] "mystructarr" -datatype[28].documenttype[0].fieldsets{[document]}.fields[16] "mystructfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[17] "mystructmap" -datatype[28].documenttype[0].fieldsets{[document]}.fields[18] "pos" -datatype[28].documenttype[0].fieldsets{[document]}.fields[19] "setfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[20] "setfield2" -datatype[28].documenttype[0].fieldsets{[document]}.fields[21] "setfield3" -datatype[28].documenttype[0].fieldsets{[document]}.fields[22] "setfield4" -datatype[28].documenttype[0].fieldsets{[document]}.fields[23] "stringmapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[24] "structarrayfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[25] "structfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[26] "tagfield" +datatype[27].id -853072901 +datatype[27].documenttype[0].name "types" +datatype[27].documenttype[0].version 0 +datatype[27].documenttype[0].inherits[0].name "document" +datatype[27].documenttype[0].inherits[0].version 0 +datatype[27].documenttype[0].headerstruct 1328581348 +datatype[27].documenttype[0].bodystruct 0 +datatype[27].documenttype[0].fieldsets{[document]}.fields[0] "Folders" +datatype[27].documenttype[0].fieldsets{[document]}.fields[1] "abyte" +datatype[27].documenttype[0].fieldsets{[document]}.fields[2] "album0" +datatype[27].documenttype[0].fieldsets{[document]}.fields[3] "album1" +datatype[27].documenttype[0].fieldsets{[document]}.fields[4] "along" +datatype[27].documenttype[0].fieldsets{[document]}.fields[5] "arrarr" +datatype[27].documenttype[0].fieldsets{[document]}.fields[6] "arrayfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[7] "arraymapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[8] "complexarray" +datatype[27].documenttype[0].fieldsets{[document]}.fields[9] "doublemapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[10] "floatmapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[11] "intmapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[12] "juletre" +datatype[27].documenttype[0].fieldsets{[document]}.fields[13] "longmapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[14] "maparr" +datatype[27].documenttype[0].fieldsets{[document]}.fields[15] "mystructarr" +datatype[27].documenttype[0].fieldsets{[document]}.fields[16] "mystructfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[17] "mystructmap" +datatype[27].documenttype[0].fieldsets{[document]}.fields[18] "pos" +datatype[27].documenttype[0].fieldsets{[document]}.fields[19] "setfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[20] "setfield2" +datatype[27].documenttype[0].fieldsets{[document]}.fields[21] "setfield3" +datatype[27].documenttype[0].fieldsets{[document]}.fields[22] "setfield4" +datatype[27].documenttype[0].fieldsets{[document]}.fields[23] "stringmapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[24] "structarrayfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[25] "structfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[26] "tagfield" diff --git a/config-model/src/test/configmodel/types/documenttypes.cfg b/config-model/src/test/configmodel/types/documenttypes.cfg index 8c88ee4f4e0..8f576715a4f 100644 --- a/config-model/src/test/configmodel/types/documenttypes.cfg +++ b/config-model/src/test/configmodel/types/documenttypes.cfg @@ -3,7 +3,7 @@ documenttype[0].id -853072901 documenttype[0].name "types" documenttype[0].version 0 documenttype[0].headerstruct 1328581348 -documenttype[0].bodystruct 348447225 +documenttype[0].bodystruct 0 documenttype[0].inherits[0].id 8 documenttype[0].datatype[0].id -1865479609 documenttype[0].datatype[0].type MAP @@ -547,21 +547,6 @@ documenttype[0].datatype[25].sstruct.field[27].name "other" documenttype[0].datatype[25].sstruct.field[27].id 2443357 documenttype[0].datatype[25].sstruct.field[27].datatype 4 documenttype[0].datatype[25].sstruct.field[27].detailedtype "" -documenttype[0].datatype[26].id 348447225 -documenttype[0].datatype[26].type STRUCT -documenttype[0].datatype[26].array.element.id 0 -documenttype[0].datatype[26].map.key.id 0 -documenttype[0].datatype[26].map.value.id 0 -documenttype[0].datatype[26].wset.key.id 0 -documenttype[0].datatype[26].wset.createifnonexistent false -documenttype[0].datatype[26].wset.removeifzero false -documenttype[0].datatype[26].annotationref.annotation.id 0 -documenttype[0].datatype[26].sstruct.name "types.body" -documenttype[0].datatype[26].sstruct.version 0 -documenttype[0].datatype[26].sstruct.compression.type NONE -documenttype[0].datatype[26].sstruct.compression.level 0 -documenttype[0].datatype[26].sstruct.compression.threshold 95 -documenttype[0].datatype[26].sstruct.compression.minsize 200 documenttype[0].fieldsets{[document]}.fields[0] "Folders" documenttype[0].fieldsets{[document]}.fields[1] "abyte" documenttype[0].fieldsets{[document]}.fields[2] "album0" diff --git a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg index 4f04e901a92..283e5c2fe79 100644 --- a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg +++ b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg @@ -3,7 +3,7 @@ documenttype[0].id -1368624373 documenttype[0].name "other_doc" documenttype[0].version 0 documenttype[0].headerstruct 1631005140 -documenttype[0].bodystruct 549879017 +documenttype[0].bodystruct 0 documenttype[0].inherits[0].id 8 documenttype[0].datatype[0].id 1631005140 documenttype[0].datatype[0].type STRUCT @@ -20,26 +20,11 @@ documenttype[0].datatype[0].sstruct.compression.type NONE documenttype[0].datatype[0].sstruct.compression.level 0 documenttype[0].datatype[0].sstruct.compression.threshold 95 documenttype[0].datatype[0].sstruct.compression.minsize 200 -documenttype[0].datatype[1].id 549879017 -documenttype[0].datatype[1].type STRUCT -documenttype[0].datatype[1].array.element.id 0 -documenttype[0].datatype[1].map.key.id 0 -documenttype[0].datatype[1].map.value.id 0 -documenttype[0].datatype[1].wset.key.id 0 -documenttype[0].datatype[1].wset.createifnonexistent false -documenttype[0].datatype[1].wset.removeifzero false -documenttype[0].datatype[1].annotationref.annotation.id 0 -documenttype[0].datatype[1].sstruct.name "other_doc.body" -documenttype[0].datatype[1].sstruct.version 0 -documenttype[0].datatype[1].sstruct.compression.type NONE -documenttype[0].datatype[1].sstruct.compression.level 0 -documenttype[0].datatype[1].sstruct.compression.threshold 95 -documenttype[0].datatype[1].sstruct.compression.minsize 200 documenttype[1].id -853072901 documenttype[1].name "types" documenttype[1].version 0 documenttype[1].headerstruct 1328581348 -documenttype[1].bodystruct 348447225 +documenttype[1].bodystruct 0 documenttype[1].inherits[0].id 8 documenttype[1].datatype[0].id -1368624373 documenttype[1].datatype[0].type STRUCT @@ -75,19 +60,4 @@ documenttype[1].datatype[1].sstruct.field[0].name "doc_field" documenttype[1].datatype[1].sstruct.field[0].id 819293364 documenttype[1].datatype[1].sstruct.field[0].datatype -1368624373 documenttype[1].datatype[1].sstruct.field[0].detailedtype "" -documenttype[1].datatype[2].id 348447225 -documenttype[1].datatype[2].type STRUCT -documenttype[1].datatype[2].array.element.id 0 -documenttype[1].datatype[2].map.key.id 0 -documenttype[1].datatype[2].map.value.id 0 -documenttype[1].datatype[2].wset.key.id 0 -documenttype[1].datatype[2].wset.createifnonexistent false -documenttype[1].datatype[2].wset.removeifzero false -documenttype[1].datatype[2].annotationref.annotation.id 0 -documenttype[1].datatype[2].sstruct.name "types.body" -documenttype[1].datatype[2].sstruct.version 0 -documenttype[1].datatype[2].sstruct.compression.type NONE -documenttype[1].datatype[2].sstruct.compression.level 0 -documenttype[1].datatype[2].sstruct.compression.threshold 95 -documenttype[1].datatype[2].sstruct.compression.minsize 200 documenttype[1].fieldsets{[document]}.fields[0] "doc_field" diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg index 7b50176625d..7ae73c23685 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -29,76 +29,55 @@ datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].field[1].name "person_ref" datatype[3].structtype[0].field[1].datatype 542332920 datatype[3].structtype[0].field[1].detailedtype "" -datatype[4].id -255288561 -datatype[4].structtype[0].name "ad.body" -datatype[4].structtype[0].version 0 -datatype[4].structtype[0].compresstype NONE -datatype[4].structtype[0].compresslevel 0 -datatype[4].structtype[0].compressthreshold 95 -datatype[4].structtype[0].compressminsize 800 -datatype[5].id 2987301 -datatype[5].documenttype[0].name "ad" -datatype[5].documenttype[0].version 0 -datatype[5].documenttype[0].inherits[0].name "document" -datatype[5].documenttype[0].inherits[0].version 0 -datatype[5].documenttype[0].headerstruct 959075962 -datatype[5].documenttype[0].bodystruct -255288561 -datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" -datatype[5].documenttype[0].importedfield[0].name "my_cool_field" -datatype[5].documenttype[0].importedfield[1].name "my_swag_field" -datatype[5].documenttype[0].importedfield[2].name "my_name" -datatype[6].id -2041471955 -datatype[6].structtype[0].name "campaign.header" -datatype[6].structtype[0].version 0 -datatype[6].structtype[0].compresstype NONE -datatype[6].structtype[0].compresslevel 0 -datatype[6].structtype[0].compressthreshold 95 -datatype[6].structtype[0].compressminsize 800 -datatype[6].structtype[0].field[0].name "cool_field" -datatype[6].structtype[0].field[0].datatype 2 -datatype[6].structtype[0].field[0].detailedtype "" -datatype[6].structtype[0].field[1].name "swag_field" -datatype[6].structtype[0].field[1].datatype 4 -datatype[6].structtype[0].field[1].detailedtype "" -datatype[7].id 1448849794 -datatype[7].structtype[0].name "campaign.body" +datatype[4].id 2987301 +datatype[4].documenttype[0].name "ad" +datatype[4].documenttype[0].version 0 +datatype[4].documenttype[0].inherits[0].name "document" +datatype[4].documenttype[0].inherits[0].version 0 +datatype[4].documenttype[0].headerstruct 959075962 +datatype[4].documenttype[0].bodystruct 0 +datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +datatype[4].documenttype[0].importedfield[0].name "my_cool_field" +datatype[4].documenttype[0].importedfield[1].name "my_swag_field" +datatype[4].documenttype[0].importedfield[2].name "my_name" +datatype[5].id -2041471955 +datatype[5].structtype[0].name "campaign.header" +datatype[5].structtype[0].version 0 +datatype[5].structtype[0].compresstype NONE +datatype[5].structtype[0].compresslevel 0 +datatype[5].structtype[0].compressthreshold 95 +datatype[5].structtype[0].compressminsize 800 +datatype[5].structtype[0].field[0].name "cool_field" +datatype[5].structtype[0].field[0].datatype 2 +datatype[5].structtype[0].field[0].detailedtype "" +datatype[5].structtype[0].field[1].name "swag_field" +datatype[5].structtype[0].field[1].datatype 4 +datatype[5].structtype[0].field[1].detailedtype "" +datatype[6].id -1318255918 +datatype[6].documenttype[0].name "campaign" +datatype[6].documenttype[0].version 0 +datatype[6].documenttype[0].inherits[0].name "document" +datatype[6].documenttype[0].inherits[0].version 0 +datatype[6].documenttype[0].headerstruct -2041471955 +datatype[6].documenttype[0].bodystruct 0 +datatype[6].documenttype[0].fieldsets{[document]}.fields[0] "cool_field" +datatype[6].documenttype[0].fieldsets{[document]}.fields[1] "swag_field" +datatype[7].id 3129224 +datatype[7].structtype[0].name "person.header" datatype[7].structtype[0].version 0 datatype[7].structtype[0].compresstype NONE datatype[7].structtype[0].compresslevel 0 datatype[7].structtype[0].compressthreshold 95 datatype[7].structtype[0].compressminsize 800 -datatype[8].id -1318255918 -datatype[8].documenttype[0].name "campaign" +datatype[7].structtype[0].field[0].name "name" +datatype[7].structtype[0].field[0].datatype 2 +datatype[7].structtype[0].field[0].detailedtype "" +datatype[8].id 443162583 +datatype[8].documenttype[0].name "person" datatype[8].documenttype[0].version 0 datatype[8].documenttype[0].inherits[0].name "document" datatype[8].documenttype[0].inherits[0].version 0 -datatype[8].documenttype[0].headerstruct -2041471955 -datatype[8].documenttype[0].bodystruct 1448849794 -datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "cool_field" -datatype[8].documenttype[0].fieldsets{[document]}.fields[1] "swag_field" -datatype[9].id 3129224 -datatype[9].structtype[0].name "person.header" -datatype[9].structtype[0].version 0 -datatype[9].structtype[0].compresstype NONE -datatype[9].structtype[0].compresslevel 0 -datatype[9].structtype[0].compressthreshold 95 -datatype[9].structtype[0].compressminsize 800 -datatype[9].structtype[0].field[0].name "name" -datatype[9].structtype[0].field[0].datatype 2 -datatype[9].structtype[0].field[0].detailedtype "" -datatype[10].id -2003767395 -datatype[10].structtype[0].name "person.body" -datatype[10].structtype[0].version 0 -datatype[10].structtype[0].compresstype NONE -datatype[10].structtype[0].compresslevel 0 -datatype[10].structtype[0].compressthreshold 95 -datatype[10].structtype[0].compressminsize 800 -datatype[11].id 443162583 -datatype[11].documenttype[0].name "person" -datatype[11].documenttype[0].version 0 -datatype[11].documenttype[0].inherits[0].name "document" -datatype[11].documenttype[0].inherits[0].version 0 -datatype[11].documenttype[0].headerstruct 3129224 -datatype[11].documenttype[0].bodystruct -2003767395 -datatype[11].documenttype[0].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file +datatype[8].documenttype[0].headerstruct 3129224 +datatype[8].documenttype[0].bodystruct 0 +datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "name" diff --git a/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg index e624ffdf7f5..a613c2c034d 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg @@ -24,18 +24,11 @@ datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "self_ref" datatype[].structtype[].field[].datatype -1895788438 datatype[].structtype[].field[].detailedtype "" -datatype[].id -255288561 -datatype[].structtype[].name "ad.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 2987301 datatype[].documenttype[].name "ad" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 959075962 -datatype[].documenttype[].bodystruct -255288561 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "self_ref" diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg index 90ddb8c9f8d..2b6e2e852a3 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg @@ -29,61 +29,40 @@ datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].field[1].name "person_ref" datatype[3].structtype[0].field[1].datatype 542332920 datatype[3].structtype[0].field[1].detailedtype "" -datatype[4].id -255288561 -datatype[4].structtype[0].name "ad.body" -datatype[4].structtype[0].version 0 -datatype[4].structtype[0].compresstype NONE -datatype[4].structtype[0].compresslevel 0 -datatype[4].structtype[0].compressthreshold 95 -datatype[4].structtype[0].compressminsize 800 -datatype[5].id 2987301 -datatype[5].documenttype[0].name "ad" -datatype[5].documenttype[0].version 0 -datatype[5].documenttype[0].inherits[0].name "document" -datatype[5].documenttype[0].inherits[0].version 0 -datatype[5].documenttype[0].headerstruct 959075962 -datatype[5].documenttype[0].bodystruct -255288561 -datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" -datatype[6].id -2041471955 -datatype[6].structtype[0].name "campaign.header" -datatype[6].structtype[0].version 0 -datatype[6].structtype[0].compresstype NONE -datatype[6].structtype[0].compresslevel 0 -datatype[6].structtype[0].compressthreshold 95 -datatype[6].structtype[0].compressminsize 800 -datatype[7].id 1448849794 -datatype[7].structtype[0].name "campaign.body" +datatype[4].id 2987301 +datatype[4].documenttype[0].name "ad" +datatype[4].documenttype[0].version 0 +datatype[4].documenttype[0].inherits[0].name "document" +datatype[4].documenttype[0].inherits[0].version 0 +datatype[4].documenttype[0].headerstruct 959075962 +datatype[4].documenttype[0].bodystruct 0 +datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +datatype[5].id -2041471955 +datatype[5].structtype[0].name "campaign.header" +datatype[5].structtype[0].version 0 +datatype[5].structtype[0].compresstype NONE +datatype[5].structtype[0].compresslevel 0 +datatype[5].structtype[0].compressthreshold 95 +datatype[5].structtype[0].compressminsize 800 +datatype[6].id -1318255918 +datatype[6].documenttype[0].name "campaign" +datatype[6].documenttype[0].version 0 +datatype[6].documenttype[0].inherits[0].name "document" +datatype[6].documenttype[0].inherits[0].version 0 +datatype[6].documenttype[0].headerstruct -2041471955 +datatype[6].documenttype[0].bodystruct 0 +datatype[7].id 3129224 +datatype[7].structtype[0].name "person.header" datatype[7].structtype[0].version 0 datatype[7].structtype[0].compresstype NONE datatype[7].structtype[0].compresslevel 0 datatype[7].structtype[0].compressthreshold 95 datatype[7].structtype[0].compressminsize 800 -datatype[8].id -1318255918 -datatype[8].documenttype[0].name "campaign" +datatype[8].id 443162583 +datatype[8].documenttype[0].name "person" datatype[8].documenttype[0].version 0 datatype[8].documenttype[0].inherits[0].name "document" datatype[8].documenttype[0].inherits[0].version 0 -datatype[8].documenttype[0].headerstruct -2041471955 -datatype[8].documenttype[0].bodystruct 1448849794 -datatype[9].id 3129224 -datatype[9].structtype[0].name "person.header" -datatype[9].structtype[0].version 0 -datatype[9].structtype[0].compresstype NONE -datatype[9].structtype[0].compresslevel 0 -datatype[9].structtype[0].compressthreshold 95 -datatype[9].structtype[0].compressminsize 800 -datatype[10].id -2003767395 -datatype[10].structtype[0].name "person.body" -datatype[10].structtype[0].version 0 -datatype[10].structtype[0].compresstype NONE -datatype[10].structtype[0].compresslevel 0 -datatype[10].structtype[0].compressthreshold 95 -datatype[10].structtype[0].compressminsize 800 -datatype[11].id 443162583 -datatype[11].documenttype[0].name "person" -datatype[11].documenttype[0].version 0 -datatype[11].documenttype[0].inherits[0].name "document" -datatype[11].documenttype[0].inherits[0].version 0 -datatype[11].documenttype[0].headerstruct 3129224 -datatype[11].documenttype[0].bodystruct -2003767395 +datatype[8].documenttype[0].headerstruct 3129224 +datatype[8].documenttype[0].bodystruct 0 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg index 1807adeb68d..bab281cca36 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg @@ -27,40 +27,26 @@ datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].field[1].name "other_campaign_ref" datatype[2].structtype[0].field[1].datatype 595216861 datatype[2].structtype[0].field[1].detailedtype "" -datatype[3].id -255288561 -datatype[3].structtype[0].name "ad.body" -datatype[3].structtype[0].version 0 -datatype[3].structtype[0].compresstype NONE -datatype[3].structtype[0].compresslevel 0 -datatype[3].structtype[0].compressthreshold 95 -datatype[3].structtype[0].compressminsize 800 -datatype[4].id 2987301 -datatype[4].documenttype[0].name "ad" -datatype[4].documenttype[0].version 0 -datatype[4].documenttype[0].inherits[0].name "document" -datatype[4].documenttype[0].inherits[0].version 0 -datatype[4].documenttype[0].headerstruct 959075962 -datatype[4].documenttype[0].bodystruct -255288561 -datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "other_campaign_ref" -datatype[5].id -2041471955 -datatype[5].structtype[0].name "campaign.header" -datatype[5].structtype[0].version 0 -datatype[5].structtype[0].compresstype NONE -datatype[5].structtype[0].compresslevel 0 -datatype[5].structtype[0].compressthreshold 95 -datatype[5].structtype[0].compressminsize 800 -datatype[6].id 1448849794 -datatype[6].structtype[0].name "campaign.body" -datatype[6].structtype[0].version 0 -datatype[6].structtype[0].compresstype NONE -datatype[6].structtype[0].compresslevel 0 -datatype[6].structtype[0].compressthreshold 95 -datatype[6].structtype[0].compressminsize 800 -datatype[7].id -1318255918 -datatype[7].documenttype[0].name "campaign" -datatype[7].documenttype[0].version 0 -datatype[7].documenttype[0].inherits[0].name "document" -datatype[7].documenttype[0].inherits[0].version 0 -datatype[7].documenttype[0].headerstruct -2041471955 -datatype[7].documenttype[0].bodystruct 1448849794 +datatype[3].id 2987301 +datatype[3].documenttype[0].name "ad" +datatype[3].documenttype[0].version 0 +datatype[3].documenttype[0].inherits[0].name "document" +datatype[3].documenttype[0].inherits[0].version 0 +datatype[3].documenttype[0].headerstruct 959075962 +datatype[3].documenttype[0].bodystruct 0 +datatype[3].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +datatype[3].documenttype[0].fieldsets{[document]}.fields[1] "other_campaign_ref" +datatype[4].id -2041471955 +datatype[4].structtype[0].name "campaign.header" +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].compresstype NONE +datatype[4].structtype[0].compresslevel 0 +datatype[4].structtype[0].compressthreshold 95 +datatype[4].structtype[0].compressminsize 800 +datatype[5].id -1318255918 +datatype[5].documenttype[0].name "campaign" +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0].name "document" +datatype[5].documenttype[0].inherits[0].version 0 +datatype[5].documenttype[0].headerstruct -2041471955 +datatype[5].documenttype[0].bodystruct 0 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg index 7859703ffe0..242310b57a4 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -3,7 +3,7 @@ documenttype[0].id 2987301 documenttype[0].name "ad" documenttype[0].version 0 documenttype[0].headerstruct 959075962 -documenttype[0].bodystruct -255288561 +documenttype[0].bodystruct 0 documenttype[0].inherits[0].id 8 documenttype[0].datatype[0].id 959075962 documenttype[0].datatype[0].type STRUCT @@ -28,21 +28,6 @@ documenttype[0].datatype[0].sstruct.field[1].name "person_ref" documenttype[0].datatype[0].sstruct.field[1].id 100779805 documenttype[0].datatype[0].sstruct.field[1].datatype 542332920 documenttype[0].datatype[0].sstruct.field[1].detailedtype "" -documenttype[0].datatype[1].id -255288561 -documenttype[0].datatype[1].type STRUCT -documenttype[0].datatype[1].array.element.id 0 -documenttype[0].datatype[1].map.key.id 0 -documenttype[0].datatype[1].map.value.id 0 -documenttype[0].datatype[1].wset.key.id 0 -documenttype[0].datatype[1].wset.createifnonexistent false -documenttype[0].datatype[1].wset.removeifzero false -documenttype[0].datatype[1].annotationref.annotation.id 0 -documenttype[0].datatype[1].sstruct.name "ad.body" -documenttype[0].datatype[1].sstruct.version 0 -documenttype[0].datatype[1].sstruct.compression.type NONE -documenttype[0].datatype[1].sstruct.compression.level 0 -documenttype[0].datatype[1].sstruct.compression.threshold 95 -documenttype[0].datatype[1].sstruct.compression.minsize 200 documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" documenttype[0].fieldsets{[document]}.fields[1] "person_ref" documenttype[0].referencetype[0].id 595216861 @@ -56,7 +41,7 @@ documenttype[1].id -1318255918 documenttype[1].name "campaign" documenttype[1].version 0 documenttype[1].headerstruct -2041471955 -documenttype[1].bodystruct 1448849794 +documenttype[1].bodystruct 0 documenttype[1].inherits[0].id 8 documenttype[1].datatype[0].id -2041471955 documenttype[1].datatype[0].type STRUCT @@ -81,28 +66,13 @@ documenttype[1].datatype[0].sstruct.field[1].name "swag_field" documenttype[1].datatype[0].sstruct.field[1].id 1691224741 documenttype[1].datatype[0].sstruct.field[1].datatype 4 documenttype[1].datatype[0].sstruct.field[1].detailedtype "" -documenttype[1].datatype[1].id 1448849794 -documenttype[1].datatype[1].type STRUCT -documenttype[1].datatype[1].array.element.id 0 -documenttype[1].datatype[1].map.key.id 0 -documenttype[1].datatype[1].map.value.id 0 -documenttype[1].datatype[1].wset.key.id 0 -documenttype[1].datatype[1].wset.createifnonexistent false -documenttype[1].datatype[1].wset.removeifzero false -documenttype[1].datatype[1].annotationref.annotation.id 0 -documenttype[1].datatype[1].sstruct.name "campaign.body" -documenttype[1].datatype[1].sstruct.version 0 -documenttype[1].datatype[1].sstruct.compression.type NONE -documenttype[1].datatype[1].sstruct.compression.level 0 -documenttype[1].datatype[1].sstruct.compression.threshold 95 -documenttype[1].datatype[1].sstruct.compression.minsize 200 documenttype[1].fieldsets{[document]}.fields[0] "cool_field" documenttype[1].fieldsets{[document]}.fields[1] "swag_field" documenttype[2].id 443162583 documenttype[2].name "person" documenttype[2].version 0 documenttype[2].headerstruct 3129224 -documenttype[2].bodystruct -2003767395 +documenttype[2].bodystruct 0 documenttype[2].inherits[0].id 8 documenttype[2].datatype[0].id 3129224 documenttype[2].datatype[0].type STRUCT @@ -123,19 +93,4 @@ documenttype[2].datatype[0].sstruct.field[0].name "name" documenttype[2].datatype[0].sstruct.field[0].id 1160796772 documenttype[2].datatype[0].sstruct.field[0].datatype 2 documenttype[2].datatype[0].sstruct.field[0].detailedtype "" -documenttype[2].datatype[1].id -2003767395 -documenttype[2].datatype[1].type STRUCT -documenttype[2].datatype[1].array.element.id 0 -documenttype[2].datatype[1].map.key.id 0 -documenttype[2].datatype[1].map.value.id 0 -documenttype[2].datatype[1].wset.key.id 0 -documenttype[2].datatype[1].wset.createifnonexistent false -documenttype[2].datatype[1].wset.removeifzero false -documenttype[2].datatype[1].annotationref.annotation.id 0 -documenttype[2].datatype[1].sstruct.name "person.body" -documenttype[2].datatype[1].sstruct.version 0 -documenttype[2].datatype[1].sstruct.compression.type NONE -documenttype[2].datatype[1].sstruct.compression.level 0 -documenttype[2].datatype[1].sstruct.compression.threshold 95 -documenttype[2].datatype[1].sstruct.compression.minsize 200 -documenttype[2].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file +documenttype[2].fieldsets{[document]}.fields[0] "name" diff --git a/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg index ca568aaa43d..f925ac99a25 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg @@ -3,7 +3,7 @@ documenttype[].id 2987301 documenttype[].name "ad" documenttype[].version 0 documenttype[].headerstruct 959075962 -documenttype[].bodystruct -255288561 +documenttype[].bodystruct 0 documenttype[].inherits[].id 8 documenttype[].datatype[].id 959075962 documenttype[].datatype[].type STRUCT @@ -24,21 +24,6 @@ documenttype[].datatype[].sstruct.field[].name "self_ref" documenttype[].datatype[].sstruct.field[].id 852207313 documenttype[].datatype[].sstruct.field[].datatype -1895788438 documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id -255288561 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "ad.body" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 documenttype[].fieldsets{[document]}.fields[] "self_ref" documenttype[].referencetype[].id -1895788438 documenttype[].referencetype[].target_type_id 2987301 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg index a7fd3577789..c3aba21a498 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg @@ -3,7 +3,7 @@ documenttype[0].id 2987301 documenttype[0].name "ad" documenttype[0].version 0 documenttype[0].headerstruct 959075962 -documenttype[0].bodystruct -255288561 +documenttype[0].bodystruct 0 documenttype[0].inherits[0].id 8 documenttype[0].datatype[0].id 959075962 documenttype[0].datatype[0].type STRUCT @@ -28,21 +28,6 @@ documenttype[0].datatype[0].sstruct.field[1].name "person_ref" documenttype[0].datatype[0].sstruct.field[1].id 100779805 documenttype[0].datatype[0].sstruct.field[1].datatype 542332920 documenttype[0].datatype[0].sstruct.field[1].detailedtype "" -documenttype[0].datatype[1].id -255288561 -documenttype[0].datatype[1].type STRUCT -documenttype[0].datatype[1].array.element.id 0 -documenttype[0].datatype[1].map.key.id 0 -documenttype[0].datatype[1].map.value.id 0 -documenttype[0].datatype[1].wset.key.id 0 -documenttype[0].datatype[1].wset.createifnonexistent false -documenttype[0].datatype[1].wset.removeifzero false -documenttype[0].datatype[1].annotationref.annotation.id 0 -documenttype[0].datatype[1].sstruct.name "ad.body" -documenttype[0].datatype[1].sstruct.version 0 -documenttype[0].datatype[1].sstruct.compression.type NONE -documenttype[0].datatype[1].sstruct.compression.level 0 -documenttype[0].datatype[1].sstruct.compression.threshold 95 -documenttype[0].datatype[1].sstruct.compression.minsize 200 documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" documenttype[0].fieldsets{[document]}.fields[1] "person_ref" documenttype[0].referencetype[0].id 595216861 @@ -53,7 +38,7 @@ documenttype[1].id -1318255918 documenttype[1].name "campaign" documenttype[1].version 0 documenttype[1].headerstruct -2041471955 -documenttype[1].bodystruct 1448849794 +documenttype[1].bodystruct 0 documenttype[1].inherits[0].id 8 documenttype[1].datatype[0].id -2041471955 documenttype[1].datatype[0].type STRUCT @@ -70,26 +55,11 @@ documenttype[1].datatype[0].sstruct.compression.type NONE documenttype[1].datatype[0].sstruct.compression.level 0 documenttype[1].datatype[0].sstruct.compression.threshold 95 documenttype[1].datatype[0].sstruct.compression.minsize 200 -documenttype[1].datatype[1].id 1448849794 -documenttype[1].datatype[1].type STRUCT -documenttype[1].datatype[1].array.element.id 0 -documenttype[1].datatype[1].map.key.id 0 -documenttype[1].datatype[1].map.value.id 0 -documenttype[1].datatype[1].wset.key.id 0 -documenttype[1].datatype[1].wset.createifnonexistent false -documenttype[1].datatype[1].wset.removeifzero false -documenttype[1].datatype[1].annotationref.annotation.id 0 -documenttype[1].datatype[1].sstruct.name "campaign.body" -documenttype[1].datatype[1].sstruct.version 0 -documenttype[1].datatype[1].sstruct.compression.type NONE -documenttype[1].datatype[1].sstruct.compression.level 0 -documenttype[1].datatype[1].sstruct.compression.threshold 95 -documenttype[1].datatype[1].sstruct.compression.minsize 200 documenttype[2].id 443162583 documenttype[2].name "person" documenttype[2].version 0 documenttype[2].headerstruct 3129224 -documenttype[2].bodystruct -2003767395 +documenttype[2].bodystruct 0 documenttype[2].inherits[0].id 8 documenttype[2].datatype[0].id 3129224 documenttype[2].datatype[0].type STRUCT @@ -106,18 +76,3 @@ documenttype[2].datatype[0].sstruct.compression.type NONE documenttype[2].datatype[0].sstruct.compression.level 0 documenttype[2].datatype[0].sstruct.compression.threshold 95 documenttype[2].datatype[0].sstruct.compression.minsize 200 -documenttype[2].datatype[1].id -2003767395 -documenttype[2].datatype[1].type STRUCT -documenttype[2].datatype[1].array.element.id 0 -documenttype[2].datatype[1].map.key.id 0 -documenttype[2].datatype[1].map.value.id 0 -documenttype[2].datatype[1].wset.key.id 0 -documenttype[2].datatype[1].wset.createifnonexistent false -documenttype[2].datatype[1].wset.removeifzero false -documenttype[2].datatype[1].annotationref.annotation.id 0 -documenttype[2].datatype[1].sstruct.name "person.body" -documenttype[2].datatype[1].sstruct.version 0 -documenttype[2].datatype[1].sstruct.compression.type NONE -documenttype[2].datatype[1].sstruct.compression.level 0 -documenttype[2].datatype[1].sstruct.compression.threshold 95 -documenttype[2].datatype[1].sstruct.compression.minsize 200 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg index 46a951ae8ea..c5930449dc1 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg @@ -3,7 +3,7 @@ documenttype[0].id 2987301 documenttype[0].name "ad" documenttype[0].version 0 documenttype[0].headerstruct 959075962 -documenttype[0].bodystruct -255288561 +documenttype[0].bodystruct 0 documenttype[0].inherits[0].id 8 documenttype[0].datatype[0].id 959075962 documenttype[0].datatype[0].type STRUCT @@ -28,21 +28,6 @@ documenttype[0].datatype[0].sstruct.field[1].name "other_campaign_ref" documenttype[0].datatype[0].sstruct.field[1].id 874751172 documenttype[0].datatype[0].sstruct.field[1].datatype 595216861 documenttype[0].datatype[0].sstruct.field[1].detailedtype "" -documenttype[0].datatype[1].id -255288561 -documenttype[0].datatype[1].type STRUCT -documenttype[0].datatype[1].array.element.id 0 -documenttype[0].datatype[1].map.key.id 0 -documenttype[0].datatype[1].map.value.id 0 -documenttype[0].datatype[1].wset.key.id 0 -documenttype[0].datatype[1].wset.createifnonexistent false -documenttype[0].datatype[1].wset.removeifzero false -documenttype[0].datatype[1].annotationref.annotation.id 0 -documenttype[0].datatype[1].sstruct.name "ad.body" -documenttype[0].datatype[1].sstruct.version 0 -documenttype[0].datatype[1].sstruct.compression.type NONE -documenttype[0].datatype[1].sstruct.compression.level 0 -documenttype[0].datatype[1].sstruct.compression.threshold 95 -documenttype[0].datatype[1].sstruct.compression.minsize 200 documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" documenttype[0].fieldsets{[document]}.fields[1] "other_campaign_ref" documenttype[0].referencetype[0].id 595216861 @@ -51,7 +36,7 @@ documenttype[1].id -1318255918 documenttype[1].name "campaign" documenttype[1].version 0 documenttype[1].headerstruct -2041471955 -documenttype[1].bodystruct 1448849794 +documenttype[1].bodystruct 0 documenttype[1].inherits[0].id 8 documenttype[1].datatype[0].id -2041471955 documenttype[1].datatype[0].type STRUCT @@ -68,18 +53,3 @@ documenttype[1].datatype[0].sstruct.compression.type NONE documenttype[1].datatype[0].sstruct.compression.level 0 documenttype[1].datatype[0].sstruct.compression.threshold 95 documenttype[1].datatype[0].sstruct.compression.minsize 200 -documenttype[1].datatype[1].id 1448849794 -documenttype[1].datatype[1].type STRUCT -documenttype[1].datatype[1].array.element.id 0 -documenttype[1].datatype[1].map.key.id 0 -documenttype[1].datatype[1].map.value.id 0 -documenttype[1].datatype[1].wset.key.id 0 -documenttype[1].datatype[1].wset.createifnonexistent false -documenttype[1].datatype[1].wset.removeifzero false -documenttype[1].datatype[1].annotationref.annotation.id 0 -documenttype[1].datatype[1].sstruct.name "campaign.body" -documenttype[1].datatype[1].sstruct.version 0 -documenttype[1].datatype[1].sstruct.compression.type NONE -documenttype[1].datatype[1].sstruct.compression.level 0 -documenttype[1].datatype[1].sstruct.compression.threshold 95 -documenttype[1].datatype[1].sstruct.compression.minsize 200 diff --git a/config-model/src/test/derived/advanced/documentmanager.cfg b/config-model/src/test/derived/advanced/documentmanager.cfg index a0a59fbf7ac..4da92d82fb9 100644 --- a/config-model/src/test/derived/advanced/documentmanager.cfg +++ b/config-model/src/test/derived/advanced/documentmanager.cfg @@ -67,20 +67,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "mysummary" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id -704605648 -datatype[].structtype[].name "advanced.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 686681444 datatype[].documenttype[].name "advanced" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1337915045 -datatype[].documenttype[].bodystruct -704605648 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{titleabstract}.fields[] "title" datatype[].documenttype[].fieldsets{default}.fields[] "title" datatype[].documenttype[].fieldsets{[document]}.fields[] "attributes_src" diff --git a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg index fae6bd46ad7..aa74ecebd5b 100644 --- a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg @@ -29,20 +29,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -1503592268 -datatype[].structtype[].name "annotationsimplicitstruct.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -2099544992 datatype[].documenttype[].name "annotationsimplicitstruct" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -364910881 -datatype[].documenttype[].bodystruct -1503592268 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -269517759 annotationtype[].name "banana" annotationtype[].datatype 517946310 diff --git a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg index 21baed26dbf..e103218793d 100644 --- a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg @@ -94,20 +94,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 1181354668 -datatype[].structtype[].name "annotationsinheritance.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -748546200 datatype[].documenttype[].name "annotationsinheritance" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1406250281 -datatype[].documenttype[].bodystruct 1181354668 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -269517759 annotationtype[].name "banana" annotationtype[].datatype 517946310 diff --git a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg index 3ef71148f12..5b5b2ac348f 100644 --- a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg @@ -57,20 +57,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 1375438150 -datatype[].structtype[].name "annotationsinheritance2.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1730091890 datatype[].documenttype[].name "annotationsinheritance2" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 424382193 -datatype[].documenttype[].bodystruct 1375438150 +datatype[].documenttype[].bodystruct 0 annotationtype[].id 1769416289 annotationtype[].name "a" annotationtype[].datatype -1 diff --git a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg index e9ec2cb3715..1f71057f268 100644 --- a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg +++ b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg @@ -31,20 +31,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -570750959 -datatype[].structtype[].name "annotationspolymorphy.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1383624989 datatype[].documenttype[].name "annotationspolymorphy" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1552577796 -datatype[].documenttype[].bodystruct -570750959 +datatype[].documenttype[].bodystruct 0 annotationtype[].id 668095690 annotationtype[].name "super" annotationtype[].datatype -1 diff --git a/config-model/src/test/derived/annotationsreference/documentmanager.cfg b/config-model/src/test/derived/annotationsreference/documentmanager.cfg index 6526f56a906..737bcbf3cac 100644 --- a/config-model/src/test/derived/annotationsreference/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsreference/documentmanager.cfg @@ -65,20 +65,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 1692909067 -datatype[].structtype[].name "annotationsreference.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1448377175 datatype[].documenttype[].name "annotationsreference" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 571255414 -datatype[].documenttype[].bodystruct 1692909067 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -269517759 annotationtype[].name "banana" annotationtype[].datatype 517946310 diff --git a/config-model/src/test/derived/annotationssimple/documentmanager.cfg b/config-model/src/test/derived/annotationssimple/documentmanager.cfg index d32f0addceb..3af65e96558 100644 --- a/config-model/src/test/derived/annotationssimple/documentmanager.cfg +++ b/config-model/src/test/derived/annotationssimple/documentmanager.cfg @@ -19,20 +19,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -682121732 -datatype[].structtype[].name "annotationssimple.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1584092648 datatype[].documenttype[].name "annotationssimple" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1205708249 -datatype[].documenttype[].bodystruct -682121732 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -269517759 annotationtype[].name "banana" annotationtype[].datatype -1 diff --git a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg index c91b5c5e97e..0a1cda99a95 100644 --- a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg @@ -39,20 +39,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -1180029319 -datatype[].structtype[].name "annotationsstruct.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -263977093 datatype[].documenttype[].name "annotationsstruct" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1341437796 -datatype[].documenttype[].bodystruct -1180029319 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -160036815 annotationtype[].name "my_anno" annotationtype[].datatype -1080124700 diff --git a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg index 22b951b1b5d..fca86c58ffa 100644 --- a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg @@ -41,20 +41,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 1616435858 -datatype[].structtype[].name "annotationsstructarray.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1593733058 datatype[].documenttype[].name "annotationsstructarray" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 94945597 -datatype[].documenttype[].bodystruct 1616435858 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -160036815 annotationtype[].name "my_anno" annotationtype[].datatype -1080124700 diff --git a/config-model/src/test/derived/arrays/documentmanager.cfg b/config-model/src/test/derived/arrays/documentmanager.cfg index a2d8e2e78b4..f542a936574 100644 --- a/config-model/src/test/derived/arrays/documentmanager.cfg +++ b/config-model/src/test/derived/arrays/documentmanager.cfg @@ -42,20 +42,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "c" datatype[].structtype[].field[].datatype 1328286588 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1747896808 -datatype[].structtype[].name "arrays.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1292863364 datatype[].documenttype[].name "arrays" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1081627459 -datatype[].documenttype[].bodystruct -1747896808 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{default}.fields[] "a" datatype[].documenttype[].fieldsets{default}.fields[] "b" datatype[].documenttype[].fieldsets{default}.fields[] "c" diff --git a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg index e27c72fbe50..dc208a86913 100644 --- a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg +++ b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg @@ -109,20 +109,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "wsstring" datatype[].structtype[].field[].datatype 1328286588 datatype[].structtype[].field[].detailedtype "" -datatype[].id 932425403 -datatype[].structtype[].name "prefetch.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1458051591 datatype[].documenttype[].name "prefetch" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -109105370 -datatype[].documenttype[].bodystruct 932425403 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "multibyte" datatype[].documenttype[].fieldsets{[document]}.fields[] "multidouble" datatype[].documenttype[].fieldsets{[document]}.fields[] "multifloat" diff --git a/config-model/src/test/derived/complex/documentmanager.cfg b/config-model/src/test/derived/complex/documentmanager.cfg index 42234e52211..50d5dac1ef9 100644 --- a/config-model/src/test/derived/complex/documentmanager.cfg +++ b/config-model/src/test/derived/complex/documentmanager.cfg @@ -98,20 +98,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "exact" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1665926686 -datatype[].structtype[].name "complex.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1402929550 datatype[].documenttype[].name "complex" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1749463923 -datatype[].documenttype[].bodystruct -1665926686 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{default}.fields[] "stringfield" datatype[].documenttype[].fieldsets{default}.fields[] "title" datatype[].documenttype[].fieldsets{special}.fields[] "special1" diff --git a/config-model/src/test/derived/emptydefault/documentmanager.cfg b/config-model/src/test/derived/emptydefault/documentmanager.cfg index b6cb2d06718..e69b2c5d8c3 100644 --- a/config-model/src/test/derived/emptydefault/documentmanager.cfg +++ b/config-model/src/test/derived/emptydefault/documentmanager.cfg @@ -25,19 +25,12 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "two" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id 311791038 -datatype[].structtype[].name "emptydefault.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1663995626 datatype[].documenttype[].name "emptydefault" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 461724009 -datatype[].documenttype[].bodystruct 311791038 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "one" datatype[].documenttype[].fieldsets{[document]}.fields[] "two" diff --git a/config-model/src/test/derived/id/documentmanager.cfg b/config-model/src/test/derived/id/documentmanager.cfg index 5140abc65fa..8ee82cdd946 100644 --- a/config-model/src/test/derived/id/documentmanager.cfg +++ b/config-model/src/test/derived/id/documentmanager.cfg @@ -22,18 +22,11 @@ datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "uri" datatype[].structtype[].field[].datatype 10 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1830022377 -datatype[].structtype[].name "id.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 3225629 datatype[].documenttype[].name "id" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -531633022 -datatype[].documenttype[].bodystruct -1830022377 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "uri" diff --git a/config-model/src/test/derived/indexswitches/documentmanager.cfg b/config-model/src/test/derived/indexswitches/documentmanager.cfg index 78dbdb7ae74..ffeaab177ba 100644 --- a/config-model/src/test/derived/indexswitches/documentmanager.cfg +++ b/config-model/src/test/derived/indexswitches/documentmanager.cfg @@ -31,20 +31,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "source" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1892617122 -datatype[].structtype[].name "indexswitches.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -753375626 datatype[].documenttype[].name "indexswitches" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -555640823 -datatype[].documenttype[].bodystruct -1892617122 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{default}.fields[] "descr" datatype[].documenttype[].fieldsets{default}.fields[] "title" datatype[].documenttype[].fieldsets{[document]}.fields[] "descr" diff --git a/config-model/src/test/derived/inheritance/documentmanager.cfg b/config-model/src/test/derived/inheritance/documentmanager.cfg index b15ef13ed3f..e054019bd8f 100644 --- a/config-model/src/test/derived/inheritance/documentmanager.cfg +++ b/config-model/src/test/derived/inheritance/documentmanager.cfg @@ -25,20 +25,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "overridden" datatype[].structtype[].field[].datatype 0 datatype[].structtype[].field[].detailedtype "" -datatype[].id 978262812 -datatype[].structtype[].name "grandparent.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -154107656 datatype[].documenttype[].name "grandparent" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 990971719 -datatype[].documenttype[].bodystruct 978262812 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "onlygrandparent" datatype[].documenttype[].fieldsets{[document]}.fields[] "overridden" datatype[].id 1306663898 @@ -54,13 +47,6 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "overridden" datatype[].structtype[].field[].datatype 0 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1989003153 -datatype[].structtype[].name "mother.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -158393403 datatype[].documenttype[].name "mother" datatype[].documenttype[].version 0 @@ -69,7 +55,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1306663898 -datatype[].documenttype[].bodystruct -1989003153 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "onlygrandparent" datatype[].documenttype[].fieldsets{[document]}.fields[] "onlymother" datatype[].documenttype[].fieldsets{[document]}.fields[] "overridden" @@ -86,13 +72,6 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "overridden" datatype[].structtype[].field[].datatype 0 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1742340170 -datatype[].structtype[].name "father.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 986686494 datatype[].documenttype[].name "father" datatype[].documenttype[].version 0 @@ -101,7 +80,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 2126589281 -datatype[].documenttype[].bodystruct -1742340170 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "onlyfather" datatype[].documenttype[].fieldsets{[document]}.fields[] "onlygrandparent" datatype[].documenttype[].fieldsets{[document]}.fields[] "overridden" @@ -118,13 +97,6 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "overridden" datatype[].structtype[].field[].datatype 0 datatype[].structtype[].field[].detailedtype "" -datatype[].id -126593034 -datatype[].structtype[].name "child.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 746267614 datatype[].documenttype[].name "child" datatype[].documenttype[].version 0 @@ -135,7 +107,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "mother" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 81425825 -datatype[].documenttype[].bodystruct -126593034 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "onlychild" datatype[].documenttype[].fieldsets{[document]}.fields[] "onlyfather" datatype[].documenttype[].fieldsets{[document]}.fields[] "onlygrandparent" diff --git a/config-model/src/test/derived/inheritdiamond/documentmanager.cfg b/config-model/src/test/derived/inheritdiamond/documentmanager.cfg index c3ead0d31f8..df3f8908a60 100644 --- a/config-model/src/test/derived/inheritdiamond/documentmanager.cfg +++ b/config-model/src/test/derived/inheritdiamond/documentmanager.cfg @@ -1,11 +1,4 @@ enablecompression false -datatype[].id -126593034 -datatype[].structtype[].name "child.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 336538650 datatype[].structtype[].name "child_struct" datatype[].structtype[].version 0 @@ -40,7 +33,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "father" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 81425825 -datatype[].documenttype[].bodystruct -126593034 +datatype[].documenttype[].bodystruct 0 datatype[].id -1913265190 datatype[].structtype[].name "father_struct" datatype[].structtype[].version 0 @@ -66,27 +59,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -52742073 -datatype[].structtype[].name "father_search.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1464571117 datatype[].documenttype[].name "father_search" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1962244686 -datatype[].documenttype[].bodystruct -52742073 -datatype[].id -1852215954 -datatype[].structtype[].name "mother_search.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 +datatype[].documenttype[].bodystruct 0 datatype[].id -384824039 datatype[].structtype[].name "mother_search.header" datatype[].structtype[].version 0 @@ -109,7 +88,7 @@ datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -384824039 -datatype[].documenttype[].bodystruct -1852215954 +datatype[].documenttype[].bodystruct 0 datatype[].id 1306663898 datatype[].structtype[].name "mother.header" datatype[].structtype[].version 0 @@ -117,13 +96,6 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -1989003153 -datatype[].structtype[].name "mother.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -158393403 datatype[].documenttype[].name "mother" datatype[].documenttype[].version 0 @@ -132,7 +104,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1306663898 -datatype[].documenttype[].bodystruct -1989003153 +datatype[].documenttype[].bodystruct 0 datatype[].id -205818510 datatype[].structtype[].name "child_search.header" datatype[].structtype[].version 0 @@ -140,20 +112,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -1467672569 -datatype[].structtype[].name "child_search.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -580592339 datatype[].documenttype[].name "child_search" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -205818510 -datatype[].documenttype[].bodystruct -1467672569 +datatype[].documenttype[].bodystruct 0 datatype[].id 111553393 datatype[].structtype[].name "url" datatype[].structtype[].version 0 @@ -186,13 +151,6 @@ datatype[].structtype[].field[].name "x" datatype[].structtype[].field[].datatype 0 datatype[].structtype[].field[].name "y" datatype[].structtype[].field[].datatype 0 -datatype[].id 1845861921 -datatype[].structtype[].name "grandparent_search.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1530060044 datatype[].structtype[].name "grandparent_search.header" datatype[].structtype[].version 0 @@ -206,7 +164,7 @@ datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1530060044 -datatype[].documenttype[].bodystruct 1845861921 +datatype[].documenttype[].bodystruct 0 datatype[].id 990971719 datatype[].structtype[].name "grandparent.header" datatype[].structtype[].version 0 @@ -214,27 +172,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 978262812 -datatype[].structtype[].name "grandparent.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -154107656 datatype[].documenttype[].name "grandparent" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 990971719 -datatype[].documenttype[].bodystruct 978262812 -datatype[].id -1742340170 -datatype[].structtype[].name "father.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 +datatype[].documenttype[].bodystruct 0 datatype[].id 2126589281 datatype[].structtype[].name "father.header" datatype[].structtype[].version 0 @@ -250,4 +194,4 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 2126589281 -datatype[].documenttype[].bodystruct -1742340170 +datatype[].documenttype[].bodystruct 0 diff --git a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg index 8e2ee3bbc4e..25872641741 100644 --- a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg @@ -29,20 +29,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 978262812 -datatype[].structtype[].name "grandparent.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -154107656 datatype[].documenttype[].name "grandparent" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 990971719 -datatype[].documenttype[].bodystruct 978262812 +datatype[].documenttype[].bodystruct 0 datatype[].id 836075987 datatype[].structtype[].name "parent.header" datatype[].structtype[].version 0 @@ -50,13 +43,6 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -389494616 -datatype[].structtype[].name "parent.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1175161836 datatype[].documenttype[].name "parent" datatype[].documenttype[].version 0 @@ -65,7 +51,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 836075987 -datatype[].documenttype[].bodystruct -389494616 +datatype[].documenttype[].bodystruct 0 datatype[].id 81425825 datatype[].structtype[].name "child.header" datatype[].structtype[].version 0 @@ -76,13 +62,6 @@ datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "child_field" datatype[].structtype[].field[].datatype 1246084544 datatype[].structtype[].field[].detailedtype "" -datatype[].id -126593034 -datatype[].structtype[].name "child.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 746267614 datatype[].documenttype[].name "child" datatype[].documenttype[].version 0 @@ -91,5 +70,5 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "parent" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 81425825 -datatype[].documenttype[].bodystruct -126593034 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "child_field" diff --git a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg index 7c65a7b72f3..c9cd6fd3042 100644 --- a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg @@ -35,20 +35,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "weight" datatype[].structtype[].field[].datatype 1 datatype[].structtype[].field[].detailedtype "" -datatype[].id -389494616 -datatype[].structtype[].name "parent.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1175161836 datatype[].documenttype[].name "parent" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 836075987 -datatype[].documenttype[].bodystruct -389494616 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[]}.fields[] "weight_src" datatype[].id 81425825 datatype[].structtype[].name "child.header" @@ -60,13 +53,6 @@ datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "child_field" datatype[].structtype[].field[].datatype 1091188812 datatype[].structtype[].field[].detailedtype "" -datatype[].id -126593034 -datatype[].structtype[].name "child.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 746267614 datatype[].documenttype[].name "child" datatype[].documenttype[].version 0 @@ -75,6 +61,6 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "parent" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 81425825 -datatype[].documenttype[].bodystruct -126593034 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[]}.fields[] "child_field" datatype[].documenttype[].fieldsets{[]}.fields[] "weight_src" diff --git a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg index f5ec18c4203..faef3f6923b 100644 --- a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg +++ b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg @@ -3,7 +3,7 @@ documenttype[].id 1175161836 documenttype[].name "parent" documenttype[].version 0 documenttype[].headerstruct 836075987 -documenttype[].bodystruct -389494616 +documenttype[].bodystruct 0 documenttype[].inherits[].id 8 documenttype[].datatype[].id 1091188812 documenttype[].datatype[].type STRUCT @@ -47,27 +47,12 @@ documenttype[].datatype[].sstruct.field[].name "weight" documenttype[].datatype[].sstruct.field[].id 1001392207 documenttype[].datatype[].sstruct.field[].datatype 1 documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id -389494616 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "parent.body" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 documenttype[].fieldsets{[]}.fields[] "weight_src" documenttype[].id 746267614 documenttype[].name "child" documenttype[].version 0 documenttype[].headerstruct 81425825 -documenttype[].bodystruct -126593034 +documenttype[].bodystruct 0 documenttype[].inherits[].id 8 documenttype[].inherits[].id 1175161836 documenttype[].datatype[].id 81425825 @@ -89,20 +74,5 @@ documenttype[].datatype[].sstruct.field[].name "child_field" documenttype[].datatype[].sstruct.field[].id 1814271363 documenttype[].datatype[].sstruct.field[].datatype 1091188812 documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id -126593034 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "child.body" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 documenttype[].fieldsets{[]}.fields[] "child_field" documenttype[].fieldsets{[]}.fields[] "weight_src" diff --git a/config-model/src/test/derived/mail/documentmanager.cfg b/config-model/src/test/derived/mail/documentmanager.cfg index 2fa9e5923c9..baf122d0241 100644 --- a/config-model/src/test/derived/mail/documentmanager.cfg +++ b/config-model/src/test/derived/mail/documentmanager.cfg @@ -37,15 +37,6 @@ datatype[].structtype[].field[].name "subject" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].name "snippet" datatype[].structtype[].field[].datatype 2 -datatype[].id -1206550296 -datatype[].arraytype[].datatype 12 -datatype[].id -953584901 -datatype[].structtype[].name "mail.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "body" datatype[].structtype[].field[].datatype 12 datatype[].structtype[].field[].name "attachmentcount" @@ -60,13 +51,15 @@ datatype[].structtype[].field[].name "attachmentcontent" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].name "attachments" datatype[].structtype[].field[].datatype -1206550296 +datatype[].id -1206550296 +datatype[].arraytype[].datatype 12 datatype[].id -1081574983 datatype[].documenttype[].name "mail" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -88808602 -datatype[].documenttype[].bodystruct -953584901 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{sender}.fields[] "from" datatype[].documenttype[].fieldsets{address}.fields[] "cc" datatype[].documenttype[].fieldsets{address}.fields[] "from" diff --git a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg index 060510c3578..9ab2da3f686 100644 --- a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg +++ b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg @@ -34,20 +34,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "indexfield2" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id -480519133 -datatype[].structtype[].name "prefixexactattribute.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1812793455 datatype[].documenttype[].name "prefixexactattribute" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -739138930 -datatype[].documenttype[].bodystruct -480519133 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "attributefield1" datatype[].documenttype[].fieldsets{[document]}.fields[] "attributefield2" datatype[].documenttype[].fieldsets{[document]}.fields[] "indexfield0" diff --git a/config-model/src/test/derived/ranktypes/documentmanager.cfg b/config-model/src/test/derived/ranktypes/documentmanager.cfg index 072a0fff126..a8bb9e904dc 100644 --- a/config-model/src/test/derived/ranktypes/documentmanager.cfg +++ b/config-model/src/test/derived/ranktypes/documentmanager.cfg @@ -34,20 +34,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "identity_literal" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id 1374506021 -datatype[].structtype[].name "ranktypes.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -883421617 datatype[].documenttype[].name "ranktypes" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -471393776 -datatype[].documenttype[].bodystruct 1374506021 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "descr" datatype[].documenttype[].fieldsets{[document]}.fields[] "identity" datatype[].documenttype[].fieldsets{[document]}.fields[] "keywords" diff --git a/config-model/src/test/derived/streamingstruct/documentmanager.cfg b/config-model/src/test/derived/streamingstruct/documentmanager.cfg index 2cd35c7bdfa..63001ea38ca 100644 --- a/config-model/src/test/derived/streamingstruct/documentmanager.cfg +++ b/config-model/src/test/derived/streamingstruct/documentmanager.cfg @@ -119,20 +119,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "snippet2" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id 1858438651 -datatype[].structtype[].name "streamingstruct.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1433175737 datatype[].documenttype[].name "streamingstruct" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 731395686 -datatype[].documenttype[].bodystruct 1858438651 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "a" datatype[].documenttype[].fieldsets{[document]}.fields[] "array1" datatype[].documenttype[].fieldsets{[document]}.fields[] "array2" diff --git a/config-model/src/test/derived/structanyorder/documentmanager.cfg b/config-model/src/test/derived/structanyorder/documentmanager.cfg index c18b1cc11b0..3ffc2f22a9b 100644 --- a/config-model/src/test/derived/structanyorder/documentmanager.cfg +++ b/config-model/src/test/derived/structanyorder/documentmanager.cfg @@ -69,20 +69,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "structarrayfield" datatype[].structtype[].field[].datatype -1244829667 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1503592268 -datatype[].structtype[].name "annotationsimplicitstruct.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -2099544992 datatype[].documenttype[].name "annotationsimplicitstruct" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -364910881 -datatype[].documenttype[].bodystruct -1503592268 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "structarrayfield" datatype[].documenttype[].fieldsets{[document]}.fields[] "structfield" annotationtype[].id -269517759 diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg index bbf9759659b..acf5c7ed12f 100644 --- a/config-model/src/test/derived/tensor/documenttypes.cfg +++ b/config-model/src/test/derived/tensor/documenttypes.cfg @@ -3,7 +3,7 @@ documenttype[].id -1290043429 documenttype[].name "tensor" documenttype[].version 0 documenttype[].headerstruct 2125927172 -documenttype[].bodystruct -1903234535 +documenttype[].bodystruct 0 documenttype[].inherits[].id 8 documenttype[].datatype[].id 2125927172 documenttype[].datatype[].type STRUCT @@ -44,21 +44,6 @@ documenttype[].datatype[].sstruct.field[].name "f6" documenttype[].datatype[].sstruct.field[].id 596352344 documenttype[].datatype[].sstruct.field[].datatype 1 documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id -1903234535 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "tensor.body" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 documenttype[].fieldsets{[document]}.fields[] "f1" documenttype[].fieldsets{[document]}.fields[] "f2" documenttype[].fieldsets{[document]}.fields[] "f3" diff --git a/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg b/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg index bb5bb001036..19d00483a5a 100644 --- a/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg +++ b/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg @@ -90,20 +90,13 @@ datatype[].structtype[].field[].name "snippet" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].name "snippet2" datatype[].structtype[].field[].datatype 2 -datatype[].id 1858438651 -datatype[].structtype[].name "streamingstruct.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1433175737 datatype[].documenttype[].name "streamingstruct" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 731395686 -datatype[].documenttype[].bodystruct 1858438651 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "a" datatype[].documenttype[].fieldsets{[document]}.fields[] "array1" datatype[].documenttype[].fieldsets{[document]}.fields[] "array2" @@ -139,13 +132,6 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -1417926544 -datatype[].structtype[].name "whatever.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "f1" datatype[].structtype[].field[].datatype -995681764 datatype[].id -778211548 @@ -154,5 +140,5 @@ datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 355471259 -datatype[].documenttype[].bodystruct -1417926544 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "f1" diff --git a/config-model/src/test/derived/types/documentmanager.cfg b/config-model/src/test/derived/types/documentmanager.cfg index a4fcd4f49f6..9556f77f6d9 100644 --- a/config-model/src/test/derived/types/documentmanager.cfg +++ b/config-model/src/test/derived/types/documentmanager.cfg @@ -209,28 +209,21 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "other" datatype[].structtype[].field[].datatype 4 datatype[].structtype[].field[].detailedtype "" +datatype[].structtype[].field[].name "complexarray" +datatype[].structtype[].field[].datatype 1416345047 +datatype[].structtype[].field[].detailedtype "" datatype[].id -372512406 datatype[].maptype[].keytype 0 datatype[].maptype[].valtype 1707615575 datatype[].id 1416345047 datatype[].arraytype[].datatype -372512406 -datatype[].id 348447225 -datatype[].structtype[].name "types.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "complexarray" -datatype[].structtype[].field[].datatype 1416345047 -datatype[].structtype[].field[].detailedtype "" datatype[].id -853072901 datatype[].documenttype[].name "types" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1328581348 -datatype[].documenttype[].bodystruct 348447225 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "Folders" datatype[].documenttype[].fieldsets{[document]}.fields[] "abool" datatype[].documenttype[].fieldsets{[document]}.fields[] "abyte" diff --git a/config-model/src/test/examples/fieldoftypedocument.cfg b/config-model/src/test/examples/fieldoftypedocument.cfg index b7bb444ec93..8074d86b45f 100644 --- a/config-model/src/test/examples/fieldoftypedocument.cfg +++ b/config-model/src/test/examples/fieldoftypedocument.cfg @@ -22,51 +22,37 @@ datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "soundtrack" datatype[1].structtype[0].field[0].datatype 1412693671 datatype[1].structtype[0].field[0].detailedtype "" -datatype[2].id -820813431 -datatype[2].structtype[0].name "book.body" -datatype[2].structtype[0].version 0 -datatype[2].structtype[0].compresstype NONE -datatype[2].structtype[0].compresslevel 0 -datatype[2].structtype[0].compressthreshold 95 -datatype[2].structtype[0].compressminsize 800 -datatype[3].id -1383388565 -datatype[3].documenttype[0].name "book" -datatype[3].documenttype[0].version 0 -datatype[3].documenttype[0].inherits[0].name "document" -datatype[3].documenttype[0].inherits[0].version 0 -datatype[3].documenttype[0].headerstruct -1344444812 -datatype[3].documenttype[0].bodystruct -820813431 -datatype[3].documenttype[0].fieldsets{[document]}.fields[0] "soundtrack" -datatype[4].id -1910204744 -datatype[4].structtype[0].name "music.header" -datatype[4].structtype[0].version 0 -datatype[4].structtype[0].compresstype NONE -datatype[4].structtype[0].compresslevel 0 -datatype[4].structtype[0].compressthreshold 95 -datatype[4].structtype[0].compressminsize 800 -datatype[4].structtype[0].field[0].name "intfield" -datatype[4].structtype[0].field[0].datatype 0 -datatype[4].structtype[0].field[0].detailedtype "" -datatype[4].structtype[0].field[1].name "stringfield" -datatype[4].structtype[0].field[1].datatype 2 -datatype[4].structtype[0].field[1].detailedtype "" -datatype[4].structtype[0].field[2].name "longfield" -datatype[4].structtype[0].field[2].datatype 4 -datatype[4].structtype[0].field[2].detailedtype "" -datatype[5].id 993120973 -datatype[5].structtype[0].name "music.body" -datatype[5].structtype[0].version 0 -datatype[5].structtype[0].compresstype NONE -datatype[5].structtype[0].compresslevel 0 -datatype[5].structtype[0].compressthreshold 95 -datatype[5].structtype[0].compressminsize 800 -datatype[6].id 1412693671 -datatype[6].documenttype[0].name "music" -datatype[6].documenttype[0].version 0 -datatype[6].documenttype[0].inherits[0].name "document" -datatype[6].documenttype[0].inherits[0].version 0 -datatype[6].documenttype[0].headerstruct -1910204744 -datatype[6].documenttype[0].bodystruct 993120973 -datatype[6].documenttype[0].fieldsets{[document]}.fields[0] "intfield" -datatype[6].documenttype[0].fieldsets{[document]}.fields[1] "longfield" -datatype[6].documenttype[0].fieldsets{[document]}.fields[2] "stringfield" +datatype[2].id -1383388565 +datatype[2].documenttype[0].name "book" +datatype[2].documenttype[0].version 0 +datatype[2].documenttype[0].inherits[0].name "document" +datatype[2].documenttype[0].inherits[0].version 0 +datatype[2].documenttype[0].headerstruct -1344444812 +datatype[2].documenttype[0].bodystruct 0 +datatype[2].documenttype[0].fieldsets{[document]}.fields[0] "soundtrack" +datatype[3].id -1910204744 +datatype[3].structtype[0].name "music.header" +datatype[3].structtype[0].version 0 +datatype[3].structtype[0].compresstype NONE +datatype[3].structtype[0].compresslevel 0 +datatype[3].structtype[0].compressthreshold 95 +datatype[3].structtype[0].compressminsize 800 +datatype[3].structtype[0].field[0].name "intfield" +datatype[3].structtype[0].field[0].datatype 0 +datatype[3].structtype[0].field[0].detailedtype "" +datatype[3].structtype[0].field[1].name "stringfield" +datatype[3].structtype[0].field[1].datatype 2 +datatype[3].structtype[0].field[1].detailedtype "" +datatype[3].structtype[0].field[2].name "longfield" +datatype[3].structtype[0].field[2].datatype 4 +datatype[3].structtype[0].field[2].detailedtype "" +datatype[4].id 1412693671 +datatype[4].documenttype[0].name "music" +datatype[4].documenttype[0].version 0 +datatype[4].documenttype[0].inherits[0].name "document" +datatype[4].documenttype[0].inherits[0].version 0 +datatype[4].documenttype[0].headerstruct -1910204744 +datatype[4].documenttype[0].bodystruct 0 +datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "intfield" +datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "longfield" +datatype[4].documenttype[0].fieldsets{[document]}.fields[2] "stringfield" diff --git a/config-model/src/test/examples/structresult.cfg b/config-model/src/test/examples/structresult.cfg index eff48f18914..ceaad2e6584 100755 --- a/config-model/src/test/examples/structresult.cfg +++ b/config-model/src/test/examples/structresult.cfg @@ -54,20 +54,13 @@ datatype[4].structtype[0].field[1].detailedtype "" datatype[4].structtype[0].field[2].name "advanced" datatype[4].structtype[0].field[2].datatype 93505813 datatype[4].structtype[0].field[2].detailedtype "" -datatype[5].id 993120973 -datatype[5].structtype[0].name "music.body" -datatype[5].structtype[0].version 0 -datatype[5].structtype[0].compresstype NONE -datatype[5].structtype[0].compresslevel 0 -datatype[5].structtype[0].compressthreshold 95 -datatype[5].structtype[0].compressminsize 800 -datatype[6].id 1412693671 -datatype[6].documenttype[0].name "music" -datatype[6].documenttype[0].version 0 -datatype[6].documenttype[0].inherits[0].name "document" -datatype[6].documenttype[0].inherits[0].version 0 -datatype[6].documenttype[0].headerstruct -1910204744 -datatype[6].documenttype[0].bodystruct 993120973 -datatype[6].documenttype[0].fieldsets{[document]}.fields[0] "advanced" -datatype[6].documenttype[0].fieldsets{[document]}.fields[1] "arraystruct" -datatype[6].documenttype[0].fieldsets{[document]}.fields[2] "mystruct" +datatype[5].id 1412693671 +datatype[5].documenttype[0].name "music" +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0].name "document" +datatype[5].documenttype[0].inherits[0].version 0 +datatype[5].documenttype[0].headerstruct -1910204744 +datatype[5].documenttype[0].bodystruct 0 +datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "advanced" +datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "arraystruct" +datatype[5].documenttype[0].fieldsets{[document]}.fields[2] "mystruct" diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java index 02233608eea..1b51fd354f3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java @@ -22,6 +22,7 @@ import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; /** * Tests inheritance @@ -68,8 +69,8 @@ public class InheritanceTestCase extends AbstractExportingTestCase { DocumentmanagerConfig.Builder b = new DocumentmanagerConfig.Builder(); DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), b), outDir.getPath()); DocumentmanagerConfig dc = b.build(); - assertEquals(17, dc.datatype().size()); - assertNotNull(structType("child.body", dc)); + assertEquals(13, dc.datatype().size()); + assertNull(structType("child.body", dc)); DocumentmanagerConfig.Datatype.Structtype childHeader = structType("child.header", dc); assertEquals(childHeader.field(0).name(), "foo"); assertEquals(childHeader.field(1).name(), "bar"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 3dfcef70aba..4bba66f0fb3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -201,7 +201,6 @@ public class DocumentTypeChangeValidatorTest { return new NewDocumentType( new NewDocumentType.Name("mydoc"), headerfields, - new StructDataType("bodyfields"), new FieldSets(), Collections.emptySet(), Collections.emptySet()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index 484709a0c18..294df42bd77 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -1,7 +1,4 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ package com.yahoo.vespa.model.container.xml; @@ -11,10 +8,7 @@ import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; + import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ContainerCluster; @@ -50,7 +44,6 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) .deployLogger(logger) - .properties(new TestProperties().setHostedVespa(true)) .build()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); model.getConfig(qrStartBuilder, "container/container.0"); @@ -85,11 +78,11 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { } private static void verifyIgnoreJvmGCOptions(boolean isHosted) throws IOException, SAXException { - verifyIgnoreJvmGCOptionsIfJvmArgs(isHosted, "jvmargs", ContainerCluster.G1GC); - verifyIgnoreJvmGCOptionsIfJvmArgs(isHosted, "jvm-options", "-XX:+UseG1GC"); + verifyIgnoreJvmGCOptionsIfJvmArgs("jvmargs", ContainerCluster.G1GC); + verifyIgnoreJvmGCOptionsIfJvmArgs( "jvm-options", "-XX:+UseG1GC"); } - private static void verifyIgnoreJvmGCOptionsIfJvmArgs(boolean isHosted, String jvmOptionsName, String expectedGC) throws IOException, SAXException { + private static void verifyIgnoreJvmGCOptionsIfJvmArgs(String jvmOptionsName, String expectedGC) throws IOException, SAXException { String servicesXml = "<container version='1.0'>" + " <nodes jvm-gc-options='-XX:+UseG1GC' " + jvmOptionsName + "='-XX:+UseParNewGC'>" + @@ -102,8 +95,6 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) .deployLogger(logger) - .zone(new Zone(SystemName.cd, Environment.dev, RegionName.from("here"))) - .properties(new TestProperties().setHostedVespa(isHosted)) .build()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); model.getConfig(qrStartBuilder, "container/container.0"); @@ -117,7 +108,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { verifyIgnoreJvmGCOptions(true); } - private void verifyJvmGCOptions(boolean isHosted, String override, Zone zone, String expected) throws IOException, SAXException { + private void verifyJvmGCOptions(boolean isHosted, String featureFlagDefault, String override, String expected) throws IOException, SAXException { String servicesXml = "<container version='1.0'>" + " <nodes " + ((override == null) ? ">" : ("jvm-gc-options='" + override + "'>")) + @@ -130,8 +121,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) .deployLogger(logger) - .zone(zone) - .properties(new TestProperties().setHostedVespa(isHosted)) + .properties(new TestProperties().setJvmGCOptions(featureFlagDefault).setHostedVespa(isHosted)) .build()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); model.getConfig(qrStartBuilder, "container/container.0"); @@ -139,18 +129,16 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { assertEquals(expected, qrStartConfig.jvm().gcopts()); } - private void verifyJvmGCOptions(boolean isHosted, Zone zone, String expected) throws IOException, SAXException { - verifyJvmGCOptions(isHosted, null, zone, expected); - verifyJvmGCOptions(isHosted, "-XX:+UseG1GC", zone, "-XX:+UseG1GC"); - Zone DEV = new Zone(SystemName.dev, zone.environment(), zone.region()); - verifyJvmGCOptions(isHosted, null, DEV, ContainerCluster.G1GC); - verifyJvmGCOptions(isHosted, "-XX:+UseConcMarkSweepGC", DEV, "-XX:+UseConcMarkSweepGC"); - } - @Test public void requireThatJvmGCOptionsIsHonoured() throws IOException, SAXException { - verifyJvmGCOptions(false, Zone.defaultZone(),ContainerCluster.G1GC); - verifyJvmGCOptions(true, Zone.defaultZone(), ContainerCluster.G1GC); + verifyJvmGCOptions(false, null,null, ContainerCluster.G1GC); + verifyJvmGCOptions(true, null,null, ContainerCluster.CMS); + verifyJvmGCOptions(true, "",null, ContainerCluster.CMS); + verifyJvmGCOptions(false, "-XX:+UseConcMarkSweepGC",null, "-XX:+UseConcMarkSweepGC"); + verifyJvmGCOptions(true, "-XX:+UseConcMarkSweepGC",null, "-XX:+UseConcMarkSweepGC"); + verifyJvmGCOptions(false, null,"-XX:+UseG1GC", "-XX:+UseG1GC"); + verifyJvmGCOptions(false, "-XX:+UseConcMarkSweepGC","-XX:+UseG1GC", "-XX:+UseG1GC"); + verifyJvmGCOptions(false, null,"-XX:+UseConcMarkSweepGC", "-XX:+UseConcMarkSweepGC"); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index dd88096c62f..9625bdf7447 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -49,12 +49,10 @@ import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.metrics.ApplicationMetricsRetriever; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.RemoteSession; -import com.yahoo.vespa.config.server.session.RemoteSessionRepo; import com.yahoo.vespa.config.server.session.Session; -import com.yahoo.vespa.config.server.session.SessionFactory; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; @@ -79,6 +77,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -216,7 +215,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Slime deployLog = createDeployLog(); DeployLogger logger = new DeployHandlerLogger(deployLog.get().setArray("log"), prepareParams.isVerbose(), applicationId); try (ActionTimer timer = timerFor(applicationId, "deployment.prepareMillis")) { - ConfigChangeActions actions = session.prepare(logger, prepareParams, currentActiveApplicationSet, tenant.getPath(), now); + SessionRepository sessionRepository = tenant.getSessionRepository(); + ConfigChangeActions actions = sessionRepository.prepareLocalSession(session, logger, prepareParams, + currentActiveApplicationSet, tenant.getPath(), now); logConfigChangeActions(actions, logger); log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); return new PrepareResult(sessionId, actions, deployLog); @@ -304,8 +305,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye LocalSession activeSession = getActiveLocalSession(tenant, application); if (activeSession == null) return Optional.empty(); TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - LocalSession newSession = tenant.getSessionFactory().createSessionFromExisting(activeSession, logger, true, timeoutBudget); - tenant.getLocalSessionRepo().addSession(newSession); + LocalSession newSession = tenant.getSessionRepository().createSessionFromExisting(activeSession, logger, true, timeoutBudget); + tenant.getSessionRepository().addSession(newSession); return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock, false /* don't validate as this is already deployed */, bootstrap)); @@ -489,7 +490,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); long sessionId = getSessionIdForApplication(tenant, applicationId); - RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId); + RemoteSession session = tenant.getSessionRepo().getRemoteSession(sessionId); if (session == null) throw new NotFoundException("Remote session " + sessionId + " not found"); return session.ensureApplicationLoaded().getForVersionOrLatest(version, clock.instant()); } catch (NotFoundException e) { @@ -526,10 +527,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private boolean localSessionHasBeenDeleted(ApplicationId applicationId, long sessionId, Duration waitTime) { - RemoteSessionRepo remoteSessionRepo = tenantRepository.getTenant(applicationId.tenant()).getRemoteSessionRepo(); + SessionRepository sessionRepository = tenantRepository.getTenant(applicationId.tenant()).getSessionRepo(); Instant end = Instant.now().plus(waitTime); do { - if (remoteSessionRepo.getSession(sessionId) == null) return true; + if (sessionRepository.getRemoteSession(sessionId) == null) return true; try { Thread.sleep(10); } catch (InterruptedException e) { /* ignored */} } while (Instant.now().isBefore(end)); @@ -635,11 +636,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye boolean internalRedeploy, TimeoutBudget timeoutBudget) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); - LocalSessionRepo localSessionRepo = tenant.getLocalSessionRepo(); - SessionFactory sessionFactory = tenant.getSessionFactory(); + SessionRepository sessionRepository = tenant.getSessionRepository(); RemoteSession fromSession = getExistingSession(tenant, applicationId); - LocalSession session = sessionFactory.createSessionFromExisting(fromSession, logger, internalRedeploy, timeoutBudget); - localSessionRepo.addSession(session); + LocalSession session = sessionRepository.createSessionFromExisting(fromSession, logger, internalRedeploy, timeoutBudget); + sessionRepository.addSession(session); return session.getSessionId(); } @@ -658,15 +658,17 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); tenant.getApplicationRepo().createApplication(applicationId); Optional<Long> activeSessionId = tenant.getApplicationRepo().activeSessionOf(applicationId); - LocalSession session = tenant.getSessionFactory().createSession(applicationDirectory, applicationId, - timeoutBudget, activeSessionId); - tenant.getLocalSessionRepo().addSession(session); + LocalSession session = tenant.getSessionRepository().createSession(applicationDirectory, + applicationId, + timeoutBudget, + activeSessionId); + tenant.getSessionRepository().addSession(session); return session.getSessionId(); } public void deleteExpiredLocalSessions() { Map<Tenant, List<LocalSession>> sessionsPerTenant = new HashMap<>(); - tenantRepository.getAllTenants().forEach(tenant -> sessionsPerTenant.put(tenant, tenant.getLocalSessionRepo().getSessions())); + tenantRepository.getAllTenants().forEach(tenant -> sessionsPerTenant.put(tenant, tenant.getSessionRepository().getLocalSessions())); Set<ApplicationId> applicationIds = new HashSet<>(); sessionsPerTenant.values().forEach(sessionList -> sessionList.forEach(s -> applicationIds.add(s.getApplicationId()))); @@ -677,7 +679,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (activeSession != null) activeSessions.put(applicationId, activeSession.getSessionId()); }); - sessionsPerTenant.keySet().forEach(tenant -> tenant.getLocalSessionRepo().deleteExpiredSessions(activeSessions)); + sessionsPerTenant.keySet().forEach(tenant -> tenant.getSessionRepository().deleteExpiredSessions(activeSessions)); } public int deleteExpiredRemoteSessions(Duration expiryTime) { @@ -687,7 +689,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { return tenantRepository.getAllTenants() .stream() - .map(tenant -> tenant.getRemoteSessionRepo().deleteExpiredSessions(clock, expiryTime)) + .map(tenant -> tenant.getSessionRepo().deleteExpiredRemoteSessions(clock, expiryTime)) .mapToInt(i -> i) .sum(); } @@ -739,6 +741,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return configserverConfig; } + public ApplicationId getApplicationIdForHostname(String hostname) { + Optional<ApplicationId> applicationId = tenantRepository.getAllTenantNames().stream() + .map(tenantName -> tenantRepository.getTenant(tenantName).getApplicationRepo().getApplicationIdForHostName(hostname)) + .filter(Objects::nonNull) + .findFirst(); + return applicationId.orElse(null); + } + private void validateThatLocalSessionIsNotActive(Tenant tenant, long sessionId) { LocalSession session = getLocalSession(tenant, sessionId); if (Session.Status.ACTIVATE.equals(session.getStatus())) { @@ -747,20 +757,20 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private LocalSession getLocalSession(Tenant tenant, long sessionId) { - LocalSession session = tenant.getLocalSessionRepo().getSession(sessionId); + LocalSession session = tenant.getSessionRepository().getLocalSession(sessionId); if (session == null) throw new NotFoundException("Session " + sessionId + " was not found"); return session; } private RemoteSession getRemoteSession(Tenant tenant, long sessionId) { - RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId); + RemoteSession session = tenant.getSessionRepo().getRemoteSession(sessionId); if (session == null) throw new NotFoundException("Session " + sessionId + " was not found"); return session; } - private Optional<ApplicationSet> getCurrentActiveApplicationSet(Tenant tenant, ApplicationId appId) { + public Optional<ApplicationSet> getCurrentActiveApplicationSet(Tenant tenant, ApplicationId appId) { Optional<ApplicationSet> currentActiveApplicationSet = Optional.empty(); TenantApplications applicationRepo = tenant.getApplicationRepo(); try { @@ -805,7 +815,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private RemoteSession getActiveSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); if (applicationRepo.activeApplications().contains(applicationId)) { - return tenant.getRemoteSessionRepo().getSession(applicationRepo.requireActiveSessionOf(applicationId)); + return tenant.getSessionRepo().getRemoteSession(applicationRepo.requireActiveSessionOf(applicationId)); } return null; } @@ -813,7 +823,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public LocalSession getActiveLocalSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); if (applicationRepo.activeApplications().contains(applicationId)) { - return tenant.getLocalSessionRepo().getSession(applicationRepo.requireActiveSessionOf(applicationId)); + return tenant.getSessionRepository().getLocalSession(applicationRepo.requireActiveSessionOf(applicationId)); } return null; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index a4dfec708d6..4feaf51c3e0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -213,7 +213,7 @@ public class TenantApplications implements RequestHandler, ReloadHandler, HostVa break; } // We may have lost events and may need to remove applications. - // New applications are added when session is added, not here. See RemoteSessionRepo. + // New applications are added when session is added, not here. See SessionRepository. removeUnusedApplications(); }); } @@ -419,5 +419,17 @@ public class TenantApplications implements RequestHandler, ReloadHandler, HostVa reloadListener.verifyHostsAreAvailable(tenant, newHosts); } + public HostValidator<ApplicationId> getHostValidator() { + return this; + } + + public HostRegistry<ApplicationId> getApplicationHostRegistry() { + return hostRegistry; + } + + public ApplicationId getApplicationIdForHostName(String hostname) { + return hostRegistry.getKeyForHost(hostname); + } + public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index d90a79795cf..39fac90d242 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.config.server.ActivationConflictException; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -118,7 +119,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { .isBootstrap(isBootstrap); dockerImageRepository.ifPresent(params::dockerImageRepository); athenzDomain.ifPresent(params::athenzDomain); - session.prepare(logger, params.build(), Optional.empty(), tenant.getPath(), clock.instant()); + Optional<ApplicationSet> activeApplicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, session.getApplicationId()); + tenant.getSessionRepository().prepareLocalSession(session, logger, params.build(), activeApplicationSet, + tenant.getPath(), clock.instant()); this.prepared = true; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index c925157b980..74a9e72e255 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -153,7 +153,7 @@ public class ModelContextImpl implements ModelContext { private final double defaultTermwiseLimit; private final double threadPoolSizeFactor; private final double queueSizefactor; - private final String docprocLoadBalancerType; + private final String jvmGCOPtions; private final Optional<AthenzDomain> athenzDomain; private final Optional<ApplicationRoles> applicationRoles; private final int jdiscHealthCheckProxyClientTimeout; @@ -195,7 +195,7 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - docprocLoadBalancerType = Flags.DOCPROC_LOADBALANCER_TYPE.bindTo(flagSource) + jvmGCOPtions = Flags.JVM_GC_OPTIONS.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.athenzDomain = athenzDomain; this.applicationRoles = applicationRoles; @@ -257,11 +257,6 @@ public class ModelContextImpl implements ModelContext { } @Override - public String docprocLoadBalancerType() { - return docprocLoadBalancerType; - } - - @Override public boolean useDistributorBtreeDb() { return useDistributorBtreeDb; } @@ -280,6 +275,8 @@ public class ModelContextImpl implements ModelContext { } @Override public Duration jdiscHealthCheckProxyClientTimeout() { return Duration.ofMillis(jdiscHealthCheckProxyClientTimeout); } + @Override public String jvmGCOptions() { return jvmGCOPtions; } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 0865b72dbbf..07a686a63bd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -177,4 +177,10 @@ public class FileDirectory { destChannel.transferFrom(sourceChannel, 0, sourceChannel.size()); } } + + @Override + public String toString() { + return "root dir: " + root.getAbsolutePath(); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java index 2c888df6658..1ea41b85983 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -3,61 +3,37 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingMatch; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.host.HostRegistries; -import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; -import java.util.logging.Level; - - /** * Handler for getting tenant and application for a given hostname. * * @author hmusum - * @since 5.19 */ public class HostHandler extends HttpHandler { - final HostRegistries hostRegistries; - private final Zone zone; + private final ApplicationRepository applicationRepository; @Inject - public HostHandler(HttpHandler.Context ctx, - GlobalComponentRegistry globalComponentRegistry) { + public HostHandler(HttpHandler.Context ctx, ApplicationRepository applicationRepository) { super(ctx); - this.hostRegistries = globalComponentRegistry.getHostRegistries(); - this.zone = globalComponentRegistry.getZone(); + this.applicationRepository = applicationRepository; } @Override public HttpResponse handleGET(HttpRequest request) { String hostname = getBindingMatch(request).group(2); - log.log(Level.FINE, "hostname=" + hostname); - - HostRegistry<TenantName> tenantHostRegistry = hostRegistries.getTenantHostRegistry(); - log.log(Level.FINE, "hosts in tenant host registry '" + tenantHostRegistry + "' " + tenantHostRegistry.getAllHosts()); - TenantName tenant = tenantHostRegistry.getKeyForHost(hostname); - if (tenant == null) return createError(hostname); - log.log(Level.FINE, "tenant=" + tenant); - HostRegistry<ApplicationId> applicationIdHostRegistry = hostRegistries.getApplicationHostRegistry(tenant); - ApplicationId applicationId; - if (applicationIdHostRegistry == null) return createError(hostname); - applicationId = applicationIdHostRegistry.getKeyForHost(hostname); - log.log(Level.FINE, "applicationId=" + applicationId); - if (applicationId == null) { - return createError(hostname); - } else { - log.log(Level.FINE, "hosts in application host registry '" + applicationIdHostRegistry + "' " + applicationIdHostRegistry.getAllHosts()); - return new HostResponse(Response.Status.OK, applicationId, zone); - } + ApplicationId applicationId = applicationRepository.getApplicationIdForHostname(hostname); + return (applicationId == null) + ? createError(hostname) + : new HostResponse(Response.Status.OK, applicationId, applicationRepository.zone()); } private HttpErrorResponse createError(String hostname) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 3ea7959c212..4e6a541793d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -37,7 +37,7 @@ public class ConfigServerMaintenance extends AbstractComponent { //tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval); fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig); sessionsMaintainer = new SessionsMaintainer(applicationRepository, curator, defaults.defaultInterval); - applicationPackageMaintainer = new ApplicationPackageMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig, flagSource); + applicationPackageMaintainer = new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofMinutes(1), configserverConfig, flagSource); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index 56e32f7d802..73e7b36c381 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -4,25 +4,11 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.transaction.AbstractTransaction; -import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; -import com.yahoo.vespa.config.server.host.HostValidator; -import com.yahoo.vespa.curator.Curator; - -import java.io.File; -import java.time.Instant; -import java.util.Optional; -import java.util.logging.Level; /** * A LocalSession is a session that has been created locally on this configserver. A local session can be edited and @@ -37,43 +23,18 @@ public class LocalSession extends Session { protected final ApplicationPackage applicationPackage; private final TenantApplications applicationRepo; - private final SessionPreparer sessionPreparer; - private final File serverDBSessionDir; - private final SessionZooKeeperClient sessionZooKeeperClient; - private final HostValidator<ApplicationId> hostValidator; /** - * Create a session. This involves loading the application, validating it and distributing it. + * Creates a session. This involves loading the application, validating it and distributing it. * * @param sessionId The session id for this session. */ - public LocalSession(TenantName tenant, long sessionId, SessionPreparer sessionPreparer, - ApplicationPackage applicationPackage, SessionZooKeeperClient sessionZooKeeperClient, - File serverDBSessionDir, TenantApplications applicationRepo, - HostValidator<ApplicationId> hostValidator) { + public LocalSession(TenantName tenant, long sessionId, ApplicationPackage applicationPackage, + SessionZooKeeperClient sessionZooKeeperClient, + TenantApplications applicationRepo) { super(tenant, sessionId, sessionZooKeeperClient); - this.serverDBSessionDir = serverDBSessionDir; this.applicationPackage = applicationPackage; - this.sessionZooKeeperClient = sessionZooKeeperClient; this.applicationRepo = applicationRepo; - this.sessionPreparer = sessionPreparer; - this.hostValidator = hostValidator; - } - - public ConfigChangeActions prepare(DeployLogger logger, - PrepareParams params, - Optional<ApplicationSet> currentActiveApplicationSet, - Path tenantPath, - Instant now) { - applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( - logger.log(Level.FINE, "Created application " + params.getApplicationId()); - Curator.CompletionWaiter waiter = zooKeeperClient.createPrepareWaiter(); - ConfigChangeActions actions = sessionPreparer.prepare(hostValidator, logger, params, - currentActiveApplicationSet, tenantPath, now, - serverDBSessionDir, applicationPackage, sessionZooKeeperClient); - setPrepared(); - waiter.awaitCompletion(params.getTimeoutBudget().timeLeft()); - return actions; } public ApplicationFile getApplicationFile(Path relativePath, Mode mode) { @@ -83,22 +44,22 @@ public class LocalSession extends Session { return applicationPackage.getFile(relativePath); } - private void setPrepared() { + void setPrepared() { setStatus(Session.Status.PREPARE); } private Transaction createSetStatusTransaction(Status status) { - return zooKeeperClient.createWriteStatusTransaction(status); + return sessionZooKeeperClient.createWriteStatusTransaction(status); } private void setStatus(Session.Status newStatus) { - zooKeeperClient.writeStatus(newStatus); + sessionZooKeeperClient.writeStatus(newStatus); } public Transaction createActivateTransaction() { - zooKeeperClient.createActiveWaiter(); + sessionZooKeeperClient.createActiveWaiter(); Transaction transaction = createSetStatusTransaction(Status.ACTIVATE); - transaction.add(applicationRepo.createPutTransaction(zooKeeperClient.readApplicationId(), getSessionId()).operations()); + transaction.add(applicationRepo.createPutTransaction(sessionZooKeeperClient.readApplicationId(), getSessionId()).operations()); return transaction; } @@ -110,79 +71,16 @@ public class LocalSession extends Session { return applicationPackage.getMetaData().getPreviousActiveGeneration(); } - /** Add transactions to delete this session to the given nested transaction */ - public void delete(NestedTransaction transaction) { - transaction.add(zooKeeperClient.deleteTransaction(), FileTransaction.class); - transaction.add(FileTransaction.from(FileOperations.delete(serverDBSessionDir.getAbsolutePath()))); - } - public void waitUntilActivated(TimeoutBudget timeoutBudget) { - zooKeeperClient.getActiveWaiter().awaitCompletion(timeoutBudget.timeLeft()); + sessionZooKeeperClient.getActiveWaiter().awaitCompletion(timeoutBudget.timeLeft()); } public enum Mode { READ, WRITE } - public ApplicationMetaData getMetaData() { - return applicationPackage.getMetaData(); - } - - // The rest of this class should be moved elsewhere ... - - private static class FileTransaction extends AbstractTransaction { - - public static FileTransaction from(FileOperation operation) { - FileTransaction transaction = new FileTransaction(); - transaction.add(operation); - return transaction; - } - - @Override - public void prepare() { } + public ApplicationMetaData getMetaData() { return applicationPackage.getMetaData(); } - @Override - public void commit() { - for (Operation operation : operations()) - ((FileOperation)operation).commit(); - } - - } - - /** Factory for file operations */ - private static class FileOperations { - - /** Creates an operation which recursively deletes the given path */ - public static DeleteOperation delete(String pathToDelete) { - return new DeleteOperation(pathToDelete); - } - - } - - private interface FileOperation extends Transaction.Operation { - - void commit(); - - } - - /** - * Recursively deletes this path and everything below. - * Succeeds with no action if the path does not exist. - */ - private static class DeleteOperation implements FileOperation { - - private final String pathToDelete; - - DeleteOperation(String pathToDelete) { - this.pathToDelete = pathToDelete; - } - - @Override - public void commit() { - // TODO: Check delete access in prepare() - IOUtils.recursiveDeleteDir(new File(pathToDelete)); - } - - } + public ApplicationPackage getApplicationPackage() { return applicationPackage; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java deleted file mode 100644 index e23552dee44..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; - -import java.io.File; -import java.io.FilenameFilter; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * - * Contains state for the local instance of the configserver. - * - * @author Ulf Lilleengen - */ -public class LocalSessionRepo { - - private static final Logger log = Logger.getLogger(LocalSessionRepo.class.getName()); - private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); - - private final SessionCache<LocalSession> sessionCache; - private final Map<Long, LocalSessionStateWatcher> sessionStateWatchers = new HashMap<>(); - private final Duration sessionLifetime; - private final Clock clock; - private final Curator curator; - private final Executor zkWatcherExecutor; - private final TenantFileSystemDirs tenantFileSystemDirs; - - public LocalSessionRepo(TenantName tenantName, GlobalComponentRegistry componentRegistry, SessionFactory sessionFactory) { - sessionCache = new SessionCache<>(); - this.clock = componentRegistry.getClock(); - this.curator = componentRegistry.getCurator(); - this.sessionLifetime = Duration.ofSeconds(componentRegistry.getConfigserverConfig().sessionLifetime()); - this.zkWatcherExecutor = command -> componentRegistry.getZkWatcherExecutor().execute(tenantName, command); - this.tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName); - loadSessions(sessionFactory); - } - - public synchronized void addSession(LocalSession session) { - sessionCache.addSession(session); - Path sessionsPath = TenantRepository.getSessionsPath(session.getTenantName()); - long sessionId = session.getSessionId(); - Curator.FileCache fileCache = curator.createFileCache(sessionsPath.append(String.valueOf(sessionId)).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); - sessionStateWatchers.put(sessionId, new LocalSessionStateWatcher(fileCache, session, this, zkWatcherExecutor)); - } - - public LocalSession getSession(long sessionId) { - return sessionCache.getSession(sessionId); - } - - public List<LocalSession> getSessions() { - return sessionCache.getSessions(); - } - - private void loadSessions(SessionFactory sessionFactory) { - File[] sessions = tenantFileSystemDirs.sessionsPath().listFiles(sessionApplicationsFilter); - if (sessions == null) { - return; - } - for (File session : sessions) { - try { - addSession(sessionFactory.createSessionFromId(Long.parseLong(session.getName()))); - } catch (IllegalArgumentException e) { - log.log(Level.WARNING, "Could not load session '" + - session.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); - } - } - } - - public void deleteExpiredSessions(Map<ApplicationId, Long> activeSessions) { - log.log(Level.FINE, "Purging old sessions"); - try { - for (LocalSession candidate : sessionCache.getSessions()) { - Instant createTime = candidate.getCreateTime(); - log.log(Level.FINE, "Candidate session for deletion: " + candidate.getSessionId() + ", created: " + createTime); - - // Sessions with state other than ACTIVATED - if (hasExpired(candidate) && !isActiveSession(candidate)) { - deleteSession(candidate); - } else if (createTime.plus(Duration.ofDays(1)).isBefore(clock.instant())) { - // Sessions with state ACTIVATE, but which are not actually active - ApplicationId applicationId = candidate.getApplicationId(); - Long activeSession = activeSessions.get(applicationId); - if (activeSession == null || activeSession != candidate.getSessionId()) { - deleteSession(candidate); - log.log(Level.INFO, "Deleted inactive session " + candidate.getSessionId() + " created " + - createTime + " for '" + applicationId + "'"); - } - } - } - // Make sure to catch here, to avoid executor just dying in case of issues ... - } catch (Throwable e) { - log.log(Level.WARNING, "Error when purging old sessions ", e); - } - log.log(Level.FINE, "Done purging old sessions"); - } - - private boolean hasExpired(LocalSession candidate) { - return (candidate.getCreateTime().plus(sessionLifetime).isBefore(clock.instant())); - } - - private boolean isActiveSession(LocalSession candidate) { - return candidate.getStatus() == Session.Status.ACTIVATE; - } - - public void deleteSession(LocalSession session) { - long sessionId = session.getSessionId(); - log.log(Level.FINE, "Deleting local session " + sessionId); - LocalSessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); - if (watcher != null) watcher.close(); - sessionCache.removeSession(sessionId); - NestedTransaction transaction = new NestedTransaction(); - session.delete(transaction); - transaction.commit(); - } - - public void close() { - deleteAllSessions(); - tenantFileSystemDirs.delete(); - } - - private void deleteAllSessions() { - List<LocalSession> sessions = new ArrayList<>(sessionCache.getSessions()); - for (LocalSession session : sessions) { - deleteSession(session); - } - } - - @Override - public String toString() { - return getSessions().toString(); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java index 662094fc0ca..acbb1dc81ce 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java @@ -21,14 +21,14 @@ public class LocalSessionStateWatcher { private final Curator.FileCache fileCache; private final LocalSession session; - private final LocalSessionRepo localSessionRepo; + private final SessionRepository sessionRepository; private final Executor zkWatcherExecutor; LocalSessionStateWatcher(Curator.FileCache fileCache, LocalSession session, - LocalSessionRepo localSessionRepo, Executor zkWatcherExecutor) { + SessionRepository sessionRepository, Executor zkWatcherExecutor) { this.fileCache = fileCache; this.session = session; - this.localSessionRepo = localSessionRepo; + this.sessionRepository = sessionRepository; this.zkWatcherExecutor = zkWatcherExecutor; this.fileCache.start(); this.fileCache.addListener(this::nodeChanged); @@ -40,9 +40,9 @@ public class LocalSessionStateWatcher { log.log(status == Session.Status.DELETE ? Level.INFO : Level.FINE, session.logPre() + "Session change: Local session " + sessionId + " changed status to " + status); - if (status.equals(Session.Status.DELETE) && localSessionRepo.getSession(sessionId) != null) { + if (status.equals(Session.Status.DELETE) && sessionRepository.getLocalSession(sessionId) != null) { log.log(Level.FINE, session.logPre() + "Deleting session " + sessionId); - localSessionRepo.deleteSession(session); + sessionRepository.deleteLocalSession(session); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index c1179a2dd17..763c77f2088 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -50,20 +50,20 @@ public class RemoteSession extends Session { } void loadPrepared() { - Curator.CompletionWaiter waiter = zooKeeperClient.getPrepareWaiter(); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.getPrepareWaiter(); ensureApplicationLoaded(); notifyCompletion(waiter); } private ApplicationSet loadApplication() { - ApplicationPackage applicationPackage = zooKeeperClient.loadApplicationPackage(); + ApplicationPackage applicationPackage = sessionZooKeeperClient.loadApplicationPackage(); // Read hosts allocated on the config server instance which created this Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts(); - return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(), - zooKeeperClient.readDockerImageRepository(), - zooKeeperClient.readVespaVersion(), + return ApplicationSet.fromList(applicationLoader.buildModels(sessionZooKeeperClient.readApplicationId(), + sessionZooKeeperClient.readDockerImageRepository(), + sessionZooKeeperClient.readVespaVersion(), applicationPackage, new SettableOptional<>(allocatedHosts), clock.instant())); @@ -78,11 +78,11 @@ public class RemoteSession extends Session { } public Transaction createDeleteTransaction() { - return zooKeeperClient.createWriteStatusTransaction(Status.DELETE); + return sessionZooKeeperClient.createWriteStatusTransaction(Status.DELETE); } void makeActive(ReloadHandler reloadHandler) { - Curator.CompletionWaiter waiter = zooKeeperClient.getActiveWaiter(); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.getActiveWaiter(); log.log(Level.FINE, () -> logPre() + "Getting session from repo: " + getSessionId()); ApplicationSet app = ensureApplicationLoaded(); log.log(Level.FINE, () -> logPre() + "Reloading config for " + getSessionId()); @@ -93,7 +93,7 @@ public class RemoteSession extends Session { } void confirmUpload() { - Curator.CompletionWaiter waiter = zooKeeperClient.getUploadWaiter(); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.getUploadWaiter(); log.log(Level.FINE, "Notifying upload waiter for session " + getSessionId()); notifyCompletion(waiter); log.log(Level.FINE, "Done notifying upload for session " + getSessionId()); @@ -116,13 +116,13 @@ public class RemoteSession extends Session { } public void delete() { - Transaction transaction = zooKeeperClient.deleteTransaction(); + Transaction transaction = sessionZooKeeperClient.deleteTransaction(); transaction.commit(); transaction.close(); } public ApplicationMetaData getMetaData() { - return zooKeeperClient.loadApplicationPackage().getMetaData(); + return sessionZooKeeperClient.loadApplicationPackage().getMetaData(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java deleted file mode 100644 index 0e538b05931..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.google.common.collect.HashMultiset; -import com.google.common.collect.Multiset; -import com.yahoo.concurrent.StripedExecutor; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; - -import java.time.Clock; -import java.util.logging.Level; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.ReloadHandler; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; - -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Session repository for RemoteSessions. There is one such repo per tenant. - * Will watch/prepare sessions (applications) based on watched nodes in ZooKeeper. The zookeeper state watched in - * this class is shared between all config servers, so it should not modify any global state, because the operation - * will be performed on all servers. The repo can be regarded as read only from the POV of the configserver. - * - * @author Vegard Havdal - * @author Ulf Lilleengen - */ -public class RemoteSessionRepo { - - private static final Logger log = Logger.getLogger(RemoteSessionRepo.class.getName()); - - private final Curator curator; - private final Path sessionsPath; - private final SessionFactory sessionFactory; - private final Map<Long, RemoteSessionStateWatcher> sessionStateWatchers = new HashMap<>(); - private final ReloadHandler reloadHandler; - private final TenantName tenantName; - private final MetricUpdater metrics; - private final BooleanFlag distributeApplicationPackage; - private final Curator.DirectoryCache directoryCache; - private final TenantApplications applicationRepo; - private final Executor zkWatcherExecutor; - private final SessionCache<RemoteSession> sessionCache; - - public RemoteSessionRepo(GlobalComponentRegistry componentRegistry, - SessionFactory sessionFactory, - ReloadHandler reloadHandler, - TenantName tenantName, - TenantApplications applicationRepo, - FlagSource flagSource) { - this.sessionCache = new SessionCache<>(); - this.curator = componentRegistry.getCurator(); - this.sessionsPath = TenantRepository.getSessionsPath(tenantName); - this.applicationRepo = applicationRepo; - this.sessionFactory = sessionFactory; - this.reloadHandler = reloadHandler; - this.tenantName = tenantName; - this.metrics = componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); - this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); - StripedExecutor<TenantName> zkWatcherExecutor = componentRegistry.getZkWatcherExecutor(); - this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenantName, command); - initializeSessions(); - this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, componentRegistry.getZkCacheExecutor()); - this.directoryCache.addListener(this::childEvent); - this.directoryCache.start(); - } - - public RemoteSession getSession(long sessionId) { - return sessionCache.getSession(sessionId); - } - - public List<Long> getSessions() { - return getSessionList(curator.getChildren(sessionsPath)); - } - - public void addSession(RemoteSession session) { - sessionCache.addSession(session); - metrics.incAddedSessions(); - } - - public int deleteExpiredSessions(Clock clock, Duration expiryTime) { - int deleted = 0; - for (long sessionId : getSessions()) { - RemoteSession session = sessionCache.getSession(sessionId); - if (session == null) continue; // Internal sessions not in synch with zk, continue - if (session.getStatus() == Session.Status.ACTIVATE) continue; - if (sessionHasExpired(session.getCreateTime(), expiryTime, clock)) { - log.log(Level.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); - session.delete(); - deleted++; - } - } - return deleted; - } - - private boolean sessionHasExpired(Instant created, Duration expiryTime, Clock clock) { - return (created.plus(expiryTime).isBefore(clock.instant())); - } - - private List<Long> getSessionListFromDirectoryCache(List<ChildData> children) { - return getSessionList(children.stream() - .map(child -> Path.fromString(child.getPath()).getName()) - .collect(Collectors.toList())); - } - - private List<Long> getSessionList(List<String> children) { - return children.stream().map(Long::parseLong).collect(Collectors.toList()); - } - - private void initializeSessions() throws NumberFormatException { - getSessions().forEach(this::sessionAdded); - } - - private synchronized void sessionsChanged() throws NumberFormatException { - List<Long> sessions = getSessionListFromDirectoryCache(directoryCache.getCurrentData()); - checkForRemovedSessions(sessions); - checkForAddedSessions(sessions); - } - - private void checkForRemovedSessions(List<Long> sessions) { - for (RemoteSession session : sessionCache.getSessions()) - if ( ! sessions.contains(session.getSessionId())) - sessionRemoved(session.getSessionId()); - } - - private void checkForAddedSessions(List<Long> sessions) { - for (Long sessionId : sessions) - if (sessionCache.getSession(sessionId) == null) - sessionAdded(sessionId); - } - - /** - * A session for which we don't have a watcher, i.e. hitherto unknown to us. - * - * @param sessionId session id for the new session - */ - private void sessionAdded(long sessionId) { - log.log(Level.FINE, () -> "Adding session to RemoteSessionRepo: " + sessionId); - RemoteSession session = sessionFactory.createRemoteSession(sessionId); - Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); - Curator.FileCache fileCache = curator.createFileCache(sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); - fileCache.addListener(this::nodeChanged); - loadSessionIfActive(session); - addSession(session); - sessionStateWatchers.put(sessionId, new RemoteSessionStateWatcher(fileCache, reloadHandler, session, metrics, zkWatcherExecutor)); - if (distributeApplicationPackage.value()) - sessionFactory.createLocalSessionUsingDistributedApplicationPackage(sessionId); - } - - private void sessionRemoved(long sessionId) { - RemoteSessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); - if (watcher != null) watcher.close(); - sessionCache.removeSession(sessionId); - metrics.incRemovedSessions(); - } - - private void loadSessionIfActive(RemoteSession session) { - for (ApplicationId applicationId : applicationRepo.activeApplications()) { - if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { - log.log(Level.FINE, () -> "Found active application for session " + session.getSessionId() + " , loading it"); - reloadHandler.reloadConfig(session.ensureApplicationLoaded()); - log.log(Level.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); - return; - } - } - } - - public synchronized void close() { - try { - if (directoryCache != null) { - directoryCache.close(); - } - } catch (Exception e) { - log.log(Level.WARNING, "Exception when closing path cache", e); - } finally { - checkForRemovedSessions(new ArrayList<>()); - } - } - - private void nodeChanged() { - zkWatcherExecutor.execute(() -> { - Multiset<Session.Status> sessionMetrics = HashMultiset.create(); - for (RemoteSession session : sessionCache.getSessions()) { - sessionMetrics.add(session.getStatus()); - } - metrics.setNewSessions(sessionMetrics.count(Session.Status.NEW)); - metrics.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE)); - metrics.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE)); - metrics.setDeactivatedSessions(sessionMetrics.count(Session.Status.DEACTIVATE)); - }); - } - - @SuppressWarnings("unused") - private void childEvent(CuratorFramework ignored, PathChildrenCacheEvent event) { - zkWatcherExecutor.execute(() -> { - log.log(Level.FINE, () -> "Got child event: " + event); - switch (event.getType()) { - case CHILD_ADDED: - sessionsChanged(); - synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); - break; - case CHILD_REMOVED: - sessionsChanged(); - break; - case CONNECTION_RECONNECTED: - sessionsChanged(); - break; - } - }); - } - - private void synchronizeOnNew(List<Long> sessionList) { - for (long sessionId : sessionList) { - RemoteSession session = sessionCache.getSession(sessionId); - if (session == null) continue; // session might have been deleted after getting session list - log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); - session.confirmUpload(); - } - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index c553133ba12..1e832548342 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -25,12 +25,12 @@ public abstract class Session implements Comparable<Session> { private final long sessionId; protected final TenantName tenant; - protected final SessionZooKeeperClient zooKeeperClient; + protected final SessionZooKeeperClient sessionZooKeeperClient; - protected Session(TenantName tenant, long sessionId, SessionZooKeeperClient zooKeeperClient) { + protected Session(TenantName tenant, long sessionId, SessionZooKeeperClient sessionZooKeeperClient) { this.tenant = tenant; this.sessionId = sessionId; - this.zooKeeperClient = zooKeeperClient; + this.sessionZooKeeperClient = sessionZooKeeperClient; } /** * Retrieve the session id for this session. @@ -41,7 +41,7 @@ public abstract class Session implements Comparable<Session> { } public Session.Status getStatus() { - return zooKeeperClient.readStatus(); + return sessionZooKeeperClient.readStatus(); } @Override @@ -80,43 +80,43 @@ public abstract class Session implements Comparable<Session> { } public Instant getCreateTime() { - return zooKeeperClient.readCreateTime(); + return sessionZooKeeperClient.readCreateTime(); } public void setApplicationId(ApplicationId applicationId) { - zooKeeperClient.writeApplicationId(applicationId); + sessionZooKeeperClient.writeApplicationId(applicationId); } void setApplicationPackageReference(FileReference applicationPackageReference) { if (applicationPackageReference == null) throw new IllegalArgumentException(String.format( "Null application package FileReference for tenant: %s, session: %d", tenant, sessionId)); - zooKeeperClient.writeApplicationPackageReference(applicationPackageReference); + sessionZooKeeperClient.writeApplicationPackageReference(applicationPackageReference); } public void setVespaVersion(Version version) { - zooKeeperClient.writeVespaVersion(version); + sessionZooKeeperClient.writeVespaVersion(version); } public void setDockerImageRepository(Optional<DockerImage> dockerImageRepository) { - zooKeeperClient.writeDockerImageRepository(dockerImageRepository); + sessionZooKeeperClient.writeDockerImageRepository(dockerImageRepository); } public void setAthenzDomain(Optional<AthenzDomain> athenzDomain) { - zooKeeperClient.writeAthenzDomain(athenzDomain); + sessionZooKeeperClient.writeAthenzDomain(athenzDomain); } - public ApplicationId getApplicationId() { return zooKeeperClient.readApplicationId(); } + public ApplicationId getApplicationId() { return sessionZooKeeperClient.readApplicationId(); } - public FileReference getApplicationPackageReference() {return zooKeeperClient.readApplicationPackageReference(); } + public FileReference getApplicationPackageReference() {return sessionZooKeeperClient.readApplicationPackageReference(); } - public Optional<DockerImage> getDockerImageRepository() { return zooKeeperClient.readDockerImageRepository(); } + public Optional<DockerImage> getDockerImageRepository() { return sessionZooKeeperClient.readDockerImageRepository(); } - public Version getVespaVersion() { return zooKeeperClient.readVespaVersion(); } + public Version getVespaVersion() { return sessionZooKeeperClient.readVespaVersion(); } - public Optional<AthenzDomain> getAthenzDomain() { return zooKeeperClient.readAthenzDomain(); } + public Optional<AthenzDomain> getAthenzDomain() { return sessionZooKeeperClient.readAthenzDomain(); } public AllocatedHosts getAllocatedHosts() { - return zooKeeperClient.getAllocatedHosts(); + return sessionZooKeeperClient.getAllocatedHosts(); } public Transaction createDeactivateTransaction() { @@ -124,7 +124,7 @@ public abstract class Session implements Comparable<Session> { } private Transaction createSetStatusTransaction(Status status) { - return zooKeeperClient.createWriteStatusTransaction(status); + return sessionZooKeeperClient.createWriteStatusTransaction(status); } // Note: Assumes monotonically increasing session ids diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java index 8808dc0cf75..501d4918996 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java @@ -16,9 +16,7 @@ public class SessionCache<SESSIONTYPE extends Session> { private final HashMap<Long, SESSIONTYPE> sessions = new HashMap<>(); public synchronized void addSession(SESSIONTYPE session) { - if (sessions.containsKey(session.getSessionId())) - throw new IllegalArgumentException("There already exists a session with id '" + session.getSessionId() + "'"); - sessions.put(session.getSessionId(), session); + sessions.putIfAbsent(session.getSessionId(), session); } synchronized void removeSession(long id) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java deleted file mode 100644 index 8aba9fa465d..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.FileReference; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.DeployData; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; -import com.yahoo.vespa.config.server.filedistribution.FileDirectory; -import com.yahoo.vespa.config.server.host.HostValidator; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.config.server.zookeeper.SessionCounter; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.Flags; - -import java.io.File; -import java.io.IOException; -import java.time.Clock; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Serves as the factory of sessions. Takes care of copying files to the correct folder and initializing the - * session state. There is one SessionFactory per tenant. - * - * @author Ulf Lilleengen - */ -public class SessionFactory { - - private static final Logger log = Logger.getLogger(SessionFactory.class.getName()); - private static final long nonExistingActiveSession = 0; - - private final SessionPreparer sessionPreparer; - private final Curator curator; - private final ConfigCurator configCurator; - private final TenantApplications applicationRepo; - private final Path sessionsPath; - private final GlobalComponentRegistry componentRegistry; - private final HostValidator<ApplicationId> hostRegistry; - private final TenantName tenant; - private final String serverId; - private final Optional<NodeFlavors> nodeFlavors; - private final Clock clock; - private final BooleanFlag distributeApplicationPackage; - - public SessionFactory(GlobalComponentRegistry globalComponentRegistry, - TenantApplications applicationRepo, - HostValidator<ApplicationId> hostRegistry, - TenantName tenant) { - this.hostRegistry = hostRegistry; - this.tenant = tenant; - this.sessionPreparer = globalComponentRegistry.getSessionPreparer(); - this.curator = globalComponentRegistry.getCurator(); - this.configCurator = globalComponentRegistry.getConfigCurator(); - this.sessionsPath = TenantRepository.getSessionsPath(tenant); - this.applicationRepo = applicationRepo; - this.componentRegistry = globalComponentRegistry; - this.serverId = globalComponentRegistry.getConfigserverConfig().serverId(); - this.nodeFlavors = globalComponentRegistry.getZone().nodeFlavors(); - this.clock = globalComponentRegistry.getClock(); - this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE - .bindTo(globalComponentRegistry.getFlagSource()); - } - - /** - * Creates a new deployment session from an application package. - * - * @param applicationDirectory a File pointing to an application. - * @param applicationId application id for this new session. - * @param timeoutBudget Timeout for creating session and waiting for other servers. - * @return a new session - */ - public LocalSession createSession(File applicationDirectory, ApplicationId applicationId, - TimeoutBudget timeoutBudget, Optional<Long> activeSessionId) { - return create(applicationDirectory, applicationId, activeSessionId.orElse(nonExistingActiveSession), false, timeoutBudget); - } - - public RemoteSession createRemoteSession(long sessionId) { - SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(getSessionPath(sessionId)); - return new RemoteSession(tenant, sessionId, componentRegistry, sessionZKClient); - } - - private void ensureSessionPathDoesNotExist(long sessionId) { - Path sessionPath = getSessionPath(sessionId); - if (configCurator.exists(sessionPath.getAbsolute())) { - throw new IllegalArgumentException("Path " + sessionPath.getAbsolute() + " already exists in ZooKeeper"); - } - } - - private ApplicationPackage createApplication(File userDir, - File configApplicationDir, - ApplicationId applicationId, - long sessionId, - long currentlyActiveSessionId, - boolean internalRedeploy) { - long deployTimestamp = System.currentTimeMillis(); - String user = System.getenv("USER"); - if (user == null) { - user = "unknown"; - } - DeployData deployData = new DeployData(user, userDir.getAbsolutePath(), applicationId, deployTimestamp, internalRedeploy, sessionId, currentlyActiveSessionId); - return FilesApplicationPackage.fromFileWithDeployData(configApplicationDir, deployData); - } - - private LocalSession createSessionFromApplication(ApplicationPackage applicationPackage, - long sessionId, - TimeoutBudget timeoutBudget, - Clock clock) { - log.log(Level.FINE, TenantRepository.logPre(tenant) + "Creating session " + sessionId + " in ZooKeeper"); - SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(getSessionPath(sessionId)); - sessionZKClient.createNewSession(clock.instant()); - Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); - LocalSession session = new LocalSession(tenant, sessionId, sessionPreparer, applicationPackage, sessionZKClient, - getSessionAppDir(sessionId), applicationRepo, hostRegistry); - waiter.awaitCompletion(timeoutBudget.timeLeft()); - return session; - } - - /** - * Creates a new deployment session from an already existing session. - * - * @param existingSession the session to use as base - * @param logger a deploy logger where the deploy log will be written. - * @param internalRedeploy whether this session is for a system internal redeploy — not an application package change - * @param timeoutBudget timeout for creating session and waiting for other servers. - * @return a new session - */ - public LocalSession createSessionFromExisting(Session existingSession, - DeployLogger logger, - boolean internalRedeploy, - TimeoutBudget timeoutBudget) { - File existingApp = getSessionAppDir(existingSession.getSessionId()); - ApplicationId existingApplicationId = existingSession.getApplicationId(); - - long activeSessionId = getActiveSessionId(existingApplicationId); - logger.log(Level.FINE, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); - LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); - // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() - session.setApplicationId(existingApplicationId); - if (distributeApplicationPackage.value() && existingSession.getApplicationPackageReference() != null) { - session.setApplicationPackageReference(existingSession.getApplicationPackageReference()); - } - session.setVespaVersion(existingSession.getVespaVersion()); - session.setDockerImageRepository(existingSession.getDockerImageRepository()); - session.setAthenzDomain(existingSession.getAthenzDomain()); - return session; - } - - private LocalSession create(File applicationFile, ApplicationId applicationId, long currentlyActiveSessionId, - boolean internalRedeploy, TimeoutBudget timeoutBudget) { - long sessionId = getNextSessionId(); - try { - ensureSessionPathDoesNotExist(sessionId); - ApplicationPackage app = createApplicationPackage(applicationFile, applicationId, - sessionId, currentlyActiveSessionId, internalRedeploy); - return createSessionFromApplication(app, sessionId, timeoutBudget, clock); - } catch (Exception e) { - throw new RuntimeException("Error creating session " + sessionId, e); - } - } - - /** - * This method is used when creating a session based on a remote session and the distributed application package - * It does not wait for session being created on other servers - */ - private LocalSession createLocalSession(File applicationFile, ApplicationId applicationId, - long sessionId, long currentlyActiveSessionId) { - try { - ApplicationPackage applicationPackage = createApplicationPackage(applicationFile, applicationId, - sessionId, currentlyActiveSessionId, false); - SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(getSessionPath(sessionId)); - return new LocalSession(tenant, sessionId, sessionPreparer, applicationPackage, sessionZooKeeperClient, - getSessionAppDir(sessionId), applicationRepo, hostRegistry); - } catch (Exception e) { - throw new RuntimeException("Error creating session " + sessionId, e); - } - } - - private ApplicationPackage createApplicationPackage(File applicationFile, ApplicationId applicationId, - long sessionId, long currentlyActiveSessionId, - boolean internalRedeploy) throws IOException { - File userApplicationDir = getSessionAppDir(sessionId); - IOUtils.copyDirectory(applicationFile, userApplicationDir); - ApplicationPackage applicationPackage = createApplication(applicationFile, - userApplicationDir, - applicationId, - sessionId, - currentlyActiveSessionId, - internalRedeploy); - applicationPackage.writeMetaData(); - return applicationPackage; - } - - /** - * Returns a new session instance for the given session id. - */ - LocalSession createSessionFromId(long sessionId) { - File sessionDir = getAndValidateExistingSessionAppDir(sessionId); - ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); - Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId)); - SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionIdPath); - return new LocalSession(tenant, sessionId, sessionPreparer, applicationPackage, sessionZKClient, - getSessionAppDir(sessionId), applicationRepo, hostRegistry); - } - - /** - * Returns a new session instance for the given session id. - */ - LocalSession createLocalSessionUsingDistributedApplicationPackage(long sessionId) { - if (applicationRepo.hasLocalSession(sessionId)) { - log.log(Level.FINE, "Local session for session id " + sessionId + " already exists"); - return createSessionFromId(sessionId); - } - - log.log(Level.INFO, "Creating local session for session id " + sessionId); - SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(getSessionPath(sessionId)); - FileReference fileReference = sessionZKClient.readApplicationPackageReference(); - log.log(Level.FINE, "File reference for session id " + sessionId + ": " + fileReference); - if (fileReference != null) { - File rootDir = new File(Defaults.getDefaults().underVespaHome(componentRegistry.getConfigserverConfig().fileReferencesDir())); - File sessionDir = new FileDirectory(rootDir).getFile(fileReference); - if (!sessionDir.exists()) - throw new RuntimeException("File reference for session " + sessionId + " not found (" + sessionDir.getAbsolutePath() + ")"); - ApplicationId applicationId = sessionZKClient.readApplicationId(); - return createLocalSession(sessionDir, - applicationId, - sessionId, - applicationRepo.activeSessionOf(applicationId).orElse(nonExistingActiveSession)); - } - return null; - } - - // Return Optional instead of faking it with nonExistingActiveSession - private long getActiveSessionId(ApplicationId applicationId) { - List<ApplicationId> applicationIds = applicationRepo.activeApplications(); - if (applicationIds.contains(applicationId)) { - return applicationRepo.requireActiveSessionOf(applicationId); - } - return nonExistingActiveSession; - } - - private long getNextSessionId() { - return new SessionCounter(componentRegistry.getConfigCurator(), tenant).nextSessionId(); - } - - private Path getSessionPath(long sessionId) { - return sessionsPath.append(String.valueOf(sessionId)); - } - - private SessionZooKeeperClient createSessionZooKeeperClient(Path sessionPath) { - return new SessionZooKeeperClient(curator, configCurator, sessionPath, serverId, nodeFlavors); - } - - private File getAndValidateExistingSessionAppDir(long sessionId) { - File appDir = getSessionAppDir(sessionId); - if (!appDir.exists() || !appDir.isDirectory()) { - throw new IllegalArgumentException("Unable to find correct application directory for session " + sessionId); - } - return appDir; - } - - private File getSessionAppDir(long sessionId) { - return new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenant).getUserApplicationDir(sessionId); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 4a2e7cb405b..3c83263f781 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -55,6 +55,7 @@ import java.io.File; import java.io.IOException; import java.net.URI; import java.time.Instant; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; @@ -115,7 +116,7 @@ public class SessionPreparer { * @param logger For storing logs returned in response to client. * @param params parameters controlling behaviour of prepare. * @param currentActiveApplicationSet Set of currently active applications. - * @param tenantPath Zookeeper path for the tenant for this session + * @param tenantPath Zookeeper path for the tenant for this session * @return the config change actions that must be done to handle the activation of the models prepared. */ public ConfigChangeActions prepare(HostValidator<ApplicationId> hostValidator, DeployLogger logger, PrepareParams params, @@ -317,7 +318,7 @@ public class SessionPreparer { } private List<ContainerEndpoint> readEndpointsIfNull(List<ContainerEndpoint> endpoints) { - if (endpoints == null) { // endpoints is only set when prepared via HTTP + if (endpoints == null) { // endpoints are only set when prepared via HTTP endpoints = this.containerEndpointsCache.read(applicationId); } return List.copyOf(endpoints); @@ -383,7 +384,7 @@ public class SessionPreparer { */ public ConfigChangeActions getConfigChangeActions() { return new ConfigChangeActions(results.stream().map(result -> result.actions) - .flatMap(actions -> actions.stream()) + .flatMap(Collection::stream) .collect(Collectors.toList())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java new file mode 100644 index 00000000000..8317dc02e90 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -0,0 +1,659 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.DeployData; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.TenantName; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.transaction.AbstractTransaction; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.TenantApplications; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.vespa.config.server.filedistribution.FileDirectory; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.server.zookeeper.SessionCounter; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * + * Session repository for a tenant. Stores session state in zookeeper and file system. There are two + * different session types (RemoteSession and LocalSession). + * + * @author Ulf Lilleengen + * @author hmusum + * + */ +public class SessionRepository { + + private static final Logger log = Logger.getLogger(SessionRepository.class.getName()); + private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); + private static final long nonExistingActiveSession = 0; + + private final SessionCache<LocalSession> localSessionCache = new SessionCache<>(); + private final SessionCache<RemoteSession> remoteSessionCache = new SessionCache<>(); + private final Map<Long, LocalSessionStateWatcher> localSessionStateWatchers = new HashMap<>(); + private final Map<Long, RemoteSessionStateWatcher> remoteSessionStateWatchers = new HashMap<>(); + private final Duration sessionLifetime; + private final Clock clock; + private final Curator curator; + private final Executor zkWatcherExecutor; + private final TenantFileSystemDirs tenantFileSystemDirs; + private final BooleanFlag distributeApplicationPackage; + private final ReloadHandler reloadHandler; + private final MetricUpdater metrics; + private final Curator.DirectoryCache directoryCache; + private final TenantApplications applicationRepo; + private final SessionPreparer sessionPreparer; + private final Path sessionsPath; + private final TenantName tenantName; + private final GlobalComponentRegistry componentRegistry; + + public SessionRepository(TenantName tenantName, + GlobalComponentRegistry componentRegistry, + TenantApplications applicationRepo, + ReloadHandler reloadHandler, + FlagSource flagSource, + SessionPreparer sessionPreparer) { + this.tenantName = tenantName; + this.componentRegistry = componentRegistry; + this.sessionsPath = TenantRepository.getSessionsPath(tenantName); + this.clock = componentRegistry.getClock(); + this.curator = componentRegistry.getCurator(); + this.sessionLifetime = Duration.ofSeconds(componentRegistry.getConfigserverConfig().sessionLifetime()); + this.zkWatcherExecutor = command -> componentRegistry.getZkWatcherExecutor().execute(tenantName, command); + this.tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName); + this.applicationRepo = applicationRepo; + this.sessionPreparer = sessionPreparer; + this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); + this.reloadHandler = reloadHandler; + this.metrics = componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); + + loadLocalSessions(); + initializeRemoteSessions(); + this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, componentRegistry.getZkCacheExecutor()); + this.directoryCache.addListener(this::childEvent); + this.directoryCache.start(); + } + + // ---------------- Local sessions ---------------------------------------------------------------- + + public synchronized void addSession(LocalSession session) { + localSessionCache.addSession(session); + Path sessionsPath = TenantRepository.getSessionsPath(session.getTenantName()); + long sessionId = session.getSessionId(); + Curator.FileCache fileCache = curator.createFileCache(sessionsPath.append(String.valueOf(sessionId)).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); + localSessionStateWatchers.put(sessionId, new LocalSessionStateWatcher(fileCache, session, this, zkWatcherExecutor)); + } + + public LocalSession getLocalSession(long sessionId) { + return localSessionCache.getSession(sessionId); + } + + public List<LocalSession> getLocalSessions() { + return localSessionCache.getSessions(); + } + + private void loadLocalSessions() { + File[] sessions = tenantFileSystemDirs.sessionsPath().listFiles(sessionApplicationsFilter); + if (sessions == null) { + return; + } + for (File session : sessions) { + try { + addSession(createSessionFromId(Long.parseLong(session.getName()))); + } catch (IllegalArgumentException e) { + log.log(Level.WARNING, "Could not load session '" + + session.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); + } + } + } + + public ConfigChangeActions prepareLocalSession(LocalSession session, + DeployLogger logger, + PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, + Path tenantPath, + Instant now) { + applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( + logger.log(Level.FINE, "Created application " + params.getApplicationId()); + long sessionId = session.getSessionId(); + SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.createPrepareWaiter(); + ConfigChangeActions actions = sessionPreparer.prepare(applicationRepo.getHostValidator(), logger, params, + currentActiveApplicationSet, tenantPath, now, + getSessionAppDir(sessionId), + session.getApplicationPackage(), sessionZooKeeperClient); + session.setPrepared(); + waiter.awaitCompletion(params.getTimeoutBudget().timeLeft()); + return actions; + } + + public void deleteExpiredSessions(Map<ApplicationId, Long> activeSessions) { + log.log(Level.FINE, "Purging old sessions"); + try { + for (LocalSession candidate : localSessionCache.getSessions()) { + Instant createTime = candidate.getCreateTime(); + log.log(Level.FINE, "Candidate session for deletion: " + candidate.getSessionId() + ", created: " + createTime); + + // Sessions with state other than ACTIVATED + if (hasExpired(candidate) && !isActiveSession(candidate)) { + deleteLocalSession(candidate); + } else if (createTime.plus(Duration.ofDays(1)).isBefore(clock.instant())) { + // Sessions with state ACTIVATE, but which are not actually active + ApplicationId applicationId = candidate.getApplicationId(); + Long activeSession = activeSessions.get(applicationId); + if (activeSession == null || activeSession != candidate.getSessionId()) { + deleteLocalSession(candidate); + log.log(Level.INFO, "Deleted inactive session " + candidate.getSessionId() + " created " + + createTime + " for '" + applicationId + "'"); + } + } + } + // Make sure to catch here, to avoid executor just dying in case of issues ... + } catch (Throwable e) { + log.log(Level.WARNING, "Error when purging old sessions ", e); + } + log.log(Level.FINE, "Done purging old sessions"); + } + + private boolean hasExpired(LocalSession candidate) { + return (candidate.getCreateTime().plus(sessionLifetime).isBefore(clock.instant())); + } + + private boolean isActiveSession(LocalSession candidate) { + return candidate.getStatus() == Session.Status.ACTIVATE; + } + + public void deleteLocalSession(LocalSession session) { + long sessionId = session.getSessionId(); + log.log(Level.FINE, "Deleting local session " + sessionId); + LocalSessionStateWatcher watcher = localSessionStateWatchers.remove(sessionId); + if (watcher != null) watcher.close(); + localSessionCache.removeSession(sessionId); + NestedTransaction transaction = new NestedTransaction(); + deleteLocalSession(session, transaction); + transaction.commit(); + } + + /** Add transactions to delete this session to the given nested transaction */ + public void deleteLocalSession(LocalSession session, NestedTransaction transaction) { + long sessionId = session.getSessionId(); + SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); + transaction.add(sessionZooKeeperClient.deleteTransaction(), FileTransaction.class); + transaction.add(FileTransaction.from(FileOperations.delete(getSessionAppDir(sessionId).getAbsolutePath()))); + } + + public void close() { + deleteAllSessions(); + tenantFileSystemDirs.delete(); + try { + if (directoryCache != null) { + directoryCache.close(); + } + } catch (Exception e) { + log.log(Level.WARNING, "Exception when closing path cache", e); + } finally { + checkForRemovedSessions(new ArrayList<>()); + } + } + + private void deleteAllSessions() { + List<LocalSession> sessions = new ArrayList<>(localSessionCache.getSessions()); + for (LocalSession session : sessions) { + deleteLocalSession(session); + } + } + + // ---------------- Remote sessions ---------------------------------------------------------------- + + public RemoteSession getRemoteSession(long sessionId) { + return remoteSessionCache.getSession(sessionId); + } + + public List<Long> getRemoteSessions() { + return getSessionList(curator.getChildren(sessionsPath)); + } + + public void addRemoteSession(RemoteSession session) { + remoteSessionCache.addSession(session); + metrics.incAddedSessions(); + } + + public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { + int deleted = 0; + for (long sessionId : getRemoteSessions()) { + RemoteSession session = remoteSessionCache.getSession(sessionId); + if (session == null) continue; // Internal sessions not in synch with zk, continue + if (session.getStatus() == Session.Status.ACTIVATE) continue; + if (sessionHasExpired(session.getCreateTime(), expiryTime, clock)) { + log.log(Level.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); + session.delete(); + deleted++; + } + } + return deleted; + } + + private boolean sessionHasExpired(Instant created, Duration expiryTime, Clock clock) { + return (created.plus(expiryTime).isBefore(clock.instant())); + } + + private List<Long> getSessionListFromDirectoryCache(List<ChildData> children) { + return getSessionList(children.stream() + .map(child -> Path.fromString(child.getPath()).getName()) + .collect(Collectors.toList())); + } + + private List<Long> getSessionList(List<String> children) { + return children.stream().map(Long::parseLong).collect(Collectors.toList()); + } + + private void initializeRemoteSessions() throws NumberFormatException { + getRemoteSessions().forEach(this::sessionAdded); + } + + private synchronized void sessionsChanged() throws NumberFormatException { + List<Long> sessions = getSessionListFromDirectoryCache(directoryCache.getCurrentData()); + checkForRemovedSessions(sessions); + checkForAddedSessions(sessions); + } + + private void checkForRemovedSessions(List<Long> sessions) { + for (RemoteSession session : remoteSessionCache.getSessions()) + if ( ! sessions.contains(session.getSessionId())) + sessionRemoved(session.getSessionId()); + } + + private void checkForAddedSessions(List<Long> sessions) { + for (Long sessionId : sessions) + if (remoteSessionCache.getSession(sessionId) == null) + sessionAdded(sessionId); + } + + /** + * A session for which we don't have a watcher, i.e. hitherto unknown to us. + * + * @param sessionId session id for the new session + */ + public void sessionAdded(long sessionId) { + log.log(Level.FINE, () -> "Adding session to SessionRepository: " + sessionId); + RemoteSession session = createRemoteSession(sessionId); + Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); + Curator.FileCache fileCache = curator.createFileCache(sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); + fileCache.addListener(this::nodeChanged); + loadSessionIfActive(session); + addRemoteSession(session); + remoteSessionStateWatchers.put(sessionId, new RemoteSessionStateWatcher(fileCache, reloadHandler, session, metrics, zkWatcherExecutor)); + if (distributeApplicationPackage.value()) { + Optional<LocalSession> localSession = createLocalSessionUsingDistributedApplicationPackage(sessionId); + localSession.ifPresent(this::addSession); + } + } + + private void sessionRemoved(long sessionId) { + RemoteSessionStateWatcher watcher = remoteSessionStateWatchers.remove(sessionId); + if (watcher != null) watcher.close(); + remoteSessionCache.removeSession(sessionId); + metrics.incRemovedSessions(); + } + + private void loadSessionIfActive(RemoteSession session) { + for (ApplicationId applicationId : applicationRepo.activeApplications()) { + if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { + log.log(Level.FINE, () -> "Found active application for session " + session.getSessionId() + " , loading it"); + reloadHandler.reloadConfig(session.ensureApplicationLoaded()); + log.log(Level.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); + return; + } + } + } + + private void nodeChanged() { + zkWatcherExecutor.execute(() -> { + Multiset<Session.Status> sessionMetrics = HashMultiset.create(); + for (RemoteSession session : remoteSessionCache.getSessions()) { + sessionMetrics.add(session.getStatus()); + } + metrics.setNewSessions(sessionMetrics.count(Session.Status.NEW)); + metrics.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE)); + metrics.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE)); + metrics.setDeactivatedSessions(sessionMetrics.count(Session.Status.DEACTIVATE)); + }); + } + + @SuppressWarnings("unused") + private void childEvent(CuratorFramework ignored, PathChildrenCacheEvent event) { + zkWatcherExecutor.execute(() -> { + log.log(Level.FINE, () -> "Got child event: " + event); + switch (event.getType()) { + case CHILD_ADDED: + sessionsChanged(); + synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); + break; + case CHILD_REMOVED: + case CONNECTION_RECONNECTED: + sessionsChanged(); + break; + } + }); + } + + private void synchronizeOnNew(List<Long> sessionList) { + for (long sessionId : sessionList) { + RemoteSession session = remoteSessionCache.getSession(sessionId); + if (session == null) continue; // session might have been deleted after getting session list + log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); + session.confirmUpload(); + } + } + + /** + * Creates a new deployment session from an application package. + * + * @param applicationDirectory a File pointing to an application. + * @param applicationId application id for this new session. + * @param timeoutBudget Timeout for creating session and waiting for other servers. + * @return a new session + */ + public LocalSession createSession(File applicationDirectory, ApplicationId applicationId, + TimeoutBudget timeoutBudget, Optional<Long> activeSessionId) { + return create(applicationDirectory, applicationId, activeSessionId.orElse(nonExistingActiveSession), false, timeoutBudget); + } + + public RemoteSession createRemoteSession(long sessionId) { + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + return new RemoteSession(tenantName, sessionId, componentRegistry, sessionZKClient); + } + + private void ensureSessionPathDoesNotExist(long sessionId) { + Path sessionPath = getSessionPath(sessionId); + if (componentRegistry.getConfigCurator().exists(sessionPath.getAbsolute())) { + throw new IllegalArgumentException("Path " + sessionPath.getAbsolute() + " already exists in ZooKeeper"); + } + } + + private ApplicationPackage createApplication(File userDir, + File configApplicationDir, + ApplicationId applicationId, + long sessionId, + long currentlyActiveSessionId, + boolean internalRedeploy) { + long deployTimestamp = System.currentTimeMillis(); + String user = System.getenv("USER"); + if (user == null) { + user = "unknown"; + } + DeployData deployData = new DeployData(user, userDir.getAbsolutePath(), applicationId, deployTimestamp, internalRedeploy, sessionId, currentlyActiveSessionId); + return FilesApplicationPackage.fromFileWithDeployData(configApplicationDir, deployData); + } + + private LocalSession createSessionFromApplication(ApplicationPackage applicationPackage, + long sessionId, + TimeoutBudget timeoutBudget, + Clock clock) { + log.log(Level.FINE, TenantRepository.logPre(tenantName) + "Creating session " + sessionId + " in ZooKeeper"); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + sessionZKClient.createNewSession(clock.instant()); + Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); + LocalSession session = new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient, applicationRepo); + waiter.awaitCompletion(timeoutBudget.timeLeft()); + return session; + } + + /** + * Creates a new deployment session from an already existing session. + * + * @param existingSession the session to use as base + * @param logger a deploy logger where the deploy log will be written. + * @param internalRedeploy whether this session is for a system internal redeploy — not an application package change + * @param timeoutBudget timeout for creating session and waiting for other servers. + * @return a new session + */ + public LocalSession createSessionFromExisting(Session existingSession, + DeployLogger logger, + boolean internalRedeploy, + TimeoutBudget timeoutBudget) { + File existingApp = getSessionAppDir(existingSession.getSessionId()); + ApplicationId existingApplicationId = existingSession.getApplicationId(); + + long activeSessionId = getActiveSessionId(existingApplicationId); + logger.log(Level.FINE, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); + LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); + // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() + session.setApplicationId(existingApplicationId); + if (distributeApplicationPackage.value() && existingSession.getApplicationPackageReference() != null) { + session.setApplicationPackageReference(existingSession.getApplicationPackageReference()); + } + session.setVespaVersion(existingSession.getVespaVersion()); + session.setDockerImageRepository(existingSession.getDockerImageRepository()); + session.setAthenzDomain(existingSession.getAthenzDomain()); + return session; + } + + private LocalSession create(File applicationFile, ApplicationId applicationId, long currentlyActiveSessionId, + boolean internalRedeploy, TimeoutBudget timeoutBudget) { + long sessionId = getNextSessionId(); + try { + ensureSessionPathDoesNotExist(sessionId); + ApplicationPackage app = createApplicationPackage(applicationFile, applicationId, + sessionId, currentlyActiveSessionId, internalRedeploy); + return createSessionFromApplication(app, sessionId, timeoutBudget, clock); + } catch (Exception e) { + throw new RuntimeException("Error creating session " + sessionId, e); + } + } + + /** + * This method is used when creating a session based on a remote session and the distributed application package + * It does not wait for session being created on other servers + */ + private LocalSession createLocalSession(File applicationFile, ApplicationId applicationId, + long sessionId, long currentlyActiveSessionId) { + try { + ApplicationPackage applicationPackage = createApplicationPackage(applicationFile, applicationId, + sessionId, currentlyActiveSessionId, false); + SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); + return new LocalSession(tenantName, sessionId, applicationPackage, sessionZooKeeperClient, applicationRepo); + } catch (Exception e) { + throw new RuntimeException("Error creating session " + sessionId, e); + } + } + + private ApplicationPackage createApplicationPackage(File applicationFile, ApplicationId applicationId, + long sessionId, long currentlyActiveSessionId, + boolean internalRedeploy) throws IOException { + File userApplicationDir = getSessionAppDir(sessionId); + IOUtils.copyDirectory(applicationFile, userApplicationDir); + ApplicationPackage applicationPackage = createApplication(applicationFile, + userApplicationDir, + applicationId, + sessionId, + currentlyActiveSessionId, + internalRedeploy); + applicationPackage.writeMetaData(); + return applicationPackage; + } + + /** + * Returns a new session instance for the given session id. + */ + LocalSession createSessionFromId(long sessionId) { + File sessionDir = getAndValidateExistingSessionAppDir(sessionId); + ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + return new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient, applicationRepo); + } + + /** + * Returns a new session instance for the given session id. + */ + Optional<LocalSession> createLocalSessionUsingDistributedApplicationPackage(long sessionId) { + if (applicationRepo.hasLocalSession(sessionId)) { + log.log(Level.FINE, "Local session for session id " + sessionId + " already exists"); + return Optional.of(createSessionFromId(sessionId)); + } + + log.log(Level.INFO, "Creating local session for session id " + sessionId); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + FileReference fileReference = sessionZKClient.readApplicationPackageReference(); + log.log(Level.FINE, "File reference for session id " + sessionId + ": " + fileReference); + if (fileReference != null) { + File rootDir = new File(Defaults.getDefaults().underVespaHome(componentRegistry.getConfigserverConfig().fileReferencesDir())); + File sessionDir; + FileDirectory fileDirectory = new FileDirectory(rootDir); + try { + sessionDir = fileDirectory.getFile(fileReference); + } catch (IllegalArgumentException e) { + // We cannot be guaranteed that the file reference exists (it could be that it has not + // been downloaded yet), and e.g when bootstrapping we cannot throw an exception in that case + log.log(Level.INFO, "File reference for session id " + sessionId + ": " + fileReference + " not found in " + fileDirectory); + return Optional.empty(); + } + ApplicationId applicationId = sessionZKClient.readApplicationId(); + return Optional.of(createLocalSession(sessionDir, + applicationId, + sessionId, + applicationRepo.activeSessionOf(applicationId).orElse(nonExistingActiveSession))); + } + return Optional.empty(); + } + + // Return Optional instead of faking it with nonExistingActiveSession + private long getActiveSessionId(ApplicationId applicationId) { + List<ApplicationId> applicationIds = applicationRepo.activeApplications(); + if (applicationIds.contains(applicationId)) { + return applicationRepo.requireActiveSessionOf(applicationId); + } + return nonExistingActiveSession; + } + + private long getNextSessionId() { + return new SessionCounter(componentRegistry.getConfigCurator(), tenantName).nextSessionId(); + } + + private Path getSessionPath(long sessionId) { + return sessionsPath.append(String.valueOf(sessionId)); + } + + + private SessionZooKeeperClient createSessionZooKeeperClient(long sessionId) { + String serverId = componentRegistry.getConfigserverConfig().serverId(); + Optional<NodeFlavors> nodeFlavors = componentRegistry.getZone().nodeFlavors(); + Path sessionPath = getSessionPath(sessionId); + return new SessionZooKeeperClient(curator, componentRegistry.getConfigCurator(), sessionPath, serverId, nodeFlavors); + } + + private File getAndValidateExistingSessionAppDir(long sessionId) { + File appDir = getSessionAppDir(sessionId); + if (!appDir.exists() || !appDir.isDirectory()) { + throw new IllegalArgumentException("Unable to find correct application directory for session " + sessionId); + } + return appDir; + } + + private File getSessionAppDir(long sessionId) { + return new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName).getUserApplicationDir(sessionId); + } + + @Override + public String toString() { + return getLocalSessions().toString(); + } + + private static class FileTransaction extends AbstractTransaction { + + public static FileTransaction from(FileOperation operation) { + FileTransaction transaction = new FileTransaction(); + transaction.add(operation); + return transaction; + } + + @Override + public void prepare() { } + + @Override + public void commit() { + for (Operation operation : operations()) + ((FileOperation)operation).commit(); + } + + } + + /** Factory for file operations */ + private static class FileOperations { + + /** Creates an operation which recursively deletes the given path */ + public static DeleteOperation delete(String pathToDelete) { + return new DeleteOperation(pathToDelete); + } + + } + + private interface FileOperation extends Transaction.Operation { + + void commit(); + + } + + /** + * Recursively deletes this path and everything below. + * Succeeds with no action if the path does not exist. + */ + private static class DeleteOperation implements FileOperation { + + private final String pathToDelete; + + DeleteOperation(String pathToDelete) { + this.pathToDelete = pathToDelete; + } + + @Override + public void commit() { + // TODO: Check delete access in prepare() + IOUtils.recursiveDeleteDir(new File(pathToDelete)); + } + + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 33001d2996c..807629a2148 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -14,7 +14,6 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.path.Path; import com.yahoo.text.Utf8; -import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.UserConfigDefinitionRepo; import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; @@ -129,18 +128,6 @@ public class SessionZooKeeperClient { return curator.getCompletionWaiter(path, getNumberOfMembers(), serverId); } - public void delete(NestedTransaction transaction ) { - try { - log.log(Level.FINE, "Deleting " + sessionPath.getAbsolute()); - CuratorTransaction curatorTransaction = new CuratorTransaction(curator); - CuratorOperations.deleteAll(sessionPath.getAbsolute(), curator).forEach(curatorTransaction::add); - transaction.add(curatorTransaction); - transaction.commit(); - } catch (RuntimeException e) { - log.log(Level.INFO, "Error deleting session (" + sessionPath.getAbsolute() + ") from zookeeper", e); - } - } - /** Returns a transaction deleting this session on commit */ public CuratorTransaction deleteTransaction() { return CuratorTransaction.from(CuratorOperations.deleteAll(sessionPath.getAbsolute(), curator), curator); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java index f0aab8b2312..f7c8ae9d5c3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java @@ -6,9 +6,7 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; -import com.yahoo.vespa.config.server.session.RemoteSessionRepo; -import com.yahoo.vespa.config.server.session.SessionFactory; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.curator.Curator; import org.apache.zookeeper.data.Stat; @@ -28,19 +26,15 @@ public class Tenant implements TenantHandlerProvider { static final String APPLICATIONS = "applications"; private final TenantName name; - private final RemoteSessionRepo remoteSessionRepo; private final Path path; - private final SessionFactory sessionFactory; - private final LocalSessionRepo localSessionRepo; + private final SessionRepository sessionRepository; private final TenantApplications applicationRepo; private final RequestHandler requestHandler; private final ReloadHandler reloadHandler; private final Curator curator; Tenant(TenantName name, - SessionFactory sessionFactory, - LocalSessionRepo localSessionRepo, - RemoteSessionRepo remoteSessionRepo, + SessionRepository sessionRepository, RequestHandler requestHandler, ReloadHandler reloadHandler, TenantApplications applicationRepo, @@ -49,9 +43,7 @@ public class Tenant implements TenantHandlerProvider { this.path = TenantRepository.getTenantPath(name); this.requestHandler = requestHandler; this.reloadHandler = reloadHandler; - this.remoteSessionRepo = remoteSessionRepo; - this.sessionFactory = sessionFactory; - this.localSessionRepo = localSessionRepo; + this.sessionRepository = sessionRepository; this.applicationRepo = applicationRepo; this.curator = curator; } @@ -74,13 +66,8 @@ public class Tenant implements TenantHandlerProvider { return requestHandler; } - /** - * The RemoteSessionRepo for this - * - * @return repo - */ - public RemoteSessionRepo getRemoteSessionRepo() { - return remoteSessionRepo; + public SessionRepository getSessionRepo() { + return sessionRepository; } public TenantName getName() { @@ -91,13 +78,7 @@ public class Tenant implements TenantHandlerProvider { return path; } - public SessionFactory getSessionFactory() { - return sessionFactory; - } - - public LocalSessionRepo getLocalSessionRepo() { - return localSessionRepo; - } + public SessionRepository getSessionRepository() { return sessionRepository; } @Override public String toString() { @@ -140,9 +121,8 @@ public class Tenant implements TenantHandlerProvider { * Called by watchers as a reaction to {@link #delete()}. */ void close() { - remoteSessionRepo.close(); // Closes watchers and clears memory. applicationRepo.close(); // Closes watchers. - localSessionRepo.close(); // Closes watchers, clears memory, and deletes local files and ZK session state. + sessionRepository.close(); // Closes watchers, clears memory, and deletes local files and ZK session state. } /** Deletes the tenant tree from ZooKeeper (application and session status for the tenant) and triggers {@link #close()}. */ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index d34f89a179b..304fbb6786a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -14,9 +14,7 @@ import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; -import com.yahoo.vespa.config.server.session.RemoteSessionRepo; -import com.yahoo.vespa.config.server.session.SessionFactory; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -49,7 +47,7 @@ import java.util.stream.Collectors; * This component will monitor the set of tenants in the config server by watching in ZooKeeper. * It will set up Tenant objects accordingly, which will manage the config sessions per tenant. * This class will read the preexisting set of tenants from ZooKeeper at startup. (For now it will also - * create a default tenant since that will be used for API that do no know about tenants or have not yet + * create a default tenant since that will be used for APIs that do no know about tenants or have not yet * implemented support for it). * * This instance is called from two different threads, the http handler threads and the zookeeper watcher threads. @@ -223,16 +221,12 @@ public class TenantRepository { requestHandler = applicationRepo; if (reloadHandler == null) reloadHandler = applicationRepo; - SessionFactory sessionFactory = new SessionFactory(componentRegistry, applicationRepo, applicationRepo, tenantName); - LocalSessionRepo localSessionRepo = new LocalSessionRepo(tenantName, componentRegistry, sessionFactory); - RemoteSessionRepo remoteSessionRepo = new RemoteSessionRepo(componentRegistry, - sessionFactory, - reloadHandler, - tenantName, - applicationRepo, - componentRegistry.getFlagSource()); + SessionRepository sessionRepository = new SessionRepository(tenantName, componentRegistry, + applicationRepo, reloadHandler, + componentRegistry.getFlagSource(), + componentRegistry.getSessionPreparer()); log.log(Level.INFO, "Creating tenant '" + tenantName + "'"); - Tenant tenant = new Tenant(tenantName, sessionFactory, localSessionRepo, remoteSessionRepo, requestHandler, + Tenant tenant = new Tenant(tenantName, sessionRepository, requestHandler, reloadHandler, applicationRepo, componentRegistry.getCurator()); notifyNewTenant(tenant); tenants.putIfAbsent(tenantName, tenant); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java index 35e8b0917cf..20ac4b65c64 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -3,16 +3,11 @@ package com.yahoo.vespa.config.server.zookeeper; import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.io.IOUtils; -import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.Curator; -import java.io.File; -import java.io.FilenameFilter; -import java.io.IOException; -import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; /** * A (stateful) curator wrapper for the config server. This simplifies Curator method calls used by the config server @@ -49,8 +44,6 @@ public class ConfigCurator { /** Path for session state */ public static final String SESSIONSTATE_ZK_SUBPATH = "/sessionState"; - private static final FilenameFilter acceptsAllFileNameFilter = (dir, name) -> true; - private final Curator curator; public static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigCurator.class.getName()); @@ -229,65 +222,6 @@ public class ConfigCurator { putData(path, name, data); } - /** - * Takes for instance the dir /app and puts the contents into the given ZK path. Ignores files starting with dot, - * and dirs called CVS. - * - * @param dir directory which holds the summary class part files - * @param path zookeeper path - * @param filenameFilter A FilenameFilter which decides which files in dir are fed to zookeeper - * @param recurse recurse subdirectories - */ - void feedZooKeeper(File dir, String path, FilenameFilter filenameFilter, boolean recurse) { - try { - if (filenameFilter == null) { - filenameFilter = acceptsAllFileNameFilter; - } - if (!dir.isDirectory()) { - log.fine(dir.getCanonicalPath() + " is not a directory. Not feeding the files into ZooKeeper."); - return; - } - for (File file : listFiles(dir, filenameFilter)) { - if (file.getName().startsWith(".")) continue; //.svn , .git ... - if ("CVS".equals(file.getName())) continue; - if (file.isFile()) { - byte[] contents = IOUtils.readFileBytes(file); - putData(path, file.getName(), contents); - } else if (recurse && file.isDirectory()) { - createNode(path, file.getName()); - feedZooKeeper(file, path + '/' + file.getName(), filenameFilter, recurse); - } - } - } - catch (IOException e) { - throw new RuntimeException("Exception feeding ZooKeeper at path " + path, e); - } - } - - /** - * Same as normal listFiles, but use the filter only for normal files - * - * @param dir directory to list files in - * @param filter A FilenameFilter which decides which files in dir are listed - * @return an array of Files - */ - protected File[] listFiles(File dir, FilenameFilter filter) { - File[] rawList = dir.listFiles(); - List<File> ret = new ArrayList<>(); - if (rawList != null) { - for (File f : rawList) { - if (f.isDirectory()) { - ret.add(f); - } else { - if (filter.accept(dir, f.getName())) { - ret.add(f); - } - } - } - } - return ret.toArray(new File[0]); - } - /** Deletes the node at the given path, and any children it may have. If the node does not exist this does nothing */ public void deleteRecurse(String path) { try { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index a3d072be38b..f0aa38a228e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.ConfigInstance; +import com.yahoo.config.FileReference; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.model.NullConfigModelRegistry; @@ -33,11 +34,12 @@ import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployTester; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.session.LocalSession; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.Session; @@ -45,8 +47,12 @@ import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModelFactory; import org.hamcrest.core.Is; import org.jetbrains.annotations.NotNull; @@ -103,6 +109,7 @@ public class ApplicationRepositoryTest { private SessionHandlerTest.MockProvisioner provisioner; private OrchestratorMock orchestrator; private TimeoutBudget timeoutBudget; + private ConfigCurator configCurator; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -110,19 +117,23 @@ public class ApplicationRepositoryTest { @Rule public ExpectedException exceptionRule = ExpectedException.none(); - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); - @Before public void setup() throws IOException { + setup(new InMemoryFlagSource()); + } + + public void setup(FlagSource flagSource) throws IOException { Curator curator = new MockCurator(); + configCurator = ConfigCurator.create(curator); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .curator(curator) .configServerConfig(new ConfigserverConfig.Builder() .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) - .configServerDBDir(tempFolder.newFolder("configserverdb").getAbsolutePath()) - .configDefinitionsDir(tempFolder.newFolder("configdefinitions").getAbsolutePath()) + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build()) + .flagSource(flagSource) .build(); tenantRepository = new TenantRepository(componentRegistry, false); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); @@ -146,7 +157,7 @@ public class ApplicationRepositoryTest { TenantName tenantName = applicationId().tenant(); Tenant tenant = tenantRepository.getTenant(tenantName); - LocalSession session = tenant.getLocalSessionRepo().getSession(tenant.getApplicationRepo() + LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo() .requireActiveSessionOf(applicationId())); session.getAllocatedHosts(); } @@ -178,7 +189,7 @@ public class ApplicationRepositoryTest { TenantName tenantName = applicationId().tenant(); Tenant tenant = tenantRepository.getTenant(tenantName); - LocalSession session = tenant.getLocalSessionRepo().getSession( + LocalSession session = tenant.getSessionRepository().getLocalSession( tenant.getApplicationRepo().requireActiveSessionOf(applicationId())); assertEquals(firstSessionId, session.getMetaData().getPreviousActiveGeneration()); } @@ -291,24 +302,32 @@ public class ApplicationRepositoryTest { @Test public void delete() { + TenantName tenantName = applicationId().tenant(); + Tenant tenant = tenantRepository.getTenant(tenantName); { PrepareResult result = deployApp(testApp); long sessionId = result.sessionId(); - Tenant tenant = tenantRepository.getTenant(applicationId().tenant()); - LocalSession applicationData = tenant.getLocalSessionRepo().getSession(sessionId); + LocalSession applicationData = tenant.getSessionRepository().getLocalSession(sessionId); assertNotNull(applicationData); assertNotNull(applicationData.getApplicationId()); - assertNotNull(tenant.getRemoteSessionRepo().getSession(sessionId)); + assertNotNull(tenant.getSessionRepo().getLocalSession(sessionId)); assertNotNull(applicationRepository.getActiveSession(applicationId())); + String sessionNode = TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId)).getAbsolute(); + assertTrue(configCurator.exists(sessionNode)); + TenantFileSystemDirs tenantFileSystemDirs = tenant.getApplicationRepo().getTenantFileSystemDirs(); + File sessionFile = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); + assertTrue(sessionFile.exists()); // Delete app and verify that it has been deleted from repos and provisioner assertTrue(applicationRepository.delete(applicationId())); assertNull(applicationRepository.getActiveSession(applicationId())); - assertNull(tenant.getLocalSessionRepo().getSession(sessionId)); - assertNull(tenant.getRemoteSessionRepo().getSession(sessionId)); + assertNull(tenant.getSessionRepository().getLocalSession(sessionId)); + assertNull(tenant.getSessionRepo().getLocalSession(sessionId)); assertTrue(provisioner.removed); assertEquals(tenant.getName(), provisioner.lastApplicationId.tenant()); assertEquals(applicationId(), provisioner.lastApplicationId); + assertFalse(configCurator.exists(sessionNode)); + assertFalse(sessionFile.exists()); assertFalse(applicationRepository.delete(applicationId())); } @@ -345,8 +364,7 @@ public class ApplicationRepositoryTest { // A new delete should cleanup and be successful RemoteSession activeSession = applicationRepository.getActiveSession(applicationId()); assertNull(activeSession); - Tenant tenant = tenantRepository.getTenant(applicationId().tenant()); - assertNull(tenant.getRemoteSessionRepo().getSession(prepareResult.sessionId())); + assertNull(tenant.getSessionRepo().getLocalSession(prepareResult.sessionId())); assertTrue(applicationRepository.delete(applicationId())); } @@ -379,14 +397,14 @@ public class ApplicationRepositoryTest { assertNotEquals(activeSessionId, deployment3session); // No change to active session id assertEquals(activeSessionId, tester.tenant().getApplicationRepo().requireActiveSessionOf(tester.applicationId())); - LocalSessionRepo localSessionRepo = tester.tenant().getLocalSessionRepo(); - assertEquals(3, localSessionRepo.getSessions().size()); + SessionRepository sessionRepository = tester.tenant().getSessionRepository(); + assertEquals(3, sessionRepository.getLocalSessions().size()); clock.advance(Duration.ofHours(1)); // longer than session lifetime // All sessions except 3 should be removed after the call to deleteExpiredLocalSessions tester.applicationRepository().deleteExpiredLocalSessions(); - Collection<LocalSession> sessions = localSessionRepo.getSessions(); + Collection<LocalSession> sessions = sessionRepository.getLocalSessions(); assertEquals(1, sessions.size()); ArrayList<LocalSession> localSessions = new ArrayList<>(sessions); LocalSession localSession = localSessions.get(0); @@ -400,9 +418,9 @@ public class ApplicationRepositoryTest { assertTrue(deployment4.isPresent()); deployment4.get().prepare(); // session 5 (not activated) - assertEquals(2, localSessionRepo.getSessions().size()); - localSessionRepo.deleteSession(localSession); - assertEquals(1, localSessionRepo.getSessions().size()); + assertEquals(2, sessionRepository.getLocalSessions().size()); + sessionRepository.deleteLocalSession(localSession); + assertEquals(1, sessionRepository.getLocalSessions().size()); // Check that trying to expire when there are no active sessions works tester.applicationRepository().deleteExpiredLocalSessions(); @@ -457,7 +475,7 @@ public class ApplicationRepositoryTest { TenantName tenantName = applicationId().tenant(); Tenant tenant = tenantRepository.getTenant(tenantName); - LocalSession session = tenant.getLocalSessionRepo().getSession(tenant.getApplicationRepo().requireActiveSessionOf(applicationId())); + LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo().requireActiveSessionOf(applicationId())); List<NetworkPorts.Allocation> list = new ArrayList<>(); list.add(new NetworkPorts.Allocation(8080, "container", "container/container.0", "http")); @@ -658,6 +676,14 @@ public class ApplicationRepositoryTest { resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); } + @Test + public void testDistributionOfApplicationPackage() throws IOException { + FlagSource flagSource = new InMemoryFlagSource() + .withBooleanFlag(Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.id(), true); + setup(flagSource); + applicationRepository.deploy(app1, prepareParams()); + } + private ApplicationRepository createApplicationRepository() { return new ApplicationRepository(tenantRepository, provisioner, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index ec5648757f1..bc16f44b405 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -60,6 +60,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private final StripedExecutor<TenantName> zkWatcherExecutor; private final ExecutorService zkCacheExecutor; private final SecretStore secretStore; + private final FlagSource flagSource; private TestComponentRegistry(Curator curator, ConfigCurator configCurator, Metrics metrics, ModelFactoryRegistry modelFactoryRegistry, @@ -74,7 +75,8 @@ public class TestComponentRegistry implements GlobalComponentRegistry { TenantListener tenantListener, Zone zone, Clock clock, - SecretStore secretStore) { + SecretStore secretStore, + FlagSource flagSource) { this.curator = curator; this.configCurator = configCurator; this.metrics = metrics; @@ -94,6 +96,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { this.zkWatcherExecutor = new StripedExecutor<>(new InThreadExecutorService()); this.zkCacheExecutor = new InThreadExecutorService(); this.secretStore = secretStore; + this.flagSource = flagSource; } public static class Builder { @@ -115,6 +118,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private Optional<Provisioner> hostProvisioner = Optional.empty(); private Zone zone = Zone.defaultZone(); private Clock clock = Clock.systemUTC(); + private FlagSource flagSource = new InMemoryFlagSource(); public Builder configServerConfig(ConfigserverConfig configserverConfig) { this.configserverConfig = configserverConfig; @@ -161,6 +165,11 @@ public class TestComponentRegistry implements GlobalComponentRegistry { return this; } + public Builder flagSource(FlagSource flagSource) { + this.flagSource = flagSource; + return this; + } + public TestComponentRegistry build() { final PermanentApplicationPackage permApp = this.permanentApplicationPackage .orElse(new PermanentApplicationPackage(configserverConfig)); @@ -172,11 +181,11 @@ public class TestComponentRegistry implements GlobalComponentRegistry { SessionPreparer sessionPreparer = new SessionPreparer(modelFactoryRegistry, fileDistributionProvider, hostProvisionerProvider, permApp, configserverConfig, defRepo, curator, - zone, new InMemoryFlagSource(), secretStore); + zone, flagSource, secretStore); return new TestComponentRegistry(curator, ConfigCurator.create(curator), metrics, modelFactoryRegistry, permApp, fileDistributionProvider, hostRegistries, configserverConfig, sessionPreparer, hostProvisioner, defRepo, reloadListener, tenantListener, - zone, clock, secretStore); + zone, clock, secretStore, flagSource); } } @@ -221,7 +230,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { } @Override - public FlagSource getFlagSource() { return new InMemoryFlagSource(); } + public FlagSource getFlagSource() { return flagSource; } @Override public ExecutorService getZkCacheExecutor() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg deleted file mode 100644 index 8b43ff9c793..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg +++ /dev/null @@ -1,78 +0,0 @@ -accesslog "/home/vespa/logs/vespa/foo.log" -partialsd "sd" -partialsd2 "global2" -asyncfetchocc 10 -a 0 -b 1 -c 2 -d 3 -e 4 -onlyindef 45 -listenport 13700 -rangecheck2 10 -rangecheck3 10 -kanon -78.56 -rangecheck1 10.0 -testref search/cluster.music/c0/r0/indexer.4 -testref2 some/babbel -mode BATCH -functionmodules[0] -storage[2] -storage[0].feeder[1] -storage[0].feeder[0] "test" -storage[1].id search/cluster.music/c0/r0/indexer.4 -storage[1].id2 pjatt -storage[1].feeder[2] -storage[1].feeder[0] "me" -storage[1].feeder[1] "now" -search[3] -search[0].feeder[1] -search[0].feeder[0] "foofeeder" -search[1].feeder[4] -search[1].feeder[0] "barfeeder1_1" -search[1].feeder[1] "barfeeder2" -search[1].feeder[2] "" -search[1].feeder[3] "barfeeder2_1" -search[2].feeder[2] -search[2].feeder[0] "" -search[2].feeder[1] "bazfeeder" -f[1] -f[0].a "A" -f[0].b "B" -f[0].c "C" -f[0].h "H" -f[0].f "F" -config[1] -config[0].role "rtx" -config[0].usewrapper false -config[0].id search/cluster.music/rtx/0 -routingtable[1] -routingtable[0].hop[3] -routingtable[0].hop[0].name "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].hop[0].selector "docproc/cluster.music.indexing/*/chain.music.indexing" -routingtable[0].hop[0].recipient[0] -routingtable[0].hop[1].name "search/cluster.music" -routingtable[0].hop[1].selector "search/cluster.music/[SearchColumn]/[SearchRow]/feed-destination" -routingtable[0].hop[1].recipient[1] -routingtable[0].hop[1].recipient[0] "search/cluster.music/c0/r0/feed-destination" -routingtable[0].hop[2].selector "[DocumentRouteSelector]" -routingtable[0].hop[2].name "indexing" -routingtable[0].hop[2].recipient[1] -routingtable[0].hop[2].recipient[0] "search/cluster.music" -speciallog[1] -speciallog[0].filehandler.name "QueryAccessLog" -speciallog[0].filehandler.pattern "logs/vespa/qrs/QueryAccessLog.%Y%m%d%H%M%S" -speciallog[0].filehandler.rotation "0 1 ..." -speciallog[0].cachehandler.name "QueryAccessLog" -speciallog[0].name "QueryAccessLog" -speciallog[0].type "file" -speciallog[0].cachehandler.size 1000 -rulebase[4] -rulebase[0].name "cjk" -rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# ä½³:\u4f73\n# 能:\u80fd\n# ç´¢:\u7d22\n# å°¼:\u5c3c\n# æƒ :\u60e0\n# æ™®:\u666e\n\n@default\n\naç´¢ -> ç´¢a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,æƒ æ™®,佳能;\n" -rulebase[1].name "common" -rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" -rulebase[2].name "egyik" -rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" -rulebase[3].name "masik" -rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg deleted file mode 100644 index 927ff8a26c9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg +++ /dev/null @@ -1,48 +0,0 @@ -accesslog "/home/vespa/logs/vespa/foo.log" -partialsd "global" -partialsd2 "global2" -asyncfetchocc 10 -a 0 -b 1 -c 67 -d 89 -e 4 -onlyindef 45 -listenport 13700 -rangecheck2 10 -rangecheck3 10 -rangecheck1 10.0 -mode BATCH -functionmodules[0] -storage[0] -search[3] -search[0].feeder[1] -search[0].feeder[0] "foofeeder" -search[1].feeder[4] -search[1].feeder[0] "barfeeder1_1" -search[1].feeder[1] "sportsfeeder1" -search[1].feeder[2] "" -search[1].feeder[3] "barfeeder2_1" -search[2].feeder[2] -search[2].feeder[0] "" -search[2].feeder[1] "bazfeeder" -f[0] -config[0] -routingtable[0] -speciallog[1] -speciallog[0].filehandler.name "QueryAccessLog" -speciallog[0].filehandler.pattern "logs/vespa/qrs/QueryAccessLog.%Y%m%d%H%M%S" -speciallog[0].filehandler.rotation "0 1 ..." -speciallog[0].cachehandler.name "QueryAccessLog" -speciallog[0].name "QueryAccessLog" -speciallog[0].type "file" -speciallog[0].cachehandler.size 1000 -rulebase[4] -rulebase[0].name "cjk" -rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# ä½³:\u4f73\n# 能:\u80fd\n# ç´¢:\u7d22\n# å°¼:\u5c3c\n# æƒ :\u60e0\n# æ™®:\u666e\n\n@default\n\naç´¢ -> ç´¢a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,æƒ æ™®,佳能;\n" -rulebase[1].name "common" -rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" -rulebase[2].name "egyik" -rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" -rulebase[3].name "masik" -rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar b/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar Binary files differdeleted file mode 100644 index 69f6e335092..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar +++ /dev/null diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml b/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml deleted file mode 100644 index b7924e73e7a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml +++ /dev/null @@ -1,18 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services version="1.0"> - - <admin version="2.0"> - <adminserver hostalias="node1"/> - </admin> - - <content version="1.0"> - <redundancy>1</redundancy> - <documents> - <document type="music" mode="index"/> - </documents> - <nodes>> - <node hostalias="node1" distribution-key="0"/> - </nodes> - </content> -</services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java index e64921e3ea0..cf7740f133f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java @@ -1,25 +1,32 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.http.HttpFetcher; -import com.yahoo.vespa.config.server.http.StaticResponse; import com.yahoo.vespa.config.server.http.RequestTimeoutException; +import com.yahoo.vespa.config.server.http.StaticResponse; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import java.net.URL; +import java.util.Arrays; +import java.util.Collections; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; +import static com.yahoo.vespa.config.server.application.MockModel.createServiceInfo; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertSame; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class HttpProxyTest { + private final HttpFetcher fetcher = mock(HttpFetcher.class); private final HttpProxy proxy = new HttpProxy(fetcher); @@ -29,7 +36,7 @@ public class HttpProxyTest { @Before public void setup() { - Model modelMock = MockModel.createClusterController(hostname, port); + Model modelMock = createClusterController(); when(applicationMock.getModel()).thenReturn(modelMock); } @@ -52,7 +59,7 @@ public class HttpProxyTest { // The HttpResponse returned by the fetcher IS the same object as the one returned by the proxy, // when everything goes well. - assertTrue(actualResponse == response); + assertSame(actualResponse, response); } @Test(expected = RequestTimeoutException.class) @@ -62,4 +69,20 @@ public class HttpProxyTest { proxy.get(applicationMock, hostname, CLUSTERCONTROLLER_CONTAINER.serviceName, "clustercontroller-status/v1/clusterName"); } + + private static MockModel createClusterController() { + ServiceInfo container = createServiceInfo( + hostname, + "foo", // name + CLUSTERCONTROLLER_CONTAINER.serviceName, + ClusterSpec.Type.container, + port, + "state http external query"); + ServiceInfo serviceNoStatePort = createServiceInfo(hostname, "storagenode", "storagenode", + ClusterSpec.Type.content, 1234, "rpc"); + HostInfo hostInfo = new HostInfo(hostname, Arrays.asList(container, serviceNoStatePort)); + + return new MockModel(Collections.singleton(hostInfo)); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java index 0da96f9f01d..c9f25451ea4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java @@ -23,8 +23,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; - /** * Model with two services, one that does not have a state port * @@ -45,22 +43,6 @@ public class MockModel implements Model { return new HostInfo(hostname, Arrays.asList(container, serviceNoStatePort)); } - // TODO: Move to caller - static MockModel createClusterController(String hostname, int statePort) { - ServiceInfo container = createServiceInfo( - hostname, - "foo", // name - CLUSTERCONTROLLER_CONTAINER.serviceName, - ClusterSpec.Type.container, - statePort, - "state http external query"); - ServiceInfo serviceNoStatePort = createServiceInfo(hostname, "storagenode", "storagenode", - ClusterSpec.Type.content, 1234, "rpc"); - HostInfo hostInfo = new HostInfo(hostname, Arrays.asList(container, serviceNoStatePort)); - - return new MockModel(Collections.singleton(hostInfo)); - } - static MockModel createConfigProxies(List<String> hostnames, int rpcPort) { Set<HostInfo> hostInfos = new HashSet<>(); hostnames.forEach(hostname -> { @@ -71,7 +53,7 @@ public class MockModel implements Model { return new MockModel(hostInfos); } - static private ServiceInfo createServiceInfo( + static ServiceInfo createServiceInfo( String hostname, String name, String type, @@ -121,4 +103,5 @@ public class MockModel implements Model { public AllocatedHosts allocatedHosts() { throw new UnsupportedOperationException(); } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index b2091d6e537..7a8bad3d199 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -249,7 +249,7 @@ public class DeployTester { public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { Tenant tenant = tenant(); - LocalSession session = tenant.getLocalSessionRepo().getSession(tenant.getApplicationRepo() + LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo() .requireActiveSessionOf(applicationId)); return session.getAllocatedHosts(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index 36467a2ca64..c07c7316930 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -33,11 +33,11 @@ public class RedeployTest { assertTrue(deployment.isPresent()); long activeSessionIdBefore = tester.applicationRepository().getActiveSession(tester.applicationId()).getSessionId(); - assertEquals(tester.applicationId(), tester.tenant().getLocalSessionRepo().getSession(activeSessionIdBefore).getApplicationId()); + assertEquals(tester.applicationId(), tester.tenant().getSessionRepository().getLocalSession(activeSessionIdBefore).getApplicationId()); deployment.get().activate(); long activeSessionIdAfter = tester.applicationRepository().getActiveSession(tester.applicationId()).getSessionId(); assertEquals(activeSessionIdAfter, activeSessionIdBefore + 1); - assertEquals(tester.applicationId(), tester.tenant().getLocalSessionRepo().getSession(activeSessionIdAfter).getApplicationId()); + assertEquals(tester.applicationId(), tester.tenant().getSessionRepository().getLocalSession(activeSessionIdAfter).getApplicationId()); } /** No deployment is done because there is no local active session. */ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java deleted file mode 100644 index b6d9ab5d618..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import org.junit.Test; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; - -import static com.yahoo.jdisc.http.HttpResponse.Status.*; -import static com.yahoo.jdisc.http.HttpRequest.Method.*; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author hmusum - * @since 5.1.14 - */ -public class SessionExampleHandlerTest { - private static final String URI = "http://localhost:19071/session/example"; - - @Test - public void basicPut() throws IOException { - final SessionExampleHandler handler = new SessionExampleHandler(Executors.newCachedThreadPool()); - final HttpRequest request = HttpRequest.createTestRequest(URI, PUT); - HttpResponse response = handler.handle(request); - assertThat(response.getStatus(), is(OK)); - assertThat(SessionHandlerTest.getRenderedString(response), is("{\"test\":\"PUT received\"}")); - } - - @Test - public void invalidMethod() { - final SessionExampleHandler handler = new SessionExampleHandler(Executors.newCachedThreadPool()); - final HttpRequest request = HttpRequest.createTestRequest(URI, GET); - HttpResponse response = handler.handle(request); - assertThat(response.getStatus(), is(METHOD_NOT_ALLOWED)); - } - - - /** - * A handler that prepares a session given by an id in the request. - * - * @author hmusum - * @since 5.1.14 - */ - public static class SessionExampleHandler extends ThreadedHttpRequestHandler { - - public SessionExampleHandler(Executor executor) { - super(executor, null); - } - - @Override - public HttpResponse handle(HttpRequest request) { - final com.yahoo.jdisc.http.HttpRequest.Method method = request.getMethod(); - switch (method) { - case PUT: - return handlePUT(request); - case GET: - return new SessionExampleResponse(METHOD_NOT_ALLOWED, "Method '" + method + "' is not supported"); - default: - return new SessionExampleResponse(INTERNAL_SERVER_ERROR); - } - } - - @SuppressWarnings({"UnusedDeclaration"}) - HttpResponse handlePUT(HttpRequest request) { - return new SessionExampleResponse(OK, "PUT received"); - } - - private static class SessionExampleResponse extends HttpResponse { - private final Slime slime = new Slime(); - private final Cursor root = slime.setObject(); - private final String message; - - - private SessionExampleResponse(int status) { - this(status, ""); - headers().put("Cache-Control","max-age=120"); - } - - private SessionExampleResponse(int status, String message) { - super(status); - this.message = message; - } - - @Override - public void render(OutputStream outputStream) throws IOException { - root.setString("test", message); - new JsonFormat(true).encode(outputStream, slime); - } - } - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index 91a40bd6083..0b9a780d9e1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -3,11 +3,9 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; @@ -18,14 +16,9 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; -import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.session.DummyTransaction; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.MockSessionZKClient; -import com.yahoo.vespa.config.server.session.PrepareParams; -import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.Session; import java.io.ByteArrayOutputStream; @@ -36,7 +29,6 @@ import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Optional; /** * Base class for session handler tests @@ -90,13 +82,11 @@ public class SessionHandlerTest { public static class MockLocalSession extends LocalSession { public Session.Status status; - private ConfigChangeActions actions = new ConfigChangeActions(); private Instant createTime = Instant.now(); private ApplicationId applicationId; - private Optional<DockerImage> dockerImageRepository; public MockLocalSession(long sessionId, ApplicationPackage app) { - super(TenantName.defaultName(), sessionId, null, app, new MockSessionZKClient(app), null, null, new HostRegistry<>()); + super(TenantName.defaultName(), sessionId, app, new MockSessionZKClient(app), null); } public MockLocalSession(long sessionId, ApplicationPackage app, ApplicationId applicationId) { @@ -104,13 +94,6 @@ public class SessionHandlerTest { this.applicationId = applicationId; } - @Override - public ConfigChangeActions prepare(DeployLogger logger, PrepareParams params, Optional<ApplicationSet> application, Path tenantPath, Instant now) { - status = Session.Status.PREPARE; - this.dockerImageRepository = params.dockerImageRepository(); - return actions; - } - public void setStatus(Session.Status status) { this.status = status; } @@ -140,13 +123,6 @@ public class SessionHandlerTest { return createTime; } - @Override - public void delete(NestedTransaction transaction) { } - - @Override - public Optional<DockerImage> getDockerImageRepository() { - return dockerImageRepository; - } } public enum Cmd { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 078dc47af51..4cf81d22e3c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -54,13 +54,13 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { session2 = new MockLocalSession(2, FilesApplicationPackage.fromFile(new File("src/test/apps/content"))); Tenant tenant1 = tenantRepository.getTenant(tenantName1); - tenant1.getLocalSessionRepo().addSession(session2); + tenant1.getSessionRepository().addSession(session2); tenant1.getApplicationRepo().createApplication(idTenant1); tenant1.getApplicationRepo().createPutTransaction(idTenant1, 2).commit(); MockLocalSession session3 = new MockLocalSession(3, FilesApplicationPackage.fromFile(new File("src/test/apps/content2"))); Tenant tenant2 = tenantRepository.getTenant(tenantName2); - tenant2.getLocalSessionRepo().addSession(session3); + tenant2.getSessionRepository().addSession(session3); tenant2.getApplicationRepo().createApplication(idTenant2); tenant2.getApplicationRepo().createPutTransaction(idTenant2, 3).commit(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index ff8f7a291ad..0a5221b2c97 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -286,13 +286,13 @@ public class ApplicationHandlerTest { private void deleteAndAssertOKResponseMocked(ApplicationId applicationId, boolean fullAppIdInUrl) throws IOException { long sessionId = tenantRepository.getTenant(applicationId.tenant()).getApplicationRepo().requireActiveSessionOf(applicationId); deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, fullAppIdInUrl); - assertNull(tenantRepository.getTenant(applicationId.tenant()).getLocalSessionRepo().getSession(sessionId)); + assertNull(tenantRepository.getTenant(applicationId.tenant()).getSessionRepository().getLocalSession(sessionId)); } private void deleteAndAssertOKResponse(Tenant tenant, ApplicationId applicationId) throws IOException { long sessionId = tenant.getApplicationRepo().requireActiveSessionOf(applicationId); deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, true); - assertNull(tenant.getLocalSessionRepo().getSession(sessionId)); + assertNull(tenant.getSessionRepository().getLocalSession(sessionId)); } private void deleteAndAssertResponse(ApplicationId applicationId, Zone zone, int expectedStatus, HttpErrorResponse.errorCodes errorCode, boolean fullAppIdInUrl) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 2eaa5d75ba7..37181abfcf4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -12,8 +12,9 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; +import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.host.HostRegistries; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -29,14 +30,13 @@ import org.junit.Test; import java.io.File; import java.io.IOException; +import java.time.Clock; import java.util.Collections; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - /** * @author hmusum */ +// TODO: Try to move testing to ApplicationRepositoryTest and avoid all the low-level setup code here public class HostHandlerTest { private static final String urlPrefix = "http://myhost:14000/application/v2/host/"; private static File testApp = new File("src/test/apps/app"); @@ -44,49 +44,45 @@ public class HostHandlerTest { private HostHandler handler; private final static TenantName mytenant = TenantName.from("mytenant"); private final static String hostname = "testhost"; + private final static Zone zone = Zone.defaultZone(); private TenantRepository tenantRepository; - private HostRegistries hostRegistries; - private HostHandler hostHandler; static void addMockApplication(Tenant tenant, ApplicationId applicationId, long sessionId) { tenant.getApplicationRepo().createApplication(applicationId); tenant.getApplicationRepo().createPutTransaction(applicationId, sessionId).commit(); ApplicationPackage app = FilesApplicationPackage.fromFile(testApp); - tenant.getLocalSessionRepo().addSession(new SessionHandlerTest.MockLocalSession(sessionId, app, applicationId)); + tenant.getSessionRepository().addSession(new SessionHandlerTest.MockLocalSession(sessionId, app, applicationId)); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .modelFactoryRegistry(new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())))) .build(); - tenant.getRemoteSessionRepo().addSession(new RemoteSession(tenant.getName(), sessionId, componentRegistry, new MockSessionZKClient(app))); + tenant.getSessionRepo().addRemoteSession(new RemoteSession(tenant.getName(), sessionId, componentRegistry, new MockSessionZKClient(app))); } @Before public void setup() { - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); - tenantRepository = new TenantRepository(componentRegistry, false); - tenantRepository.addTenant(mytenant); - handler = createHostHandler(); - } - - private HostHandler createHostHandler() { final HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); hostRegistry.update(mytenant, Collections.singletonList(hostname)); - TestComponentRegistry testComponentRegistry = new TestComponentRegistry.Builder().build(); - hostRegistries = testComponentRegistry.getHostRegistries(); - hostRegistries.createApplicationHostRegistry(mytenant).update(ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()), Collections.singletonList(hostname)); - hostRegistries.getTenantHostRegistry().update(mytenant, Collections.singletonList(hostname)); - hostHandler = new HostHandler( - HostHandler.testOnlyContext(), - testComponentRegistry); - return hostHandler; + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .zone(zone) + .build(); + tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(mytenant); + Tenant tenant = tenantRepository.getTenant(mytenant); + HostRegistry<ApplicationId> applicationHostRegistry = tenant.getApplicationRepo().getApplicationHostRegistry(); + applicationHostRegistry.update(ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()), Collections.singletonList(hostname)); + ApplicationRepository applicationRepository = new ApplicationRepository(tenantRepository, + new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), + Clock.systemUTC()); + handler = new HostHandler(HostHandler.testOnlyContext(), applicationRepository); } @Test public void require_correct_tenant_and_application_for_hostname() throws Exception { - assertThat(hostRegistries, is(hostHandler.hostRegistries)); long sessionId = 1; ApplicationId id = ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()); addMockApplication(tenantRepository.getTenant(mytenant), id, sessionId); - assertApplicationForHost(hostname, mytenant, id, Zone.defaultZone()); + assertApplicationForHost(hostname, mytenant, id, zone); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 88bf6fb7172..f639843ac08 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -43,7 +43,7 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { public void setupHandler() throws Exception { tenantRepository = new TenantRepository(componentRegistry, false); tenantRepository.addTenant(tenant); - tenantRepository.getTenant(tenant).getLocalSessionRepo().addSession(new MockLocalSession(1L, FilesApplicationPackage.fromFile(createTestApp()))); + tenantRepository.getTenant(tenant).getSessionRepository().addSession(new MockLocalSession(1L, FilesApplicationPackage.fromFile(createTestApp()))); handler = createHandler(); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; baseUrl = "http://foo:1337/application/v2/tenant/" + tenant + "/session/1/content/"; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg deleted file mode 100644 index d3970ee48eb..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg +++ /dev/null @@ -1,44 +0,0 @@ -include: search/cluster.music -include: search/cluster.music -c 2 -storage[2] -storage[0].feeder[1] -storage[0].feeder[0] "test" -storage[1].feeder[2] -storage[1].feeder[0] "me" -storage[1].feeder[1] now -storage[1].id :parent: -storage[1].id2 pjatt -testref :parent: -testref2 some/babbel -config[1] -config[0].role "rtx" -#config[0].usewrapper false -config[0].id search/cluster.music/rtx/0 -f[1] -f[0].a "A" -f[0].b "B" -f[0].c "C" -f[0].h "H" -f[0].f "F" -f[0].notindef "notindef" -routingtable[1] -routingtable[0].hop[3] -routingtable[0].hop[0].name "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].hop[0].selector "docproc/cluster.music.indexing/*/chain.music.indexing" -routingtable[0].hop[1].name "search/cluster.music" -routingtable[0].hop[1].selector "search/cluster.music/[SearchColumn]/[SearchRow]/feed-destination" -routingtable[0].hop[1].recipient[1] -routingtable[0].hop[1].recipient[0] "search/cluster.music/c0/r0/feed-destination" -routingtable[0].hop[2].selector "[DocumentRouteSelector]" -routingtable[0].hop[2].name "indexing" -routingtable[0].hop[2].notindef "not in def" -routingtable[0].hop[2].recipient[1] -routingtable[0].hop[2].recipient[0] "search/cluster.music" -notindef "dfsd" -nopenotindef[0] "boo" -nadaindef[0].naw 98 -mode NOTINDEF -rangecheck1 100 -rangecheck2 10000 -rangecheck3 20 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg deleted file mode 100644 index 727a5052ed6..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg +++ /dev/null @@ -1,2 +0,0 @@ -include: search/cluster.sports -c 67 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg deleted file mode 100644 index f4996027f60..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg +++ /dev/null @@ -1 +0,0 @@ -model vespa diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg deleted file mode 100644 index d75d76810f9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg +++ /dev/null @@ -1,2 +0,0 @@ -foo "bar" -gaz -78 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg deleted file mode 100644 index 7ccdb73eb9a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg +++ /dev/null @@ -1,2 +0,0 @@ -include: search/cluster.logical/* -include: search/cluster.video/* diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg deleted file mode 100644 index 5b07d3a2890..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg +++ /dev/null @@ -1 +0,0 @@ -include: search/cluster.music diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg deleted file mode 100644 index f3acd4cf8b9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg +++ /dev/null @@ -1,5 +0,0 @@ -asyncfetchocc 9 -d 3 -kanon -78.56 - -partialsd "sd" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg deleted file mode 100644 index 5d8a01a18ea..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg +++ /dev/null @@ -1,2 +0,0 @@ -d 89 -search[1].feeder[1] "sportsfeeder1" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg deleted file mode 100644 index f6c35df398d..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg +++ /dev/null @@ -1 +0,0 @@ -gaff -89 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg deleted file mode 100644 index c3d9b1e45a1..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg +++ /dev/null @@ -1,8 +0,0 @@ -classes[1] -classes[logical].id 1906788747 -classes[logical].name logical -classes[logical].fields[2] -classes[logical].fields[0].name sddocnameNAM -classes[logical].fields[0].type longstring -classes[logical].fields[1].name title -classes[logical].fields[1].type longstringSTRIN diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg deleted file mode 100644 index 12a21671b4a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg +++ /dev/null @@ -1,8 +0,0 @@ -classes[1] -classes[music].id 1906788746 -classes[music].name music -classes[music].fields[2] -classes[music].fields[0].name sddocnameNAME -classes[music].fields[0].type longstring -classes[music].fields[1].name title -classes[music].fields[1].type longstringSTRING diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg deleted file mode 100644 index 4001c59adbc..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg +++ /dev/null @@ -1,14 +0,0 @@ -classes[0] -classes[smallsum614540714].id 614540714 -classes[smallsum614540714].name smallsum -classes[smallsum614540714].fields[5] -classes[smallsum614540714].fields[0].name s_13 -classes[smallsum614540714].fields[0].type longstring -classes[smallsum614540714].fields[1].name ranklog -classes[smallsum614540714].fields[1].type longstring -classes[smallsum614540714].fields[2].name rankfeatures -classes[smallsum614540714].fields[2].type longstring -classes[smallsum614540714].fields[3].name summaryfeatures -classes[smallsum614540714].fields[3].type longstring -classes[smallsum614540714].fields[4].name sddocname -classes[smallsum614540714].fields[4].type longstring diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg deleted file mode 100644 index 33d07b99ab6..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg +++ /dev/null @@ -1,14 +0,0 @@ -classes[0] -classes[smallsum507688128].id 507688128 -classes[smallsum507688128].name smallsum -classes[smallsum507688128].fields[5] -classes[smallsum507688128].fields[0].name title -classes[smallsum507688128].fields[0].type longstring -classes[smallsum507688128].fields[1].name ranklog -classes[smallsum507688128].fields[1].type longstring -classes[smallsum507688128].fields[2].name rankfeatures -classes[smallsum507688128].fields[2].type longstring -classes[smallsum507688128].fields[3].name summaryfeatures -classes[smallsum507688128].fields[3].type longstring -classes[smallsum507688128].fields[4].name sddocname -classes[smallsum507688128].fields[4].type longstring diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg deleted file mode 100644 index de9fbdd39f4..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg +++ /dev/null @@ -1,7 +0,0 @@ -rec 56 -national 77 -ilscript[1] -ilscript[music].name music -ilscript[music].doctype music -ilscript[music].content[1] -ilscript[music].content[0] "input year | summary s_3 | tokenize \"stemming,normalizing\" { index f_3 | index f_4; };" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg deleted file mode 100644 index e95f976a43a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg +++ /dev/null @@ -1,7 +0,0 @@ -ursive -50 -teatern 78 -ilscript[1] -ilscript[father].name father -ilscript[father].doctype father -ilscript[father].content[6] -ilscript[father].content[0] "input year | summary s_3 | tokenize \"stemming,normalizing\" { index f_3 | index f_5; };" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg deleted file mode 100644 index cea943d5bc9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg +++ /dev/null @@ -1,2 +0,0 @@ -include: search/cluster.music/conf1.sd.derived -include: search/cluster.music/conf2.sd.derived diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java deleted file mode 100644 index a758698d3b5..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.MockReloadHandler; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.host.HostRegistry; -import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; - -/** - * @author Ulf Lilleengen - */ -public class LocalSessionRepoTest { - - private File testApp = new File("src/test/apps/app"); - private LocalSessionRepo repo; - private static final TenantName tenantName = TenantName.defaultName(); - - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Before - public void setupSessions() throws Exception { - setupSessions(tenantName, true); - } - - private void setupSessions(TenantName tenantName, boolean createInitialSessions) throws Exception { - File configserverDbDir = temporaryFolder.newFolder().getAbsoluteFile(); - if (createInitialSessions) { - Path sessionsPath = Paths.get(configserverDbDir.getAbsolutePath(), "tenants", tenantName.value(), "sessions"); - IOUtils.copyDirectory(testApp, sessionsPath.resolve("1").toFile()); - IOUtils.copyDirectory(testApp, sessionsPath.resolve("2").toFile()); - IOUtils.copyDirectory(testApp, sessionsPath.resolve("3").toFile()); - } - GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder() - .curator(new MockCurator()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(configserverDbDir.getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .sessionLifetime(5) - .build()) - .build(); - SessionFactory sessionFactory = new SessionFactory(globalComponentRegistry, - TenantApplications.create(globalComponentRegistry, tenantName), - new HostRegistry<>(), - tenantName); - repo = new LocalSessionRepo(tenantName, globalComponentRegistry, sessionFactory); - } - - @Test - public void require_that_sessions_can_be_loaded_from_disk() { - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNotNull(repo.getSession(3L)); - assertNull(repo.getSession(4L)); - } - - @Test - public void require_that_all_sessions_are_deleted() { - repo.close(); - assertNull(repo.getSession(1L)); - assertNull(repo.getSession(2L)); - assertNull(repo.getSession(3L)); - } - - @Test - public void require_that_sessions_belong_to_a_tenant() { - // tenant is "default" - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNotNull(repo.getSession(3L)); - assertNull(repo.getSession(4L)); - - // tenant is "newTenant" - try { - setupSessions(TenantName.from("newTenant"), false); - } catch (Exception e) { - fail(); - } - assertNull(repo.getSession(1L)); - - repo.addSession(new SessionHandlerTest.MockLocalSession(1L, FilesApplicationPackage.fromFile(testApp))); - repo.addSession(new SessionHandlerTest.MockLocalSession(2L, FilesApplicationPackage.fromFile(testApp))); - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNull(repo.getSession(3L)); - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index b072f20414f..c1377ae439b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -11,27 +12,25 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.slime.Slime; -import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; -import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; -import java.nio.file.Files; +import java.io.IOException; import java.time.Instant; import java.util.Collections; import java.util.Optional; -import static com.yahoo.yolean.Exceptions.uncheck; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; @@ -43,18 +42,29 @@ import static org.junit.Assert.assertTrue; public class LocalSessionTest { private static final File testApp = new File("src/test/apps/app"); + private static final TenantName tenantName = TenantName.from("test_tenant"); + private static final Path tenantPath = Path.createRoot(); - private Path tenantPath = Path.createRoot(); + private TenantRepository tenantRepository; private Curator curator; private ConfigCurator configCurator; - private TenantFileSystemDirs tenantFileSystemDirs; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void setupTest() { + public void setupTest() throws IOException { curator = new MockCurator(); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .build()) + .build(); + tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(tenantName); configCurator = ConfigCurator.create(curator); - tenantFileSystemDirs = new TenantFileSystemDirs(uncheck(() -> Files.createTempDirectory("serverdb")).toFile(), - TenantName.from("test_tenant")); } @Test @@ -96,35 +106,16 @@ public class LocalSessionTest { assertFalse(f2.exists()); } - @Test - public void require_that_session_can_be_deleted() throws Exception { - TenantName tenantName = TenantName.defaultName(); - LocalSession session = createSession(tenantName, 3); - String sessionNode = TenantRepository.getSessionsPath(tenantName).append(String.valueOf(3)).getAbsolute(); - assertTrue(configCurator.exists(sessionNode)); - assertTrue(new File(tenantFileSystemDirs.sessionsPath(), "3").exists()); - NestedTransaction transaction = new NestedTransaction(); - session.delete(transaction); - transaction.commit(); - assertFalse(configCurator.exists(sessionNode)); - assertFalse(new File(tenantFileSystemDirs.sessionsPath(), "3").exists()); - } - @Test(expected = IllegalStateException.class) public void require_that_no_provision_info_throws_exception() throws Exception { createSession(TenantName.defaultName(), 3).getAllocatedHosts(); } private LocalSession createSession(TenantName tenant, long sessionId) throws Exception { - SessionTest.MockSessionPreparer preparer = new SessionTest.MockSessionPreparer(); - return createSession(tenant, sessionId, preparer); - } - - private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer) throws Exception { - return createSession(tenant, sessionId, preparer, Optional.empty()); + return createSession(tenant, sessionId, Optional.empty()); } - private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer, + private LocalSession createSession(TenantName tenant, long sessionId, Optional<AllocatedHosts> allocatedHosts) throws Exception { SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenant, sessionId, allocatedHosts); zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); @@ -134,13 +125,9 @@ public class LocalSessionTest { zkClient.write(allocatedHosts.get()); } zkClient.write(Collections.singletonMap(new Version(0, 0, 0), new MockFileRegistry())); - File sessionDir = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); - sessionDir.createNewFile(); - TenantApplications applications = TenantApplications.create( - new TestComponentRegistry.Builder().curator(curator).build(), tenant); + TenantApplications applications = tenantRepository.getTenant(tenantName).getApplicationRepo(); applications.createApplication(zkc.readApplicationId()); - return new LocalSession(tenant, sessionId, preparer, FilesApplicationPackage.fromFile(testApp), - zkc, sessionDir, applications, new HostRegistry<>()); + return new LocalSession(tenant, sessionId, FilesApplicationPackage.fromFile(testApp), zkc, applications); } private void doPrepare(LocalSession session) { @@ -148,12 +135,13 @@ public class LocalSessionTest { } private void doPrepare(LocalSession session, PrepareParams params) { - session.prepare(getLogger(), params, Optional.empty(), tenantPath, Instant.now()); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + sessionRepository.prepareLocalSession(session, getLogger(), params, Optional.empty(), tenantPath, Instant.now()); } private DeployHandlerLogger getLogger() { return new DeployHandlerLogger(new Slime().get(), false, - new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build()); + new ApplicationId.Builder().tenant(tenantName).applicationName("testapp").build()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java deleted file mode 100644 index 468dd5a15a7..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.tenant.Tenant; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.function.LongPredicate; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; - -/** - * @author Ulf Lilleengen - */ -public class RemoteSessionRepoTest { - - private static final TenantName tenantName = TenantName.defaultName(); - - private RemoteSessionRepo remoteSessionRepo; - private Curator curator; - TenantRepository tenantRepository; - - @Before - public void setupFacade() { - curator = new MockCurator(); - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) - .build(); - tenantRepository = new TenantRepository(componentRegistry, false); - tenantRepository.addTenant(tenantName); - this.remoteSessionRepo = tenantRepository.getTenant(tenantName).getRemoteSessionRepo(); - curator.create(TenantRepository.getTenantPath(tenantName).append("/applications")); - curator.create(TenantRepository.getSessionsPath(tenantName)); - createSession(1L, false); - createSession(2L, false); - } - - private void createSession(long sessionId, boolean wait) { - createSession(sessionId, wait, tenantName); - } - - private void createSession(long sessionId, boolean wait, TenantName tenantName) { - Path sessionsPath = TenantRepository.getSessionsPath(tenantName); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, sessionsPath.append(String.valueOf(sessionId))); - zkc.createNewSession(Instant.now()); - if (wait) { - Curator.CompletionWaiter waiter = zkc.getUploadWaiter(); - waiter.awaitCompletion(Duration.ofSeconds(120)); - } - } - - @Test - public void testInitialize() { - assertSessionExists(1L); - assertSessionExists(2L); - } - - @Test - public void testCreateSession() { - createSession(3L, true); - assertSessionExists(3L); - } - - @Test - public void testSessionStateChange() throws Exception { - long sessionId = 3L; - createSession(sessionId, true); - assertSessionStatus(sessionId, Session.Status.NEW); - assertStatusChange(sessionId, Session.Status.PREPARE); - assertStatusChange(sessionId, Session.Status.ACTIVATE); - - Path session = TenantRepository.getSessionsPath(tenantName).append("" + sessionId); - curator.delete(session); - assertSessionRemoved(sessionId); - assertNull(remoteSessionRepo.getSession(sessionId)); - } - - // If reading a session throws an exception it should be handled and not prevent other applications - // from loading. In this test we just show that we end up with one session in remote session - // repo even if it had bad data (by making getSessionIdForApplication() in FailingTenantApplications - // throw an exception). - @Test - public void testBadApplicationRepoOnActivate() { - long sessionId = 3L; - TenantName mytenant = TenantName.from("mytenant"); - curator.set(TenantRepository.getApplicationsPath(mytenant).append("mytenant:appX:default"), new byte[0]); // Invalid data - tenantRepository.addTenant(mytenant); - Tenant tenant = tenantRepository.getTenant(mytenant); - curator.create(TenantRepository.getSessionsPath(mytenant)); - remoteSessionRepo = tenant.getRemoteSessionRepo(); - assertThat(remoteSessionRepo.getSessions().size(), is(0)); - createSession(sessionId, true, mytenant); - assertThat(remoteSessionRepo.getSessions().size(), is(1)); - } - - private void assertStatusChange(long sessionId, Session.Status status) throws Exception { - Path statePath = TenantRepository.getSessionsPath(tenantName).append("" + sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); - curator.create(statePath); - curator.framework().setData().forPath(statePath.getAbsolute(), Utf8.toBytes(status.toString())); - assertSessionStatus(sessionId, status); - } - - private void assertSessionRemoved(long sessionId) { - waitFor(p -> remoteSessionRepo.getSession(sessionId) == null, sessionId); - assertNull(remoteSessionRepo.getSession(sessionId)); - } - - private void assertSessionExists(long sessionId) { - assertSessionStatus(sessionId, Session.Status.NEW); - } - - private void assertSessionStatus(long sessionId, Session.Status status) { - waitFor(p -> remoteSessionRepo.getSession(sessionId) != null && - remoteSessionRepo.getSession(sessionId).getStatus() == status, sessionId); - assertNotNull(remoteSessionRepo.getSession(sessionId)); - assertThat(remoteSessionRepo.getSession(sessionId).getStatus(), is(status)); - } - - private void waitFor(LongPredicate predicate, long sessionId) { - long endTime = System.currentTimeMillis() + 60_000; - boolean ok; - do { - ok = predicate.test(sessionId); - try { - Thread.sleep(10); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } while (System.currentTimeMillis() < endTime && !ok); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java new file mode 100644 index 00000000000..b9e872261c5 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java @@ -0,0 +1,217 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.OrchestratorMock; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.function.LongPredicate; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; + +/** + * @author Ulf Lilleengen + */ +public class SessionRepositoryTest { + + private static final TenantName tenantName = TenantName.defaultName(); + private static final ApplicationId applicationId = ApplicationId.from(tenantName.value(), "testApp", "default"); + private static final File testApp = new File("src/test/apps/app"); + + private MockCurator curator; + private TenantRepository tenantRepository; + private ApplicationRepository applicationRepository; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + public void setup() throws Exception { + setup(new InMemoryFlagSource()); + } + + private void setup(FlagSource flagSource) throws Exception { + curator = new MockCurator(); + File configserverDbDir = temporaryFolder.newFolder().getAbsoluteFile(); + GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .configServerDBDir(configserverDbDir.getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .sessionLifetime(5) + .build()) + .flagSource(flagSource) + .build(); + tenantRepository = new TenantRepository(globalComponentRegistry, false); + tenantRepository.addTenant(SessionRepositoryTest.tenantName); + applicationRepository = new ApplicationRepository(tenantRepository, + new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), + Clock.systemUTC()); + } + + @Test + public void require_that_local_sessions_are_created_and_deleted() throws Exception { + setup(); + long firstSessionId = deploy(); + long secondSessionId = deploy(); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + assertNotNull(sessionRepository.getLocalSession(firstSessionId)); + assertNotNull(sessionRepository.getLocalSession(secondSessionId)); + assertNull(sessionRepository.getLocalSession(secondSessionId + 1)); + + sessionRepository.close(); + // All created sessions are deleted + assertNull(sessionRepository.getLocalSession(firstSessionId)); + assertNull(sessionRepository.getLocalSession(secondSessionId)); + } + + @Test + public void require_that_local_sessions_belong_to_a_tenant() throws Exception { + setup(); + // tenant is "default" + + long firstSessionId = deploy(); + long secondSessionId = deploy(); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + assertNotNull(sessionRepository.getLocalSession(firstSessionId)); + assertNotNull(sessionRepository.getLocalSession(secondSessionId)); + assertNull(sessionRepository.getLocalSession(secondSessionId + 1)); + + // tenant is "newTenant" + TenantName newTenant = TenantName.from("newTenant"); + tenantRepository.addTenant(newTenant); + long sessionId = deploy(ApplicationId.from(newTenant.value(), "testapp", "default")); + SessionRepository sessionRepository2 = tenantRepository.getTenant(newTenant).getSessionRepository(); + assertNotNull(sessionRepository2.getLocalSession(sessionId)); + } + + @Test + public void testInitialize() throws Exception { + setup(); + createSession(10L, false); + createSession(11L, false); + assertRemoteSessionExists(10L); + assertRemoteSessionExists(11L); + } + + @Test + public void testSessionStateChange() throws Exception { + setup(); + long sessionId = 3L; + createSession(sessionId, true); + assertRemoteSessionStatus(sessionId, Session.Status.NEW); + assertStatusChange(sessionId, Session.Status.PREPARE); + assertStatusChange(sessionId, Session.Status.ACTIVATE); + + com.yahoo.path.Path session = TenantRepository.getSessionsPath(tenantName).append("" + sessionId); + curator.delete(session); + assertSessionRemoved(sessionId); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + assertNull(sessionRepository.getRemoteSession(sessionId)); + } + + // If reading a session throws an exception it should be handled and not prevent other applications + // from loading. In this test we just show that we end up with one session in remote session + // repo even if it had bad data (by making getSessionIdForApplication() in FailingTenantApplications + // throw an exception). + @Test + public void testBadApplicationRepoOnActivate() throws Exception { + setup(); + long sessionId = 3L; + TenantName mytenant = TenantName.from("mytenant"); + curator.set(TenantRepository.getApplicationsPath(mytenant).append("mytenant:appX:default"), new byte[0]); // Invalid data + tenantRepository.addTenant(mytenant); + Tenant tenant = tenantRepository.getTenant(mytenant); + curator.create(TenantRepository.getSessionsPath(mytenant)); + SessionRepository sessionRepository = tenant.getSessionRepo(); + assertThat(sessionRepository.getRemoteSessions().size(), is(0)); + createSession(sessionId, true, mytenant); + assertThat(sessionRepository.getRemoteSessions().size(), is(1)); + } + + private void createSession(long sessionId, boolean wait) { + createSession(sessionId, wait, tenantName); + } + + private void createSession(long sessionId, boolean wait, TenantName tenantName) { + com.yahoo.path.Path sessionsPath = TenantRepository.getSessionsPath(tenantName); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, sessionsPath.append(String.valueOf(sessionId))); + zkc.createNewSession(Instant.now()); + if (wait) { + Curator.CompletionWaiter waiter = zkc.getUploadWaiter(); + waiter.awaitCompletion(Duration.ofSeconds(120)); + } + } + + private void assertStatusChange(long sessionId, Session.Status status) throws Exception { + com.yahoo.path.Path statePath = TenantRepository.getSessionsPath(tenantName).append("" + sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); + curator.create(statePath); + curator.framework().setData().forPath(statePath.getAbsolute(), Utf8.toBytes(status.toString())); + assertRemoteSessionStatus(sessionId, status); + } + + private void assertSessionRemoved(long sessionId) { + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + waitFor(p -> sessionRepository.getRemoteSession(sessionId) == null, sessionId); + assertNull(sessionRepository.getRemoteSession(sessionId)); + } + + private void assertRemoteSessionExists(long sessionId) { + assertRemoteSessionStatus(sessionId, Session.Status.NEW); + } + + private void assertRemoteSessionStatus(long sessionId, Session.Status status) { + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + waitFor(p -> sessionRepository.getRemoteSession(sessionId) != null && + sessionRepository.getRemoteSession(sessionId).getStatus() == status, sessionId); + assertNotNull(sessionRepository.getRemoteSession(sessionId)); + assertThat(sessionRepository.getRemoteSession(sessionId).getStatus(), is(status)); + } + + private void waitFor(LongPredicate predicate, long sessionId) { + long endTime = System.currentTimeMillis() + 60_000; + boolean ok; + do { + ok = predicate.test(sessionId); + try { + Thread.sleep(10); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } while (System.currentTimeMillis() < endTime && !ok); + } + + private long deploy() { + return deploy(applicationId); + } + + private long deploy(ApplicationId applicationId) { + applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId).build()); + return applicationRepository.getActiveSession(applicationId).getSessionId(); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg deleted file mode 100644 index 0bc17bae65e..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg +++ /dev/null @@ -1,18 +0,0 @@ -asyncfetchocc 10 -e 4 -search[2].feeder[1] "bazfeeder" -search[1].feeder[0] "barfeeder1_1" -search[1].feeder[3] "barfeeder2_1" -onlyindef 45 - -speciallog[0].filehandler.rotation "0 1 ..." - -rulebase[4] -rulebase[0].name "cjk" -rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# ä½³:\u4f73\n# 能:\u80fd\n# ç´¢:\u7d22\n# å°¼:\u5c3c\n# æƒ :\u60e0\n# æ™®:\u666e\n\n@default\n\naç´¢ -> ç´¢a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,æƒ æ™®,佳能;\n" -rulebase[1].name "common" -rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" -rulebase[2].name "egyik" -rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" -rulebase[3].name "masik" -rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg deleted file mode 100644 index 88b50384058..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg +++ /dev/null @@ -1 +0,0 @@ -usercfgwithid 86 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg deleted file mode 100644 index b34c4ed311e..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg +++ /dev/null @@ -1 +0,0 @@ -foo "test" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg deleted file mode 100644 index 12c5b53de7d..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg +++ /dev/null @@ -1 +0,0 @@ -theint 34 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg deleted file mode 100644 index 73ed41667f9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg +++ /dev/null @@ -1 +0,0 @@ -keepsuccess true diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java index a73818cda12..a34c17dc909 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java @@ -24,7 +24,7 @@ public class ZKApplicationFileTest extends ApplicationFileTest { private void feed(ConfigCurator zk, File dirToFeed) { assertTrue(dirToFeed.isDirectory()); String appPath = "/0"; - zk.feedZooKeeper(dirToFeed, appPath + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); + ZKApplicationPackageTest.feedZooKeeper(zk, dirToFeed, appPath + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); zk.putData(appPath, ZKApplicationPackage.fileRegistryNode, "dummyfiles"); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 4397e087fb7..4b4605cce7d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -22,9 +22,12 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.io.Reader; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.regex.Pattern; @@ -37,6 +40,7 @@ import static org.junit.Assert.assertTrue; public class ZKApplicationPackageTest { + private static final FilenameFilter acceptsAllFileNameFilter = (dir, name) -> true; private static final String APP = "src/test/apps/zkapp"; private static final String TEST_FLAVOR_NAME = "test-flavor"; private static final Optional<Flavor> TEST_FLAVOR = new MockNodeFlavors().getFlavor(TEST_FLAVOR_NAME); @@ -97,7 +101,7 @@ public class ZKApplicationPackageTest { private void feed(ConfigCurator zk, File dirToFeed) throws IOException { assertTrue(dirToFeed.isDirectory()); - zk.feedZooKeeper(dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); + feedZooKeeper(zk, dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"id\":\"foo:foo:default\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData); zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles"); @@ -115,4 +119,63 @@ public class ZKApplicationPackageTest { } } + /** + * Takes for instance the dir /app and puts the contents into the given ZK path. Ignores files starting with dot, + * and dirs called CVS. + * + * @param dir directory which holds the summary class part files + * @param path zookeeper path + * @param filenameFilter A FilenameFilter which decides which files in dir are fed to zookeeper + * @param recurse recurse subdirectories + */ + static void feedZooKeeper(ConfigCurator zk, File dir, String path, FilenameFilter filenameFilter, boolean recurse) { + try { + if (filenameFilter == null) { + filenameFilter = acceptsAllFileNameFilter; + } + if (!dir.isDirectory()) { + throw new IllegalArgumentException(dir + " is not a directory"); + } + for (File file : listFiles(dir, filenameFilter)) { + if (file.getName().startsWith(".")) continue; //.svn , .git ... + if ("CVS".equals(file.getName())) continue; + if (file.isFile()) { + String contents = IOUtils.readFile(file); + zk.putData(path, file.getName(), contents); + } else if (recurse && file.isDirectory()) { + zk.createNode(path, file.getName()); + feedZooKeeper(zk, file, path + '/' + file.getName(), filenameFilter, recurse); + } + } + } + catch (IOException e) { + throw new RuntimeException("Exception feeding ZooKeeper at path " + path, e); + } + } + + /** + * Same as normal listFiles, but use the filter only for normal files + * + * @param dir directory to list files in + * @param filter A FilenameFilter which decides which files in dir are listed + * @return an array of Files + */ + protected static File[] listFiles(File dir, FilenameFilter filter) { + File[] rawList = dir.listFiles(); + List<File> ret = new ArrayList<>(); + if (rawList != null) { + for (File f : rawList) { + if (f.isDirectory()) { + ret.add(f); + } else { + if (filter.accept(dir, f.getName())) { + ret.add(f); + } + } + } + } + return ret.toArray(new File[0]); + } + + } diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 3e6276f3f6e..dac33d2d431 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -880,55 +880,5 @@ "public bridge synthetic java.lang.Object clone()" ], "fields": [] - }, - "ai.vespa.cloud.Environment": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static ai.vespa.cloud.Environment[] values()", - "public static ai.vespa.cloud.Environment valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum ai.vespa.cloud.Environment dev", - "public static final enum ai.vespa.cloud.Environment perf", - "public static final enum ai.vespa.cloud.Environment test", - "public static final enum ai.vespa.cloud.Environment staging", - "public static final enum ai.vespa.cloud.Environment prod" - ] - }, - "ai.vespa.cloud.SystemInfo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.cloud.config.ConfigserverConfig)", - "public void <init>(ai.vespa.cloud.Zone)", - "public ai.vespa.cloud.Zone zone()" - ], - "fields": [] - }, - "ai.vespa.cloud.Zone": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(ai.vespa.cloud.Environment, java.lang.String)", - "public ai.vespa.cloud.Environment environment()", - "public java.lang.String region()", - "public java.lang.String toString()", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public static ai.vespa.cloud.Zone from(java.lang.String)" - ], - "fields": [] } }
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 0b42b3a481b..1d6e1a0893d 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -35,9 +35,6 @@ public class LogHandler extends ThreadedHttpRequestHandler { .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MAX); return new HttpResponse(200) { - { - headers().add("Content-Encoding", "gzip"); - } @Override public void render(OutputStream outputStream) { logReader.writeLogs(outputStream, from, to); diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index e3fef4e0e44..3cf849a6835 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -1,12 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; +import com.google.common.collect.Iterators; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.yolean.Exceptions; import java.io.BufferedReader; import java.io.BufferedWriter; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -19,18 +19,21 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Comparator; +import java.util.Iterator; import java.util.List; -import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Comparator.comparing; /** * @author olaaun @@ -39,6 +42,9 @@ import static java.util.Comparator.comparing; */ class LogReader { + static final Pattern logArchivePathPattern = Pattern.compile("(\\d{4})/(\\d{2})/(\\d{2})/(\\d{2})-\\d+(.gz)?"); + static final Pattern vespaLogPathPattern = Pattern.compile("vespa\\.log(?:-(\\d{4})-(\\d{2})-(\\d{2})\\.(\\d{2})-(\\d{2})-(\\d{2})(?:.gz)?)?"); + private final Path logDirectory; private final Pattern logFilePattern; @@ -51,61 +57,110 @@ class LogReader { this.logFilePattern = logFilePattern; } - void writeLogs(OutputStream outputStream, Instant from, Instant to) { + void writeLogs(OutputStream out, Instant from, Instant to) { + double fromSeconds = from.getEpochSecond() + from.getNano() / 1e9; + double toSeconds = to.getEpochSecond() + to.getNano() / 1e9; + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out)); try { - List<Path> logs = getMatchingFiles(from, to); - for (int i = 0; i < logs.size(); i++) { - Path log = logs.get(i); - boolean zipped = log.toString().endsWith(".gz"); - try (InputStream in = Files.newInputStream(log)) { - InputStream inProxy; - - // If the log needs filtering, possibly unzip (and rezip) it, and filter its lines on timestamp. - if (i == 0 || i == logs.size() - 1) { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(in) : in, UTF_8)); - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(zipped ? new GZIPOutputStream(buffer) : buffer, UTF_8))) { - for (String line; (line = reader.readLine()) != null; ) { - String[] parts = line.split("\t"); - if (parts.length != 7) - continue; - - Instant at = Instant.EPOCH.plus((long) (Double.parseDouble(parts[0]) * 1_000_000), ChronoUnit.MICROS); - if (at.isAfter(from) && ! at.isAfter(to)) { - writer.write(line); - writer.newLine(); - } - } - } - inProxy = new ByteArrayInputStream(buffer.toByteArray()); + for (List<Path> logs : getMatchingFiles(from, to)) { + List<LogLineIterator> logLineIterators = new ArrayList<>(); + try { + // Logs in each sub-list contain entries covering the same time interval, so do a merge sort while reading + for (Path log : logs) + logLineIterators.add(new LogLineIterator(log, fromSeconds, toSeconds)); + + Iterator<LineWithTimestamp> lines = Iterators.mergeSorted(logLineIterators, + Comparator.comparingDouble(LineWithTimestamp::timestamp)); + while (lines.hasNext()) { + writer.write(lines.next().line()); + writer.newLine(); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + finally { + for (LogLineIterator ll : logLineIterators) { + try { ll.close(); } catch (IOException ignored) { } } - else - inProxy = in; - - // At the point when logs switch to un-zipped, replace the output stream with a zipping proxy. - if ( ! zipped && ! (outputStream instanceof GZIPOutputStream)) - outputStream = new GZIPOutputStream(outputStream); - - inProxy.transferTo(outputStream); } } } - catch (IOException e) { - throw new UncheckedIOException(e); - } finally { + Exceptions.uncheck(writer::flush); + } + } + + private static class LogLineIterator implements Iterator<LineWithTimestamp>, AutoCloseable { + + private final BufferedReader reader; + private final double from; + private final double to; + private LineWithTimestamp next; + + private LogLineIterator(Path log, double from, double to) throws IOException { + boolean zipped = log.toString().endsWith(".gz"); + InputStream in = Files.newInputStream(log); + this.reader = new BufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(in) : in, UTF_8)); + this.from = from; + this.to = to; + this.next = readNext(); + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public LineWithTimestamp next() { + LineWithTimestamp current = next; + next = readNext(); + return current; + } + + @Override + public void close() throws IOException { + reader.close(); + } + + private LineWithTimestamp readNext() { try { - outputStream.close(); + for (String line; (line = reader.readLine()) != null; ) { + String[] parts = line.split("\t"); + if (parts.length != 7) + continue; + + double timestamp = Double.parseDouble(parts[0]); + if (timestamp > to) + return null; + + if (timestamp >= from) + return new LineWithTimestamp(line, timestamp); + } + return null; } catch (IOException e) { throw new UncheckedIOException(e); } } + } - /** Returns log files which may have relevant entries, sorted by modification time — the first and last must be filtered. */ - private List<Path> getMatchingFiles(Instant from, Instant to) { - Map<Path, Instant> paths = new HashMap<>(); + private static class LineWithTimestamp { + final String line; + final double timestamp; + LineWithTimestamp(String line, double timestamp) { + this.line = line; + this.timestamp = timestamp; + } + String line() { return line; } + double timestamp() { return timestamp; } + } + + /** Returns log files which may have relevant entries, grouped and sorted by {@link #extractTimestamp(Path)} — the first and last group must be filtered. */ + private List<List<Path>> getMatchingFiles(Instant from, Instant to) { + List<Path> paths = new ArrayList<>(); try { Files.walkFileTree(logDirectory, new SimpleFileVisitor<>() { @@ -117,7 +172,7 @@ class LogReader { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { if (logFilePattern.matcher(file.getFileName().toString()).matches()) - paths.put(file, attrs.lastModifiedTime().toInstant()); + paths.add(file); return FileVisitResult.CONTINUE; } @@ -132,15 +187,54 @@ class LogReader { throw new UncheckedIOException(e); } - List<Path> sorted = new ArrayList<>(); - for (var entries = paths.entrySet().stream().sorted(comparing(Map.Entry::getValue)).iterator(); entries.hasNext(); ) { - var entry = entries.next(); - if (entry.getValue().isAfter(from)) - sorted.add(entry.getKey()); - if (entry.getValue().isAfter(to)) + var logsByTimestamp = paths.stream() + .collect(Collectors.groupingBy(this::extractTimestamp, + TreeMap::new, + Collectors.toList())); + + List<List<Path>> sorted = new ArrayList<>(); + for (var entry : logsByTimestamp.entrySet()) { + if (entry.getKey().isAfter(from)) + sorted.add(entry.getValue()); + if (entry.getKey().isAfter(to)) break; } return sorted; } + /** Extracts a timestamp after all entries in the log file with the given path. */ + Instant extractTimestamp(Path path) { + String relativePath = logDirectory.relativize(path).toString(); + Matcher matcher = logArchivePathPattern.matcher(relativePath); + if (matcher.matches()) { + return ZonedDateTime.of(Integer.parseInt(matcher.group(1)), + Integer.parseInt(matcher.group(2)), + Integer.parseInt(matcher.group(3)), + Integer.parseInt(matcher.group(4)), + 0, + 0, + 0, + ZoneId.of("UTC")) + .toInstant() + .plus(Duration.ofHours(1)); + } + matcher = vespaLogPathPattern.matcher(relativePath); + if (matcher.matches()) { + if (matcher.group(1) == null) + return Instant.MAX; + + return ZonedDateTime.of(Integer.parseInt(matcher.group(1)), + Integer.parseInt(matcher.group(2)), + Integer.parseInt(matcher.group(3)), + Integer.parseInt(matcher.group(4)), + Integer.parseInt(matcher.group(5)), + Integer.parseInt(matcher.group(6)), + 0, + ZoneId.of("UTC")) + .toInstant() + .plus(Duration.ofSeconds(1)); + } + throw new IllegalArgumentException("Unrecognized file pattern for file at '" + path + "'"); + } + } diff --git a/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java b/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java index 38f5b72336b..16cf741813c 100644 --- a/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java +++ b/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java @@ -5,7 +5,7 @@ import com.yahoo.protect.Process; /** * An injectable terminator of the Java vm. - * Components that encounters conditions where the vm should be terminator should + * Components that encounters conditions where the vm should be terminated should * request an instance of this injected. That makes termination testable * as tests can create subclasses of this which register the termination request * rather than terminating. diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index 01dcb885a97..ab0d0d54675 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -47,12 +47,12 @@ public class LogHandlerTest { } @Override - protected void writeLogs(OutputStream outputStream, Instant from, Instant to) { + protected void writeLogs(OutputStream out, Instant from, Instant to) { try { if (to.isAfter(Instant.ofEpochMilli(1000))) { - outputStream.write("newer log".getBytes()); + out.write("newer log".getBytes()); } else { - outputStream.write("older log".getBytes()); + out.write("older log".getBytes()); } } catch (Exception e) {} } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index c68facf4f01..3f7a78e13be 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -12,7 +12,7 @@ import java.io.OutputStream; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.FileTime; +import java.time.Duration; import java.time.Instant; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; @@ -26,47 +26,56 @@ public class LogReaderTest { private final FileSystem fileSystem = TestFileSystem.create(); private final Path logDirectory = fileSystem.getPath("/opt/vespa/logs"); - private static final String log1 = "0.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"; - private static final String log2 = "0.2\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)\n"; + private static final String logv11 = "3600.2\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tfourth\n"; + private static final String logv = "90000.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tlast\n"; + private static final String log100 = "0.2\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tsecond\n"; + private static final String log101 = "0.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"; + private static final String log110 = "3600.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tthird\n"; + private static final String log200 = "86400.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)\n"; @Before public void setup() throws IOException { - Files.createDirectories(logDirectory.resolve("subfolder")); - - Files.setLastModifiedTime( - Files.write(logDirectory.resolve("log1.log.gz"), compress(log1)), - FileTime.from(Instant.ofEpochMilli(123))); - Files.setLastModifiedTime( - Files.write(logDirectory.resolve("subfolder/log2.log"), log2.getBytes(UTF_8)), - FileTime.from(Instant.ofEpochMilli(234))); - + // Log archive paths and file names indicate what hour they contain logs for, with the start of that hour. + // Multiple entries may exist for each hour. + Files.createDirectories(logDirectory.resolve("1970/01/01")); + Files.write(logDirectory.resolve("1970/01/01/00-0.gz"), compress(log100)); + Files.write(logDirectory.resolve("1970/01/01/00-1"), log101.getBytes(UTF_8)); + Files.write(logDirectory.resolve("1970/01/01/01-0.gz"), compress(log110)); + + Files.createDirectories(logDirectory.resolve("1970/01/02")); + Files.write(logDirectory.resolve("1970/01/02/00-0"), log200.getBytes(UTF_8)); + + // Vespa log file names are the second-truncated timestamp of the last entry. + // The current log file has no timestamp suffix. + Files.write(logDirectory.resolve("vespa.log-1970-01-01.01-00-00"), logv11.getBytes(UTF_8)); + Files.write(logDirectory.resolve("vespa.log"), logv.getBytes(UTF_8)); } @Test public void testThatLogsOutsideRangeAreExcluded() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(160)); + logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050)); - assertEquals("", decompress(baos.toByteArray())); + assertEquals(log100 + logv11 + log110, baos.toString(UTF_8)); } @Test public void testThatLogsNotMatchingRegexAreExcluded() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*2\\.log")); - logReader.writeLogs(baos, Instant.ofEpochMilli(0), Instant.ofEpochMilli(300)); + LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*-1.*")); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2))); - assertEquals(log2, decompress(baos.toByteArray())); + assertEquals(log101 + logv11, baos.toString(UTF_8)); } @Test public void testZippedStreaming() throws IOException { ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(zippedBaos, Instant.ofEpochMilli(0), Instant.ofEpochMilli(300)); + logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2))); - assertEquals(log1 + log2, decompress(zippedBaos.toByteArray())); + assertEquals(log101 + log100 + logv11 + log110 + log200 + logv, zippedBaos.toString(UTF_8)); } private byte[] compress(String input) throws IOException { @@ -77,10 +86,4 @@ public class LogReaderTest { return baos.toByteArray(); } - private String decompress(byte[] input) throws IOException { - if (input.length == 0) return ""; - byte[] decompressed = new GZIPInputStream(new ByteArrayInputStream(input)).readAllBytes(); - return new String(decompressed); - } - } diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index be6a12526f5..1c4c9e9521b 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -446,7 +446,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.28.v20200408</jetty.version> + <jetty.version>9.4.30.v20200611</jetty.version> <org.lz4.version>1.7.1</org.lz4.version> <org.json.version>20090211</org.json.version> <slf4j.version>1.7.5</slf4j.version> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 1bb06ab9694..dd2d9ceb188 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -187,6 +187,11 @@ <artifactId>config-bundle</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-zone-api</artifactId> + <version>${project.version}</version> + </dependency> <!-- NOTE: Dependencies below are added explicitly to exclude transitive deps that are not provided runtime by the container, and hence make them invisible to user projects' build classpath. diff --git a/container-disc/pom.xml b/container-disc/pom.xml index 15b8cd08808..48872d0665b 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -106,6 +106,12 @@ <artifactId>vespalog</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-zone-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <!-- WARNING: These are only here to make bundlification work --> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -184,6 +190,7 @@ container-search-and-docproc-jar-with-dependencies.jar, container-search-gui-jar-with-dependencies.jar, docprocs-jar-with-dependencies.jar, + hosted-zone-api-jar-with-dependencies.jar, jdisc-security-filters-jar-with-dependencies.jar, jdisc_http_service-jar-with-dependencies.jar, model-evaluation-jar-with-dependencies.jar, diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java new file mode 100644 index 00000000000..0bb3832ddf5 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java @@ -0,0 +1,27 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import ai.vespa.cloud.Environment; +import ai.vespa.cloud.SystemInfo; +import ai.vespa.cloud.Zone; +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.container.di.componentgraph.Provider; + +/** + * Provides information about the system in which this container is running. + * This is available and can be injected when running in a cloud environment. + * + * @author bratseth + */ +public class SystemInfoProvider extends AbstractComponent implements Provider<SystemInfo> { + + private final SystemInfo instance; + + @Inject public SystemInfoProvider(ConfigserverConfig config) { + this.instance = new SystemInfo(new Zone(Environment.valueOf(config.environment()), config.region())); + } + + @Override public SystemInfo get() { return instance; } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index b54446c071e..59324079e6f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -5,6 +5,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.dns; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; @@ -22,11 +23,11 @@ public class MemoryNameService implements NameService { return Collections.unmodifiableSet(records); } - private void add(Record record) { - if (records.stream().anyMatch(r -> r.type().equals(record.type()) && - r.name().equals(record.name()) && - r.data().equals(record.data()))) { - throw new IllegalArgumentException("Record already exists: " + record); + public void add(Record record) { + Optional<Record> conflict = records.stream().filter(r -> conflicts(r, record)).findFirst(); + if (conflict.isPresent()) { + throw new AssertionError("'" + record + "' conflicts with existing record '" + + conflict.get() + "'"); } records.add(record); } @@ -45,8 +46,9 @@ public class MemoryNameService implements NameService { .map(target -> new Record(Record.Type.ALIAS, name, target.asData())) .collect(Collectors.toList()); // Satisfy idempotency contract of interface - removeRecords(records); - records.forEach(this::add); + records.stream() + .filter(r -> !this.records.contains(r)) + .forEach(this::add); return records; } @@ -108,4 +110,15 @@ public class MemoryNameService implements NameService { this.records.removeAll(records); } + /** + * Returns whether record r1 and r2 can co-exist in a name service. This attempts to enforce the same constraints as + * most real name services. + */ + private static boolean conflicts(Record r1, Record r2) { + if (!r1.name().equals(r2.name())) return false; // Distinct names never conflict + if (r1.type() == Record.Type.ALIAS && r1.type() == r2.type()) // ALIAS records only require distinct data + return r1.data().equals(r2.data()); + return true; // Anything else is considered a conflict + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java index 295f8e8fd98..dfdd273b6f5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java @@ -75,4 +75,8 @@ public class MockUserManagement implements UserManagement { return List.copyOf(get(role)); } + @Override + public List<Role> listRoles(UserId userId) { + return List.of(); + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java index 8a549b505c7..bfb617a75b6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java @@ -34,4 +34,6 @@ public interface UserManagement { /** Returns all users in the given role, or throws if the role does not exist. */ List<User> listUsers(Role role); + /** Returns all roles of which the given user is part, or throws if the user does not exist */ + List<Role> listRoles(UserId user); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index fc904b9d1a0..9a5a0ad0e77 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -48,15 +48,10 @@ enum Policy { .on(PathGroup.user) .in(SystemName.main, SystemName.cd, SystemName.dev)), - /** Access to create a tenant in select systems. */ + /** Access to create a tenant. */ tenantCreate(Privilege.grant(Action.create) .on(PathGroup.tenant) - .in(SystemName.main, SystemName.cd, SystemName.dev)), // TODO SystemName.all() - - /** Access to create a tenant in public */ - tenantCreatePublic(Privilege.grant(Action.create) - .on(PathGroup.tenant) - .in(SystemName.PublicCd, SystemName.Public)), + .in(SystemName.all())), /** Full access to tenant information and settings. */ tenantDelete(Privilege.grant(Action.delete) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index ad7b3f68440..bf5ba4001fa 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -22,8 +22,7 @@ public enum RoleDefinition { hostedOperator(Policy.operator), /** Machina autem exspiravit. */ - hostedSupporter(Policy.supporter, - Policy.tenantCreatePublic), + hostedSupporter(Policy.supporter), /** Base role which every user is part of. */ everyone(Policy.classifiedRead, diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index 2da93c5ceca..10d4732984c 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import org.junit.Test; -import java.awt.event.AdjustmentEvent; import java.net.URI; import java.util.List; import java.util.stream.Stream; @@ -59,8 +58,9 @@ public class RoleTest { assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/application/v4/tenant/t2/application/a2"))); assertFalse(mainEnforcer.allows(role, Action.delete, URI.create("/application/v4/tenant/t8/application/a6/instance/i1/environment/dev/region/r1"))); - // Check that we are allowed to create tenants in public - assertTrue(publicEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t1"))); + // Check that we are allowed to create tenants in public. + // hostedSupporter isn't actually allowed to create tenants - but any logged in user will be a member of the "everyone" role. + assertTrue(publicEnforcer.allows(Role.everyone(), Action.create, URI.create("/application/v4/tenant/t1"))); } @Test diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 463ffb58460..c441188b1be 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointList; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; @@ -84,13 +85,14 @@ public class RoutingController { /** Returns zone-scoped endpoints for given deployment */ public EndpointList endpointsOf(DeploymentId deployment) { var endpoints = new LinkedHashSet<Endpoint>(); + boolean isSystemApplication = SystemApplication.matching(deployment.applicationId()).isPresent(); // Avoid reading application more than once per call to this var application = Suppliers.memoize(() -> controller.applications().requireApplication(TenantAndApplicationId.from(deployment.applicationId()))); for (var policy : routingPolicies.get(deployment).values()) { if (!policy.status().isActive()) continue; for (var routingMethod : controller.zoneRegistry().routingMethods(policy.id().zone())) { - if (routingMethod.isDirect() && !canRouteDirectlyTo(deployment, application.get())) continue; - endpoints.add(policy.endpointIn(controller.system(), routingMethod)); + if (routingMethod.isDirect() && !isSystemApplication && !canRouteDirectlyTo(deployment, application.get())) continue; + endpoints.add(policy.endpointIn(controller.system(), routingMethod, controller.zoneRegistry())); } } return EndpointList.copyOf(endpoints); @@ -98,6 +100,7 @@ public class RoutingController { /** Returns global-scoped endpoints for given instance */ public EndpointList endpointsOf(ApplicationId instance) { + if (SystemApplication.matching(instance).isPresent()) return EndpointList.copyOf(List.of()); return endpointsOf(controller.applications().requireApplication(TenantAndApplicationId.from(instance)), instance.instance()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index ed5add8b98a..f8982c96637 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -36,20 +36,18 @@ public class Endpoint { private final RoutingMethod routingMethod; private final boolean tls; - private Endpoint(String name, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system, - Port port, boolean legacy, RoutingMethod routingMethod) { + private Endpoint(String name, URI url, List<ZoneId> zones, Scope scope, Port port, boolean legacy, + RoutingMethod routingMethod) { Objects.requireNonNull(name, "name must be non-null"); - Objects.requireNonNull(application, "application must be non-null"); Objects.requireNonNull(zones, "zones must be non-null"); Objects.requireNonNull(scope, "scope must be non-null"); - Objects.requireNonNull(system, "system must be non-null"); Objects.requireNonNull(port, "port must be non-null"); Objects.requireNonNull(routingMethod, "routingMethod must be non-null"); if (scope == Scope.zone && zones.size() != 1) { throw new IllegalArgumentException("A single zone must be given for zone-scoped endpoints"); } this.name = name; - this.url = createUrl(name, application, zones, scope, system, port, legacy, routingMethod); + this.url = url; this.zones = List.copyOf(zones); this.scope = scope; this.legacy = legacy; @@ -57,6 +55,20 @@ public class Endpoint { this.tls = port.tls; } + private Endpoint(String name, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system, + Port port, boolean legacy, RoutingMethod routingMethod) { + this(name, + createUrl(name, + Objects.requireNonNull(application, "application must be non-null"), + zones, + scope, + Objects.requireNonNull(system, "system must be non-null"), + port, + legacy, + routingMethod), + zones, scope, port, legacy, routingMethod); + } + /** * Returns the name of this endpoint (the first component of the DNS name). Depending on the endpoint type, this * can be one of the following: @@ -286,6 +298,14 @@ public class Endpoint { return new EndpointBuilder(application); } + /** Create an endpoint for given system application */ + public static Endpoint of(SystemApplication systemApplication, ZoneId zone, URI url) { + if (!systemApplication.hasEndpoint()) throw new IllegalArgumentException(systemApplication + " has no endpoint"); + RoutingMethod routingMethod = RoutingMethod.exclusive; + Port port = url.getPort() == -1 ? Port.tls() : Port.tls(url.getPort()); // System application endpoints are always TLS + return new Endpoint("", url, List.of(zone), Scope.zone, port, false, routingMethod); + } + public static class EndpointBuilder { private final ApplicationId application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 243bca8c027..e1acf867744 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -8,7 +8,9 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -71,11 +73,27 @@ public enum SystemApplication { return nodeType.isDockerHost(); } + /** Returns whether this has an endpoint */ + public boolean hasEndpoint() { + return this == configServer; + } + + /** Returns the endpoint of this, if any */ + public Optional<Endpoint> endpointIn(ZoneId zone, ZoneRegistry zoneRegistry) { + if (!hasEndpoint()) return Optional.empty(); + return Optional.of(Endpoint.of(this, zone, zoneRegistry.getConfigServerVipUri(zone))); + } + /** All known system applications */ public static List<SystemApplication> all() { return List.of(values()); } + /** Returns the system application matching given id, if any */ + public static Optional<SystemApplication> matching(ApplicationId id) { + return Arrays.stream(values()).filter(app -> app.id().equals(id)).findFirst(); + } + @Override public String toString() { return String.format("system application %s of type %s", id, nodeType); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 0ac9e17c24f..092143255d4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -167,10 +167,11 @@ public class DeploymentTrigger { public JobId reTrigger(ApplicationId applicationId, JobType jobType) { Application application = applications().requireApplication(TenantAndApplicationId.from(applicationId)); Instance instance = application.require(applicationId.instance()); - DeploymentStatus status = jobs.deploymentStatus(application); JobId job = new JobId(instance.id(), jobType); - JobStatus jobStatus = status.jobs().get(new JobId(applicationId, jobType)).get(); - Versions versions = jobStatus.lastTriggered().get().versions(); + JobStatus jobStatus = jobs.jobStatus(new JobId(applicationId, jobType)); + Versions versions = jobStatus.lastTriggered() + .orElseThrow(() -> new IllegalArgumentException(job + " has never been triggered")) + .versions(); trigger(deploymentJob(instance, versions, jobType, jobStatus, clock.instant())); return job; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 6f3e868dc1a..ca695a2d234 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -43,6 +43,7 @@ public class ControllerMaintenance extends AbstractComponent { private final CloudEventReporter cloudEventReporter; private final RotationStatusUpdater rotationStatusUpdater; private final ResourceTagMaintainer resourceTagMaintainer; + private final SystemRoutingPolicyMaintainer systemRoutingPolicyMaintainer; @Inject @SuppressWarnings("unused") // instantiated by Dependency Injection @@ -71,6 +72,7 @@ public class ControllerMaintenance extends AbstractComponent { cloudEventReporter = new CloudEventReporter(controller, Duration.ofDays(1)); rotationStatusUpdater = new RotationStatusUpdater(controller, maintenanceInterval); resourceTagMaintainer = new ResourceTagMaintainer(controller, Duration.ofMinutes(30), controller.serviceRegistry().resourceTagger()); + systemRoutingPolicyMaintainer = new SystemRoutingPolicyMaintainer(controller, Duration.ofMinutes(10)); } public Upgrader upgrader() { return upgrader; } @@ -97,6 +99,7 @@ public class ControllerMaintenance extends AbstractComponent { cloudEventReporter.close(); rotationStatusUpdater.close(); resourceTagMaintainer.close(); + systemRoutingPolicyMaintainer.close(); } /** Create one OS upgrader per cloud found in the zone registry of controller */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java new file mode 100644 index 00000000000..6c271ed0470 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java @@ -0,0 +1,40 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; + +import java.time.Duration; + +/** + * This maintains {@link RoutingPolicy}'s for {@link SystemApplication}s. In contrast to regular applications, this + * refreshes policies at an interval, not on deployment. + * + * @author mpolden + */ +public class SystemRoutingPolicyMaintainer extends ControllerMaintainer { + + private final BooleanFlag featureFlag; + + public SystemRoutingPolicyMaintainer(Controller controller, Duration interval) { + super(controller, interval); + this.featureFlag = Flags.CONFIGSERVER_PROVISION_LB.bindTo(controller.flagSource()); + } + + @Override + protected void maintain() { + for (var zone : controller().zoneRegistry().zones().all().ids()) { + for (var application : SystemApplication.values()) { + if (!application.hasEndpoint()) continue; + if (!featureFlag.with(FetchVector.Dimension.ZONE_ID, zone.value()).value()) continue; + controller().routing().policies().refresh(application.id(), DeploymentSpec.empty, zone); + } + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index df6b77ccb9e..1b3c216c1f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -157,10 +157,8 @@ public class UserApiHandler extends LoggingRequestHandler { root.setBool("isPublic", controller.system().isPublic()); root.setBool("isCd", controller.system().isCd()); - - if(enable_public_signup_flow.with(FetchVector.Dimension.CONSOLE_USER_EMAIL, user.email()).value()) { - root.setBool(enable_public_signup_flow.id().toString(), true); - } + root.setBool(enable_public_signup_flow.id().toString(), + enable_public_signup_flow.with(FetchVector.Dimension.CONSOLE_USER_EMAIL, user.email()).value()); toSlime(root.setObject("user"), user); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 033539f64c4..a429c444e0b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -74,15 +75,15 @@ public class RoutingPolicies { * load balancers for given application have changed. */ public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { - var loadBalancers = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer() - .getLoadBalancers(application, zone), - deploymentSpec); + var allocation = new LoadBalancerAllocation(application, zone, controller.serviceRegistry().configServer() + .getLoadBalancers(application, zone), + deploymentSpec); var inactiveZones = inactiveZones(application, deploymentSpec); try (var lock = db.lockRoutingPolicies()) { - removeGlobalDnsUnreferencedBy(loadBalancers, lock); - storePoliciesOf(loadBalancers, lock); - removePoliciesUnreferencedBy(loadBalancers, lock); - updateGlobalDnsOf(get(loadBalancers.deployment.applicationId()).values(), inactiveZones, lock); + removeGlobalDnsUnreferencedBy(allocation, lock); + storePoliciesOf(allocation, lock); + removePoliciesUnreferencedBy(allocation, lock); + updateGlobalDnsOf(get(allocation.deployment.applicationId()).values(), inactiveZones, lock); } } @@ -155,13 +156,13 @@ public class RoutingPolicies { } /** Store routing policies for given load balancers */ - private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var policies = new LinkedHashMap<>(get(loadBalancers.deployment.applicationId())); - for (LoadBalancer loadBalancer : loadBalancers.list) { - var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), loadBalancers.deployment.zoneId()); + private void storePoliciesOf(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) { + var policies = new LinkedHashMap<>(get(allocation.deployment.applicationId())); + for (LoadBalancer loadBalancer : allocation.loadBalancers) { + var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), allocation.deployment.zoneId()); var existingPolicy = policies.get(policyId); var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.dnsZone(), - loadBalancers.endpointIdsOf(loadBalancer), + allocation.endpointIdsOf(loadBalancer), new Status(isActive(loadBalancer), GlobalRouting.DEFAULT_STATUS)); // Preserve global routing status for existing policy if (existingPolicy != null) { @@ -170,53 +171,62 @@ public class RoutingPolicies { updateZoneDnsOf(newPolicy); policies.put(newPolicy.id(), newPolicy); } - db.writeRoutingPolicies(loadBalancers.deployment.applicationId(), policies); + db.writeRoutingPolicies(allocation.deployment.applicationId(), policies); } /** Update zone DNS record for given policy */ private void updateZoneDnsOf(RoutingPolicy policy) { - var name = RecordName.from(policy.endpointIn(controller.system(), RoutingMethod.exclusive).dnsName()); + var name = RecordName.from(policy.endpointIn(controller.system(), RoutingMethod.exclusive, controller.zoneRegistry()) + .dnsName()); var data = RecordData.fqdn(policy.canonicalName().value()); - nameUpdaterIn(policy.id().zone()).createCname(name, data); + NameUpdater nameUpdater = nameUpdaterIn(policy.id().zone()); + if (policy.id().owner().equals(SystemApplication.configServer.id())) { + // TODO(mpolden): Remove this after transition is complete. Before automatic provisioning of config server + // load balancers, the DNS records for the config server LB were of type A. It's not possible + // to change the type of an existing record, we therefore remove the A record before creating + // a CNAME. + nameUpdater.removeRecords(Record.Type.A, name); + } + nameUpdater.createCname(name, data); } /** Remove policies and zone DNS records unreferenced by given load balancers */ - private void removePoliciesUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var policies = get(loadBalancers.deployment.applicationId()); + private void removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) { + var policies = get(allocation.deployment.applicationId()); var newPolicies = new LinkedHashMap<>(policies); - var activeLoadBalancers = loadBalancers.list.stream().map(LoadBalancer::hostname).collect(Collectors.toSet()); + var activeIds = allocation.asPolicyIds(); for (var policy : policies.values()) { // Leave active load balancers and irrelevant zones alone - if (activeLoadBalancers.contains(policy.canonicalName()) || - !policy.id().zone().equals(loadBalancers.deployment.zoneId())) continue; + if (activeIds.contains(policy.id()) || + !policy.id().zone().equals(allocation.deployment.zoneId())) continue; - var dnsName = policy.endpointIn(controller.system(), RoutingMethod.exclusive).dnsName(); - nameUpdaterIn(loadBalancers.deployment.zoneId()).removeRecords(Record.Type.CNAME, RecordName.from(dnsName)); + var dnsName = policy.endpointIn(controller.system(), RoutingMethod.exclusive, controller.zoneRegistry()).dnsName(); + nameUpdaterIn(allocation.deployment.zoneId()).removeRecords(Record.Type.CNAME, RecordName.from(dnsName)); newPolicies.remove(policy.id()); } - db.writeRoutingPolicies(loadBalancers.deployment.applicationId(), newPolicies); + db.writeRoutingPolicies(allocation.deployment.applicationId(), newPolicies); } /** Remove unreferenced global endpoints from DNS */ - private void removeGlobalDnsUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var zonePolicies = get(loadBalancers.deployment).values(); + private void removeGlobalDnsUnreferencedBy(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) { + var zonePolicies = get(allocation.deployment).values(); var removalCandidates = new HashSet<>(routingTableFrom(zonePolicies).keySet()); - var activeRoutingIds = routingIdsFrom(loadBalancers); + var activeRoutingIds = routingIdsFrom(allocation); removalCandidates.removeAll(activeRoutingIds); for (var id : removalCandidates) { var endpoints = controller.routing().endpointsOf(id.application()) .not().requiresRotation() .named(id.endpointId()); - var nameUpdater = nameUpdaterIn(loadBalancers.deployment.zoneId()); + var nameUpdater = nameUpdaterIn(allocation.deployment.zoneId()); endpoints.forEach(endpoint -> nameUpdater.removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()))); } } /** Compute routing IDs from given load balancers */ - private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) { + private static Set<RoutingId> routingIdsFrom(LoadBalancerAllocation allocation) { Set<RoutingId> routingIds = new LinkedHashSet<>(); - for (var loadBalancer : loadBalancers.list) { - for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) { + for (var loadBalancer : allocation.loadBalancers) { + for (var endpointId : allocation.endpointIdsOf(loadBalancer)) { routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); } } @@ -257,19 +267,28 @@ public class RoutingPolicies { } /** Load balancers allocated to a deployment */ - private static class AllocatedLoadBalancers { + private static class LoadBalancerAllocation { private final DeploymentId deployment; - private final List<LoadBalancer> list; + private final List<LoadBalancer> loadBalancers; private final DeploymentSpec deploymentSpec; - private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, + private LoadBalancerAllocation(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, DeploymentSpec deploymentSpec) { this.deployment = new DeploymentId(application, zone); - this.list = List.copyOf(loadBalancers); + this.loadBalancers = List.copyOf(loadBalancers); this.deploymentSpec = deploymentSpec; } + /** Returns the policy IDs of the load balancers contained in this */ + private Set<RoutingPolicyId> asPolicyIds() { + return loadBalancers.stream() + .map(lb -> new RoutingPolicyId(lb.application(), + lb.cluster(), + deployment.zoneId())) + .collect(Collectors.toUnmodifiableSet()); + } + /** Compute all endpoint IDs for given load balancer */ private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { if (!deployment.zoneId().environment().isProduction()) { // Only production deployments have configurable endpoints diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 37027e8a8c9..c56a5f0bd66 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -5,9 +5,11 @@ import com.google.common.collect.ImmutableSortedSet; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import java.util.Objects; import java.util.Optional; @@ -68,7 +70,10 @@ public class RoutingPolicy { } /** Returns the endpoint of this */ - public Endpoint endpointIn(SystemName system, RoutingMethod routingMethod) { + public Endpoint endpointIn(SystemName system, RoutingMethod routingMethod, ZoneRegistry zoneRegistry) { + Optional<Endpoint> infraEndpoint = SystemApplication.matching(id.owner()) + .flatMap(app -> app.endpointIn(id.zone(), zoneRegistry)); + if (infraEndpoint.isPresent()) return infraEndpoint.get(); return Endpoint.of(id.owner()) .target(id.cluster(), id.zone()) .on(Port.fromRoutingMethod(routingMethod)) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java new file mode 100644 index 00000000000..a908b341039 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.security; + +import com.yahoo.vespa.hosted.controller.api.role.Role; + +import java.security.Principal; +import java.util.Collections; +import java.util.Set; + +/** + * Like {@link Credentials}, but we know the principal is authenticated by Auth0. + * Also includes the set of roles for which the principal is a member. + * + * @author andreer + */ +public class Auth0Credentials extends Credentials { + + private final Set<Role> roles; + + public Auth0Credentials(Principal user, Set<Role> roles) { + super(user); + this.roles = Collections.unmodifiableSet(roles); + } + + /** The set of roles set in the auth0 cookie, extracted by CloudAccessControlRequests. */ + public Set<Role> getRolesFromCookie() { + return roles; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index f34c9c67baa..dc3dbabcc07 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -3,6 +3,10 @@ package com.yahoo.vespa.hosted.controller.security; import com.google.inject.Inject; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; @@ -15,24 +19,32 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; +import javax.ws.rs.ForbiddenException; import java.util.List; +import static com.yahoo.vespa.hosted.controller.api.role.RoleDefinition.*; + /** * @author jonmv + * @author andreer */ public class CloudAccessControl implements AccessControl { private static final BillingInfo defaultBillingInfo = new BillingInfo("customer", "Vespa"); private final UserManagement userManagement; + private final BooleanFlag enablePublicSignup; @Inject - public CloudAccessControl(UserManagement userManagement) { + public CloudAccessControl(UserManagement userManagement, FlagSource flagSource) { this.userManagement = userManagement; + this.enablePublicSignup = Flags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource); } @Override public CloudTenant createTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing) { + requireTenantCreationAllowed((Auth0Credentials) credentials); + CloudTenantSpec spec = (CloudTenantSpec) tenantSpec; CloudTenant tenant = CloudTenant.create(spec.tenant(), defaultBillingInfo); @@ -48,6 +60,36 @@ public class CloudAccessControl implements AccessControl { return tenant; } + private void requireTenantCreationAllowed(Auth0Credentials auth0Credentials) { + if (allowedByPrivilegedRole(auth0Credentials)) return; + + if (!allowedByFeatureFlag(auth0Credentials)) { + throw new ForbiddenException("You are not currently permitted to create tenants. Please contact the Vespa team to request access."); + } + + if(administeredTenants(auth0Credentials) >= 3) { + throw new ForbiddenException("You are already administering 3 tenants. If you need more, please contact the Vespa team."); + } + } + + private boolean allowedByPrivilegedRole(Auth0Credentials auth0Credentials) { + return auth0Credentials.getRolesFromCookie().stream() + .map(Role::definition) + .anyMatch(rd -> rd == hostedOperator || rd == hostedSupporter); + } + + private boolean allowedByFeatureFlag(Auth0Credentials auth0Credentials) { + return enablePublicSignup.with(FetchVector.Dimension.CONSOLE_USER_EMAIL, auth0Credentials.user().getName()).value(); + } + + private long administeredTenants(Auth0Credentials auth0Credentials) { + // We have to verify the roles with auth0 to ensure the user is not using an "old" cookie to make too many tenants. + return userManagement.listRoles(new UserId(auth0Credentials.user().getName())).stream() + .map(Role::definition) + .filter(rd -> rd == administrator) + .count(); + } + @Override public Tenant updateTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing, List<Application> applications) { throw new UnsupportedOperationException("Update is not supported here, as it would entail changing the tenant name."); @@ -55,7 +97,7 @@ public class CloudAccessControl implements AccessControl { @Override public void deleteTenant(TenantName tenant, Credentials credentials) { - // Probably terminate customer subscription? + // TODO: allow only if 0 resources, 0 balance for (TenantRole role : Roles.tenantRoles(tenant)) userManagement.deleteRole(role); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java index c0f8f585216..cc26ed1427a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java @@ -4,25 +4,37 @@ package com.yahoo.vespa.hosted.controller.security; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.slime.Inspector; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import java.util.Optional; +import java.util.Set; + /** * Extracts access control data for {@link CloudTenant}s from HTTP requests. * * @author jonmv + * @author andreer */ public class CloudAccessControlRequests implements AccessControlRequests { @Override public CloudTenantSpec specification(TenantName tenant, Inspector requestObject) { - // TODO extract marketplace token. - return new CloudTenantSpec(tenant, "token"); + return new CloudTenantSpec(tenant, "token"); // TODO: remove token } @Override public Credentials credentials(TenantName tenant, Inspector requestObject, HttpRequest request) { - // TODO Include roles, if this is to be used for displaying accessible data. - return new Credentials(request.getUserPrincipal()); + return new Auth0Credentials(request.getUserPrincipal(), getUserRoles(request)); + } + + private static Set<Role> getUserRoles(HttpRequest request) { + var securityContext = Optional.ofNullable(request.context().get(SecurityContext.ATTRIBUTE_NAME)) + .filter(SecurityContext.class::isInstance) + .map(SecurityContext.class::cast) + .orElseThrow(() -> new IllegalArgumentException("Attribute '" + SecurityContext.ATTRIBUTE_NAME + "' was not set on request")); + return securityContext.roles(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 189615ad763..35093c22f42 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -41,6 +41,7 @@ import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; +import com.yahoo.vespa.hosted.controller.security.Auth0Credentials; import com.yahoo.vespa.hosted.controller.security.CloudTenantSpec; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; @@ -54,6 +55,7 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.OptionalLong; @@ -302,7 +304,7 @@ public final class ControllerTester { private TenantName createCloudTenant(String tenantName) { TenantName tenant = TenantName.from(tenantName); TenantSpec spec = new CloudTenantSpec(tenant, "token"); - controller().tenants().create(spec, new Credentials(new SimplePrincipal("dev"))); + controller().tenants().create(spec, new Auth0Credentials(new SimplePrincipal("dev"), Collections.emptySet())); return tenant; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java new file mode 100644 index 00000000000..8d6316d447f --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java @@ -0,0 +1,61 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +/** + * @author mpolden + */ +public class SystemRoutingPolicyMaintainerTest { + + @Test + public void maintain() { + var tester = new ControllerTester(); + var updater = new SystemRoutingPolicyMaintainer(tester.controller(), Duration.ofDays(1)); + var dispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofDays(1), Integer.MAX_VALUE); + + var zone = ZoneId.from("prod", "us-west-1"); + tester.zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(zone)); + tester.configServer().putLoadBalancers(zone, List.of(new LoadBalancer("lb1", + SystemApplication.configServer.id(), + ClusterSpec.Id.from("config"), + HostName.from("lb1.example.com"), + LoadBalancer.State.active, + Optional.of("dns-zone-1")))); + + // Nothing happens without feature flag + updater.run(); + dispatcher.run(); + assertEquals(Set.of(), tester.nameService().records()); + + // Record is created + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.CONFIGSERVER_PROVISION_LB.id(), true); + updater.run(); + dispatcher.run(); + Set<Record> records = tester.nameService().records(); + assertEquals(1, records.size()); + Record record = records.iterator().next(); + assertSame(Record.Type.CNAME, record.type()); + assertEquals("cfg.prod.us-west-1.test.vip", record.name().asString()); + assertEquals("lb1.example.com.", record.data().asString()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 5c5dc4b5fe6..a1b06262241 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -12,11 +14,13 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; +import com.yahoo.vespa.hosted.controller.security.Auth0Credentials; import com.yahoo.vespa.hosted.controller.security.CloudTenantSpec; import com.yahoo.vespa.hosted.controller.security.Credentials; import org.junit.Before; import org.junit.Test; +import java.util.Collections; import java.util.Set; import static com.yahoo.application.container.handler.Request.Method.POST; @@ -36,6 +40,8 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { @Before public void before() { tester = new ContainerTester(container, responseFiles); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withBooleanFlag(Flags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); deploymentTester = new DeploymentTester(new ControllerTester(tester)); deploymentTester.controllerTester().computeVersionStatus(); } @@ -73,6 +79,6 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { } private static Credentials credentials(String name) { - return new Credentials(() -> name); + return new Auth0Credentials(() -> name, Collections.emptySet()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index ecb5c319f44..acd542b001c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -52,6 +52,9 @@ "name": "RotationStatusUpdater" }, { + "name": "SystemRoutingPolicyMaintainer" + }, + { "name": "SystemUpgrader" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index c066c50ca20..d9ad30020db 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.controller.restapi.user; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -62,7 +64,7 @@ public class UserApiTest extends ControllerContainerCloudTest { // POST a tenant is not available to everyone. tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) .data("{\"token\":\"hello\"}"), - accessDenied, 403); + "{\"error-code\":\"FORBIDDEN\",\"message\":\"You are not currently permitted to create tenants. Please contact the Vespa team to request access.\"}", 403); // POST a tenant is available to operators. tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) @@ -200,6 +202,8 @@ public class UserApiTest extends ControllerContainerCloudTest { @Test public void userMetadataTest() { ContainerTester tester = new ContainerTester(container, responseFiles); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withBooleanFlag(Flags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); ControllerTester controller = new ControllerTester(tester); Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter(), Role.hostedAccountant()); User user = new User("dev@domail", "Joe Developer", "dev", null); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json index a69b4d2a062..36918c743fa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json @@ -1,6 +1,7 @@ { "isPublic": false, "isCd": false, + "enable-public-signup-flow": (ignore), "user": { "name": "Joe Developer", "email": "dev@domail", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json index 7d4e01073e0..27398352e53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json @@ -1,6 +1,7 @@ { "isPublic": true, "isCd": false, +"enable-public-signup-flow": (ignore), "user": { "name": "Joe Developer", "email": "dev@domail", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json index bc702f02150..b62a70d1871 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json @@ -1,6 +1,7 @@ { "isPublic": (ignore), "isCd": (ignore), + "enable-public-signup-flow": (ignore), "user": { "name": "Joe Developer", "email": "dev@domail", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index e44a1364185..760d80d230c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; @@ -138,13 +139,18 @@ public class RoutingPoliciesTest { @Test public void zone_routing_policies() { + zone_routing_policies(false); + zone_routing_policies(true); + } + + private void zone_routing_policies(boolean sharedRoutingLayer) { var tester = new RoutingPoliciesTester(); var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); // Deploy application int clustersPerZone = 2; - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Deployment creates records and policies for all clusters in all zones @@ -163,7 +169,7 @@ public class RoutingPoliciesTest { assertEquals(4, tester.policiesOf(context1.instanceId()).size()); // Add 1 cluster in each zone and deploy - tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), zone1, zone2); + tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), sharedRoutingLayer, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -177,7 +183,7 @@ public class RoutingPoliciesTest { assertEquals(6, tester.policiesOf(context1.instanceId()).size()); // Deploy another application - tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), zone1, zone2); + tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), sharedRoutingLayer, zone1, zone2); context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -195,7 +201,7 @@ public class RoutingPoliciesTest { assertEquals(4, tester.policiesOf(context2.instanceId()).size()); // Deploy removes cluster from app1 - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -574,6 +580,23 @@ public class RoutingPoliciesTest { tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); } + @Test + public void config_server_routing_policy() { + var tester = new RoutingPoliciesTester(); + var app = SystemApplication.configServer.id(); + RecordName name = RecordName.from("cfg.prod.us-west-1.test.vip"); + tester.controllerTester().nameService().add(new Record(Record.Type.A, name, RecordData.from("192.0.2.1"))); + + tester.provisionLoadBalancers(1, app, zone1); + tester.routingPolicies().refresh(app, DeploymentSpec.empty, zone1); + new NameServiceDispatcher(tester.tester.controller(), Duration.ofDays(1), Integer.MAX_VALUE).run(); + + List<Record> records = tester.controllerTester().nameService().findRecords(Record.Type.CNAME, name); + assertEquals(1, records.size()); + assertEquals(RecordData.from("lb-0--hosted-vespa:zone-config-servers:default--prod.us-west-1."), + records.get(0).data()); + } + /** Returns an application package builder that satisfies requirements for a directly routed endpoint */ private static ApplicationPackageBuilder applicationPackageBuilder() { return new ApplicationPackageBuilder() @@ -581,15 +604,21 @@ public class RoutingPoliciesTest { .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION); } - private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count) { + private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, boolean shared, int count) { List<LoadBalancer> loadBalancers = new ArrayList<>(); for (int i = 0; i < count; i++) { + HostName lbHostname; + if (shared) { + lbHostname = HostName.from("shared-lb--" + zone.value()); + } else { + lbHostname = HostName.from("lb-" + i + "--" + application.serializedForm() + + "--" + zone.value()); + } loadBalancers.add( new LoadBalancer("LB-" + i + "-Z-" + zone.value(), application, ClusterSpec.Id.from("c" + i), - HostName.from("lb-" + i + "--" + application.serializedForm() + - "--" + zone.value()), + lbHostname, LoadBalancer.State.active, Optional.of("dns-zone-1"))); } @@ -626,13 +655,17 @@ public class RoutingPoliciesTest { tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones()); } - private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { + private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, boolean shared, ZoneId... zones) { for (ZoneId zone : zones) { tester.configServer().removeLoadBalancers(application, zone); - tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone)); + tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, shared, clustersPerZone)); } } + private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { + provisionLoadBalancers(clustersPerZone, application, false, zones); + } + private Collection<RoutingPolicy> policiesOf(ApplicationId instance) { return tester.controller().curator().readRoutingPolicies(instance).values(); } diff --git a/default_build_settings.cmake b/default_build_settings.cmake index cc51bbde852..5ba7494d1e1 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -31,7 +31,7 @@ endfunction() function(setup_vespa_default_build_settings_centos_8) message("-- Setting up default build settings for centos 8") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_darwin) diff --git a/dist/vespa.spec b/dist/vespa.spec index 690d4123de4..1e1658d3d55 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -62,11 +62,7 @@ BuildRequires: vespa-icu-devel >= 65.1.0-1 %endif %if 0%{?el8} BuildRequires: cmake >= 3.11.4-3 -%if 0%{?centos} -BuildRequires: llvm-devel >= 8.0.1 -%else BuildRequires: llvm-devel >= 9.0.1 -%endif BuildRequires: boost-devel >= 1.66 BuildRequires: openssl-devel BuildRequires: vespa-gtest >= 1.8.1-1 @@ -136,7 +132,9 @@ Requires: perl-LWP-Protocol-https Requires: perl-Net-INET6Glue Requires: perl-Pod-Usage Requires: perl-URI +%if ! 0%{?el7} Requires: valgrind +%endif Requires: Judy Requires: xxhash Requires: xxhash-libs >= 0.7.3 @@ -161,18 +159,14 @@ Requires: vespa-openssl >= 1.1.1g-1 Requires: vespa-icu >= 65.1.0-1 Requires: vespa-protobuf >= 3.7.0-4 Requires: vespa-telegraf >= 1.1.1-1 +Requires: vespa-valgrind >= 3.16.0-1 %define _vespa_llvm_version 7 %define _extra_link_directory /usr/lib64/llvm7.0/lib;%{_vespa_deps_prefix}/lib64 %define _extra_include_directory /usr/include/llvm7.0;%{_vespa_deps_prefix}/include;/usr/include/openblas %endif %if 0%{?el8} -%if 0%{?centos} -Requires: llvm-libs >= 8.0.1 -%define _vespa_llvm_version 8 -%else Requires: llvm-libs >= 9.0.1 %define _vespa_llvm_version 9 -%endif Requires: vespa-protobuf >= 3.7.0-4 Requires: openssl-libs %define _extra_link_directory %{_vespa_deps_prefix}/lib64 @@ -596,6 +590,7 @@ fi %{_prefix}/lib/jars/docprocs-jar-with-dependencies.jar %{_prefix}/lib/jars/flags-jar-with-dependencies.jar %{_prefix}/lib/jars/hk2-*.jar +%{_prefix}/lib/jars/hosted-zone-api-jar-with-dependencies.jar %{_prefix}/lib/jars/jackson-*.jar %{_prefix}/lib/jars/javassist-*.jar %{_prefix}/lib/jars/javax.*.jar diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java index e825db4e21d..e1482dead8d 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java @@ -239,7 +239,7 @@ public class ProxyDocument extends Document implements DocumentOperationWrapper @Override @SuppressWarnings("deprecation") public Struct getBody() { - return doc.getBody(); + return null; } @Override diff --git a/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java index 96ea96b9e78..617cd01e6e1 100644 --- a/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java +++ b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java @@ -306,7 +306,6 @@ public class SchemaMappingAndAccessesTest { assertEquals(mapped.getId().toString(), "id:map:album::2"); assertEquals(doc.getId().toString(), "id:map:album::2"); assertEquals(doc.getHeader(), mapped.getHeader()); - assertEquals(doc.getBody(), mapped.getBody()); assertEquals(doc.getSerializedSize(), mapped.getSerializedSize()); ByteArrayOutputStream bos = new ByteArrayOutputStream(); ByteArrayOutputStream bos2 = new ByteArrayOutputStream(); diff --git a/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg b/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg index 3347c3127b5..65ce1b56811 100644 --- a/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg +++ b/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg @@ -4,16 +4,15 @@ datatype[0].id -1407012075 datatype[0].structtype[1] datatype[0].structtype[0].name "outerdoc.body" datatype[0].structtype[0].version 0 -datatype[0].structtype[0].field[1] -datatype[0].structtype[0].field[0].datatype -2035324352 -datatype[0].structtype[0].field[0].name "innerdocuments" datatype[1].id -1686125086 datatype[1].structtype[1] datatype[1].structtype[0].name "docindoc.header" datatype[1].structtype[0].version 0 -datatype[1].structtype[0].field[1] +datatype[1].structtype[0].field[2] datatype[1].structtype[0].field[0].datatype 2 datatype[1].structtype[0].field[0].name "name" +datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].name "content" datatype[2].id -2035324352 datatype[2].arraytype[1] datatype[2].arraytype[0].datatype 1447635645 @@ -21,6 +20,9 @@ datatype[3].id -2040625920 datatype[3].structtype[1] datatype[3].structtype[0].name "outerdoc.header" datatype[3].structtype[0].version 0 +datatype[3].structtype[0].field[1] +datatype[3].structtype[0].field[0].datatype -2035324352 +datatype[3].structtype[0].field[0].name "innerdocuments" datatype[4].id 1447635645 datatype[4].documenttype[1] datatype[4].documenttype[0].bodystruct 2030224503 @@ -37,6 +39,3 @@ datatype[6].id 2030224503 datatype[6].structtype[1] datatype[6].structtype[0].name "docindoc.body" datatype[6].structtype[0].version 0 -datatype[6].structtype[0].field[1] -datatype[6].structtype[0].field[0].datatype 2 -datatype[6].structtype[0].field[0].name "content" diff --git a/document/abi-spec.json b/document/abi-spec.json index 3764015b917..7a0637db1aa 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -426,7 +426,9 @@ ], "methods": [ "public void <init>(java.lang.String)", + "public void <init>(java.lang.String, com.yahoo.document.StructDataType)", "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType)", + "public void <init>(java.lang.String, com.yahoo.document.StructDataType, java.util.Set)", "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType, java.util.Set)", "public void <init>(java.lang.String, java.util.Set)", "public com.yahoo.document.DocumentType clone()", @@ -435,7 +437,6 @@ "public boolean isValueCompatible(com.yahoo.document.datatypes.FieldValue)", "public com.yahoo.document.StructDataType contentStruct()", "public com.yahoo.document.StructDataType getHeaderType()", - "public com.yahoo.document.StructDataType getBodyType()", "protected void register(com.yahoo.document.DocumentTypeManager, java.util.List)", "public boolean isA(java.lang.String)", "public void addField(com.yahoo.document.Field)", diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java index 375d8d962a5..568fe9265d5 100644 --- a/document/src/main/java/com/yahoo/document/Document.java +++ b/document/src/main/java/com/yahoo/document/Document.java @@ -44,7 +44,6 @@ public class Document extends StructuredFieldValue { public static final short SERIALIZED_VERSION = 8; private DocumentId docId; private Struct header; - private Struct body; private Long lastModified = null; /** @@ -75,7 +74,6 @@ public class Document extends StructuredFieldValue { public Document(Document doc) { this(doc.getDataType(), doc.getId()); header = doc.header; - body = doc.body; lastModified = doc.lastModified; } @@ -104,7 +102,7 @@ public class Document extends StructuredFieldValue { /** @deprecated do not use: Use getField(), getFieldValue() or iterator() instead */ @Deprecated // TODO: Remove on Vespa 8 - public Struct getBody() { return body; } + public Struct getBody() { return null; } @Override public void assign(Object o) { @@ -116,14 +114,11 @@ public class Document extends StructuredFieldValue { Document doc = (Document) super.clone(); doc.docId = docId.clone(); doc.header = header.clone(); - doc.body = body.clone(); return doc; } - @SuppressWarnings("deprecation") private void setNewType(DocumentType type) { header = type.contentStruct().createFieldValue(); - body = type.getBodyType().createFieldValue(); } public void setDataType(DataType type) { @@ -178,9 +173,6 @@ public class Document extends StructuredFieldValue { public Field getField(String fieldName) { Field field = header.getField(fieldName); if (field == null) { - field = body.getField(fieldName); - } - if (field == null) { for(DocumentType parent : getDataType().getInheritedTypes()) { field = parent.getField(fieldName); if (field != null) { @@ -193,68 +185,27 @@ public class Document extends StructuredFieldValue { @Override public FieldValue getFieldValue(Field field) { - FieldValue fv = header.getFieldValue(field); - if (fv == null) { - fv = body.getFieldValue(field); - } - return fv; + return header.getFieldValue(field); } @Override - @SuppressWarnings("deprecation") protected void doSetFieldValue(Field field, FieldValue value) { - if (field.isHeader()) { - header.setFieldValue(field, value); - } else { - body.setFieldValue(field, value); - } + header.setFieldValue(field, value); } @Override public FieldValue removeFieldValue(Field field) { - FieldValue removed = header.removeFieldValue(field); - if (removed == null) { - removed = body.removeFieldValue(field); - } - return removed; + return header.removeFieldValue(field); } @Override public void clear() { header.clear(); - body.clear(); } @Override public Iterator<Map.Entry<Field, FieldValue>> iterator() { - return new Iterator<>() { - - private Iterator<Map.Entry<Field, FieldValue>> headerIt = header.iterator(); - private Iterator<Map.Entry<Field, FieldValue>> bodyIt = body.iterator(); - - public boolean hasNext() { - if (headerIt != null) { - if (headerIt.hasNext()) { - return true; - } else { - headerIt = null; - } - } - return bodyIt.hasNext(); - } - - public Map.Entry<Field, FieldValue> next() { - return (headerIt == null ? bodyIt.next() : headerIt.next()); - } - - public void remove() { - if (headerIt == null) { - bodyIt.remove(); - } else { - headerIt.remove(); - } - } - }; + return header.iterator(); } public String toString() { @@ -302,7 +253,7 @@ public class Document extends StructuredFieldValue { if (!(o instanceof Document)) return false; Document other = (Document) o; return (super.equals(o) && docId.equals(other.docId) && - header.equals(other.header) && body.equals(other.body)); + header.equals(other.header)); } @Override @@ -347,7 +298,7 @@ public class Document extends StructuredFieldValue { @Override public int getFieldCount() { - return header.getFieldCount() + body.getFieldCount(); + return header.getFieldCount(); } public void serialize(DocumentWriter writer) { @@ -393,7 +344,6 @@ public class Document extends StructuredFieldValue { return comp; } - comp = body.compareTo(otherValue.body); return comp; } diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java index 23559878fbb..f73fd634e0e 100755 --- a/document/src/main/java/com/yahoo/document/DocumentType.java +++ b/document/src/main/java/com/yahoo/document/DocumentType.java @@ -38,7 +38,6 @@ public class DocumentType extends StructuredDataType { public static final String DOCUMENT = "[document]"; public static final int classId = registerClass(Ids.document + 58, DocumentType.class); private StructDataType headerType; - private StructDataType bodyType; private List<DocumentType> inherits = new ArrayList<>(1); private Map<String, Set<Field>> fieldSets = new HashMap<>(); private final Set<String> importedFieldNames; @@ -52,7 +51,7 @@ public class DocumentType extends StructuredDataType { * @param name The name of the new document type */ public DocumentType(String name) { - this(name, createHeaderStructType(name), createBodyStructType(name)); + this(name, createHeaderStructType(name)); } /** @@ -62,37 +61,46 @@ public class DocumentType extends StructuredDataType { * * @param name The name of the new document type * @param headerType The type of the header struct - * @param bodyType The type of the body struct */ + public DocumentType(String name, StructDataType headerType) { + this(name, headerType, Collections.emptySet()); + } + + /** + * @deprecated //TODO Will be removed on Vespa 8 + */ + @Deprecated public DocumentType(String name, StructDataType headerType, StructDataType bodyType) { - this(name, headerType, bodyType, Collections.emptySet()); + this(name, headerType, Collections.emptySet()); } - public DocumentType(String name, StructDataType headerType, - StructDataType bodyType, Set<String> importedFieldNames) { + public DocumentType(String name, StructDataType headerType, Set<String> importedFieldNames) { super(name); this.headerType = headerType; - this.bodyType = bodyType; this.importedFieldNames = Collections.unmodifiableSet(importedFieldNames); } + /** + * @deprecated //TODO Will be removed on Vespa 8 + */ + @Deprecated + public DocumentType(String name, StructDataType headerType, + StructDataType bodyType, Set<String> importedFieldNames) { + this(name, headerType, importedFieldNames); + } + public DocumentType(String name, Set<String> importedFieldNames) { - this(name, createHeaderStructType(name), createBodyStructType(name), importedFieldNames); + this(name, createHeaderStructType(name), importedFieldNames); } private static StructDataType createHeaderStructType(String name) { return new StructDataType(name + ".header"); } - private static StructDataType createBodyStructType(String name) { - return new StructDataType(name + ".body"); - } - @Override public DocumentType clone() { DocumentType type = (DocumentType) super.clone(); type.headerType = headerType.clone(); - type.bodyType = bodyType.clone(); type.inherits = new ArrayList<>(inherits.size()); type.inherits.addAll(inherits); return type; @@ -136,14 +144,7 @@ public class DocumentType extends StructuredDataType { return contentStruct(); } - @Deprecated // TODO: Remove on Vespa 8 - /** @deprecated use contentStruct instead */ - public StructDataType getBodyType() { - return bodyType; - } - @Override - @SuppressWarnings("deprecation") protected void register(DocumentTypeManager manager, List<DataType> seenTypes) { seenTypes.add(this); for (DocumentType type : getInheritedTypes()) { @@ -153,23 +154,17 @@ public class DocumentType extends StructuredDataType { } // Get parent fields into fields specified in this type StructDataType header = headerType.clone(); - StructDataType body = bodyType.clone(); header.clearFields(); - body.clearFields(); for (Field field : getAllUniqueFields()) { - (field.isHeader() ? header : body).addField(field); + header.addField(field); } headerType.assign(header); - bodyType.assign(body); if (!seenTypes.contains(headerType)) { headerType.register(manager, seenTypes); } - if (!seenTypes.contains(bodyType)) { - bodyType.register(manager, seenTypes); - } manager.registerSingleType(this); } @@ -194,7 +189,6 @@ public class DocumentType extends StructuredDataType { * * @param field the field to add */ - @SuppressWarnings("deprecation") public void addField(Field field) { if (isRegistered()) { throw new IllegalStateException("You cannot add fields to a document type that is already registered."); @@ -227,7 +221,7 @@ public class DocumentType extends StructuredDataType { } /** - * Adds a new body field to this document type and returns the new field object + * Adds a new field to this document type and returns the new field object * * @param name The name of the field to add * @param type The datatype of the field to add @@ -346,9 +340,6 @@ public class DocumentType extends StructuredDataType { */ public Field getField(String name) { Field field = headerType.getField(name); - if (field == null) { - field = bodyType.getField(name); - } if (field == null && !isRegistered()) { for (DocumentType inheritedType : inherits) { field = inheritedType.getField(name); @@ -361,9 +352,6 @@ public class DocumentType extends StructuredDataType { @Override public Field getField(int id) { Field field = headerType.getField(id); - if (field == null) { - field = bodyType.getField(id); - } if (field == null && !isRegistered()) { for (DocumentType inheritedType : inherits) { field = inheritedType.getField(id); @@ -384,7 +372,7 @@ public class DocumentType extends StructuredDataType { } public int getFieldCount() { - return headerType.getFieldCount() + bodyType.getFieldCount(); + return headerType.getFieldCount(); } public Set<String> getImportedFieldNames() { @@ -407,9 +395,6 @@ public class DocumentType extends StructuredDataType { } Field field = headerType.removeField(name); if (field == null) { - field = bodyType.removeField(name); - } - if (field == null) { for (DocumentType inheritedType : inherits) { field = inheritedType.removeField(name); if (field != null) break; @@ -433,7 +418,6 @@ public class DocumentType extends StructuredDataType { } collection.addAll(headerType.getFields()); - collection.addAll(bodyType.getFields()); return ImmutableList.copyOf(collection); } @@ -483,39 +467,14 @@ public class DocumentType extends StructuredDataType { * @return An iterator for iterating the fields in this documenttype. */ public Iterator<Field> fieldIteratorThisTypeOnly() { - return new Iterator<>() { - Iterator<Field> headerIt = headerType.getFields().iterator(); - Iterator<Field> bodyIt = bodyType.getFields().iterator(); - - public boolean hasNext() { - if (headerIt != null) { - if (headerIt.hasNext()) return true; - headerIt = null; - } - return bodyIt.hasNext(); - } - - public Field next() { - return (headerIt != null ? headerIt.next() : bodyIt.next()); - } - - - public void remove() { - if (headerIt != null) { - headerIt.remove(); - } else { - bodyIt.remove(); - } - } - }; + return headerType.getFields().iterator(); } public boolean equals(Object o) { if (!(o instanceof DocumentType)) return false; DocumentType other = (DocumentType) o; // Ignore whether one of them have added inheritance to super Document.0 type - if (super.equals(o) && headerType.equals(other.headerType) && - bodyType.equals(other.bodyType)) { + if (super.equals(o) && headerType.equals(other.headerType)) { if ((inherits.size() > 1 || other.inherits.size() > 1) || (inherits.size() == 1 && other.inherits.size() == 1)) { return inherits.equals(other.inherits); @@ -527,7 +486,7 @@ public class DocumentType extends StructuredDataType { } public int hashCode() { - return super.hashCode() + headerType.hashCode() + bodyType.hashCode() + inherits.hashCode(); + return super.hashCode() + headerType.hashCode() + inherits.hashCode(); } @Override @@ -543,7 +502,6 @@ public class DocumentType extends StructuredDataType { public void visitMembers(ObjectVisitor visitor) { super.visitMembers(visitor); visitor.visit("headertype", headerType); - visitor.visit("bodytype", bodyType); visitor.visit("inherits", inherits); } } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 1d81d9e6e78..c802d2307c0 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -142,14 +142,10 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub @SuppressWarnings("deprecation") private static void registerDocumentType(DocumentTypeManager manager, DocumentmanagerConfig.Datatype.Documenttype doc) { StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct(), ""); - StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct(), ""); - for (Field field : body.getFields()) { - field.setHeader(false); - } var importedFields = doc.importedfield().stream() .map(f -> f.name()) .collect(Collectors.toUnmodifiableSet()); - DocumentType type = new DocumentType(doc.name(), header, body, importedFields); + DocumentType type = new DocumentType(doc.name(), header, importedFields); for (Object j : doc.inherits()) { DocumentmanagerConfig.Datatype.Documenttype.Inherits parent = (DocumentmanagerConfig.Datatype.Documenttype.Inherits) j; diff --git a/document/src/main/java/com/yahoo/document/TestAndSetCondition.java b/document/src/main/java/com/yahoo/document/TestAndSetCondition.java index def1c05a011..6a189fc2969 100644 --- a/document/src/main/java/com/yahoo/document/TestAndSetCondition.java +++ b/document/src/main/java/com/yahoo/document/TestAndSetCondition.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import com.google.common.annotations.Beta; - import java.util.Optional; /** @@ -14,7 +12,6 @@ import java.util.Optional; * * @author Vegard Sjonfjell */ -@Beta public class TestAndSetCondition { public static final TestAndSetCondition NOT_PRESENT_CONDITION = new TestAndSetCondition(); diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java index 27327daab47..cac05fb7879 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java @@ -118,14 +118,12 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu doc.setId(documentId); Struct h = doc.getHeader(); - Struct b = doc.getBody(); h.clear(); - b.clear(); if ((content & 0x2) != 0) { - readHeaderBody(h, b); + readHeaderBody(h); } if ((content & 0x4) != 0) { - readHeaderBody(b, h); + readHeaderBody(h); } if (dataLength != (position() - dataPos)) { @@ -326,7 +324,7 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu buf = bigBuf; } - private void readHeaderBody(Struct primary, Struct alternate) { + private void readHeaderBody(Struct primary) { primary.setVersion(version); if (version < 8) { @@ -371,24 +369,14 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu buf = GrowableByteBuffer.wrap(destination); StructDataType priType = primary.getDataType(); - StructDataType altType = alternate.getDataType(); for (int i=0; i<numberOfFields; ++i) { int posBefore = position(); - Struct s = null; Integer f_id = fieldIdsAndLengths.get(i).first; Field structField = priType.getField(f_id); if (structField != null) { - s = primary; - } else { - structField = altType.getField(f_id); - if (structField != null) { - s = alternate; - } - } - if (s != null) { FieldValue value = structField.getDataType().createFieldValue(); value.deserialize(structField, this); - s.setFieldValue(structField, value); + primary.setFieldValue(structField, value); } //jump to beginning of next field: position(posBefore + fieldIdsAndLengths.get(i).second.intValue()); diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index 630f204c44d..3fca853b4d1 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -96,27 +96,20 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume doc.getId().serialize(this); - Struct head = doc.getHeader(); - Struct body = doc.getBody(); - boolean hasHead = (head.getFieldCount() != 0); - boolean hasBody = (body.getFieldCount() != 0); + boolean hasHead = (doc.getFieldCount() != 0); byte contents = 0x01; // Indicating we have document type which we always have if (hasHead) { contents |= 0x2; // Indicate we have header } - if (hasBody) { - contents |= 0x4; // Indicate we have a body - } + buf.put(contents); doc.getDataType().serialize(this); if (hasHead) { - head.serialize(null, this); - } - if (hasBody) { - body.serialize(null, this); + doc.getHeader().serialize(null, this); } + int finalPos = buf.position(); buf.position(lenPos); buf.putInt(finalPos - lenPos - 4); // Don't include the length itself or the version diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java index 5db98f26141..9dc5b7c2480 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java @@ -106,7 +106,6 @@ public final class XmlDocumentWriter implements DocumentWriter { buffer.addAttribute("lastmodifiedtime", lastModified); } write(null, value.getHeader()); - write(null, value.getBody()); buffer.endTag(); } diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java index 85bc4d032ff..8c6444fb853 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java @@ -55,7 +55,6 @@ public class XmlSerializationHelper { xml.addAttribute("lastmodifiedtime", lastModified); } doc.getHeader().printXml(xml); - doc.getBody().printXml(xml); } public static void printDoubleXml(DoubleFieldValue d, XmlStream xml) { diff --git a/document/src/test/document/documentmanager.cfg b/document/src/test/document/documentmanager.cfg index d77c2b17460..e4c581304ce 100644 --- a/document/src/test/document/documentmanager.cfg +++ b/document/src/test/document/documentmanager.cfg @@ -5,10 +5,13 @@ datatype[0].weightedsettype[0] datatype[0].structtype[1] datatype[0].structtype[0].name foobar.header datatype[0].structtype[0].version 9 -datatype[0].structtype[0].field[1] +datatype[0].structtype[0].field[2] datatype[0].structtype[0].field[0].name foobarfield1 datatype[0].structtype[0].field[0].id[0] datatype[0].structtype[0].field[0].datatype 4 +datatype[0].structtype[0].field[1].name foobarfield0 +datatype[0].structtype[0].field[1].id[0] +datatype[0].structtype[0].field[1].datatype 2 datatype[0].documenttype[0] datatype[1].id 278604398 datatype[1].arraytype[0] @@ -16,10 +19,6 @@ datatype[1].weightedsettype[0] datatype[1].structtype[1] datatype[1].structtype[0].name foobar.body datatype[1].structtype[0].version 9 -datatype[1].structtype[0].field[1] -datatype[1].structtype[0].field[0].name foobarfield0 -datatype[1].structtype[0].field[0].id[0] -datatype[1].structtype[0].field[0].datatype 2 datatype[1].documenttype[0] datatype[2].id 378030104 datatype[2].arraytype[0] @@ -48,7 +47,6 @@ datatype[4].weightedsettype[0] datatype[4].structtype[1] datatype[4].structtype[0].name banana.body datatype[4].structtype[0].version 234 -datatype[4].structtype[0].field[0] datatype[4].documenttype[0] datatype[5].id 556449802 datatype[5].arraytype[0] @@ -68,7 +66,13 @@ datatype[6].weightedsettype[0] datatype[6].structtype[1] datatype[6].structtype[0].name customtypes.header datatype[6].structtype[0].version 3 -datatype[6].structtype[0].field[0] +datatype[6].structtype[0].field[2] +datatype[6].structtype[0].field[0].name arrayfloat +datatype[6].structtype[0].field[0].id[0] +datatype[6].structtype[0].field[0].datatype 99 +datatype[6].structtype[0].field[1].name arrayarrayfloat +datatype[6].structtype[0].field[1].id[0] +datatype[6].structtype[0].field[1].datatype 4003 datatype[6].documenttype[0] datatype[7].id 99 datatype[7].arraytype[1] @@ -88,13 +92,6 @@ datatype[9].weightedsettype[0] datatype[9].structtype[1] datatype[9].structtype[0].name customtypes.body datatype[9].structtype[0].version 3 -datatype[9].structtype[0].field[2] -datatype[9].structtype[0].field[0].name arrayfloat -datatype[9].structtype[0].field[0].id[0] -datatype[9].structtype[0].field[0].datatype 99 -datatype[9].structtype[0].field[1].name arrayarrayfloat -datatype[9].structtype[0].field[1].id[0] -datatype[9].structtype[0].field[1].datatype 4003 datatype[9].documenttype[0] datatype[10].id -1500313747 datatype[10].arraytype[0] diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java index fa47c80c6fb..b2be93bfff9 100644 --- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java @@ -116,18 +116,15 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); { doc.getDataType().contentStruct().setCompressionConfig(noncomp); - doc.getDataType().getBodyType().setCompressionConfig(noncomp); FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-uncompressed.dat", false); doc.serialize(fout); fout.close(); } { doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - doc.getDataType().getBodyType().setCompressionConfig(lz4comp); FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-lz4-9.dat", false); doc.serialize(fout); doc.getDataType().contentStruct().setCompressionConfig(noncomp); - doc.getDataType().getBodyType().setCompressionConfig(noncomp); fout.close(); } } diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index dcd4622b3f4..be6544563ed 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -753,12 +753,10 @@ public class DocumentTestCase extends DocumentTestCaseBase { CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - doc.getDataType().getBodyType().setCompressionConfig(lz4comp); buf = new GrowableByteBuffer(size, 2.0f); doc.serialize(buf); doc.getDataType().contentStruct().setCompressionConfig(noncomp); - doc.getDataType().getBodyType().setCompressionConfig(noncomp); fos = new FileOutputStream("src/tests/data/serializejava-compressed.dat"); fos.write(buf.array(), 0, buf.position()); fos.close(); @@ -816,13 +814,11 @@ public class DocumentTestCase extends DocumentTestCaseBase { CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - doc.getDataType().getBodyType().setCompressionConfig(lz4comp); GrowableByteBuffer data = new GrowableByteBuffer(); doc.serialize(data); int size = doc.getSerializedSize(); doc.getDataType().contentStruct().setCompressionConfig(noncomp); - doc.getDataType().getBodyType().setCompressionConfig(noncomp); assertEquals(size, data.position()); diff --git a/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg b/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg index 3347c3127b5..65ce1b56811 100644 --- a/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg +++ b/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg @@ -4,16 +4,15 @@ datatype[0].id -1407012075 datatype[0].structtype[1] datatype[0].structtype[0].name "outerdoc.body" datatype[0].structtype[0].version 0 -datatype[0].structtype[0].field[1] -datatype[0].structtype[0].field[0].datatype -2035324352 -datatype[0].structtype[0].field[0].name "innerdocuments" datatype[1].id -1686125086 datatype[1].structtype[1] datatype[1].structtype[0].name "docindoc.header" datatype[1].structtype[0].version 0 -datatype[1].structtype[0].field[1] +datatype[1].structtype[0].field[2] datatype[1].structtype[0].field[0].datatype 2 datatype[1].structtype[0].field[0].name "name" +datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].name "content" datatype[2].id -2035324352 datatype[2].arraytype[1] datatype[2].arraytype[0].datatype 1447635645 @@ -21,6 +20,9 @@ datatype[3].id -2040625920 datatype[3].structtype[1] datatype[3].structtype[0].name "outerdoc.header" datatype[3].structtype[0].version 0 +datatype[3].structtype[0].field[1] +datatype[3].structtype[0].field[0].datatype -2035324352 +datatype[3].structtype[0].field[0].name "innerdocuments" datatype[4].id 1447635645 datatype[4].documenttype[1] datatype[4].documenttype[0].bodystruct 2030224503 @@ -37,6 +39,3 @@ datatype[6].id 2030224503 datatype[6].structtype[1] datatype[6].structtype[0].name "docindoc.body" datatype[6].structtype[0].version 0 -datatype[6].structtype[0].field[1] -datatype[6].structtype[0].field[0].datatype 2 -datatype[6].structtype[0].field[0].name "content" diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 5e61f8c7ba0..d1f02ae45e2 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -68,6 +68,8 @@ public class DocumentSelectorTestCase { manager.registerDocumentType(new DocumentType("andornot")); manager.registerDocumentType(new DocumentType("idid")); manager.registerDocumentType(new DocumentType("usergroup")); + manager.registerDocumentType(new DocumentType("user")); + manager.registerDocumentType(new DocumentType("group")); } @Test @@ -133,6 +135,8 @@ public class DocumentSelectorTestCase { assertParse(null, "false or false_t or falsetype"); assertParse(null, "true or and_t or andtype"); assertParse(null, "true or or_t or ortype"); + assertParse(null, "user or group"); + assertParse(null, "user.foo or group.bar"); } @Test diff --git a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java index 598369bae39..079e16915e1 100644 --- a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java @@ -68,7 +68,7 @@ public class VespaDocumentSerializerTestCase { CompressionFixture() { docType = new DocumentType("map_of_structs"); - docType.getHeaderType().setCompressionConfig(new CompressionConfig(CompressionType.LZ4)); + docType.contentStruct().setCompressionConfig(new CompressionConfig(CompressionType.LZ4)); nestedType = new StructDataType("nested_type"); nestedType.addField(new Field("str", DataType.STRING)); diff --git a/document/src/tests/data/crossplatform-java-cpp-document.cfg b/document/src/tests/data/crossplatform-java-cpp-document.cfg index 134d31b1831..3ebe56b8671 100644 --- a/document/src/tests/data/crossplatform-java-cpp-document.cfg +++ b/document/src/tests/data/crossplatform-java-cpp-document.cfg @@ -34,7 +34,10 @@ datatype[4].weightedsettype[0] datatype[4].structtype[1] datatype[4].structtype[0].name docindoc.header datatype[4].structtype[0].version 0 -datatype[4].structtype[0].field[0] +datatype[4].structtype[0].field[1] +datatype[4].structtype[0].field[0].name stringindocfield +datatype[4].structtype[0].field[0].id[0] +datatype[4].structtype[0].field[0].datatype 2 datatype[4].documenttype[0] datatype[5].id 2030224503 datatype[5].arraytype[0] @@ -42,10 +45,6 @@ datatype[5].weightedsettype[0] datatype[5].structtype[1] datatype[5].structtype[0].name docindoc.body datatype[5].structtype[0].version 0 -datatype[5].structtype[0].field[1] -datatype[5].structtype[0].field[0].name stringindocfield -datatype[5].structtype[0].field[0].id[0] -datatype[5].structtype[0].field[0].datatype 2 datatype[5].documenttype[0] datatype[6].id 1447635645 datatype[6].arraytype[0] @@ -63,7 +62,7 @@ datatype[7].weightedsettype[0] datatype[7].structtype[1] datatype[7].structtype[0].name serializetest.header datatype[7].structtype[0].version 0 -datatype[7].structtype[0].field[4] +datatype[7].structtype[0].field[15] datatype[7].structtype[0].field[0].name floatfield datatype[7].structtype[0].field[0].id[0] datatype[7].structtype[0].field[0].datatype 1 @@ -76,6 +75,39 @@ datatype[7].structtype[0].field[2].datatype 4 datatype[7].structtype[0].field[3].name urifield datatype[7].structtype[0].field[3].id[0] datatype[7].structtype[0].field[3].datatype 10 +datatype[7].structtype[0].field[4].name intfield +datatype[7].structtype[0].field[4].id[0] +datatype[7].structtype[0].field[4].datatype 0 +datatype[7].structtype[0].field[5].name rawfield +datatype[7].structtype[0].field[5].id[0] +datatype[7].structtype[0].field[5].datatype 3 +datatype[7].structtype[0].field[6].name doublefield +datatype[7].structtype[0].field[6].id[0] +datatype[7].structtype[0].field[6].datatype 5 +datatype[7].structtype[0].field[7].name contentfield +datatype[7].structtype[0].field[7].id[0] +datatype[7].structtype[0].field[7].datatype 2 +datatype[7].structtype[0].field[8].name bytefield +datatype[7].structtype[0].field[8].id[0] +datatype[7].structtype[0].field[8].datatype 16 +datatype[7].structtype[0].field[9].name arrayoffloatfield +datatype[7].structtype[0].field[9].id[0] +datatype[7].structtype[0].field[9].datatype 1001 +datatype[7].structtype[0].field[10].name arrayofarrayoffloatfield +datatype[7].structtype[0].field[10].id[0] +datatype[7].structtype[0].field[10].datatype 2001 +datatype[7].structtype[0].field[11].name docfield +datatype[7].structtype[0].field[11].id[0] +datatype[7].structtype[0].field[11].datatype 8 +datatype[7].structtype[0].field[12].name wsfield +datatype[7].structtype[0].field[12].id[0] +datatype[7].structtype[0].field[12].datatype 437829 +datatype[7].structtype[0].field[13].name mapfield +datatype[7].structtype[0].field[13].id[0] +datatype[7].structtype[0].field[13].datatype 9999 +datatype[7].structtype[0].field[14].name boolfield +datatype[7].structtype[0].field[14].id[0] +datatype[7].structtype[0].field[14].datatype 6 datatype[7].documenttype[0] datatype[8].id 1026122976 datatype[8].arraytype[0] @@ -83,40 +115,7 @@ datatype[8].weightedsettype[0] datatype[8].structtype[1] datatype[8].structtype[0].name serializetest.body datatype[8].structtype[0].version 0 -datatype[8].structtype[0].field[11] -datatype[8].structtype[0].field[0].name intfield -datatype[8].structtype[0].field[0].id[0] -datatype[8].structtype[0].field[0].datatype 0 -datatype[8].structtype[0].field[1].name rawfield -datatype[8].structtype[0].field[1].id[0] -datatype[8].structtype[0].field[1].datatype 3 -datatype[8].structtype[0].field[2].name doublefield -datatype[8].structtype[0].field[2].id[0] -datatype[8].structtype[0].field[2].datatype 5 -datatype[8].structtype[0].field[3].name contentfield -datatype[8].structtype[0].field[3].id[0] -datatype[8].structtype[0].field[3].datatype 2 -datatype[8].structtype[0].field[4].name bytefield -datatype[8].structtype[0].field[4].id[0] -datatype[8].structtype[0].field[4].datatype 16 -datatype[8].structtype[0].field[5].name arrayoffloatfield -datatype[8].structtype[0].field[5].id[0] -datatype[8].structtype[0].field[5].datatype 1001 -datatype[8].structtype[0].field[6].name arrayofarrayoffloatfield -datatype[8].structtype[0].field[6].id[0] -datatype[8].structtype[0].field[6].datatype 2001 -datatype[8].structtype[0].field[7].name docfield -datatype[8].structtype[0].field[7].id[0] -datatype[8].structtype[0].field[7].datatype 8 -datatype[8].structtype[0].field[8].name wsfield -datatype[8].structtype[0].field[8].id[0] -datatype[8].structtype[0].field[8].datatype 437829 -datatype[8].structtype[0].field[9].name mapfield -datatype[8].structtype[0].field[9].id[0] -datatype[8].structtype[0].field[9].datatype 9999 -datatype[8].structtype[0].field[10].name boolfield -datatype[8].structtype[0].field[10].id[0] -datatype[8].structtype[0].field[10].datatype 6 +datatype[8].structtype[0].field[0] datatype[8].documenttype[0] datatype[9].id 1306012852 datatype[9].arraytype[0] diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 9ac402f56ef..b75d094459b 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -76,6 +76,12 @@ namespace { void DocumentSelectParserTest::SetUp() { DocumenttypesConfigBuilderHelper builder(TestDocRepo::getDefaultConfig()); + builder.document(1234567, "with_imported", + Struct("with_imported.header"), + Struct("with_imported.body")) + .imported_field("my_imported_field"); + // Additional document types with names that are (or include) identifiers + // that lex to specific tokens. builder.document(535424777, "notandor", Struct("notandor.header"), Struct("notandor.body")); builder.document(1348665801, "ornotand", @@ -87,10 +93,10 @@ void DocumentSelectParserTest::SetUp() builder.document(-1673092522, "usergroup", Struct("usergroup.header"), Struct("usergroup.body")); - builder.document(1234567, "with_imported", - Struct("with_imported.header"), - Struct("with_imported.body")) - .imported_field("my_imported_field"); + builder.document(875463456, "user", + Struct("user.header"), Struct("user.body")); + builder.document(567463442, "group", + Struct("group.header"), Struct("group.body")); _repo = std::make_unique<DocumentTypeRepo>(builder.config()); _parser = std::make_unique<select::Parser>(*_repo, _bucketIdFactory); @@ -442,6 +448,8 @@ TEST_F(DocumentSelectParserTest, testParseTerminals) verifyParse("andornot"); verifyParse("idid"); verifyParse("usergroup"); + verifyParse("user"); + verifyParse("group"); } TEST_F(DocumentSelectParserTest, testParseBranches) @@ -463,6 +471,7 @@ TEST_F(DocumentSelectParserTest, testParseBranches) verifyParse("not andornot"); verifyParse("idid or not usergroup"); verifyParse("not(andornot or idid)", "not (andornot or idid)"); + verifyParse("not user or not group"); } template <typename ContainsType> @@ -1440,6 +1449,14 @@ TEST_F(DocumentSelectParserTest, test_ambiguous_field_spec_expression_is_handled parse_to_tree("(testdoctype1.foo) AND (testdoctype1.bar)")); } +TEST_F(DocumentSelectParserTest, special_tokens_are_allowed_as_freestanding_identifier_names) { + createDocs(); + EXPECT_EQ("(NOT (DOCTYPE user))", parse_to_tree("not user")); + EXPECT_EQ("(== (ID id.user) (FIELD user user))", parse_to_tree("id.user == user.user")); + EXPECT_EQ("(NOT (DOCTYPE group))", parse_to_tree("not group")); + EXPECT_EQ("(== (ID id.group) (FIELD group group))", parse_to_tree("id.group == group.group")); +} + TEST_F(DocumentSelectParserTest, test_can_build_field_value_from_field_expr_node) { using select::FieldExprNode; diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index 092a29d9293..d53fec43e5d 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -88,7 +88,7 @@ datatype[].documenttype[].inherits[].version int default=0 datatype[].documenttype[].headerstruct int ## Specify a document field id. Must be unique within the document type. -datatype[].documenttype[].bodystruct int +datatype[].documenttype[].bodystruct int default=0 ## Field sets datatype[].documenttype[].fieldsets{}.fields[] string diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index 0f0a9e3e37c..d02e9fe49f2 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -18,7 +18,7 @@ documenttype[].version int default=0 documenttype[].headerstruct int ## Specify a document field id. Must be unique within the document type. -documenttype[].bodystruct int +documenttype[].bodystruct int default=0 ## Specify a document type to inherit (id) documenttype[].inherits[].id int diff --git a/document/src/vespa/document/select/grammar/parser.yy b/document/src/vespa/document/select/grammar/parser.yy index 76b7cb7eeba..9d5b5825330 100644 --- a/document/src/vespa/document/select/grammar/parser.yy +++ b/document/src/vespa/document/select/grammar/parser.yy @@ -219,8 +219,11 @@ doc_type } ; + /* We allow most otherwise reserved tokens to be used as identifiers. */ ident : IDENTIFIER { $$ = $1; } + | USER { $$ = $1; } + | GROUP { $$ = $1; } | SCHEME { $$ = $1; } | TYPE { $$ = $1; } | NAMESPACE { $$ = $1; } diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 8f49b51fa1c..bb4deed2914 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -957,6 +957,28 @@ ], "fields": [] }, + "com.yahoo.documentapi.local.LocalVisitorSession": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.documentapi.VisitorSession" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.documentapi.local.LocalDocumentAccess, com.yahoo.documentapi.VisitorParameters)", + "public boolean isDone()", + "public com.yahoo.documentapi.ProgressToken getProgress()", + "public com.yahoo.messagebus.Trace getTrace()", + "public boolean waitUntilDone(long)", + "public void ack(com.yahoo.documentapi.AckToken)", + "public void abort()", + "public com.yahoo.documentapi.VisitorResponse getNext()", + "public com.yahoo.documentapi.VisitorResponse getNext(int)", + "public void destroy()" + ], + "fields": [] + }, "com.yahoo.documentapi.messagebus.MessageBusAsyncSession": { "superClass": "java.lang.Object", "interfaces": [ diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java index 202929130c7..c69a8fb48de 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java @@ -3,6 +3,7 @@ package com.yahoo.documentapi.local; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; +import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.AsyncParameters; import com.yahoo.documentapi.AsyncSession; import com.yahoo.documentapi.DocumentAccess; @@ -43,8 +44,8 @@ public class LocalDocumentAccess extends DocumentAccess { } @Override - public VisitorSession createVisitorSession(VisitorParameters parameters) { - throw new UnsupportedOperationException("Not supported yet"); + public VisitorSession createVisitorSession(VisitorParameters parameters) throws ParseException { + return new LocalVisitorSession(this, parameters); } @Override diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalVisitorSession.java new file mode 100644 index 00000000000..e107be94008 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalVisitorSession.java @@ -0,0 +1,165 @@ +package com.yahoo.documentapi.local; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentGet; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.Field; +import com.yahoo.document.fieldset.FieldCollection; +import com.yahoo.document.fieldset.FieldSet; +import com.yahoo.document.fieldset.FieldSetRepo; +import com.yahoo.document.select.DocumentSelector; +import com.yahoo.document.select.Result; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.AckToken; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorDataHandler; +import com.yahoo.documentapi.VisitorDataQueue; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorResponse; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.messagebus.Trace; +import com.yahoo.yolean.Exceptions; + +import java.util.Comparator; +import java.util.Map; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Local visitor session that copies and iterates through all items in the local document access. + * Each document must be ack'ed for the session to be done visiting. + * Only document puts are sent by this session, and this is done from a separate thread. + * + * @author jonmv + */ +public class LocalVisitorSession implements VisitorSession { + + private enum State { RUNNING, FAILURE, ABORTED, SUCCESS } + + private final VisitorDataHandler data; + private final VisitorControlHandler control; + private final Map<DocumentId, Document> outstanding; + private final DocumentSelector selector; + private final FieldSet fieldSet; + private final AtomicReference<State> state; + + public LocalVisitorSession(LocalDocumentAccess access, VisitorParameters parameters) throws ParseException { + if (parameters.getResumeToken() != null) + throw new UnsupportedOperationException("Continuation via progress tokens is not supported"); + + if (parameters.getRemoteDataHandler() != null) + throw new UnsupportedOperationException("Remote data handlers are not supported"); + + this.selector = new DocumentSelector(parameters.getDocumentSelection()); + this.fieldSet = new FieldSetRepo().parse(access.getDocumentTypeManager(), parameters.fieldSet()); + + this.data = parameters.getLocalDataHandler() == null ? new VisitorDataQueue() : parameters.getLocalDataHandler(); + this.data.reset(); + this.data.setSession(this); + + this.control = parameters.getControlHandler() == null ? new VisitorControlHandler() : parameters.getControlHandler(); + this.control.reset(); + this.control.setSession(this); + + this.outstanding = new ConcurrentSkipListMap<>(Comparator.comparing(DocumentId::toString)); + this.outstanding.putAll(access.documents); + this.state = new AtomicReference<>(State.RUNNING); + + start(); + } + + void start() { + new Thread(() -> { + try { + // Iterate through all documents and pass on to data handler + outstanding.forEach((id, document) -> { + if (state.get() != State.RUNNING) + return; + + if (selector.accepts(new DocumentPut(document)) != Result.TRUE) + return; + + Document copy = new Document(document.getDataType(), document.getId()); + new FieldSetRepo().copyFields(document, copy, fieldSet); + + data.onMessage(new PutDocumentMessage(new DocumentPut(copy)), + new AckToken(id)); + }); + // Transition to a terminal state when done + state.updateAndGet(current -> { + switch (current) { + case RUNNING: + control.onDone(VisitorControlHandler.CompletionCode.SUCCESS, "Success"); + return State.SUCCESS; + case ABORTED: + control.onDone(VisitorControlHandler.CompletionCode.ABORTED, "Aborted by user"); + return State.ABORTED; + default: + control.onDone(VisitorControlHandler.CompletionCode.FAILURE, "Unexpected state '" + current + "'");; + return State.FAILURE; + } + }); + } + // Transition to failure terminal state on error + catch (Exception e) { + state.set(State.FAILURE); + outstanding.clear(); + control.onDone(VisitorControlHandler.CompletionCode.FAILURE, Exceptions.toMessageString(e)); + } + finally { + data.onDone(); + } + }).start(); + } + + @Override + public boolean isDone() { + return outstanding.isEmpty() // All documents ack'ed + && control.isDone(); // Control handler has been notified + } + + @Override + public ProgressToken getProgress() { + throw new UnsupportedOperationException("Progress tokens are not supported"); + } + + @Override + public Trace getTrace() { + throw new UnsupportedOperationException("Traces are not supported"); + } + + @Override + public boolean waitUntilDone(long timeoutMs) throws InterruptedException { + return control.waitUntilDone(timeoutMs); + } + + @Override + public void ack(AckToken token) { + outstanding.remove((DocumentId) token.ackObject); + } + + @Override + public void abort() { + state.updateAndGet(current -> current == State.RUNNING ? State.ABORTED : current); + outstanding.clear(); + } + + @Override + public VisitorResponse getNext() { + return data.getNext(); + } + + @Override + public VisitorResponse getNext(int timeoutMilliseconds) throws InterruptedException { + return data.getNext(timeoutMilliseconds); + } + + @Override + public void destroy() { + abort(); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AdaptiveLoadBalancer.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AdaptiveLoadBalancer.java index 6a68a6e122b..621064c178e 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AdaptiveLoadBalancer.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AdaptiveLoadBalancer.java @@ -30,14 +30,10 @@ class AdaptiveLoadBalancer extends LoadBalancer { entry = choices.get(0); metrics = getNodeMetrics(entry); } else { - int candA = 0; - int candB = 1; - if (choices.size() > 2) { - candA = random.nextInt(choices.size()); + int candA = random.nextInt(choices.size()); + int candB = random.nextInt(choices.size()); + while (candB == candA) { candB = random.nextInt(choices.size()); - while (candB == candA) { - candB = random.nextInt(choices.size()); - } } entry = choices.get(candA); Mirror.Entry entryB = choices.get(candB); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 16ab9a017d0..e49cf021fe3 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; -import com.google.common.annotations.Beta; import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.component.VersionSpecification; @@ -147,7 +146,6 @@ public class DocumentProtocol implements Protocol { /** * Test and set condition (selection) failed. */ - @Beta public static final int ERROR_TEST_AND_SET_CONDITION_FAILED = ErrorCode.APP_FATAL_ERROR + 1013; /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LegacyLoadBalancer.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LegacyLoadBalancer.java deleted file mode 100644 index c1e580794b4..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LegacyLoadBalancer.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol; - -import com.yahoo.jrt.slobrok.api.Mirror; - -import java.util.List; - -/** - * Load balances over a set of nodes based on statistics gathered from those nodes. - * - * @author thomasg - */ -class LegacyLoadBalancer extends LoadBalancer { - - static class LegacyNodeMetrics extends NodeMetrics { - double weight = 1.0; - } - - private double position = 0.0; - - public LegacyLoadBalancer(String cluster) { - super(cluster); - } - - /** - * The load balancing operation: Returns a node choice from the given choices, - * based on previously gathered statistics on the nodes, and a running "position" - * which is increased by 1 on each call to this. - * - * @param choices the node choices, represented as Slobrok entries - * @return the chosen node, or null only if the given choices were zero - */ - @Override - Node getRecipient(List<Mirror.Entry> choices) { - if (choices.isEmpty()) return null; - - double weightSum = 0.0; - Node selectedNode = null; - synchronized (this) { - for (Mirror.Entry entry : choices) { - LegacyNodeMetrics nodeMetrics = (LegacyNodeMetrics)getNodeMetrics(entry); - - weightSum += nodeMetrics.weight; - - if (weightSum > position) { - selectedNode = new Node(entry, nodeMetrics); - break; - } - } - if (selectedNode == null) { // Position>sum of all weights: Wrap around (but keep the remainder for some reason) - position -= weightSum; - selectedNode = new Node(choices.get(0), getNodeMetrics(choices.get(0))); - } - position += 1.0; - selectedNode.metrics.incSend(); - } - return selectedNode; - } - - @Override - protected NodeMetrics createNodeMetrics() { - return new LegacyNodeMetrics(); - } - - /** Scale weights such that ratios are preserved */ - private void increaseWeights() { - for (NodeMetrics nodeMetrics : getNodeWeights()) { - LegacyNodeMetrics n = (LegacyNodeMetrics) nodeMetrics; - if (n == null) continue; - double want = n.weight * 1.01010101010101010101; - n.weight = Math.max(1.0, want); - } - } - - @Override - void received(Node node, boolean busy) { - if (busy) { - synchronized (this) { - LegacyNodeMetrics n = (LegacyNodeMetrics) node.metrics; - double wantWeight = n.weight - 0.01; - if (wantWeight < 1.0) { - increaseWeights(); - n.weight = 1.0; - } else { - n.weight = wantWeight; - } - node.metrics.incBusy(); - } - } - } - -} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java index 4f955f3649e..3c670299f3e 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java @@ -26,7 +26,6 @@ import java.util.Map; public class LoadBalancerPolicy extends SlobrokPolicy { private final String session; private final String pattern; - private final LoadBalancer loadBalancer; LoadBalancerPolicy(String param) { @@ -48,19 +47,12 @@ public class LoadBalancerPolicy extends SlobrokPolicy { } pattern = cluster + "/*/" + session; - String type = params.get("type"); - if ("adaptive".equals(type)) { - loadBalancer = new AdaptiveLoadBalancer(cluster); - } else if ("legacy".equals(type)) { - loadBalancer = new LegacyLoadBalancer(cluster); - } else { - loadBalancer = new LegacyLoadBalancer(cluster); - } + loadBalancer = new AdaptiveLoadBalancer(cluster); } @Override public void select(RoutingContext context) { - LegacyLoadBalancer.Node node = getRecipient(context); + LoadBalancer.Node node = getRecipient(context); if (node != null) { context.setContext(node); @@ -77,7 +69,7 @@ public class LoadBalancerPolicy extends SlobrokPolicy { @return Returns a hop representing the TCP address of the target, or null if none could be found. */ - private LegacyLoadBalancer.Node getRecipient(RoutingContext context) { + private LoadBalancer.Node getRecipient(RoutingContext context) { List<Mirror.Entry> lastLookup = lookup(context, pattern); return loadBalancer.getRecipient(lastLookup); } @@ -85,7 +77,7 @@ public class LoadBalancerPolicy extends SlobrokPolicy { public void merge(RoutingContext context) { RoutingNodeIterator it = context.getChildIterator(); Reply reply = it.removeReply(); - LegacyLoadBalancer.Node target = (LegacyLoadBalancer.Node)context.getContext(); + LoadBalancer.Node target = (LoadBalancer.Node)context.getContext(); boolean busy = false; for (int i = 0; i < reply.getNumErrors(); i++) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/TestAndSetMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/TestAndSetMessage.java index 5aaf2bde423..280881308f7 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/TestAndSetMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/TestAndSetMessage.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; -import com.google.common.annotations.Beta; import com.yahoo.document.TestAndSetCondition; /** @@ -9,7 +8,6 @@ import com.yahoo.document.TestAndSetCondition; * * @author Vegard Sjonfjell */ -@Beta public abstract class TestAndSetMessage extends DocumentMessage { public abstract void setCondition(TestAndSetCondition condition); public abstract TestAndSetCondition getCondition(); diff --git a/documentapi/src/test/cfg/documentmanager.cfg b/documentapi/src/test/cfg/documentmanager.cfg index eec3a6a06a0..75f205337a1 100644 --- a/documentapi/src/test/cfg/documentmanager.cfg +++ b/documentapi/src/test/cfg/documentmanager.cfg @@ -5,7 +5,10 @@ datatype[0].weightedsettype[0] datatype[0].structtype[1] datatype[0].structtype[0].name music.header datatype[0].structtype[0].version 0 -datatype[0].structtype[0].field[0] +datatype[0].structtype[0].field[1] +datatype[0].structtype[0].field[0].name artist +datatype[0].structtype[0].field[0].id[0] +datatype[0].structtype[0].field[0].datatype 2 datatype[0].documenttype[0] datatype[1].id 993120973 datatype[1].arraytype[0] @@ -13,10 +16,6 @@ datatype[1].weightedsettype[0] datatype[1].structtype[1] datatype[1].structtype[0].name music.body datatype[1].structtype[0].version 0 -datatype[1].structtype[0].field[1] -datatype[1].structtype[0].field[0].name artist -datatype[1].structtype[0].field[0].id[0] -datatype[1].structtype[0].field[0].datatype 2 datatype[1].documenttype[0] datatype[2].id 1412693671 datatype[2].arraytype[0] diff --git a/documentapi/src/test/java/com/yahoo/documentapi/local/LocalDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/local/LocalDocumentApiTestCase.java new file mode 100644 index 00000000000..d1361e50973 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/local/LocalDocumentApiTestCase.java @@ -0,0 +1,242 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.local; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.AsyncSession; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessParams; +import com.yahoo.documentapi.DocumentResponse; +import com.yahoo.documentapi.DumpVisitorDataHandler; +import com.yahoo.documentapi.Response; +import com.yahoo.documentapi.Result; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.test.AbstractDocumentApiTestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.CountDownLatch; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * Runs the superclass tests on this implementation + * + * @author bratseth + */ +public class LocalDocumentApiTestCase extends AbstractDocumentApiTestCase { + + protected LocalDocumentAccess access; + + @Override + protected DocumentAccess access() { + return access; + } + + @Before + public void setUp() { + DocumentAccessParams params = new DocumentAccessParams(); + params.setDocumentManagerConfigId("file:src/test/cfg/documentmanager.cfg"); + access = new LocalDocumentAccess(params); + } + + @After + public void shutdownAccess() { + access.shutdown(); + } + + @Test + public void testNoExceptionFromAsync() { + AsyncSession session = access.createAsyncSession(new AsyncParameters()); + + DocumentType type = access.getDocumentTypeManager().getDocumentType("music"); + DocumentUpdate docUp = new DocumentUpdate(type, new DocumentId("id:ns:music::2")); + + Result result = session.update(docUp); + assertTrue(result.isSuccess()); + Response response = session.getNext(); + assertEquals(result.getRequestId(), response.getRequestId()); + assertFalse(response.isSuccess()); + session.destroy(); + } + + @Test + public void testAsyncFetch() { + AsyncSession session = access.createAsyncSession(new AsyncParameters()); + List<DocumentId> ids = new ArrayList<>(); + ids.add(new DocumentId("id:music:music::1")); + ids.add(new DocumentId("id:music:music::2")); + ids.add(new DocumentId("id:music:music::3")); + for (DocumentId id : ids) + session.put(new Document(access.getDocumentTypeManager().getDocumentType("music"), id)); + int timeout = 100; + + long startTime = System.currentTimeMillis(); + Set<Long> outstandingRequests = new HashSet<>(); + for (DocumentId id : ids) { + Result result = session.get(id); + if ( ! result.isSuccess()) + throw new IllegalStateException("Failed requesting document " + id, result.getError().getCause()); + outstandingRequests.add(result.getRequestId()); + } + + List<Document> documents = new ArrayList<>(); + try { + while ( ! outstandingRequests.isEmpty()) { + int timeSinceStart = (int)(System.currentTimeMillis() - startTime); + Response response = session.getNext(timeout - timeSinceStart); + if (response == null) + throw new RuntimeException("Timed out waiting for documents"); // or return what you have + if ( ! outstandingRequests.contains(response.getRequestId())) continue; // Stale: Ignore + + if (response.isSuccess()) + documents.add(((DocumentResponse)response).getDocument()); + outstandingRequests.remove(response.getRequestId()); + } + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted while waiting for documents", e); + } + + assertEquals(3, documents.size()); + for (Document document : documents) + assertNotNull(document); + } + + @Test + public void testFeedingAndVisiting() throws InterruptedException, ParseException { + DocumentType musicType = access().getDocumentTypeManager().getDocumentType("music"); + Document doc1 = new Document(musicType, "id:ns:music::1"); doc1.setFieldValue("artist", "one"); + Document doc2 = new Document(musicType, "id:ns:music::2"); doc2.setFieldValue("artist", "two"); + Document doc3 = new Document(musicType, "id:ns:music::3"); + + // Select all music documents where the "artist" field is set + VisitorParameters parameters = new VisitorParameters("music.artist"); + parameters.setFieldSet("music:artist"); + VisitorControlHandler control = new VisitorControlHandler(); + parameters.setControlHandler(control); + Set<Document> received = new ConcurrentSkipListSet<>(); + parameters.setLocalDataHandler(new DumpVisitorDataHandler() { + @Override public void onDocument(Document doc, long timeStamp) { + received.add(doc); + } + @Override public void onRemove(DocumentId id) { + throw new IllegalStateException("Not supposed to get here"); + } + }); + + // Visit when there are no documents completes immediately + access.createVisitorSession(parameters).waitUntilDone(0); + assertSame(VisitorControlHandler.CompletionCode.SUCCESS, + control.getResult().getCode()); + assertEquals(Set.of(), + received); + + // Sync-put some documents + SyncSession out = access.createSyncSession(new SyncParameters.Builder().build()); + out.put(new DocumentPut(doc1)); + out.put(new DocumentPut(doc2)); + out.put(new DocumentPut(doc3)); + assertEquals(Map.of(doc1.getId(), doc1, + doc2.getId(), doc2, + doc3.getId(), doc3), + access.documents); + + // Expect a subset of documents to be returned, based on the selection + access.createVisitorSession(parameters).waitUntilDone(0); + assertSame(VisitorControlHandler.CompletionCode.SUCCESS, + control.getResult().getCode()); + assertEquals(Set.of(doc1, doc2), + received); + + // Remove doc2 and set artist for doc3, to see changes are reflected in subsequent visits + out.remove(new DocumentRemove(doc2.getId())); + out.update(new DocumentUpdate(musicType, doc3.getId()).addFieldUpdate(FieldUpdate.createAssign(musicType.getField("artist"), + new StringFieldValue("three")))); + assertEquals(Map.of(doc1.getId(), doc1, + doc3.getId(), doc3), + access.documents); + assertEquals("three", + ((StringFieldValue) doc3.getFieldValue("artist")).getString()); + + // Visit the documents again, retrieving none of the document fields + parameters.setFieldSet("[id]"); + received.clear(); + access.createVisitorSession(parameters).waitUntilDone(0); + assertSame(VisitorControlHandler.CompletionCode.SUCCESS, + control.getResult().getCode()); + assertEquals(Set.of(new Document(musicType, doc1.getId()), new Document(musicType, doc3.getId())), + received); + + // Visit the documents again, throwing an exception in the data handler on doc3 + received.clear(); + parameters.setLocalDataHandler(new DumpVisitorDataHandler() { + @Override public void onDocument(Document doc, long timeStamp) { + if (doc3.getId().equals(doc.getId())) + throw new RuntimeException("SEGFAULT"); + received.add(doc); + } + @Override public void onRemove(DocumentId id) { + throw new IllegalStateException("Not supposed to get here"); + } + }); + access.createVisitorSession(parameters).waitUntilDone(0); + assertSame(VisitorControlHandler.CompletionCode.FAILURE, + control.getResult().getCode()); + assertEquals("SEGFAULT", + control.getResult().getMessage()); + assertEquals(Set.of(new Document(musicType, doc1.getId())), + received); + + // Visit the documents again, aborting after the first document + received.clear(); + CountDownLatch visitLatch = new CountDownLatch(1); + CountDownLatch abortLatch = new CountDownLatch(1); + parameters.setLocalDataHandler(new DumpVisitorDataHandler() { + @Override public void onDocument(Document doc, long timeStamp) { + received.add(doc); + abortLatch.countDown(); + try { + visitLatch.await(); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + @Override public void onRemove(DocumentId id) { throw new IllegalStateException("Not supposed to get here"); } + }); + VisitorSession visit = access.createVisitorSession(parameters); + abortLatch.await(); + control.abort(); + visitLatch.countDown(); + visit.waitUntilDone(0); + assertSame(VisitorControlHandler.CompletionCode.ABORTED, + control.getResult().getCode()); + assertEquals(Set.of(new Document(musicType, doc1.getId())), + received); + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/local/test/LocalDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/local/test/LocalDocumentApiTestCase.java deleted file mode 100644 index 252bf739951..00000000000 --- a/documentapi/src/test/java/com/yahoo/documentapi/local/test/LocalDocumentApiTestCase.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.local.test; - -import com.yahoo.document.*; -import com.yahoo.documentapi.*; -import com.yahoo.documentapi.local.LocalDocumentAccess; -import com.yahoo.documentapi.test.AbstractDocumentApiTestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static org.junit.Assert.*; - -/** - * Runs the superclass tests on this implementation - * - * @author bratseth - */ -public class LocalDocumentApiTestCase extends AbstractDocumentApiTestCase { - - protected DocumentAccess access; - - @Override - protected DocumentAccess access() { - return access; - } - - @Before - public void setUp() { - DocumentAccessParams params = new DocumentAccessParams(); - params.setDocumentManagerConfigId("file:src/test/cfg/documentmanager.cfg"); - access = new LocalDocumentAccess(params); - } - - @After - public void shutdownAccess() { - access.shutdown(); - } - - @Test - public void testNoExceptionFromAsync() { - AsyncSession session = access.createAsyncSession(new AsyncParameters()); - - DocumentType type = access.getDocumentTypeManager().getDocumentType("music"); - DocumentUpdate docUp = new DocumentUpdate(type, new DocumentId("id:ns:music::2")); - - Result result = session.update(docUp); - assertTrue(result.isSuccess()); - Response response = session.getNext(); - assertEquals(result.getRequestId(), response.getRequestId()); - assertFalse(response.isSuccess()); - session.destroy(); - } - - @Test - public void testAsyncFetch() { - AsyncSession session = access.createAsyncSession(new AsyncParameters()); - List<DocumentId> ids = new ArrayList<>(); - ids.add(new DocumentId("id:music:music::1")); - ids.add(new DocumentId("id:music:music::2")); - ids.add(new DocumentId("id:music:music::3")); - for (DocumentId id : ids) - session.put(new Document(access.getDocumentTypeManager().getDocumentType("music"), id)); - int timeout = 100; - - long startTime = System.currentTimeMillis(); - Set<Long> outstandingRequests = new HashSet<>(); - for (DocumentId id : ids) { - Result result = session.get(id); - if ( ! result.isSuccess()) - throw new IllegalStateException("Failed requesting document " + id, result.getError().getCause()); - outstandingRequests.add(result.getRequestId()); - } - - List<Document> documents = new ArrayList<>(); - try { - while ( ! outstandingRequests.isEmpty()) { - int timeSinceStart = (int)(System.currentTimeMillis() - startTime); - Response response = session.getNext(timeout - timeSinceStart); - if (response == null) - throw new RuntimeException("Timed out waiting for documents"); // or return what you have - if ( ! outstandingRequests.contains(response.getRequestId())) continue; // Stale: Ignore - - if (response.isSuccess()) - documents.add(((DocumentResponse)response).getDocument()); - outstandingRequests.remove(response.getRequestId()); - } - } - catch (InterruptedException e) { - throw new RuntimeException("Interrupted while waiting for documents", e); - } - - assertEquals(3, documents.size()); - for (Document document : documents) - assertNotNull(document); - } - -} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerTestCase.java index 088259b74ac..582bd53d8e7 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerTestCase.java @@ -33,7 +33,7 @@ public class LoadBalancerTestCase { } private static void assertIllegalArgument(String clusterName, String recipient, String expectedMessage) { - LegacyLoadBalancer policy = new LegacyLoadBalancer(clusterName); + LoadBalancer policy = new AdaptiveLoadBalancer(clusterName); try { fail("Expected exception, got index " + policy.getIndex(recipient) + "."); } catch (IllegalArgumentException e) { @@ -44,9 +44,9 @@ public class LoadBalancerTestCase { @Test public void testLoadBalancerCreation() { LoadBalancerPolicy lbp = new LoadBalancerPolicy("cluster=docproc/cluster.mobile.indexing;session=chain.mobile.indexing"); - assertTrue(lbp.getLoadBalancer() instanceof LegacyLoadBalancer); + assertTrue(lbp.getLoadBalancer() instanceof AdaptiveLoadBalancer); lbp = new LoadBalancerPolicy("cluster=docproc/cluster.mobile.indexing;session=chain.mobile.indexing;type=legacy"); - assertTrue(lbp.getLoadBalancer() instanceof LegacyLoadBalancer); + assertTrue(lbp.getLoadBalancer() instanceof AdaptiveLoadBalancer); lbp = new LoadBalancerPolicy("cluster=docproc/cluster.mobile.indexing;session=chain.mobile.indexing;type=adaptive"); assertTrue(lbp.getLoadBalancer() instanceof AdaptiveLoadBalancer); } @@ -110,64 +110,6 @@ public class LoadBalancerTestCase { assertEquals(1019, weights.get(2).pending()); } - @Test - public void testLegacyLoadBalancer() { - LoadBalancer lb = new LegacyLoadBalancer("foo"); - - List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1"), - new Mirror.Entry("foo/1/default", "tcp/bar:2"), - new Mirror.Entry("foo/2/default", "tcp/bar:3")); - List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); - - { - for (int i = 0; i < 99; i++) { - LoadBalancer.Node node = lb.getRecipient(entries); - assertEquals("foo/" + (i % 3) + "/default" , node.entry.getName()); - } - - assertEquals(33, weights.get(0).sent()); - assertEquals(33, weights.get(1).sent()); - assertEquals(33, weights.get(2).sent()); - - weights.get(0).reset(); - weights.get(1).reset(); - weights.get(2).reset(); - } - - { - // Simulate that one node is overloaded. It returns busy twice as often as the others. - for (int i = 0; i < 100; i++) { - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), true); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), false); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), false); - - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/2/default", "tcp/bar:3"), weights.get(2)), true); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/2/default", "tcp/bar:3"), weights.get(2)), false); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/2/default", "tcp/bar:3"), weights.get(2)), false); - - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/1/default", "tcp/bar:2"), weights.get(1)), true); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/1/default", "tcp/bar:2"), weights.get(1)), true); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/1/default", "tcp/bar:2"), weights.get(1)), false); - } - - assertEquals(421, (int)(100 * ((LegacyLoadBalancer.LegacyNodeMetrics)weights.get(0)).weight / ((LegacyLoadBalancer.LegacyNodeMetrics)weights.get(1)).weight)); - assertEquals(100, (int)(100 * ((LegacyLoadBalancer.LegacyNodeMetrics)weights.get(1)).weight)); - assertEquals(421, (int)(100 * ((LegacyLoadBalancer.LegacyNodeMetrics)weights.get(2)).weight / ((LegacyLoadBalancer.LegacyNodeMetrics)weights.get(1)).weight)); - } - - - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/1/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - } - private void verifyLoadBalancerOneItemOnly(LoadBalancer lb) { List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1") ); @@ -181,7 +123,6 @@ public class LoadBalancerTestCase { } @Test public void testLoadBalancerOneItemOnly() { - verifyLoadBalancerOneItemOnly(new LegacyLoadBalancer("foo")); verifyLoadBalancerOneItemOnly(new AdaptiveLoadBalancer("foo")); } } diff --git a/documentapi/test/cfg/testdoc.cfg b/documentapi/test/cfg/testdoc.cfg index 89bea273b6e..f218ddcd3b4 100644 --- a/documentapi/test/cfg/testdoc.cfg +++ b/documentapi/test/cfg/testdoc.cfg @@ -17,7 +17,7 @@ datatype[1].weightedsettype[0] datatype[1].structtype[1] datatype[1].structtype[0].name testdoc.header datatype[1].structtype[0].version 0 -datatype[1].structtype[0].field[4] +datatype[1].structtype[0].field[10] datatype[1].structtype[0].field[0].name floatfield datatype[1].structtype[0].field[0].id[0] datatype[1].structtype[0].field[0].datatype 1 @@ -30,6 +30,24 @@ datatype[1].structtype[0].field[2].datatype 4 datatype[1].structtype[0].field[3].name urifield datatype[1].structtype[0].field[3].id[0] datatype[1].structtype[0].field[3].datatype 10 +datatype[1].structtype[0].field[4].name intfield +datatype[1].structtype[0].field[4].id[0] +datatype[1].structtype[0].field[4].datatype 0 +datatype[1].structtype[0].field[5].name rawfield +datatype[1].structtype[0].field[5].id[0] +datatype[1].structtype[0].field[5].datatype 3 +datatype[1].structtype[0].field[6].name doublefield +datatype[1].structtype[0].field[6].id[0] +datatype[1].structtype[0].field[6].datatype 5 +datatype[1].structtype[0].field[7].name contentfield +datatype[1].structtype[0].field[7].id[0] +datatype[1].structtype[0].field[7].datatype 2 +datatype[1].structtype[0].field[8].name bytefield +datatype[1].structtype[0].field[8].id[0] +datatype[1].structtype[0].field[8].datatype 16 +datatype[1].structtype[0].field[9].name foo +datatype[1].structtype[0].field[9].id[0] +datatype[1].structtype[0].field[9].datatype 666999 datatype[1].documenttype[0] datatype[2].id 1878320748 datatype[2].arraytype[0] @@ -37,25 +55,6 @@ datatype[2].weightedsettype[0] datatype[2].structtype[1] datatype[2].structtype[0].name testdoc.body datatype[2].structtype[0].version 0 -datatype[2].structtype[0].field[6] -datatype[2].structtype[0].field[0].name intfield -datatype[2].structtype[0].field[0].id[0] -datatype[2].structtype[0].field[0].datatype 0 -datatype[2].structtype[0].field[1].name rawfield -datatype[2].structtype[0].field[1].id[0] -datatype[2].structtype[0].field[1].datatype 3 -datatype[2].structtype[0].field[2].name doublefield -datatype[2].structtype[0].field[2].id[0] -datatype[2].structtype[0].field[2].datatype 5 -datatype[2].structtype[0].field[3].name contentfield -datatype[2].structtype[0].field[3].id[0] -datatype[2].structtype[0].field[3].datatype 2 -datatype[2].structtype[0].field[4].name bytefield -datatype[2].structtype[0].field[4].id[0] -datatype[2].structtype[0].field[4].datatype 16 -datatype[2].structtype[0].field[5].name foo -datatype[2].structtype[0].field[5].id[0] -datatype[2].structtype[0].field[5].datatype 666999 datatype[2].documenttype[0] datatype[3].id -1175657560 datatype[3].arraytype[0] @@ -73,7 +72,10 @@ datatype[4].weightedsettype[0] datatype[4].structtype[1] datatype[4].structtype[0].name other.header datatype[4].structtype[0].version 0 -datatype[4].structtype[0].field[0] +datatype[4].structtype[0].field[1] +datatype[4].structtype[0].field[0].name intfield +datatype[4].structtype[0].field[0].id[0] +datatype[4].structtype[0].field[0].datatype 0 datatype[4].documenttype[0] datatype[5].id -72846462 datatype[5].arraytype[0] @@ -81,10 +83,6 @@ datatype[5].weightedsettype[0] datatype[5].structtype[1] datatype[5].structtype[0].name other.body datatype[5].structtype[0].version 0 -datatype[5].structtype[0].field[1] -datatype[5].structtype[0].field[0].name intfield -datatype[5].structtype[0].field[0].id[0] -datatype[5].structtype[0].field[0].datatype 0 datatype[5].documenttype[0] datatype[6].id -1146158894 datatype[6].arraytype[0] diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 29bee2e9e3e..91b786e20f5 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -870,25 +870,21 @@ public class DocumentGenPluginTest { final Book book = getBook(); assertEquals(book.getMystruct().getD1(), (Double)56.777); assertEquals(book.getMystruct().getCompressionType(), CompressionType.NONE); - assertEquals(book.getBody().getFieldCount(), 0); - assertEquals(book.getHeader().getFieldCount(), 13); + assertEquals(book.getFieldCount(), 13); assertEquals(book.getMystruct().getFieldCount(), 4); assertEquals(book.getContent().get(0), 3); assertEquals(book.getContent().get(1), 4); assertEquals(book.getContent().get(2), 5); final Document des = roundtripSerialize(book, typeManagerForBookType()); - assertEquals(des.getBody().getFieldCount(), 0); - assertEquals(des.getHeader().getFieldCount(), 13); + assertEquals(des.getFieldCount(), 13); assertEquals(des.getDataType().getName(), "book"); assertEquals(((Raw) des.getFieldValue("content")).getByteBuffer().get(0), 3); assertEquals(((Raw) des.getFieldValue("content")).getByteBuffer().get(1), 4); assertEquals(((Raw) des.getFieldValue("content")).getByteBuffer().get(2), 5); assertEquals(des.getFieldValue("author").toString(), "Herman Melville"); assertEquals(des.getFieldValue("title").toString(), "Moby Dick - Or The Whale"); - assertEquals(des.getHeader().getFieldValue("title").toString(), "Moby Dick - Or The Whale"); - assertNull(des.getBody().getFieldValue("title")); - assertEquals(des.getHeader().getFieldValue("author").toString(), "Herman Melville"); - assertNull(des.getBody().getFieldValue("author")); + assertEquals(des.getFieldValue("title").toString(), "Moby Dick - Or The Whale"); + assertEquals(des.getFieldValue("author").toString(), "Herman Melville"); Struct mystruct = (Struct)des.getFieldValue("mystruct"); FieldValue d1 = mystruct.getFieldValue("d1"); diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index b68440795d4..3a9aabc83ba 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -17,6 +17,7 @@ vespa_define_module( src/tests/eval/function src/tests/eval/function_speed src/tests/eval/gbdt + src/tests/eval/inline_operation src/tests/eval/interpreted_function src/tests/eval/node_tools src/tests/eval/node_types @@ -37,8 +38,10 @@ vespa_define_module( src/tests/tensor/dense_matmul_function src/tests/tensor/dense_multi_matmul_function src/tests/tensor/dense_number_join_function + src/tests/tensor/dense_pow_as_map_optimizer src/tests/tensor/dense_remove_dimension_optimizer src/tests/tensor/dense_replace_type_function + src/tests/tensor/dense_simple_expand_function src/tests/tensor/dense_simple_join_function src/tests/tensor/dense_simple_map_function src/tests/tensor/dense_single_reduce_function diff --git a/eval/src/tests/ann/nns-l2.h b/eval/src/tests/ann/nns-l2.h index 82a95741200..de24df50b6c 100644 --- a/eval/src/tests/ann/nns-l2.h +++ b/eval/src/tests/ann/nns-l2.h @@ -36,7 +36,7 @@ template <typename FltType = float> struct L2DistCalc { const vespalib::hwaccelrated::IAccelrated & _hw; - L2DistCalc() : _hw(vespalib::hwaccelrated::IAccelrated::getAccelrator()) {} + L2DistCalc() : _hw(vespalib::hwaccelrated::IAccelrated::getAccelerator()) {} using Arr = vespalib::ArrayRef<FltType>; using ConstArr = vespalib::ConstArrayRef<FltType>; diff --git a/eval/src/tests/eval/compile_cache/compile_cache_test.cpp b/eval/src/tests/eval/compile_cache/compile_cache_test.cpp index 1de56e605c9..a0dad889d9a 100644 --- a/eval/src/tests/eval/compile_cache/compile_cache_test.cpp +++ b/eval/src/tests/eval/compile_cache/compile_cache_test.cpp @@ -5,12 +5,16 @@ #include <vespa/eval/eval/test/eval_spec.h> #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/util/stringfmt.h> #include <thread> #include <set> using namespace vespalib; using namespace vespalib::eval; +using vespalib::make_string_short::fmt; + struct MyExecutor : public Executor { std::vector<Executor::Task::UP> tasks; Executor::Task::UP execute(Executor::Task::UP task) override { @@ -157,7 +161,7 @@ TEST("require that cache usage works") { } TEST("require that async cache usage works") { - ThreadStackExecutor executor(8, 256*1024); + auto executor = std::make_shared<ThreadStackExecutor>(8, 256*1024); auto binding = CompileCache::bind(executor); CompileCache::Token::UP token_a = CompileCache::compile(*Function::parse("x+y"), PassParams::SEPARATE); EXPECT_EQUAL(5.0, token_a->get().get_function<2>()(2.0, 3.0)); @@ -166,7 +170,6 @@ TEST("require that async cache usage works") { CompileCache::Token::UP token_c = CompileCache::compile(*Function::parse("x+y"), PassParams::SEPARATE); EXPECT_EQUAL(5.0, token_c->get().get_function<2>()(2.0, 3.0)); EXPECT_EQUAL(CompileCache::num_cached(), 2u); - executor.sync(); // wait for compile threads to drop all compile cache tokens token_a.reset(); TEST_DO(verify_cache(2, 2)); token_b.reset(); @@ -176,24 +179,24 @@ TEST("require that async cache usage works") { } TEST("require that compile tasks are run in the most recently bound executor") { - MyExecutor exe1; - MyExecutor exe2; + auto exe1 = std::make_shared<MyExecutor>(); + auto exe2 = std::make_shared<MyExecutor>(); auto token0 = CompileCache::compile(*Function::parse("a+b"), PassParams::SEPARATE); EXPECT_EQUAL(CompileCache::num_bound(), 0u); - EXPECT_EQUAL(exe1.tasks.size(), 0u); - EXPECT_EQUAL(exe2.tasks.size(), 0u); + EXPECT_EQUAL(exe1->tasks.size(), 0u); + EXPECT_EQUAL(exe2->tasks.size(), 0u); { auto bind1 = CompileCache::bind(exe1); auto token1 = CompileCache::compile(*Function::parse("a-b"), PassParams::SEPARATE); EXPECT_EQUAL(CompileCache::num_bound(), 1u); - EXPECT_EQUAL(exe1.tasks.size(), 1u); - EXPECT_EQUAL(exe2.tasks.size(), 0u); + EXPECT_EQUAL(exe1->tasks.size(), 1u); + EXPECT_EQUAL(exe2->tasks.size(), 0u); { auto bind2 = CompileCache::bind(exe2); auto token2 = CompileCache::compile(*Function::parse("a*b"), PassParams::SEPARATE); EXPECT_EQUAL(CompileCache::num_bound(), 2u); - EXPECT_EQUAL(exe1.tasks.size(), 1u); - EXPECT_EQUAL(exe2.tasks.size(), 1u); + EXPECT_EQUAL(exe1->tasks.size(), 1u); + EXPECT_EQUAL(exe2->tasks.size(), 1u); } EXPECT_EQUAL(CompileCache::num_bound(), 1u); } @@ -201,9 +204,9 @@ TEST("require that compile tasks are run in the most recently bound executor") { } TEST("require that executors may be unbound in any order") { - MyExecutor exe1; - MyExecutor exe2; - MyExecutor exe3; + auto exe1 = std::make_shared<MyExecutor>(); + auto exe2 = std::make_shared<MyExecutor>(); + auto exe3 = std::make_shared<MyExecutor>(); auto bind1 = CompileCache::bind(exe1); auto bind2 = CompileCache::bind(exe2); auto bind3 = CompileCache::bind(exe3); @@ -213,13 +216,13 @@ TEST("require that executors may be unbound in any order") { bind3.reset(); EXPECT_EQUAL(CompileCache::num_bound(), 1u); auto token = CompileCache::compile(*Function::parse("a+b"), PassParams::SEPARATE); - EXPECT_EQUAL(exe1.tasks.size(), 1u); - EXPECT_EQUAL(exe2.tasks.size(), 0u); - EXPECT_EQUAL(exe3.tasks.size(), 0u); + EXPECT_EQUAL(exe1->tasks.size(), 1u); + EXPECT_EQUAL(exe2->tasks.size(), 0u); + EXPECT_EQUAL(exe3->tasks.size(), 0u); } TEST("require that the same executor can be bound multiple times") { - MyExecutor exe1; + auto exe1 = std::make_shared<MyExecutor>(); auto bind1 = CompileCache::bind(exe1); auto bind2 = CompileCache::bind(exe1); auto bind3 = CompileCache::bind(exe1); @@ -230,7 +233,7 @@ TEST("require that the same executor can be bound multiple times") { EXPECT_EQUAL(CompileCache::num_bound(), 1u); auto token = CompileCache::compile(*Function::parse("a+b"), PassParams::SEPARATE); EXPECT_EQUAL(CompileCache::num_bound(), 1u); - EXPECT_EQUAL(exe1.tasks.size(), 1u); + EXPECT_EQUAL(exe1->tasks.size(), 1u); } struct CompileCheck : test::EvalSpec::EvalTest { @@ -286,9 +289,9 @@ TEST_F("compile sequentially, then run all conformance tests", test::EvalSpec()) TEST_F("compile concurrently (8 threads), then run all conformance tests", test::EvalSpec()) { f1.add_all_cases(); - ThreadStackExecutor executor(8, 256*1024); + auto executor = std::make_shared<ThreadStackExecutor>(8, 256*1024); auto binding = CompileCache::bind(executor); - while (executor.num_idle_workers() < 8) { + while (executor->num_idle_workers() < 8) { std::this_thread::sleep_for(1ms); } for (size_t i = 0; i < 2; ++i) { @@ -305,6 +308,43 @@ TEST_F("compile concurrently (8 threads), then run all conformance tests", test: } } +struct MyCompileTask : public Executor::Task { + size_t seed; + size_t loop; + MyCompileTask(size_t seed_in, size_t loop_in) : seed(seed_in), loop(loop_in) {} + void run() override { + for (size_t i = 0; i < loop; ++i) { + // use custom constant to make a unique function that needs compilation + auto token = CompileCache::compile(*Function::parse(fmt("%zu", seed + i)), PassParams::SEPARATE); + } + } +}; + +TEST_MT_FF("require that deadlock is avoided with blocking executor", 8, std::shared_ptr<Executor>(nullptr), TimeBomb(300)) { + size_t loop = 16; + if (thread_id == 0) { + auto t0 = steady_clock::now(); + f1 = std::make_shared<BlockingThreadStackExecutor>(2, 256*1024, 3); + auto binding = CompileCache::bind(f1); + TEST_BARRIER(); // #1 + for (size_t i = 0; i < num_threads; ++i) { + f1->execute(std::make_unique<MyCompileTask>(i * loop, loop)); + } + TEST_BARRIER(); // #2 + auto t1 = steady_clock::now(); + fprintf(stderr, "deadlock test took %" PRIu64 " ms\n", count_ms(t1 - t0)); + + } else { + TEST_BARRIER(); // #1 + size_t seed = (10000 + (thread_id * loop)); + for (size_t i = 0; i < loop; ++i) { + // use custom constant to make a unique function that needs compilation + auto token = CompileCache::compile(*Function::parse(fmt("%zu", seed + i)), PassParams::SEPARATE); + } + TEST_BARRIER(); // #2 + } +} + //----------------------------------------------------------------------------- TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/eval/inline_operation/CMakeLists.txt b/eval/src/tests/eval/inline_operation/CMakeLists.txt new file mode 100644 index 00000000000..04cdbca3abf --- /dev/null +++ b/eval/src/tests/eval/inline_operation/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_inline_operation_test_app TEST + SOURCES + inline_operation_test.cpp + DEPENDS + vespaeval + gtest +) +vespa_add_test(NAME eval_inline_operation_test_app COMMAND eval_inline_operation_test_app) diff --git a/eval/src/tests/eval/inline_operation/inline_operation_test.cpp b/eval/src/tests/eval/inline_operation/inline_operation_test.cpp new file mode 100644 index 00000000000..fe9396398da --- /dev/null +++ b/eval/src/tests/eval/inline_operation/inline_operation_test.cpp @@ -0,0 +1,356 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/inline_operation.h> +#include <vespa/eval/eval/function.h> +#include <vespa/vespalib/util/typify.h> +#include <vespa/vespalib/gtest/gtest.h> + +using vespalib::typify_invoke; +using namespace vespalib::eval; +using namespace vespalib::eval::operation; + +const int my_value = 42; +struct AsValue { template <typename T> static int invoke() { return my_value; } }; +struct AsRef { template <typename T> static const int &invoke() { return my_value; } }; + +template <typename T> void test_op1(op1_t ref, double a, double expect) { + bool need_ref = std::is_same_v<T,CallOp1>; + T op = need_ref ? T(ref) : T(nullptr); + EXPECT_DOUBLE_EQ(ref(a), expect); + EXPECT_DOUBLE_EQ(op(a), expect); +}; + +template <typename T> void test_op2(op2_t ref, double a, double b, double expect) { + bool need_ref = std::is_same_v<T,CallOp2>; + T op = need_ref ? T(ref) : T(nullptr); + EXPECT_DOUBLE_EQ(ref(a, b), expect); + EXPECT_DOUBLE_EQ(op(a, b), expect); +}; + +op1_t as_op1(const vespalib::string &str) { + auto fun = Function::parse({"a"}, str); + auto res = lookup_op1(*fun); + EXPECT_TRUE(res.has_value()); + return res.value(); +} + +op2_t as_op2(const vespalib::string &str) { + auto fun = Function::parse({"a", "b"}, str); + auto res = lookup_op2(*fun); + EXPECT_TRUE(res.has_value()); + return res.value(); +} + +TEST(InlineOperationTest, op1_lambdas_are_recognized) { + EXPECT_EQ(as_op1("-a"), &Neg::f); + EXPECT_EQ(as_op1("!a"), &Not::f); + EXPECT_EQ(as_op1("cos(a)"), &Cos::f); + EXPECT_EQ(as_op1("sin(a)"), &Sin::f); + EXPECT_EQ(as_op1("tan(a)"), &Tan::f); + EXPECT_EQ(as_op1("cosh(a)"), &Cosh::f); + EXPECT_EQ(as_op1("sinh(a)"), &Sinh::f); + EXPECT_EQ(as_op1("tanh(a)"), &Tanh::f); + EXPECT_EQ(as_op1("acos(a)"), &Acos::f); + EXPECT_EQ(as_op1("asin(a)"), &Asin::f); + EXPECT_EQ(as_op1("atan(a)"), &Atan::f); + EXPECT_EQ(as_op1("exp(a)"), &Exp::f); + EXPECT_EQ(as_op1("log10(a)"), &Log10::f); + EXPECT_EQ(as_op1("log(a)"), &Log::f); + EXPECT_EQ(as_op1("sqrt(a)"), &Sqrt::f); + EXPECT_EQ(as_op1("ceil(a)"), &Ceil::f); + EXPECT_EQ(as_op1("fabs(a)"), &Fabs::f); + EXPECT_EQ(as_op1("floor(a)"), &Floor::f); + EXPECT_EQ(as_op1("isNan(a)"), &IsNan::f); + EXPECT_EQ(as_op1("relu(a)"), &Relu::f); + EXPECT_EQ(as_op1("sigmoid(a)"), &Sigmoid::f); + EXPECT_EQ(as_op1("elu(a)"), &Elu::f); + //------------------------------------------- + EXPECT_EQ(as_op1("1/a"), &Inv::f); + EXPECT_EQ(as_op1("1.0/a"), &Inv::f); + EXPECT_EQ(as_op1("a*a"), &Square::f); + EXPECT_EQ(as_op1("a^2"), &Square::f); + EXPECT_EQ(as_op1("a^2.0"), &Square::f); + EXPECT_EQ(as_op1("pow(a,2)"), &Square::f); + EXPECT_EQ(as_op1("pow(a,2.0)"), &Square::f); + EXPECT_EQ(as_op1("a*a*a"), &Cube::f); + EXPECT_EQ(as_op1("(a*a)*a"), &Cube::f); + EXPECT_EQ(as_op1("a*(a*a)"), &Cube::f); + EXPECT_EQ(as_op1("a^3"), &Cube::f); + EXPECT_EQ(as_op1("a^3.0"), &Cube::f); + EXPECT_EQ(as_op1("pow(a,3)"), &Cube::f); + EXPECT_EQ(as_op1("pow(a,3.0)"), &Cube::f); +} + +TEST(InlineOperationTest, op1_lambdas_are_recognized_with_different_parameter_names) { + EXPECT_EQ(lookup_op1(*Function::parse({"x"}, "-x")).value(), &Neg::f); + EXPECT_EQ(lookup_op1(*Function::parse({"x"}, "!x")).value(), &Not::f); +} + +TEST(InlineOperationTest, non_op1_lambdas_are_not_recognized) { + EXPECT_FALSE(lookup_op1(*Function::parse({"a"}, "a*a+3")).has_value()); + EXPECT_FALSE(lookup_op1(*Function::parse({"a", "b"}, "a+b")).has_value()); +} + +TEST(InlineOperationTest, op2_lambdas_are_recognized) { + EXPECT_EQ(as_op2("a+b"), &Add::f); + EXPECT_EQ(as_op2("a-b"), &Sub::f); + EXPECT_EQ(as_op2("a*b"), &Mul::f); + EXPECT_EQ(as_op2("a/b"), &Div::f); + EXPECT_EQ(as_op2("a%b"), &Mod::f); + EXPECT_EQ(as_op2("a^b"), &Pow::f); + EXPECT_EQ(as_op2("a==b"), &Equal::f); + EXPECT_EQ(as_op2("a!=b"), &NotEqual::f); + EXPECT_EQ(as_op2("a~=b"), &Approx::f); + EXPECT_EQ(as_op2("a<b"), &Less::f); + EXPECT_EQ(as_op2("a<=b"), &LessEqual::f); + EXPECT_EQ(as_op2("a>b"), &Greater::f); + EXPECT_EQ(as_op2("a>=b"), &GreaterEqual::f); + EXPECT_EQ(as_op2("a&&b"), &And::f); + EXPECT_EQ(as_op2("a||b"), &Or::f); + EXPECT_EQ(as_op2("atan2(a,b)"), &Atan2::f); + EXPECT_EQ(as_op2("ldexp(a,b)"), &Ldexp::f); + EXPECT_EQ(as_op2("pow(a,b)"), &Pow::f); + EXPECT_EQ(as_op2("fmod(a,b)"), &Mod::f); + EXPECT_EQ(as_op2("min(a,b)"), &Min::f); + EXPECT_EQ(as_op2("max(a,b)"), &Max::f); +} + +TEST(InlineOperationTest, op2_lambdas_are_recognized_with_different_parameter_names) { + EXPECT_EQ(lookup_op2(*Function::parse({"x", "y"}, "x+y")).value(), &Add::f); + EXPECT_EQ(lookup_op2(*Function::parse({"x", "y"}, "x-y")).value(), &Sub::f); +} + +TEST(InlineOperationTest, non_op2_lambdas_are_not_recognized) { + EXPECT_FALSE(lookup_op2(*Function::parse({"a"}, "-a")).has_value()); + EXPECT_FALSE(lookup_op2(*Function::parse({"a", "b"}, "b+a")).has_value()); +} + +TEST(InlineOperationTest, generic_op1_wrapper_works) { + CallOp1 op(Neg::f); + EXPECT_EQ(op(3), -3); + EXPECT_EQ(op(-5), 5); +} + +TEST(InlineOperationTest, generic_op2_wrapper_works) { + CallOp2 op(Add::f); + EXPECT_EQ(op(2,3), 5); + EXPECT_EQ(op(3,7), 10); +} + +TEST(InlineOperationTest, op1_typifier_forwards_return_value_correctly) { + auto a = typify_invoke<1,TypifyOp1,AsValue>(Neg::f); + auto b = typify_invoke<1,TypifyOp1,AsRef>(Neg::f); + EXPECT_EQ(a, my_value); + EXPECT_EQ(b, my_value); + bool same_memory = (&(typify_invoke<1,TypifyOp1,AsRef>(Neg::f)) == &my_value); + EXPECT_EQ(same_memory, true); +} + +TEST(InlineOperationTest, op2_typifier_forwards_return_value_correctly) { + auto a = typify_invoke<1,TypifyOp2,AsValue>(Add::f); + auto b = typify_invoke<1,TypifyOp2,AsRef>(Add::f); + EXPECT_EQ(a, my_value); + EXPECT_EQ(b, my_value); + bool same_memory = (&(typify_invoke<1,TypifyOp2,AsRef>(Add::f)) == &my_value); + EXPECT_EQ(same_memory, true); +} + +TEST(InlineOperationTest, inline_op1_example_works) { + op1_t ignored = nullptr; + InlineOp1<Inv> op(ignored); + EXPECT_EQ(op(2.0), 0.5); + EXPECT_EQ(op(4.0f), 0.25f); + EXPECT_EQ(op(8.0), 0.125); +} + +TEST(InlineOperationTest, inline_op2_example_works) { + op2_t ignored = nullptr; + InlineOp2<Add> op(ignored); + EXPECT_EQ(op(2.0, 3.0), 5.0); + EXPECT_EQ(op(3.0, 7.0), 10.0); +} + +TEST(InlineOperationTest, parameter_swap_wrapper_works) { + CallOp2 op(Sub::f); + SwapArgs2<CallOp2> swap_op(Sub::f); + EXPECT_EQ(op(2,3), -1); + EXPECT_EQ(swap_op(2,3), 1); + EXPECT_EQ(op(3,7), -4); + EXPECT_EQ(swap_op(3,7), 4); +} + +//----------------------------------------------------------------------------- + +TEST(InlineOperationTest, op1_cube_is_inlined) { + TypifyOp1::resolve(Cube::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Cube>>; + op1_t ref = Cube::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 2.0, 8.0); + test_op1<T>(ref, 3.0, 27.0); + test_op1<T>(ref, 7.0, 343.0); + }); +} + +TEST(InlineOperationTest, op1_exp_is_inlined) { + TypifyOp1::resolve(Exp::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Exp>>; + op1_t ref = Exp::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 2.0, std::exp(2.0)); + test_op1<T>(ref, 3.0, std::exp(3.0)); + test_op1<T>(ref, 7.0, std::exp(7.0)); + }); +} + +TEST(InlineOperationTest, op1_inv_is_inlined) { + TypifyOp1::resolve(Inv::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Inv>>; + op1_t ref = Inv::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 2.0, 1.0/2.0); + test_op1<T>(ref, 4.0, 1.0/4.0); + test_op1<T>(ref, 8.0, 1.0/8.0); + }); +} + +TEST(InlineOperationTest, op1_sqrt_is_inlined) { + TypifyOp1::resolve(Sqrt::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Sqrt>>; + op1_t ref = Sqrt::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 2.0, sqrt(2.0)); + test_op1<T>(ref, 4.0, sqrt(4.0)); + test_op1<T>(ref, 64.0, sqrt(64.0)); + }); +} + +TEST(InlineOperationTest, op1_square_is_inlined) { + TypifyOp1::resolve(Square::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Square>>; + op1_t ref = Square::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 2.0, 4.0); + test_op1<T>(ref, 3.0, 9.0); + test_op1<T>(ref, 7.0, 49.0); + }); +} + +TEST(InlineOperationTest, op1_tanh_is_inlined) { + TypifyOp1::resolve(Tanh::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Tanh>>; + op1_t ref = Tanh::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 0.1, std::tanh(0.1)); + test_op1<T>(ref, 0.3, std::tanh(0.3)); + test_op1<T>(ref, 0.7, std::tanh(0.7)); + }); +} + +TEST(InlineOperationTest, op1_neg_is_not_inlined) { + TypifyOp1::resolve(Neg::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,CallOp1>; + op1_t ref = Neg::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 3.0, -3.0); + test_op1<T>(ref, 5.0, -5.0); + test_op1<T>(ref, -2.0, 2.0); + }); +} + +//----------------------------------------------------------------------------- + +TEST(InlineOperationTest, op2_add_is_inlined) { + TypifyOp2::resolve(Add::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp2<Add>>; + op2_t ref = Add::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 2.0, 2.0, 4.0); + test_op2<T>(ref, 3.0, 8.0, 11.0); + test_op2<T>(ref, 7.0, 1.0, 8.0); + }); +} + +TEST(InlineOperationTest, op2_div_is_inlined) { + TypifyOp2::resolve(Div::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp2<Div>>; + op2_t ref = Div::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 2.0, 2.0, 1.0); + test_op2<T>(ref, 3.0, 8.0, 3.0 / 8.0); + test_op2<T>(ref, 7.0, 5.0, 7.0 / 5.0); + }); +} + +TEST(InlineOperationTest, op2_mul_is_inlined) { + TypifyOp2::resolve(Mul::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp2<Mul>>; + op2_t ref = Mul::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 2.0, 2.0, 4.0); + test_op2<T>(ref, 3.0, 8.0, 24.0); + test_op2<T>(ref, 7.0, 5.0, 35.0); + }); +} + +TEST(InlineOperationTest, op2_pow_is_inlined) { + TypifyOp2::resolve(Pow::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp2<Pow>>; + op2_t ref = Pow::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 2.0, 2.0, std::pow(2.0, 2.0)); + test_op2<T>(ref, 3.0, 8.0, std::pow(3.0, 8.0)); + test_op2<T>(ref, 7.0, 5.0, std::pow(7.0, 5.0)); + }); +} + +TEST(InlineOperationTest, op2_sub_is_inlined) { + TypifyOp2::resolve(Sub::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp2<Sub>>; + op2_t ref = Sub::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 3.0, 2.0, 1.0); + test_op2<T>(ref, 3.0, 8.0, -5.0); + test_op2<T>(ref, 7.0, 5.0, 2.0); + }); +} + +TEST(InlineOperationTest, op2_mod_is_not_inlined) { + TypifyOp2::resolve(Mod::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,CallOp2>; + op2_t ref = Mod::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 3.0, 2.0, std::fmod(3.0, 2.0)); + test_op2<T>(ref, 3.0, 8.0, std::fmod(3.0, 8.0)); + test_op2<T>(ref, 7.0, 5.0, std::fmod(7.0, 5.0)); + }); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/eval/tensor_function/tensor_function_test.cpp b/eval/src/tests/eval/tensor_function/tensor_function_test.cpp index 1eb9912abd2..b205205f52e 100644 --- a/eval/src/tests/eval/tensor_function/tensor_function_test.cpp +++ b/eval/src/tests/eval/tensor_function/tensor_function_test.cpp @@ -38,7 +38,7 @@ struct EvalCtx { ictx = std::make_unique<InterpretedFunction::Context>(*ifun); return ifun->eval(*ictx, SimpleObjectParams(params)); } - const TensorFunction &compile(const tensor_function::Node &expr) { + const TensorFunction &compile(const TensorFunction &expr) { return engine.optimize(expr, stash); } Value::UP make_double(double value) { @@ -391,15 +391,15 @@ TEST("require that if_node works") { TEST("require that if_node result is mutable only when both children produce mutable results") { Stash stash; - const Node &cond = inject(DoubleValue::double_type(), 0, stash); - const Node &a = inject(ValueType::from_spec("tensor(x[2])"), 0, stash); - const Node &b = inject(ValueType::from_spec("tensor(x[3])"), 0, stash); - const Node &c = inject(ValueType::from_spec("tensor(x[5])"), 0, stash); - const Node &tmp = concat(a, b, "x", stash); // will be mutable - const Node &if_con_con = if_node(cond, c, c, stash); - const Node &if_mut_con = if_node(cond, tmp, c, stash); - const Node &if_con_mut = if_node(cond, c, tmp, stash); - const Node &if_mut_mut = if_node(cond, tmp, tmp, stash); + const TensorFunction &cond = inject(DoubleValue::double_type(), 0, stash); + const TensorFunction &a = inject(ValueType::from_spec("tensor(x[2])"), 0, stash); + const TensorFunction &b = inject(ValueType::from_spec("tensor(x[3])"), 0, stash); + const TensorFunction &c = inject(ValueType::from_spec("tensor(x[5])"), 0, stash); + const TensorFunction &tmp = concat(a, b, "x", stash); // will be mutable + const TensorFunction &if_con_con = if_node(cond, c, c, stash); + const TensorFunction &if_mut_con = if_node(cond, tmp, c, stash); + const TensorFunction &if_con_mut = if_node(cond, c, tmp, stash); + const TensorFunction &if_mut_mut = if_node(cond, tmp, tmp, stash); EXPECT_EQUAL(if_con_con.result_type(), c.result_type()); EXPECT_EQUAL(if_con_mut.result_type(), c.result_type()); EXPECT_EQUAL(if_mut_con.result_type(), c.result_type()); @@ -412,13 +412,13 @@ TEST("require that if_node result is mutable only when both children produce mut TEST("require that if_node gets expected result type") { Stash stash; - const Node &a = inject(DoubleValue::double_type(), 0, stash); - const Node &b = inject(ValueType::from_spec("tensor(x[2])"), 0, stash); - const Node &c = inject(ValueType::from_spec("tensor(x[3])"), 0, stash); - const Node &d = inject(ValueType::from_spec("error"), 0, stash); - const Node &if_same = if_node(a, b, b, stash); - const Node &if_different = if_node(a, b, c, stash); - const Node &if_with_error = if_node(a, b, d, stash); + const TensorFunction &a = inject(DoubleValue::double_type(), 0, stash); + const TensorFunction &b = inject(ValueType::from_spec("tensor(x[2])"), 0, stash); + const TensorFunction &c = inject(ValueType::from_spec("tensor(x[3])"), 0, stash); + const TensorFunction &d = inject(ValueType::from_spec("error"), 0, stash); + const TensorFunction &if_same = if_node(a, b, b, stash); + const TensorFunction &if_different = if_node(a, b, c, stash); + const TensorFunction &if_with_error = if_node(a, b, d, stash); EXPECT_EQUAL(if_same.result_type(), ValueType::from_spec("tensor(x[2])")); EXPECT_EQUAL(if_different.result_type(), ValueType::from_spec("error")); EXPECT_EQUAL(if_with_error.result_type(), ValueType::from_spec("error")); @@ -426,10 +426,10 @@ TEST("require that if_node gets expected result type") { TEST("require that push_children works") { Stash stash; - std::vector<Node::Child::CREF> refs; - const Node &a = inject(DoubleValue::double_type(), 0, stash); - const Node &b = inject(DoubleValue::double_type(), 1, stash); - const Node &c = const_value(stash.create<DoubleValue>(1.0), stash); + std::vector<TensorFunction::Child::CREF> refs; + const TensorFunction &a = inject(DoubleValue::double_type(), 0, stash); + const TensorFunction &b = inject(DoubleValue::double_type(), 1, stash); + const TensorFunction &c = const_value(stash.create<DoubleValue>(1.0), stash); a.push_children(refs); b.push_children(refs); c.push_children(refs); diff --git a/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp b/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp index a571837b8e9..92fdbfade46 100644 --- a/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp +++ b/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp @@ -67,7 +67,6 @@ TEST("require that matmul can be optimized") { TEST("require that matmul with lambda can be optimized") { TEST_DO(verify_optimized("reduce(join(a2d3,b5d3,f(x,y)(x*y)),sum,d)", 2, 3, 5, true, true)); - TEST_DO(verify_optimized("reduce(join(a2d3,b5d3,f(x,y)(y*x)),sum,d)", 2, 3, 5, true, true)); } TEST("require that expressions similar to matmul are not optimized") { @@ -75,6 +74,7 @@ TEST("require that expressions similar to matmul are not optimized") { TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,sum,b)")); TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,prod,d)")); TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,sum)")); + TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(y*x)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(x+y)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(x*x)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(y*y)),sum,d)")); diff --git a/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp b/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp index c0823248538..f9c563c9bf8 100644 --- a/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp +++ b/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp @@ -78,7 +78,6 @@ TEST("require that single multi matmul can be optimized") { TEST("require that multi matmul with lambda can be optimized") { TEST_DO(verify_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(x*y)),sum,d)", 2, 3, 5, 6, true, true)); - TEST_DO(verify_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(y*x)),sum,d)", 2, 3, 5, 6, true, true)); } TEST("require that expressions similar to multi matmul are not optimized") { @@ -86,6 +85,7 @@ TEST("require that expressions similar to multi matmul are not optimized") { TEST_DO(verify_not_optimized("reduce(A2B1C3a2d3*A2B1C3b5d3,sum,b)")); TEST_DO(verify_not_optimized("reduce(A2B1C3a2d3*A2B1C3b5d3,prod,d)")); TEST_DO(verify_not_optimized("reduce(A2B1C3a2d3*A2B1C3b5d3,sum)")); + TEST_DO(verify_not_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(y*x)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(x+y)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(x*x)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(y*y)),sum,d)")); diff --git a/eval/src/tests/tensor/dense_pow_as_map_optimizer/CMakeLists.txt b/eval/src/tests/tensor/dense_pow_as_map_optimizer/CMakeLists.txt new file mode 100644 index 00000000000..cf5103f8b4f --- /dev/null +++ b/eval/src/tests/tensor/dense_pow_as_map_optimizer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_dense_pow_as_map_optimizer_test_app TEST + SOURCES + dense_pow_as_map_optimizer_test.cpp + DEPENDS + vespaeval + gtest +) +vespa_add_test(NAME eval_dense_pow_as_map_optimizer_test_app COMMAND eval_dense_pow_as_map_optimizer_test_app) diff --git a/eval/src/tests/tensor/dense_pow_as_map_optimizer/dense_pow_as_map_optimizer_test.cpp b/eval/src/tests/tensor/dense_pow_as_map_optimizer/dense_pow_as_map_optimizer_test.cpp new file mode 100644 index 00000000000..38d9ac8aeef --- /dev/null +++ b/eval/src/tests/tensor/dense_pow_as_map_optimizer/dense_pow_as_map_optimizer_test.cpp @@ -0,0 +1,92 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/dense/dense_simple_map_function.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval::operation; +using namespace vespalib::eval::tensor_function; +using namespace vespalib::eval::test; +using namespace vespalib::eval; +using namespace vespalib::tensor; +//using namespace vespalib; + +const TensorEngine &prod_engine = DefaultTensorEngine::ref(); + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add("a", spec(1.5)) + .add("b", spec(2.5)) + .add("sparse", spec({x({"a"})}, N())) + .add("mixed", spec({x({"a"}),y(5)}, N())) + .add_matrix("x", 5, "y", 3); +} +EvalFixture::ParamRepo param_repo = make_params(); + +void verify_optimized(const vespalib::string &expr, op1_t op1, bool inplace = false) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true, true); + EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseSimpleMapFunction>(); + ASSERT_EQ(info.size(), 1u); + EXPECT_TRUE(info[0]->result_is_mutable()); + EXPECT_EQ(info[0]->function(), op1); + EXPECT_EQ(info[0]->inplace(), inplace); + ASSERT_EQ(fixture.num_params(), 1); + if (inplace) { + EXPECT_EQ(fixture.get_param(0), fixture.result()); + } else { + EXPECT_TRUE(!(fixture.get_param(0) == fixture.result())); + } +} + +void verify_not_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true); + EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<Map>(); + EXPECT_TRUE(info.empty()); +} + +TEST(PowAsMapTest, squared_dense_tensor_is_optimized) { + verify_optimized("x5y3^2.0", Square::f); + verify_optimized("pow(x5y3,2.0)", Square::f); + verify_optimized("join(x5y3,2.0,f(x,y)(x^y))", Square::f); + verify_optimized("join(x5y3,2.0,f(x,y)(pow(x,y)))", Square::f); + verify_optimized("join(x5y3f,2.0,f(x,y)(pow(x,y)))", Square::f); + verify_optimized("join(@x5y3,2.0,f(x,y)(pow(x,y)))", Square::f, true); + verify_optimized("join(@x5y3f,2.0,f(x,y)(pow(x,y)))", Square::f, true); +} + +TEST(PowAsMapTest, cubed_dense_tensor_is_optimized) { + verify_optimized("x5y3^3.0", Cube::f); + verify_optimized("pow(x5y3,3.0)", Cube::f); + verify_optimized("join(x5y3,3.0,f(x,y)(x^y))", Cube::f); + verify_optimized("join(x5y3,3.0,f(x,y)(pow(x,y)))", Cube::f); + verify_optimized("join(x5y3f,3.0,f(x,y)(pow(x,y)))", Cube::f); + verify_optimized("join(@x5y3,3.0,f(x,y)(pow(x,y)))", Cube::f, true); + verify_optimized("join(@x5y3f,3.0,f(x,y)(pow(x,y)))", Cube::f, true); +} + +TEST(PowAsMapTest, hypercubed_dense_tensor_is_not_optimized) { + verify_not_optimized("join(x5y3,4.0,f(x,y)(pow(x,y)))"); +} + +TEST(PowAsMapTest, scalar_join_is_not_optimized) { + verify_not_optimized("join(a,2.0,f(x,y)(pow(x,y)))"); +} + +TEST(PowAsMapTest, sparse_join_is_not_optimized) { + verify_not_optimized("join(sparse,2.0,f(x,y)(pow(x,y)))"); +} + +TEST(PowAsMapTest, mixed_join_is_not_optimized) { + verify_not_optimized("join(mixed,2.0,f(x,y)(pow(x,y)))"); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/dense_simple_expand_function/CMakeLists.txt b/eval/src/tests/tensor/dense_simple_expand_function/CMakeLists.txt new file mode 100644 index 00000000000..4e7409a6139 --- /dev/null +++ b/eval/src/tests/tensor/dense_simple_expand_function/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_dense_simple_expand_function_test_app TEST + SOURCES + dense_simple_expand_function_test.cpp + DEPENDS + vespaeval + gtest +) +vespa_add_test(NAME eval_dense_simple_expand_function_test_app COMMAND eval_dense_simple_expand_function_test_app) diff --git a/eval/src/tests/tensor/dense_simple_expand_function/dense_simple_expand_function_test.cpp b/eval/src/tests/tensor/dense_simple_expand_function/dense_simple_expand_function_test.cpp new file mode 100644 index 00000000000..4b870bc0153 --- /dev/null +++ b/eval/src/tests/tensor/dense_simple_expand_function/dense_simple_expand_function_test.cpp @@ -0,0 +1,130 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/eval/simple_tensor.h> +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/dense/dense_simple_expand_function.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; +using namespace vespalib::eval::tensor_function; +using namespace vespalib::tensor; + +using Inner = DenseSimpleExpandFunction::Inner; + +const TensorEngine &prod_engine = DefaultTensorEngine::ref(); + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add("a", spec(1.5)) + .add("sparse", spec({x({"a"})}, N())) + .add("mixed", spec({y({"a"}),z(5)}, N())) + .add_vector("a", 5) + .add_vector("b", 3) + .add_cube("A", 1, "a", 5, "c", 1) + .add_cube("B", 1, "b", 3, "c", 1) + .add_matrix("a", 5, "c", 3) + .add_matrix("x", 3, "y", 2) + .add_cube("a", 1, "b", 1, "c", 1) + .add_cube("x", 1, "y", 1, "z", 1); +} + +EvalFixture::ParamRepo param_repo = make_params(); + +void verify_optimized(const vespalib::string &expr, Inner inner) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true, true); + EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseSimpleExpandFunction>(); + ASSERT_EQ(info.size(), 1u); + EXPECT_TRUE(info[0]->result_is_mutable()); + EXPECT_EQ(info[0]->inner(), inner); + ASSERT_EQ(fixture.num_params(), 2); + EXPECT_TRUE(!(fixture.get_param(0) == fixture.result())); + EXPECT_TRUE(!(fixture.get_param(1) == fixture.result())); +} + +void verify_not_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true); + EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseSimpleExpandFunction>(); + EXPECT_TRUE(info.empty()); +} + +TEST(ExpandTest, simple_expand_is_optimized) { + verify_optimized("join(a5,b3,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(b3,a5,f(x,y)(x*y))", Inner::LHS); +} + +TEST(ExpandTest, multiple_dimensions_are_supported) { + verify_optimized("join(a5,x3y2,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(x3y2,a5,f(x,y)(x*y))", Inner::LHS); + verify_optimized("join(a5c3,x3y2,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(x3y2,a5c3,f(x,y)(x*y))", Inner::LHS); +} + +TEST(ExpandTest, trivial_dimensions_are_ignored) { + verify_optimized("join(A1a5c1,B1b3c1,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(B1b3c1,A1a5c1,f(x,y)(x*y))", Inner::LHS); +} + +TEST(ExpandTest, simple_expand_handles_asymmetric_operations_correctly) { + verify_optimized("join(a5,b3,f(x,y)(x-y))", Inner::RHS); + verify_optimized("join(b3,a5,f(x,y)(x-y))", Inner::LHS); + verify_optimized("join(a5,b3,f(x,y)(x/y))", Inner::RHS); + verify_optimized("join(b3,a5,f(x,y)(x/y))", Inner::LHS); +} + +TEST(ExpandTest, simple_expand_can_have_various_cell_types) { + verify_optimized("join(a5,b3f,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(a5f,b3,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(a5f,b3f,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(b3,a5f,f(x,y)(x*y))", Inner::LHS); + verify_optimized("join(b3f,a5,f(x,y)(x*y))", Inner::LHS); + verify_optimized("join(b3f,a5f,f(x,y)(x*y))", Inner::LHS); +} + +TEST(ExpandTest, simple_expand_is_never_inplace) { + verify_optimized("join(@a5,@b3,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(@b3,@a5,f(x,y)(x*y))", Inner::LHS); +} + +TEST(ExpandTest, interleaved_dimensions_are_not_optimized) { + verify_not_optimized("join(a5c3,b3,f(x,y)(x*y))"); + verify_not_optimized("join(b3,a5c3,f(x,y)(x*y))"); +} + +TEST(ExpandTest, matching_dimensions_are_not_expanding) { + verify_not_optimized("join(a5c3,a5,f(x,y)(x*y))"); + verify_not_optimized("join(a5,a5c3,f(x,y)(x*y))"); +} + +TEST(ExpandTest, scalar_is_not_expanding) { + verify_not_optimized("join(a5,a,f(x,y)(x*y))"); +} + +TEST(ExpandTest, unit_tensor_is_not_expanding) { + verify_not_optimized("join(a5,x1y1z1,f(x,y)(x+y))"); + verify_not_optimized("join(x1y1z1,a5,f(x,y)(x+y))"); + verify_not_optimized("join(a1b1c1,x1y1z1,f(x,y)(x+y))"); +} + +TEST(ExpandTest, sparse_expand_is_not_optimized) { + verify_not_optimized("join(a5,sparse,f(x,y)(x*y))"); + verify_not_optimized("join(sparse,a5,f(x,y)(x*y))"); +} + +TEST(ExpandTest, mixed_expand_is_not_optimized) { + verify_not_optimized("join(a5,mixed,f(x,y)(x*y))"); + verify_not_optimized("join(mixed,a5,f(x,y)(x*y))"); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp index 0b924451907..3ecc3f66cda 100644 --- a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp @@ -130,13 +130,13 @@ TEST("require that xw product gives same results as reference join/reduce") { TEST("require that various variants of xw product can be optimized") { TEST_DO(verify_optimized("reduce(join(y3,x2y3,f(x,y)(x*y)),sum,y)", 3, 2, true)); - TEST_DO(verify_optimized("reduce(join(y3,x2y3,f(x,y)(y*x)),sum,y)", 3, 2, true)); } TEST("require that expressions similar to xw product are not optimized") { TEST_DO(verify_not_optimized("reduce(y3*x2y3,sum,x)")); TEST_DO(verify_not_optimized("reduce(y3*x2y3,prod,y)")); TEST_DO(verify_not_optimized("reduce(y3*x2y3,sum)")); + TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(y*x)),sum,y)")); TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(x+y)),sum,y)")); TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(x*x)),sum,y)")); TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(y*y)),sum,y)")); diff --git a/eval/src/vespa/eval/eval/aggr.cpp b/eval/src/vespa/eval/eval/aggr.cpp index d10bbc4abb8..8efb0ec9fe7 100644 --- a/eval/src/vespa/eval/eval/aggr.cpp +++ b/eval/src/vespa/eval/eval/aggr.cpp @@ -71,15 +71,11 @@ Aggregator::~Aggregator() Aggregator & Aggregator::create(Aggr aggr, Stash &stash) { - switch (aggr) { - case Aggr::AVG: return stash.create<Wrapper<aggr::Avg<double>>>(); - case Aggr::COUNT: return stash.create<Wrapper<aggr::Count<double>>>(); - case Aggr::PROD: return stash.create<Wrapper<aggr::Prod<double>>>(); - case Aggr::SUM: return stash.create<Wrapper<aggr::Sum<double>>>(); - case Aggr::MAX: return stash.create<Wrapper<aggr::Max<double>>>(); - case Aggr::MIN: return stash.create<Wrapper<aggr::Min<double>>>(); - } - LOG_ABORT("should not be reached"); + return TypifyAggr::resolve(aggr, [&stash](auto t)->Aggregator& + { + using T = typename decltype(t)::template templ<double>; + return stash.create<Wrapper<T>>(); + }); } std::vector<Aggr> diff --git a/eval/src/vespa/eval/eval/aggr.h b/eval/src/vespa/eval/eval/aggr.h index 8dea54d8abc..169f0b1d2af 100644 --- a/eval/src/vespa/eval/eval/aggr.h +++ b/eval/src/vespa/eval/eval/aggr.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/vespalib/util/typify.h> #include <vespa/vespalib/stllike/string.h> #include <vector> #include <map> @@ -118,5 +119,21 @@ public: }; } // namespave vespalib::eval::aggr + +struct TypifyAggr { + template <template<typename> typename TT> using Result = TypifyResultSimpleTemplate<TT>; + template <typename F> static decltype(auto) resolve(Aggr aggr, F &&f) { + switch (aggr) { + case Aggr::AVG: return f(Result<aggr::Avg>()); + case Aggr::COUNT: return f(Result<aggr::Count>()); + case Aggr::PROD: return f(Result<aggr::Prod>()); + case Aggr::SUM: return f(Result<aggr::Sum>()); + case Aggr::MAX: return f(Result<aggr::Max>()); + case Aggr::MIN: return f(Result<aggr::Min>()); + } + abort(); + } +}; + } // namespace vespalib::eval } // namespace vespalib diff --git a/eval/src/vespa/eval/eval/inline_operation.h b/eval/src/vespa/eval/eval/inline_operation.h new file mode 100644 index 00000000000..21516c4d94e --- /dev/null +++ b/eval/src/vespa/eval/eval/inline_operation.h @@ -0,0 +1,148 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "operation.h" +#include <vespa/vespalib/util/typify.h> +#include <cmath> + +namespace vespalib::eval::operation { + +//----------------------------------------------------------------------------- + +struct CallOp1 { + op1_t my_op1; + CallOp1(op1_t op1) : my_op1(op1) {} + double operator()(double a) const { return my_op1(a); } +}; + +template <typename T> struct InlineOp1; +template <> struct InlineOp1<Cube> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return (a * a * a); } +}; +template <> struct InlineOp1<Exp> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return exp(a); } +}; +template <> struct InlineOp1<Inv> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return (A{1}/a); } +}; +template <> struct InlineOp1<Sqrt> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return std::sqrt(a); } +}; +template <> struct InlineOp1<Square> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return (a * a); } +}; +template <> struct InlineOp1<Tanh> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return std::tanh(a); } +}; + +struct TypifyOp1 { + template <typename T> using Result = TypifyResultType<T>; + template <typename F> static decltype(auto) resolve(op1_t value, F &&f) { + if (value == Cube::f) { + return f(Result<InlineOp1<Cube>>()); + } else if (value == Exp::f) { + return f(Result<InlineOp1<Exp>>()); + } else if (value == Inv::f) { + return f(Result<InlineOp1<Inv>>()); + } else if (value == Sqrt::f) { + return f(Result<InlineOp1<Sqrt>>()); + } else if (value == Square::f) { + return f(Result<InlineOp1<Square>>()); + } else if (value == Tanh::f) { + return f(Result<InlineOp1<Tanh>>()); + } else { + return f(Result<CallOp1>()); + } + } +}; + +//----------------------------------------------------------------------------- + +struct CallOp2 { + op2_t my_op2; + CallOp2(op2_t op2) : my_op2(op2) {} + op2_t get() const { return my_op2; } + double operator()(double a, double b) const { return my_op2(a, b); } +}; + +template <typename Op2> +struct SwapArgs2 { + Op2 op2; + SwapArgs2(op2_t op2_in) : op2(op2_in) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return op2(b, a); } +}; + +template <typename T> struct InlineOp2; +template <> struct InlineOp2<Add> { + InlineOp2(op2_t) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return (a+b); } +}; +template <> struct InlineOp2<Div> { + InlineOp2(op2_t) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return (a/b); } +}; +template <> struct InlineOp2<Mul> { + InlineOp2(op2_t) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return (a*b); } +}; +template <> struct InlineOp2<Pow> { + InlineOp2(op2_t) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return std::pow(a,b); } +}; +template <> struct InlineOp2<Sub> { + InlineOp2(op2_t) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return (a-b); } +}; + +struct TypifyOp2 { + template <typename T> using Result = TypifyResultType<T>; + template <typename F> static decltype(auto) resolve(op2_t value, F &&f) { + if (value == Add::f) { + return f(Result<InlineOp2<Add>>()); + } else if (value == Div::f) { + return f(Result<InlineOp2<Div>>()); + } else if (value == Mul::f) { + return f(Result<InlineOp2<Mul>>()); + } else if (value == Pow::f) { + return f(Result<InlineOp2<Pow>>()); + } else if (value == Sub::f) { + return f(Result<InlineOp2<Sub>>()); + } else { + return f(Result<CallOp2>()); + } + } +}; + +//----------------------------------------------------------------------------- + +template <typename A, typename OP1> +void apply_op1_vec(A *dst, const A *src, size_t n, OP1 &&f) { + for (size_t i = 0; i < n; ++i) { + dst[i] = f(src[i]); + } +} + +template <typename D, typename A, typename B, typename OP2> +void apply_op2_vec_num(D *dst, const A *a, B b, size_t n, OP2 &&f) { + for (size_t i = 0; i < n; ++i) { + dst[i] = f(a[i], b); + } +} + +template <typename D, typename A, typename B, typename OP2> +void apply_op2_vec_vec(D *dst, const A *a, const B *b, size_t n, OP2 &&f) { + for (size_t i = 0; i < n; ++i) { + dst[i] = f(a[i], b[i]); + } +} + +//----------------------------------------------------------------------------- + +} diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp index 4aa18d3bb65..e2674a6e4d6 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp @@ -10,14 +10,14 @@ namespace eval { std::mutex CompileCache::_lock{}; CompileCache::Map CompileCache::_cached{}; uint64_t CompileCache::_executor_tag{0}; -std::vector<std::pair<uint64_t,Executor*>> CompileCache::_executor_stack{}; +std::vector<std::pair<uint64_t,std::shared_ptr<Executor>>> CompileCache::_executor_stack{}; const CompiledFunction & CompileCache::Value::wait_for_result() { - std::unique_lock<std::mutex> guard(_lock); - cond.wait(guard, [this](){ return bool(compiled_function); }); - return *compiled_function; + std::unique_lock<std::mutex> guard(result->lock); + result->cond.wait(guard, [this](){ return bool(result->compiled_function); }); + return *(result->compiled_function); } void @@ -30,10 +30,10 @@ CompileCache::release(Map::iterator entry) } uint64_t -CompileCache::attach_executor(Executor &executor) +CompileCache::attach_executor(std::shared_ptr<Executor> executor) { std::lock_guard<std::mutex> guard(_lock); - _executor_stack.emplace_back(++_executor_tag, &executor); + _executor_stack.emplace_back(++_executor_tag, std::move(executor)); return _executor_tag; } @@ -52,6 +52,7 @@ CompileCache::compile(const Function &function, PassParams pass_params) { Token::UP token; Executor::Task::UP task; + std::shared_ptr<Executor> executor; vespalib::string key = gen_key(function, pass_params); { std::lock_guard<std::mutex> guard(_lock); @@ -63,14 +64,15 @@ CompileCache::compile(const Function &function, PassParams pass_params) auto res = _cached.emplace(std::move(key), Value::ctor_tag()); assert(res.second); token = std::make_unique<Token>(res.first, Token::ctor_tag()); - ++(res.first->second.num_refs); - task = std::make_unique<CompileTask>(function, pass_params, - std::make_unique<Token>(res.first, Token::ctor_tag())); + task = std::make_unique<CompileTask>(function, pass_params, res.first->second.result); if (!_executor_stack.empty()) { - task = _executor_stack.back().second->execute(std::move(task)); + executor = _executor_stack.back().second; } } } + if (executor) { + task = executor->execute(std::move(task)); + } if (task) { std::thread([&task](){ task.get()->run(); }).join(); } @@ -84,7 +86,7 @@ CompileCache::wait_pending() { std::lock_guard<std::mutex> guard(_lock); for (auto entry = _cached.begin(); entry != _cached.end(); ++entry) { - if (entry->second.compiled_function.get() == nullptr) { + if (entry->second.result->cf.load(std::memory_order_acquire) == nullptr) { ++(entry->second.num_refs); pending.push_back(std::make_unique<Token>(entry, Token::ctor_tag())); } @@ -129,7 +131,7 @@ CompileCache::count_pending() std::lock_guard<std::mutex> guard(_lock); size_t pending = 0; for (const auto &entry: _cached) { - if (entry.second.compiled_function.get() == nullptr) { + if (entry.second.result->cf.load(std::memory_order_acquire) == nullptr) { ++pending; } } @@ -139,12 +141,11 @@ CompileCache::count_pending() void CompileCache::CompileTask::run() { - auto &entry = token->_entry->second; - auto result = std::make_unique<CompiledFunction>(*function, pass_params); - std::lock_guard<std::mutex> guard(_lock); - entry.compiled_function = std::move(result); - entry.cf.store(entry.compiled_function.get(), std::memory_order_release); - entry.cond.notify_all(); + auto compiled = std::make_unique<CompiledFunction>(*function, pass_params); + std::lock_guard<std::mutex> guard(result->lock); + result->compiled_function = std::move(compiled); + result->cf.store(result->compiled_function.get(), std::memory_order_release); + result->cond.notify_all(); } } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.h b/eval/src/vespa/eval/eval/llvm/compile_cache.h index 09b5b2060f5..61d0cc83d94 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.h +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.h @@ -23,16 +23,22 @@ class CompileCache { private: using Key = vespalib::string; - struct Value { - size_t num_refs; + struct Result { + using SP = std::shared_ptr<Result>; std::atomic<const CompiledFunction *> cf; + std::mutex lock; std::condition_variable cond; CompiledFunction::UP compiled_function; + Result() : cf(nullptr), lock(), cond(), compiled_function(nullptr) {} + }; + struct Value { + size_t num_refs; + Result::SP result; struct ctor_tag {}; - Value(ctor_tag) : num_refs(1), cf(nullptr), cond(), compiled_function() {} + Value(ctor_tag) : num_refs(1), result(std::make_shared<Result>()) {} const CompiledFunction &wait_for_result(); const CompiledFunction &get() { - const CompiledFunction *ptr = cf.load(std::memory_order_acquire); + const CompiledFunction *ptr = result->cf.load(std::memory_order_acquire); if (ptr == nullptr) { return wait_for_result(); } @@ -43,10 +49,10 @@ private: static std::mutex _lock; static Map _cached; static uint64_t _executor_tag; - static std::vector<std::pair<uint64_t,Executor*>> _executor_stack; + static std::vector<std::pair<uint64_t,std::shared_ptr<Executor>>> _executor_stack; static void release(Map::iterator entry); - static uint64_t attach_executor(Executor &executor); + static uint64_t attach_executor(std::shared_ptr<Executor> executor); static void detach_executor(uint64_t tag); public: @@ -54,7 +60,6 @@ public: { private: friend class CompileCache; - friend class CompileTask; struct ctor_tag {}; CompileCache::Map::iterator _entry; public: @@ -79,14 +84,15 @@ public: ExecutorBinding &operator=(ExecutorBinding &&) = delete; ExecutorBinding &operator=(const ExecutorBinding &) = delete; using UP = std::unique_ptr<ExecutorBinding>; - explicit ExecutorBinding(Executor &executor, ctor_tag) : _tag(attach_executor(executor)) {} + explicit ExecutorBinding(std::shared_ptr<Executor> executor, ctor_tag) + : _tag(attach_executor(std::move(executor))) {} ~ExecutorBinding() { detach_executor(_tag); } }; static Token::UP compile(const Function &function, PassParams pass_params); static void wait_pending(); - static ExecutorBinding::UP bind(Executor &executor) { - return std::make_unique<ExecutorBinding>(executor, ExecutorBinding::ctor_tag()); + static ExecutorBinding::UP bind(std::shared_ptr<Executor> executor) { + return std::make_unique<ExecutorBinding>(std::move(executor), ExecutorBinding::ctor_tag()); } static size_t num_cached(); static size_t num_bound(); @@ -97,9 +103,9 @@ private: struct CompileTask : public Executor::Task { std::shared_ptr<Function const> function; PassParams pass_params; - Token::UP token; - CompileTask(const Function &function_in, PassParams pass_params_in, Token::UP token_in) - : function(function_in.shared_from_this()), pass_params(pass_params_in), token(std::move(token_in)) {} + Result::SP result; + CompileTask(const Function &function_in, PassParams pass_params_in, Result::SP result_in) + : function(function_in.shared_from_this()), pass_params(pass_params_in), result(std::move(result_in)) {} void run() override; }; }; diff --git a/eval/src/vespa/eval/eval/make_tensor_function.cpp b/eval/src/vespa/eval/eval/make_tensor_function.cpp index 3a73a3b8784..e80633b5c41 100644 --- a/eval/src/vespa/eval/eval/make_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/make_tensor_function.cpp @@ -15,25 +15,6 @@ namespace vespalib::eval { namespace { using namespace nodes; -using map_fun_t = double (*)(double); -using join_fun_t = double (*)(double, double); - -//----------------------------------------------------------------------------- - -// TODO(havardpe): generic function pointer resolving for all single -// operation lambdas. - -template <typename OP2> -bool is_op2(const Function &lambda) { - if (lambda.num_params() == 2) { - if (auto op2 = as<OP2>(lambda.root())) { - auto sym1 = as<Symbol>(op2->lhs()); - auto sym2 = as<Symbol>(op2->rhs()); - return (sym1 && sym2 && (sym1->id() != sym2->id())); - } - } - return false; -} //----------------------------------------------------------------------------- @@ -41,7 +22,7 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { Stash &stash; const TensorEngine &tensor_engine; const NodeTypes &types; - std::vector<tensor_function::Node::CREF> stack; + std::vector<TensorFunction::CREF> stack; TensorFunctionBuilder(Stash &stash_in, const TensorEngine &tensor_engine_in, const NodeTypes &types_in) : stash(stash_in), tensor_engine(tensor_engine_in), types(types_in), stack() {} @@ -63,13 +44,13 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { stack.back() = tensor_function::reduce(a, aggr, dimensions, stash); } - void make_map(const Node &, map_fun_t function) { + void make_map(const Node &, operation::op1_t function) { assert(stack.size() >= 1); const auto &a = stack.back().get(); stack.back() = tensor_function::map(a, function, stash); } - void make_join(const Node &, join_fun_t function) { + void make_join(const Node &, operation::op2_t function) { assert(stack.size() >= 2); const auto &b = stack.back().get(); stack.pop_back(); @@ -77,7 +58,7 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { stack.back() = tensor_function::join(a, b, function, stash); } - void make_merge(const Node &, join_fun_t function) { + void make_merge(const Node &, operation::op2_t function) { assert(stack.size() >= 2); const auto &b = stack.back().get(); stack.pop_back(); @@ -113,7 +94,7 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void make_create(const TensorCreate &node) { assert(stack.size() >= node.num_children()); - std::map<TensorSpec::Address, tensor_function::Node::CREF> spec; + std::map<TensorSpec::Address, TensorFunction::CREF> spec; for (size_t idx = node.num_children(); idx-- > 0; ) { spec.emplace(node.get_child_address(idx), stack.back()); stack.pop_back(); @@ -134,8 +115,8 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void make_peek(const TensorPeek &node) { assert(stack.size() >= node.num_children()); - const tensor_function::Node ¶m = stack[stack.size()-node.num_children()]; - std::map<vespalib::string, std::variant<TensorSpec::Label, tensor_function::Node::CREF>> spec; + const TensorFunction ¶m = stack[stack.size()-node.num_children()]; + std::map<vespalib::string, std::variant<TensorSpec::Label, TensorFunction::CREF>> spec; for (auto pos = node.dim_list().rbegin(); pos != node.dim_list().rend(); ++pos) { if (pos->second.is_expr()) { spec.emplace(pos->first, stack.back()); @@ -203,14 +184,16 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { abort(); } void visit(const TensorMap &node) override { - const auto &token = stash.create<CompileCache::Token::UP>(CompileCache::compile(node.lambda(), PassParams::SEPARATE)); - make_map(node, token.get()->get().get_function<1>()); + if (auto op1 = operation::lookup_op1(node.lambda())) { + make_map(node, op1.value()); + } else { + const auto &token = stash.create<CompileCache::Token::UP>(CompileCache::compile(node.lambda(), PassParams::SEPARATE)); + make_map(node, token.get()->get().get_function<1>()); + } } void visit(const TensorJoin &node) override { - if (is_op2<Mul>(node.lambda())) { - make_join(node, operation::Mul::f); - } else if (is_op2<Add>(node.lambda())) { - make_join(node, operation::Add::f); + if (auto op2 = operation::lookup_op2(node.lambda())) { + make_join(node, op2.value()); } else { const auto &token = stash.create<CompileCache::Token::UP>(CompileCache::compile(node.lambda(), PassParams::SEPARATE)); make_join(node, token.get()->get().get_function<2>()); diff --git a/eval/src/vespa/eval/eval/operation.cpp b/eval/src/vespa/eval/eval/operation.cpp index fa0a99de461..fa8be4d20bc 100644 --- a/eval/src/vespa/eval/eval/operation.cpp +++ b/eval/src/vespa/eval/eval/operation.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "operation.h" +#include "function.h" +#include "key_gen.h" #include <vespa/vespalib/util/approx.h> #include <algorithm> @@ -47,5 +49,111 @@ double IsNan::f(double a) { return std::isnan(a) ? 1.0 : 0.0; } double Relu::f(double a) { return std::max(a, 0.0); } double Sigmoid::f(double a) { return 1.0 / (1.0 + std::exp(-1.0 * a)); } double Elu::f(double a) { return (a < 0) ? std::exp(a) - 1 : a; } +//----------------------------------------------------------------------------- +double Inv::f(double a) { return (1.0 / a); } +double Square::f(double a) { return (a * a); } +double Cube::f(double a) { return (a * a * a); } + +namespace { + +template <typename T> +void add_op(std::map<vespalib::string,T> &map, const Function &fun, T op) { + assert(!fun.has_error()); + auto key = gen_key(fun, PassParams::SEPARATE); + auto res = map.emplace(key, op); + assert(res.second); +} + +template <typename T> +std::optional<T> lookup_op(const std::map<vespalib::string,T> &map, const Function &fun) { + auto key = gen_key(fun, PassParams::SEPARATE); + auto pos = map.find(key); + if (pos != map.end()) { + return pos->second; + } + return std::nullopt; +} + +void add_op1(std::map<vespalib::string,op1_t> &map, const vespalib::string &expr, op1_t op) { + add_op(map, *Function::parse({"a"}, expr), op); +} + +void add_op2(std::map<vespalib::string,op2_t> &map, const vespalib::string &expr, op2_t op) { + add_op(map, *Function::parse({"a", "b"}, expr), op); +} + +std::map<vespalib::string,op1_t> make_op1_map() { + std::map<vespalib::string,op1_t> map; + add_op1(map, "-a", Neg::f); + add_op1(map, "!a", Not::f); + add_op1(map, "cos(a)", Cos::f); + add_op1(map, "sin(a)", Sin::f); + add_op1(map, "tan(a)", Tan::f); + add_op1(map, "cosh(a)", Cosh::f); + add_op1(map, "sinh(a)", Sinh::f); + add_op1(map, "tanh(a)", Tanh::f); + add_op1(map, "acos(a)", Acos::f); + add_op1(map, "asin(a)", Asin::f); + add_op1(map, "atan(a)", Atan::f); + add_op1(map, "exp(a)", Exp::f); + add_op1(map, "log10(a)", Log10::f); + add_op1(map, "log(a)", Log::f); + add_op1(map, "sqrt(a)", Sqrt::f); + add_op1(map, "ceil(a)", Ceil::f); + add_op1(map, "fabs(a)", Fabs::f); + add_op1(map, "floor(a)", Floor::f); + add_op1(map, "isNan(a)", IsNan::f); + add_op1(map, "relu(a)", Relu::f); + add_op1(map, "sigmoid(a)", Sigmoid::f); + add_op1(map, "elu(a)", Elu::f); + //------------------------------------- + add_op1(map, "1/a", Inv::f); + add_op1(map, "a*a", Square::f); + add_op1(map, "a^2", Square::f); + add_op1(map, "pow(a,2)", Square::f); + add_op1(map, "(a*a)*a", Cube::f); + add_op1(map, "a*(a*a)", Cube::f); + add_op1(map, "a^3", Cube::f); + add_op1(map, "pow(a,3)", Cube::f); + return map; +} + +std::map<vespalib::string,op2_t> make_op2_map() { + std::map<vespalib::string,op2_t> map; + add_op2(map, "a+b", Add::f); + add_op2(map, "a-b", Sub::f); + add_op2(map, "a*b", Mul::f); + add_op2(map, "a/b", Div::f); + add_op2(map, "a%b", Mod::f); + add_op2(map, "a^b", Pow::f); + add_op2(map, "a==b", Equal::f); + add_op2(map, "a!=b", NotEqual::f); + add_op2(map, "a~=b", Approx::f); + add_op2(map, "a<b", Less::f); + add_op2(map, "a<=b", LessEqual::f); + add_op2(map, "a>b", Greater::f); + add_op2(map, "a>=b", GreaterEqual::f); + add_op2(map, "a&&b", And::f); + add_op2(map, "a||b", Or::f); + add_op2(map, "atan2(a,b)", Atan2::f); + add_op2(map, "ldexp(a,b)", Ldexp::f); + add_op2(map, "pow(a,b)", Pow::f); + add_op2(map, "fmod(a,b)", Mod::f); + add_op2(map, "min(a,b)", Min::f); + add_op2(map, "max(a,b)", Max::f); + return map; +} + +} // namespace <unnamed> + +std::optional<op1_t> lookup_op1(const Function &fun) { + static const std::map<vespalib::string,op1_t> map = make_op1_map(); + return lookup_op(map, fun); +} + +std::optional<op2_t> lookup_op2(const Function &fun) { + static const std::map<vespalib::string,op2_t> map = make_op2_map(); + return lookup_op(map, fun); +} } diff --git a/eval/src/vespa/eval/eval/operation.h b/eval/src/vespa/eval/eval/operation.h index fa99f51a308..02d3322f867 100644 --- a/eval/src/vespa/eval/eval/operation.h +++ b/eval/src/vespa/eval/eval/operation.h @@ -1,6 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <optional> + +namespace vespalib::eval { class Function; } namespace vespalib::eval::operation { @@ -45,5 +48,15 @@ struct IsNan { static double f(double a); }; struct Relu { static double f(double a); }; struct Sigmoid { static double f(double a); }; struct Elu { static double f(double a); }; +//----------------------------------------------------------------------------- +struct Inv { static double f(double a); }; +struct Square { static double f(double a); }; +struct Cube { static double f(double a); }; + +using op1_t = double (*)(double); +using op2_t = double (*)(double, double); + +std::optional<op1_t> lookup_op1(const Function &fun); +std::optional<op2_t> lookup_op2(const Function &fun); } diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp index 1aa18417b87..85079b7a8e3 100644 --- a/eval/src/vespa/eval/eval/tensor_function.cpp +++ b/eval/src/vespa/eval/eval/tensor_function.cpp @@ -545,48 +545,48 @@ If::visit_children(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- -const Node &const_value(const Value &value, Stash &stash) { +const TensorFunction &const_value(const Value &value, Stash &stash) { return stash.create<ConstValue>(value); } -const Node &inject(const ValueType &type, size_t param_idx, Stash &stash) { +const TensorFunction &inject(const ValueType &type, size_t param_idx, Stash &stash) { return stash.create<Inject>(type, param_idx); } -const Node &reduce(const Node &child, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) { +const TensorFunction &reduce(const TensorFunction &child, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) { ValueType result_type = child.result_type().reduce(dimensions); return stash.create<Reduce>(result_type, child, aggr, dimensions); } -const Node &map(const Node &child, map_fun_t function, Stash &stash) { +const TensorFunction &map(const TensorFunction &child, map_fun_t function, Stash &stash) { ValueType result_type = child.result_type(); return stash.create<Map>(result_type, child, function); } -const Node &join(const Node &lhs, const Node &rhs, join_fun_t function, Stash &stash) { +const TensorFunction &join(const TensorFunction &lhs, const TensorFunction &rhs, join_fun_t function, Stash &stash) { ValueType result_type = ValueType::join(lhs.result_type(), rhs.result_type()); return stash.create<Join>(result_type, lhs, rhs, function); } -const Node &merge(const Node &lhs, const Node &rhs, join_fun_t function, Stash &stash) { +const TensorFunction &merge(const TensorFunction &lhs, const TensorFunction &rhs, join_fun_t function, Stash &stash) { ValueType result_type = ValueType::merge(lhs.result_type(), rhs.result_type()); return stash.create<Merge>(result_type, lhs, rhs, function); } -const Node &concat(const Node &lhs, const Node &rhs, const vespalib::string &dimension, Stash &stash) { +const TensorFunction &concat(const TensorFunction &lhs, const TensorFunction &rhs, const vespalib::string &dimension, Stash &stash) { ValueType result_type = ValueType::concat(lhs.result_type(), rhs.result_type(), dimension); return stash.create<Concat>(result_type, lhs, rhs, dimension); } -const Node &create(const ValueType &type, const std::map<TensorSpec::Address,Node::CREF> &spec, Stash &stash) { +const TensorFunction &create(const ValueType &type, const std::map<TensorSpec::Address,TensorFunction::CREF> &spec, Stash &stash) { return stash.create<Create>(type, spec); } -const Node &lambda(const ValueType &type, const std::vector<size_t> &bindings, const Function &function, NodeTypes node_types, Stash &stash) { +const TensorFunction &lambda(const ValueType &type, const std::vector<size_t> &bindings, const Function &function, NodeTypes node_types, Stash &stash) { return stash.create<Lambda>(type, bindings, function, std::move(node_types)); } -const Node &peek(const Node ¶m, const std::map<vespalib::string, std::variant<TensorSpec::Label, Node::CREF>> &spec, Stash &stash) { +const TensorFunction &peek(const TensorFunction ¶m, const std::map<vespalib::string, std::variant<TensorSpec::Label, TensorFunction::CREF>> &spec, Stash &stash) { std::vector<vespalib::string> dimensions; for (const auto &dim_spec: spec) { dimensions.push_back(dim_spec.first); @@ -595,12 +595,12 @@ const Node &peek(const Node ¶m, const std::map<vespalib::string, std::varian return stash.create<Peek>(result_type, param, spec); } -const Node &rename(const Node &child, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash) { +const TensorFunction &rename(const TensorFunction &child, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash) { ValueType result_type = child.result_type().rename(from, to); return stash.create<Rename>(result_type, child, from, to); } -const Node &if_node(const Node &cond, const Node &true_child, const Node &false_child, Stash &stash) { +const TensorFunction &if_node(const TensorFunction &cond, const TensorFunction &true_child, const TensorFunction &false_child, Stash &stash) { ValueType result_type = ValueType::either(true_child.result_type(), false_child.result_type()); return stash.create<If>(result_type, cond, true_child, false_child); } diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index 6743f37eeb1..20631108775 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -52,6 +52,7 @@ class Tensor; **/ struct TensorFunction { + using CREF = std::reference_wrapper<const TensorFunction>; TensorFunction(const TensorFunction &) = delete; TensorFunction &operator=(const TensorFunction &) = delete; TensorFunction(TensorFunction &&) = delete; @@ -132,7 +133,6 @@ class Node : public TensorFunction private: ValueType _result_type; public: - using CREF = std::reference_wrapper<const Node>; Node(const ValueType &result_type_in) : _result_type(result_type_in) {} const ValueType &result_type() const final override { return _result_type; } }; @@ -310,7 +310,7 @@ class Create : public Node private: std::map<TensorSpec::Address, Child> _spec; public: - Create(const ValueType &result_type_in, const std::map<TensorSpec::Address, Node::CREF> &spec_in) + Create(const ValueType &result_type_in, const std::map<TensorSpec::Address, TensorFunction::CREF> &spec_in) : Super(result_type_in), _spec() { for (const auto &cell: spec_in) { @@ -359,8 +359,8 @@ private: Child _param; std::map<vespalib::string, MyLabel> _spec; public: - Peek(const ValueType &result_type_in, const Node ¶m, - const std::map<vespalib::string, std::variant<TensorSpec::Label, Node::CREF>> &spec) + Peek(const ValueType &result_type_in, const TensorFunction ¶m, + const std::map<vespalib::string, std::variant<TensorSpec::Label, TensorFunction::CREF>> &spec) : Super(result_type_in), _param(param), _spec() { for (const auto &dim: spec) { @@ -369,7 +369,7 @@ public: [&](const TensorSpec::Label &label) { _spec.emplace(dim.first, label); }, - [&](const Node::CREF &ref) { + [&](const TensorFunction::CREF &ref) { _spec.emplace(dim.first, ref.get()); } }, dim.second); @@ -432,18 +432,18 @@ public: //----------------------------------------------------------------------------- -const Node &const_value(const Value &value, Stash &stash); -const Node &inject(const ValueType &type, size_t param_idx, Stash &stash); -const Node &reduce(const Node &child, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash); -const Node &map(const Node &child, map_fun_t function, Stash &stash); -const Node &join(const Node &lhs, const Node &rhs, join_fun_t function, Stash &stash); -const Node &merge(const Node &lhs, const Node &rhs, join_fun_t function, Stash &stash); -const Node &concat(const Node &lhs, const Node &rhs, const vespalib::string &dimension, Stash &stash); -const Node &create(const ValueType &type, const std::map<TensorSpec::Address, Node::CREF> &spec, Stash &stash); -const Node &lambda(const ValueType &type, const std::vector<size_t> &bindings, const Function &function, NodeTypes node_types, Stash &stash); -const Node &peek(const Node ¶m, const std::map<vespalib::string, std::variant<TensorSpec::Label, Node::CREF>> &spec, Stash &stash); -const Node &rename(const Node &child, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash); -const Node &if_node(const Node &cond, const Node &true_child, const Node &false_child, Stash &stash); +const TensorFunction &const_value(const Value &value, Stash &stash); +const TensorFunction &inject(const ValueType &type, size_t param_idx, Stash &stash); +const TensorFunction &reduce(const TensorFunction &child, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash); +const TensorFunction &map(const TensorFunction &child, map_fun_t function, Stash &stash); +const TensorFunction &join(const TensorFunction &lhs, const TensorFunction &rhs, join_fun_t function, Stash &stash); +const TensorFunction &merge(const TensorFunction &lhs, const TensorFunction &rhs, join_fun_t function, Stash &stash); +const TensorFunction &concat(const TensorFunction &lhs, const TensorFunction &rhs, const vespalib::string &dimension, Stash &stash); +const TensorFunction &create(const ValueType &type, const std::map<TensorSpec::Address, TensorFunction::CREF> &spec, Stash &stash); +const TensorFunction &lambda(const ValueType &type, const std::vector<size_t> &bindings, const Function &function, NodeTypes node_types, Stash &stash); +const TensorFunction &peek(const TensorFunction ¶m, const std::map<vespalib::string, std::variant<TensorSpec::Label, TensorFunction::CREF>> &spec, Stash &stash); +const TensorFunction &rename(const TensorFunction &child, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash); +const TensorFunction &if_node(const TensorFunction &cond, const TensorFunction &true_child, const TensorFunction &false_child, Stash &stash); } // namespace vespalib::eval::tensor_function } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index 3e91240048b..a8ae9c44bb0 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/vespalib/util/typify.h> #include <vespa/vespalib/stllike/string.h> #include <vector> @@ -104,4 +105,15 @@ template <typename CT> inline ValueType::CellType get_cell_type(); template <> inline ValueType::CellType get_cell_type<double>() { return ValueType::CellType::DOUBLE; } template <> inline ValueType::CellType get_cell_type<float>() { return ValueType::CellType::FLOAT; } +struct TypifyCellType { + template <typename T> using Result = TypifyResultType<T>; + template <typename F> static decltype(auto) resolve(ValueType::CellType value, F &&f) { + switch(value) { + case ValueType::CellType::DOUBLE: return f(Result<double>()); + case ValueType::CellType::FLOAT: return f(Result<float>()); + } + abort(); + } +}; + } // namespace diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index d9fcbaa3e2a..ca14e40e4d0 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -18,8 +18,10 @@ #include "dense/dense_remove_dimension_optimizer.h" #include "dense/dense_lambda_peek_optimizer.h" #include "dense/dense_lambda_function.h" +#include "dense/dense_simple_expand_function.h" #include "dense/dense_simple_join_function.h" #include "dense/dense_number_join_function.h" +#include "dense/dense_pow_as_map_optimizer.h" #include "dense/dense_simple_map_function.h" #include "dense/vector_from_doubles_function.h" #include "dense/dense_tensor_create_function.h" @@ -176,7 +178,7 @@ DefaultTensorEngine::to_spec(const Value &value) const struct CallDenseTensorBuilder { template <typename CT> static Value::UP - call(const ValueType &type, const TensorSpec &spec) + invoke(const ValueType &type, const TensorSpec &spec) { TypedDenseTensorBuilder<CT> builder(type); for (const auto &cell: spec.cells()) { @@ -191,6 +193,8 @@ struct CallDenseTensorBuilder { } }; +using MyTypify = eval::TypifyCellType; + Value::UP DefaultTensorEngine::from_spec(const TensorSpec &spec) const { @@ -201,7 +205,7 @@ DefaultTensorEngine::from_spec(const TensorSpec &spec) const double value = spec.cells().empty() ? 0.0 : spec.cells().begin()->second.value; return std::make_unique<DoubleValue>(value); } else if (type.is_dense()) { - return dispatch_0<CallDenseTensorBuilder>(type.cell_type(), type, spec); + return typify_invoke<1,MyTypify,CallDenseTensorBuilder>(type.cell_type(), type, spec); } else if (type.is_sparse()) { DirectSparseTensorBuilder builder(type); SparseTensorAddressBuilder address_builder; @@ -285,6 +289,7 @@ DefaultTensorEngine::optimize(const TensorFunction &expr, Stash &stash) const } while (!nodes.empty()) { const Child &child = nodes.back().get(); + child.set(DenseSimpleExpandFunction::optimize(child.get(), stash)); child.set(DenseAddDimensionOptimizer::optimize(child.get(), stash)); child.set(DenseRemoveDimensionOptimizer::optimize(child.get(), stash)); child.set(VectorFromDoublesFunction::optimize(child.get(), stash)); @@ -293,6 +298,7 @@ DefaultTensorEngine::optimize(const TensorFunction &expr, Stash &stash) const child.set(DenseLambdaPeekOptimizer::optimize(child.get(), stash)); child.set(DenseLambdaFunction::optimize(child.get(), stash)); child.set(DenseFastRenameOptimizer::optimize(child.get(), stash)); + child.set(DensePowAsMapOptimizer::optimize(child.get(), stash)); child.set(DenseSimpleMapFunction::optimize(child.get(), stash)); child.set(DenseSimpleJoinFunction::optimize(child.get(), stash)); child.set(DenseNumberJoinFunction::optimize(child.get(), stash)); @@ -449,7 +455,7 @@ const Value &concat_vectors(const Value &a, const Value &b, const vespalib::stri struct CallConcatVectors { template <typename OCT> - static const Value &call(const Value &a, const Value &b, const vespalib::string &dimension, size_t vector_size, Stash &stash) { + static const Value &invoke(const Value &a, const Value &b, const vespalib::string &dimension, size_t vector_size, Stash &stash) { return concat_vectors<OCT>(a, b, dimension, vector_size, stash); } }; @@ -461,7 +467,7 @@ DefaultTensorEngine::concat(const Value &a, const Value &b, const vespalib::stri size_t b_size = vector_size(b.type(), dimension); if ((a_size > 0) && (b_size > 0)) { CellType result_cell_type = ValueType::unify_cell_types(a.type(), b.type()); - return dispatch_0<CallConcatVectors>(result_cell_type, a, b, dimension, (a_size + b_size), stash); + return typify_invoke<1,MyTypify,CallConcatVectors>(result_cell_type, a, b, dimension, (a_size + b_size), stash); } return to_default(simple_engine().concat(to_simple(a, stash), to_simple(b, stash), dimension, stash), stash); } diff --git a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt index 244e288b90a..c4b8138148c 100644 --- a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt @@ -12,8 +12,10 @@ vespa_add_library(eval_tensor_dense OBJECT dense_matmul_function.cpp dense_multi_matmul_function.cpp dense_number_join_function.cpp + dense_pow_as_map_optimizer.cpp dense_remove_dimension_optimizer.cpp dense_replace_type_function.cpp + dense_simple_expand_function.cpp dense_simple_join_function.cpp dense_simple_map_function.cpp dense_single_reduce_function.cpp diff --git a/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp index 9b93f5e7d72..84da53c8488 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp @@ -25,7 +25,7 @@ void my_cell_range_op(eval::InterpretedFunction::State &state, uint64_t param) { struct MyCellRangeOp { template <typename CT> - static auto get_fun() { return my_cell_range_op<CT>; } + static auto invoke() { return my_cell_range_op<CT>; } }; } // namespace vespalib::tensor::<unnamed> @@ -46,7 +46,9 @@ DenseCellRangeFunction::compile_self(const TensorEngine &, Stash &) const { static_assert(sizeof(uint64_t) == sizeof(this)); assert(result_type().cell_type() == child().result_type().cell_type()); - auto op = select_1<MyCellRangeOp>(result_type().cell_type()); + + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<1,MyTypify,MyCellRangeOp>(result_type().cell_type()); return eval::InterpretedFunction::Instruction(op, (uint64_t)this); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp index c9ff57e4a65..9e30451cd67 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp @@ -48,7 +48,7 @@ void my_cblas_float_dot_product_op(eval::InterpretedFunction::State &state, uint struct MyDotProductOp { template <typename LCT, typename RCT> - static auto get_fun() { return my_dot_product_op<LCT,RCT>; } + static auto invoke() { return my_dot_product_op<LCT,RCT>; } }; eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct) { @@ -60,7 +60,8 @@ eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct) { return my_cblas_float_dot_product_op; } } - return select_2<MyDotProductOp>(lct, rct); + using MyTypify = eval::TypifyCellType; + return typify_invoke<2,MyTypify,MyDotProductOp>(lct, rct); } } // namespace vespalib::tensor::<unnamed> diff --git a/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp index b60d732d7a9..e373ca09e11 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp @@ -95,7 +95,7 @@ void my_compiled_lambda_op(eval::InterpretedFunction::State &state, uint64_t par struct MyCompiledLambdaOp { template <typename CT> - static auto get_fun() { return my_compiled_lambda_op<CT>; } + static auto invoke() { return my_compiled_lambda_op<CT>; } }; //----------------------------------------------------------------------------- @@ -131,7 +131,7 @@ void my_interpreted_lambda_op(eval::InterpretedFunction::State &state, uint64_t struct MyInterpretedLambdaOp { template <typename CT> - static auto get_fun() { return my_interpreted_lambda_op<CT>; } + static auto invoke() { return my_interpreted_lambda_op<CT>; } }; //----------------------------------------------------------------------------- @@ -163,15 +163,16 @@ DenseLambdaFunction::compile_self(const TensorEngine &engine, Stash &stash) cons { assert(&engine == &prod_engine); auto mode = eval_mode(); + using MyTypify = eval::TypifyCellType; if (mode == EvalMode::COMPILED) { CompiledParams ¶ms = stash.create<CompiledParams>(_lambda); - auto op = select_1<MyCompiledLambdaOp>(result_type().cell_type()); + auto op = typify_invoke<1,MyTypify,MyCompiledLambdaOp>(result_type().cell_type()); static_assert(sizeof(¶ms) == sizeof(uint64_t)); return Instruction(op, (uint64_t)(¶ms)); } else { assert(mode == EvalMode::INTERPRETED); InterpretedParams ¶ms = stash.create<InterpretedParams>(_lambda); - auto op = select_1<MyInterpretedLambdaOp>(result_type().cell_type()); + auto op = typify_invoke<1,MyTypify,MyInterpretedLambdaOp>(result_type().cell_type()); static_assert(sizeof(¶ms) == sizeof(uint64_t)); return Instruction(op, (uint64_t)(¶ms)); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp index a5f532e643a..70bdc8ae7d6 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp @@ -45,7 +45,7 @@ void my_lambda_peek_op(InterpretedFunction::State &state, uint64_t param) { struct MyLambdaPeekOp { template <typename DST_CT, typename SRC_CT> - static auto get_fun() { return my_lambda_peek_op<DST_CT, SRC_CT>; } + static auto invoke() { return my_lambda_peek_op<DST_CT, SRC_CT>; } }; } // namespace vespalib::tensor::<unnamed> @@ -64,7 +64,8 @@ InterpretedFunction::Instruction DenseLambdaPeekFunction::compile_self(const TensorEngine &, Stash &stash) const { const Self &self = stash.create<Self>(result_type(), *_idx_fun); - auto op = select_2<MyLambdaPeekOp>(result_type().cell_type(), child().result_type().cell_type()); + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<2,MyTypify,MyLambdaPeekOp>(result_type().cell_type(), child().result_type().cell_type()); static_assert(sizeof(uint64_t) == sizeof(&self)); assert(child().result_type().is_dense()); return InterpretedFunction::Instruction(op, (uint64_t)&self); diff --git a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp index 695e0fddd08..9c18cf285d4 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp @@ -80,47 +80,6 @@ void my_cblas_float_matmul_op(eval::InterpretedFunction::State &state, uint64_t state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); } -template <bool lhs_common_inner, bool rhs_common_inner> -struct MyMatMulOp { - template <typename LCT, typename RCT> - static auto get_fun() { return my_matmul_op<LCT,RCT,lhs_common_inner,rhs_common_inner>; } -}; - -template <bool lhs_common_inner, bool rhs_common_inner> -eval::InterpretedFunction::op_function my_select3(CellType lct, CellType rct) -{ - if (lct == rct) { - if (lct == ValueType::CellType::DOUBLE) { - return my_cblas_double_matmul_op<lhs_common_inner,rhs_common_inner>; - } - if (lct == ValueType::CellType::FLOAT) { - return my_cblas_float_matmul_op<lhs_common_inner,rhs_common_inner>; - } - } - return select_2<MyMatMulOp<lhs_common_inner,rhs_common_inner>>(lct, rct); -} - -template <bool lhs_common_inner> -eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct, - bool rhs_common_inner) -{ - if (rhs_common_inner) { - return my_select3<lhs_common_inner,true>(lct, rct); - } else { - return my_select3<lhs_common_inner,false>(lct, rct); - } -} - -eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, - bool lhs_common_inner, bool rhs_common_inner) -{ - if (lhs_common_inner) { - return my_select2<true>(lct, rct, rhs_common_inner); - } else { - return my_select2<false>(lct, rct, rhs_common_inner); - } -} - bool is_matrix(const ValueType &type) { return (type.is_dense() && (type.dimensions().size() == 2)); } @@ -160,6 +119,18 @@ const TensorFunction &create_matmul(const TensorFunction &a, const TensorFunctio } } +struct MyGetFun { + template<typename R1, typename R2, typename R3, typename R4> static auto invoke() { + if (std::is_same_v<R1,double> && std::is_same_v<R2,double>) { + return my_cblas_double_matmul_op<R3::value, R4::value>; + } else if (std::is_same_v<R1,float> && std::is_same_v<R2,float>) { + return my_cblas_float_matmul_op<R3::value, R4::value>; + } else { + return my_matmul_op<R1, R2, R3::value, R4::value>; + } + } +}; + } // namespace vespalib::tensor::<unnamed> DenseMatMulFunction::Self::Self(const eval::ValueType &result_type_in, @@ -197,9 +168,11 @@ DenseMatMulFunction::~DenseMatMulFunction() = default; eval::InterpretedFunction::Instruction DenseMatMulFunction::compile_self(const TensorEngine &, Stash &stash) const { + using MyTypify = TypifyValue<eval::TypifyCellType,TypifyBool>; Self &self = stash.create<Self>(result_type(), _lhs_size, _common_size, _rhs_size); - auto op = my_select(lhs().result_type().cell_type(), rhs().result_type().cell_type(), - _lhs_common_inner, _rhs_common_inner); + auto op = typify_invoke<4,MyTypify,MyGetFun>( + lhs().result_type().cell_type(), rhs().result_type().cell_type(), + _lhs_common_inner, _rhs_common_inner); return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self)); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp index 3f48607cef4..925627c5684 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp @@ -2,8 +2,10 @@ #include "dense_number_join_function.h" #include "dense_tensor_view.h" +#include <vespa/vespalib/util/typify.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/inline_operation.h> namespace vespalib::tensor { @@ -13,6 +15,7 @@ using eval::Value; using eval::ValueType; using eval::TensorFunction; using eval::TensorEngine; +using eval::TypifyCellType; using eval::as; using namespace eval::operation; @@ -26,40 +29,6 @@ using State = eval::InterpretedFunction::State; namespace { -struct CallFun { - join_fun_t function; - CallFun(join_fun_t function_in) : function(function_in) {} - double eval(double a, double b) const { return function(a, b); } -}; - -struct AddFun { - AddFun(join_fun_t) {} - template <typename A, typename B> - auto eval(A a, B b) const { return (a + b); } -}; - -struct MulFun { - MulFun(join_fun_t) {} - template <typename A, typename B> - auto eval(A a, B b) const { return (a * b); } -}; - -// needed for asymmetric operations like Sub and Div -template <typename Fun> -struct SwapFun { - Fun fun; - SwapFun(join_fun_t function_in) : fun(function_in) {} - template <typename A, typename B> - auto eval(A a, B b) const { return fun.eval(b, a); } -}; - -template <typename CT, typename Fun> -void apply_fun_1_to_n(CT *dst, const CT *pri, CT sec, size_t n, const Fun &fun) { - for (size_t i = 0; i < n; ++i) { - dst[i] = fun.eval(pri[i], sec); - } -} - template <typename CT, bool inplace> ArrayRef<CT> make_dst_cells(ConstArrayRef<CT> src_cells, Stash &stash) { if (inplace) { @@ -71,13 +40,13 @@ ArrayRef<CT> make_dst_cells(ConstArrayRef<CT> src_cells, Stash &stash) { template <typename CT, typename Fun, bool inplace, bool swap> void my_number_join_op(State &state, uint64_t param) { - using OP = typename std::conditional<swap,SwapFun<Fun>,Fun>::type; + using OP = typename std::conditional<swap,SwapArgs2<Fun>,Fun>::type; OP my_op((join_fun_t)param); const Value &tensor = state.peek(swap ? 0 : 1); CT number = state.peek(swap ? 1 : 0).as_double(); auto src_cells = DenseTensorView::typify_cells<CT>(tensor); auto dst_cells = make_dst_cells<CT, inplace>(src_cells, state.stash); - apply_fun_1_to_n(dst_cells.begin(), src_cells.begin(), number, dst_cells.size(), my_op); + apply_op2_vec_num(dst_cells.begin(), src_cells.begin(), number, dst_cells.size(), my_op); if (inplace) { state.pop_pop_push(tensor); } else { @@ -87,39 +56,13 @@ void my_number_join_op(State &state, uint64_t param) { //----------------------------------------------------------------------------- -template <typename Fun, bool inplace, bool swap> -struct MyNumberJoinOp { - template <typename CT> - static auto get_fun() { return my_number_join_op<CT,Fun,inplace,swap>; } -}; - -template <typename Fun, bool inplace> -op_function my_select_3(ValueType::CellType ct, Primary primary) { - switch (primary) { - case Primary::LHS: return select_1<MyNumberJoinOp<Fun,inplace,false>>(ct); - case Primary::RHS: return select_1<MyNumberJoinOp<Fun,inplace,true>>(ct); - } - abort(); -} - -template <typename Fun> -op_function my_select_2(ValueType::CellType ct, Primary primary, bool inplace) { - if (inplace) { - return my_select_3<Fun, true>(ct, primary); - } else { - return my_select_3<Fun, false>(ct, primary); +struct MyGetFun { + template <typename R1, typename R2, typename R3, typename R4> static auto invoke() { + return my_number_join_op<R1, R2, R3::value, R4::value>; } -} +}; -op_function my_select(ValueType::CellType ct, Primary primary, bool inplace, join_fun_t fun_hint) { - if (fun_hint == Add::f) { - return my_select_2<AddFun>(ct, primary, inplace); - } else if (fun_hint == Mul::f) { - return my_select_2<MulFun>(ct, primary, inplace); - } else { - return my_select_2<CallFun>(ct, primary, inplace); - } -} +using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool>; bool is_dense(const TensorFunction &tf) { return tf.result_type().is_dense(); } bool is_double(const TensorFunction &tf) { return tf.result_type().is_double(); } @@ -154,7 +97,8 @@ DenseNumberJoinFunction::inplace() const Instruction DenseNumberJoinFunction::compile_self(const TensorEngine &, Stash &) const { - auto op = my_select(result_type().cell_type(), _primary, inplace(), function()); + auto op = typify_invoke<4,MyTypify,MyGetFun>(result_type().cell_type(), function(), + inplace(), (_primary == Primary::RHS)); static_assert(sizeof(uint64_t) == sizeof(function())); return Instruction(op, (uint64_t)(function())); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.cpp b/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.cpp new file mode 100644 index 00000000000..f78c23c80ac --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.cpp @@ -0,0 +1,38 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_pow_as_map_optimizer.h" +#include "dense_simple_map_function.h" +#include <vespa/eval/eval/operation.h> + +namespace vespalib::tensor { + +using eval::TensorFunction; +using eval::as; + +using namespace eval::tensor_function; +using namespace eval::operation; + +const TensorFunction & +DensePowAsMapOptimizer::optimize(const TensorFunction &expr, Stash &stash) +{ + if (auto join = as<Join>(expr)) { + const TensorFunction &lhs = join->lhs(); + const TensorFunction &rhs = join->rhs(); + if ((join->function() == Pow::f) && + lhs.result_type().is_dense() && + rhs.result_type().is_double()) + { + if (auto const_value = as<ConstValue>(rhs)) { + if (const_value->value().as_double() == 2.0) { + return map(lhs, Square::f, stash); + } + if (const_value->value().as_double() == 3.0) { + return map(lhs, Cube::f, stash); + } + } + } + } + return expr; +} + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.h b/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.h new file mode 100644 index 00000000000..4849a10c070 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.h @@ -0,0 +1,18 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::tensor { + +/** + * Tensor function optimizer for converting join expressions on the + * form 'join(tensor,<small integer constant>,f(x,y)(pow(x,y))' to + * expressions on the form 'map(tensor,f(x)(x*x...))'. + **/ +struct DensePowAsMapOptimizer { + static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash); +}; + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp new file mode 100644 index 00000000000..d45e0d936a9 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp @@ -0,0 +1,141 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_simple_expand_function.h" +#include "dense_tensor_view.h" +#include <vespa/vespalib/objects/objectvisitor.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/inline_operation.h> +#include <vespa/vespalib/util/typify.h> +#include <optional> +#include <algorithm> + +namespace vespalib::tensor { + +using vespalib::ArrayRef; + +using eval::Value; +using eval::ValueType; +using eval::TensorFunction; +using eval::TensorEngine; +using eval::TypifyCellType; +using eval::as; + +using namespace eval::operation; +using namespace eval::tensor_function; + +using Inner = DenseSimpleExpandFunction::Inner; + +using op_function = eval::InterpretedFunction::op_function; +using Instruction = eval::InterpretedFunction::Instruction; +using State = eval::InterpretedFunction::State; + +namespace { + +struct ExpandParams { + const ValueType &result_type; + size_t result_size; + join_fun_t function; + ExpandParams(const ValueType &result_type_in, size_t result_size_in, join_fun_t function_in) + : result_type(result_type_in), result_size(result_size_in), function(function_in) {} +}; + +template <typename LCT, typename RCT, typename Fun, bool rhs_inner> +void my_simple_expand_op(State &state, uint64_t param) { + using ICT = typename std::conditional<rhs_inner,RCT,LCT>::type; + using OCT = typename std::conditional<rhs_inner,LCT,RCT>::type; + using DCT = typename eval::UnifyCellTypes<ICT,OCT>::type; + using OP = typename std::conditional<rhs_inner,SwapArgs2<Fun>,Fun>::type; + const ExpandParams ¶ms = *(ExpandParams*)param; + OP my_op(params.function); + auto inner_cells = DenseTensorView::typify_cells<ICT>(state.peek(rhs_inner ? 0 : 1)); + auto outer_cells = DenseTensorView::typify_cells<OCT>(state.peek(rhs_inner ? 1 : 0)); + auto dst_cells = state.stash.create_array<DCT>(params.result_size); + DCT *dst = dst_cells.begin(); + for (OCT outer_cell: outer_cells) { + apply_op2_vec_num(dst, inner_cells.begin(), outer_cell, inner_cells.size(), my_op); + dst += inner_cells.size(); + } + state.pop_pop_push(state.stash.create<DenseTensorView>(params.result_type, TypedCells(dst_cells))); +} + +//----------------------------------------------------------------------------- + +struct MyGetFun { + template <typename R1, typename R2, typename R3, typename R4> static auto invoke() { + return my_simple_expand_op<R1, R2, R3, R4::value>; + } +}; + +using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool>; + +//----------------------------------------------------------------------------- + +std::vector<ValueType::Dimension> strip_trivial(const std::vector<ValueType::Dimension> &dim_list) { + std::vector<ValueType::Dimension> result; + std::copy_if(dim_list.begin(), dim_list.end(), std::back_inserter(result), + [](const auto &dim){ return (dim.size != 1); }); + return result; +} + +std::optional<Inner> detect_simple_expand(const TensorFunction &lhs, const TensorFunction &rhs) { + std::vector<ValueType::Dimension> a = strip_trivial(lhs.result_type().dimensions()); + std::vector<ValueType::Dimension> b = strip_trivial(rhs.result_type().dimensions()); + if (a.empty() || b.empty()) { + return std::nullopt; + } else if (a.back().name < b.front().name) { + return Inner::RHS; + } else if (b.back().name < a.front().name) { + return Inner::LHS; + } else { + return std::nullopt; + } +} + +} // namespace vespalib::tensor::<unnamed> + +//----------------------------------------------------------------------------- + +DenseSimpleExpandFunction::DenseSimpleExpandFunction(const ValueType &result_type, + const TensorFunction &lhs, + const TensorFunction &rhs, + join_fun_t function_in, + Inner inner_in) + : Join(result_type, lhs, rhs, function_in), + _inner(inner_in) +{ +} + +DenseSimpleExpandFunction::~DenseSimpleExpandFunction() = default; + +Instruction +DenseSimpleExpandFunction::compile_self(const TensorEngine &, Stash &stash) const +{ + size_t result_size = result_type().dense_subspace_size(); + const ExpandParams ¶ms = stash.create<ExpandParams>(result_type(), result_size, function()); + auto op = typify_invoke<4,MyTypify,MyGetFun>(lhs().result_type().cell_type(), + rhs().result_type().cell_type(), + function(), (_inner == Inner::RHS)); + static_assert(sizeof(uint64_t) == sizeof(¶ms)); + return Instruction(op, (uint64_t)(¶ms)); +} + +const TensorFunction & +DenseSimpleExpandFunction::optimize(const TensorFunction &expr, Stash &stash) +{ + if (auto join = as<Join>(expr)) { + const TensorFunction &lhs = join->lhs(); + const TensorFunction &rhs = join->rhs(); + if (lhs.result_type().is_dense() && rhs.result_type().is_dense()) { + if (std::optional<Inner> inner = detect_simple_expand(lhs, rhs)) { + assert(expr.result_type().dense_subspace_size() == + (lhs.result_type().dense_subspace_size() * + rhs.result_type().dense_subspace_size())); + return stash.create<DenseSimpleExpandFunction>(join->result_type(), lhs, rhs, join->function(), inner.value()); + } + } + } + return expr; +} + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.h b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.h new file mode 100644 index 00000000000..b4b303901a7 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.h @@ -0,0 +1,38 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::tensor { + +/** + * Tensor function for simple expanding join operations on dense + * tensors. An expanding operation is a join between tensors resulting + * in a larger tensor where the input tensors have no matching + * dimensions (trivial dimensions are ignored). A simple expanding + * operation is an expanding operation where all the dimensions of one + * input is nested inside all the dimensions from the other input + * within the result (trivial dimensions are again ignored). + **/ +class DenseSimpleExpandFunction : public eval::tensor_function::Join +{ + using Super = eval::tensor_function::Join; +public: + enum class Inner : uint8_t { LHS, RHS }; + using join_fun_t = ::vespalib::eval::tensor_function::join_fun_t; +private: + Inner _inner; +public: + DenseSimpleExpandFunction(const eval::ValueType &result_type, + const TensorFunction &lhs, + const TensorFunction &rhs, + join_fun_t function_in, + Inner inner_in); + ~DenseSimpleExpandFunction() override; + Inner inner() const { return _inner; } + eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override; + static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash); +}; + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp index 6b0d65c0743..5f8fbcac9bb 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp @@ -5,6 +5,8 @@ #include <vespa/vespalib/objects/objectvisitor.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/inline_operation.h> +#include <vespa/vespalib/util/typify.h> #include <optional> #include <algorithm> @@ -16,6 +18,7 @@ using eval::Value; using eval::ValueType; using eval::TensorFunction; using eval::TensorEngine; +using eval::TypifyCellType; using eval::as; using namespace eval::operation; @@ -30,6 +33,18 @@ using State = eval::InterpretedFunction::State; namespace { +struct TypifyOverlap { + template <Overlap VALUE> using Result = TypifyResultValue<Overlap, VALUE>; + template <typename F> static decltype(auto) resolve(Overlap value, F &&f) { + switch (value) { + case Overlap::INNER: return f(Result<Overlap::INNER>()); + case Overlap::OUTER: return f(Result<Overlap::OUTER>()); + case Overlap::FULL: return f(Result<Overlap::FULL>()); + } + abort(); + } +}; + struct JoinParams { const ValueType &result_type; size_t factor; @@ -38,47 +53,6 @@ struct JoinParams { : result_type(result_type_in), factor(factor_in), function(function_in) {} }; -struct CallFun { - join_fun_t function; - CallFun(const JoinParams ¶ms) : function(params.function) {} - double eval(double a, double b) const { return function(a, b); } -}; - -struct AddFun { - AddFun(const JoinParams &) {} - template <typename A, typename B> - auto eval(A a, B b) const { return (a + b); } -}; - -struct MulFun { - MulFun(const JoinParams &) {} - template <typename A, typename B> - auto eval(A a, B b) const { return (a * b); } -}; - -// needed for asymmetric operations like Sub and Div -template <typename Fun> -struct SwapFun { - Fun fun; - SwapFun(const JoinParams ¶ms) : fun(params) {} - template <typename A, typename B> - auto eval(A a, B b) const { return fun.eval(b, a); } -}; - -template <typename OCT, typename PCT, typename SCT, typename Fun> -void apply_fun_1_to_n(OCT *dst, const PCT *pri, SCT sec, size_t n, const Fun &fun) { - for (size_t i = 0; i < n; ++i) { - dst[i] = fun.eval(pri[i], sec); - } -} - -template <typename OCT, typename PCT, typename SCT, typename Fun> -void apply_fun_n_to_n(OCT *dst, const PCT *pri, const SCT *sec, size_t n, const Fun &fun) { - for (size_t i = 0; i < n; ++i) { - dst[i] = fun.eval(pri[i], sec[i]); - } -} - template <typename OCT, bool pri_mut, typename PCT> ArrayRef<OCT> make_dst_cells(ConstArrayRef<PCT> pri_cells, Stash &stash) { if constexpr (pri_mut && std::is_same<PCT,OCT>::value) { @@ -93,19 +67,19 @@ void my_simple_join_op(State &state, uint64_t param) { using PCT = typename std::conditional<swap,RCT,LCT>::type; using SCT = typename std::conditional<swap,LCT,RCT>::type; using OCT = typename eval::UnifyCellTypes<PCT,SCT>::type; - using OP = typename std::conditional<swap,SwapFun<Fun>,Fun>::type; + using OP = typename std::conditional<swap,SwapArgs2<Fun>,Fun>::type; const JoinParams ¶ms = *(JoinParams*)param; - OP my_op(params); + OP my_op(params.function); auto pri_cells = DenseTensorView::typify_cells<PCT>(state.peek(swap ? 0 : 1)); auto sec_cells = DenseTensorView::typify_cells<SCT>(state.peek(swap ? 1 : 0)); auto dst_cells = make_dst_cells<OCT, pri_mut>(pri_cells, state.stash); if (overlap == Overlap::FULL) { - apply_fun_n_to_n(dst_cells.begin(), pri_cells.begin(), sec_cells.begin(), dst_cells.size(), my_op); + apply_op2_vec_vec(dst_cells.begin(), pri_cells.begin(), sec_cells.begin(), dst_cells.size(), my_op); } else if (overlap == Overlap::OUTER) { size_t offset = 0; size_t factor = params.factor; for (SCT cell: sec_cells) { - apply_fun_1_to_n(dst_cells.begin() + offset, pri_cells.begin() + offset, cell, factor, my_op); + apply_op2_vec_num(dst_cells.begin() + offset, pri_cells.begin() + offset, cell, factor, my_op); offset += factor; } } else { @@ -113,7 +87,7 @@ void my_simple_join_op(State &state, uint64_t param) { size_t offset = 0; size_t factor = params.factor; for (size_t i = 0; i < factor; ++i) { - apply_fun_n_to_n(dst_cells.begin() + offset, pri_cells.begin() + offset, sec_cells.begin(), sec_cells.size(), my_op); + apply_op2_vec_vec(dst_cells.begin() + offset, pri_cells.begin() + offset, sec_cells.begin(), sec_cells.size(), my_op); offset += sec_cells.size(); } } @@ -122,67 +96,13 @@ void my_simple_join_op(State &state, uint64_t param) { //----------------------------------------------------------------------------- -template <typename Fun, bool swap, Overlap overlap, bool pri_mut> -struct MySimpleJoinOp { - template <typename LCT, typename RCT> - static auto get_fun() { return my_simple_join_op<LCT,RCT,Fun,swap,overlap,pri_mut>; } -}; - -template <bool swap, Overlap overlap, bool pri_mut> -op_function my_select_4(ValueType::CellType lct, - ValueType::CellType rct, - join_fun_t fun_hint) -{ - if (fun_hint == Add::f) { - return select_2<MySimpleJoinOp<AddFun,swap,overlap,pri_mut>>(lct, rct); - } else if (fun_hint == Mul::f) { - return select_2<MySimpleJoinOp<MulFun,swap,overlap,pri_mut>>(lct, rct); - } else { - return select_2<MySimpleJoinOp<CallFun,swap,overlap,pri_mut>>(lct, rct); - } -} - -template <bool swap, Overlap overlap> -op_function my_select_3(ValueType::CellType lct, - ValueType::CellType rct, - bool pri_mut, - join_fun_t fun_hint) -{ - if (pri_mut) { - return my_select_4<swap, overlap, true>(lct, rct, fun_hint); - } else { - return my_select_4<swap, overlap, false>(lct, rct, fun_hint); +struct MyGetFun { + template <typename R1, typename R2, typename R3, typename R4, typename R5, typename R6> static auto invoke() { + return my_simple_join_op<R1, R2, R3, R4::value, R5::value, R6::value>; } -} - -template <bool swap> -op_function my_select_2(ValueType::CellType lct, - ValueType::CellType rct, - Overlap overlap, - bool pri_mut, - join_fun_t fun_hint) -{ - switch (overlap) { - case Overlap::INNER: return my_select_3<swap, Overlap::INNER>(lct, rct, pri_mut, fun_hint); - case Overlap::OUTER: return my_select_3<swap, Overlap::OUTER>(lct, rct, pri_mut, fun_hint); - case Overlap::FULL: return my_select_3<swap, Overlap::FULL>(lct, rct, pri_mut, fun_hint); - } - abort(); -} +}; -op_function my_select(ValueType::CellType lct, - ValueType::CellType rct, - Primary primary, - Overlap overlap, - bool pri_mut, - join_fun_t fun_hint) -{ - switch (primary) { - case Primary::LHS: return my_select_2<false>(lct, rct, overlap, pri_mut, fun_hint); - case Primary::RHS: return my_select_2<true>(lct, rct, overlap, pri_mut, fun_hint); - } - abort(); -} +using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool,TypifyOverlap>; //----------------------------------------------------------------------------- @@ -280,11 +200,10 @@ Instruction DenseSimpleJoinFunction::compile_self(const TensorEngine &, Stash &stash) const { const JoinParams ¶ms = stash.create<JoinParams>(result_type(), factor(), function()); - auto op = my_select(lhs().result_type().cell_type(), - rhs().result_type().cell_type(), - _primary, _overlap, - primary_is_mutable(), - function()); + auto op = typify_invoke<6,MyTypify,MyGetFun>(lhs().result_type().cell_type(), + rhs().result_type().cell_type(), + function(), (_primary == Primary::RHS), + _overlap, primary_is_mutable()); static_assert(sizeof(uint64_t) == sizeof(¶ms)); return Instruction(op, (uint64_t)(¶ms)); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp index 910e8296afe..b5f46fca70c 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp @@ -2,8 +2,10 @@ #include "dense_simple_map_function.h" #include "dense_tensor_view.h" +#include <vespa/vespalib/util/typify.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/inline_operation.h> namespace vespalib::tensor { @@ -13,6 +15,7 @@ using eval::Value; using eval::ValueType; using eval::TensorFunction; using eval::TensorEngine; +using eval::TypifyCellType; using eval::as; using namespace eval::operation; @@ -24,19 +27,6 @@ using State = eval::InterpretedFunction::State; namespace { -struct CallFun { - map_fun_t function; - CallFun(map_fun_t function_in) : function(function_in) {} - double eval(double a) const { return function(a); } -}; - -template <typename CT, typename Fun> -void apply_fun_to_n(CT *dst, const CT *src, size_t n, const Fun &fun) { - for (size_t i = 0; i < n; ++i) { - dst[i] = fun.eval(src[i]); - } -} - template <typename CT, bool inplace> ArrayRef<CT> make_dst_cells(ConstArrayRef<CT> src_cells, Stash &stash) { if (inplace) { @@ -52,7 +42,7 @@ void my_simple_map_op(State &state, uint64_t param) { auto const &child = state.peek(0); auto src_cells = DenseTensorView::typify_cells<CT>(child); auto dst_cells = make_dst_cells<CT, inplace>(src_cells, state.stash); - apply_fun_to_n(dst_cells.begin(), src_cells.begin(), dst_cells.size(), my_fun); + apply_op1_vec(dst_cells.begin(), src_cells.begin(), dst_cells.size(), my_fun); if (!inplace) { state.pop_push(state.stash.create<DenseTensorView>(child.type(), TypedCells(dst_cells))); } @@ -60,25 +50,13 @@ void my_simple_map_op(State &state, uint64_t param) { //----------------------------------------------------------------------------- -template <typename Fun, bool inplace> -struct MySimpleMapOp { - template <typename CT> - static auto get_fun() { return my_simple_map_op<CT,Fun,inplace>; } -}; - -template <typename Fun> -op_function my_select_2(ValueType::CellType ct, bool inplace) { - if (inplace) { - return select_1<MySimpleMapOp<Fun,true>>(ct); - } else { - return select_1<MySimpleMapOp<Fun,false>>(ct); +struct MyGetFun { + template <typename R1, typename R2, typename R3> static auto invoke() { + return my_simple_map_op<R1, R2, R3::value>; } -} +}; -op_function my_select(ValueType::CellType ct, bool inplace, map_fun_t fun_hint) { - (void) fun_hint; // ready for function inlining - return my_select_2<CallFun>(ct, inplace); -} +using MyTypify = TypifyValue<TypifyCellType,TypifyOp1,TypifyBool>; } // namespace vespalib::tensor::<unnamed> @@ -96,7 +74,7 @@ DenseSimpleMapFunction::~DenseSimpleMapFunction() = default; Instruction DenseSimpleMapFunction::compile_self(const TensorEngine &, Stash &) const { - auto op = my_select(result_type().cell_type(), inplace(), function()); + auto op = typify_invoke<3,MyTypify,MyGetFun>(result_type().cell_type(), function(), inplace()); static_assert(sizeof(uint64_t) == sizeof(function())); return Instruction(op, (uint64_t)(function())); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp index 663993b6c26..571bcb79c9f 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp @@ -2,6 +2,7 @@ #include "dense_single_reduce_function.h" #include "dense_tensor_view.h" +#include <vespa/vespalib/util/typify.h> #include <vespa/eval/eval/value.h> namespace vespalib::tensor { @@ -12,6 +13,8 @@ using eval::TensorEngine; using eval::TensorFunction; using eval::Value; using eval::ValueType; +using eval::TypifyCellType; +using eval::TypifyAggr; using eval::as; using namespace eval::tensor_function; @@ -66,28 +69,13 @@ void my_single_reduce_op(InterpretedFunction::State &state, uint64_t param) { state.pop_push(state.stash.create<DenseTensorView>(params.result_type, TypedCells(dst_cells))); } -template <typename CT> -InterpretedFunction::op_function my_select_2(Aggr aggr) { - switch (aggr) { - case Aggr::AVG: return my_single_reduce_op<CT, Avg<CT>>; - case Aggr::COUNT: return my_single_reduce_op<CT, Count<CT>>; - case Aggr::PROD: return my_single_reduce_op<CT, Prod<CT>>; - case Aggr::SUM: return my_single_reduce_op<CT, Sum<CT>>; - case Aggr::MAX: return my_single_reduce_op<CT, Max<CT>>; - case Aggr::MIN: return my_single_reduce_op<CT, Min<CT>>; +struct MyGetFun { + template <typename R1, typename R2> static auto invoke() { + return my_single_reduce_op<R1, typename R2::template templ<R1>>; } - abort(); -} +}; -InterpretedFunction::op_function my_select(CellType cell_type, Aggr aggr) { - if (cell_type == ValueType::CellType::DOUBLE) { - return my_select_2<double>(aggr); - } - if (cell_type == ValueType::CellType::FLOAT) { - return my_select_2<float>(aggr); - } - abort(); -} +using MyTypify = TypifyValue<TypifyCellType,TypifyAggr>; bool check_input_type(const ValueType &type) { return (type.is_dense() && ((type.cell_type() == CellType::FLOAT) || (type.cell_type() == CellType::DOUBLE))); @@ -109,7 +97,7 @@ DenseSingleReduceFunction::~DenseSingleReduceFunction() = default; InterpretedFunction::Instruction DenseSingleReduceFunction::compile_self(const TensorEngine &, Stash &stash) const { - auto op = my_select(result_type().cell_type(), _aggr); + auto op = typify_invoke<2,MyTypify,MyGetFun>(result_type().cell_type(), _aggr); auto ¶ms = stash.create<Params>(result_type(), child().result_type(), _dim_idx); static_assert(sizeof(uint64_t) == sizeof(¶ms)); return InterpretedFunction::Instruction(op, (uint64_t)¶ms); diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp index 3533ab20175..7e887d4df34 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp @@ -34,7 +34,7 @@ void my_tensor_create_op(eval::InterpretedFunction::State &state, uint64_t param struct MyTensorCreateOp { template <typename CT> - static auto get_fun() { return my_tensor_create_op<CT>; } + static auto invoke() { return my_tensor_create_op<CT>; } }; size_t get_index(const TensorSpec::Address &addr, const ValueType &type) { @@ -72,7 +72,9 @@ eval::InterpretedFunction::Instruction DenseTensorCreateFunction::compile_self(const TensorEngine &, Stash &) const { static_assert(sizeof(uint64_t) == sizeof(&_self)); - auto op = select_1<MyTensorCreateOp>(result_type().cell_type()); + + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<1,MyTypify,MyTensorCreateOp>(result_type().cell_type()); return eval::InterpretedFunction::Instruction(op, (uint64_t)&_self); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp index 5cb1cbfd88f..16c0b01b169 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp @@ -44,7 +44,7 @@ void my_tensor_peek_op(eval::InterpretedFunction::State &state, uint64_t param) struct MyTensorPeekOp { template <typename CT> - static auto get_fun() { return my_tensor_peek_op<CT>; } + static auto invoke() { return my_tensor_peek_op<CT>; } }; } // namespace vespalib::tensor::<unnamed> @@ -71,7 +71,8 @@ eval::InterpretedFunction::Instruction DenseTensorPeekFunction::compile_self(const TensorEngine &, Stash &) const { static_assert(sizeof(uint64_t) == sizeof(&_spec)); - auto op = select_1<MyTensorPeekOp>(_children[0].get().result_type().cell_type()); + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<1,MyTypify,MyTensorPeekOp>(_children[0].get().result_type().cell_type()); return eval::InterpretedFunction::Instruction(op, (uint64_t)&_spec); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp index a0d63a1ce1e..968308d69c9 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp @@ -76,33 +76,6 @@ void my_cblas_float_xw_product_op(eval::InterpretedFunction::State &state, uint6 state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); } -template <bool common_inner> -struct MyXWProductOp { - template <typename LCT, typename RCT> - static auto get_fun() { return my_xw_product_op<LCT,RCT,common_inner>; } -}; - -template <bool common_inner> -eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct) { - if (lct == rct) { - if (lct == ValueType::CellType::DOUBLE) { - return my_cblas_double_xw_product_op<common_inner>; - } - if (lct == ValueType::CellType::FLOAT) { - return my_cblas_float_xw_product_op<common_inner>; - } - } - return select_2<MyXWProductOp<common_inner>>(lct, rct); -} - -eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool common_inner) { - if (common_inner) { - return my_select2<true>(lct, rct); - } else { - return my_select2<false>(lct, rct); - } -} - bool isDenseTensor(const ValueType &type, size_t d) { return (type.is_dense() && (type.dimensions().size() == d)); } @@ -132,6 +105,18 @@ const TensorFunction &createDenseXWProduct(const ValueType &res, const TensorFun common_inner); } +struct MyXWProductOp { + template<typename R1, typename R2, typename R3> static auto invoke() { + if (std::is_same_v<R1,double> && std::is_same_v<R2,double>) { + return my_cblas_double_xw_product_op<R3::value>; + } else if (std::is_same_v<R1,float> && std::is_same_v<R2,float>) { + return my_cblas_float_xw_product_op<R3::value>; + } else { + return my_xw_product_op<R1, R2, R3::value>; + } + } +}; + } // namespace vespalib::tensor::<unnamed> DenseXWProductFunction::Self::Self(const eval::ValueType &result_type_in, @@ -160,8 +145,10 @@ eval::InterpretedFunction::Instruction DenseXWProductFunction::compile_self(const TensorEngine &, Stash &stash) const { Self &self = stash.create<Self>(result_type(), _vector_size, _result_size); - auto op = my_select(lhs().result_type().cell_type(), - rhs().result_type().cell_type(), _common_inner); + using MyTypify = TypifyValue<eval::TypifyCellType,vespalib::TypifyBool>; + auto op = typify_invoke<3,MyTypify,MyXWProductOp>(lhs().result_type().cell_type(), + rhs().result_type().cell_type(), + _common_inner); return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self)); } diff --git a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp index 7a4b5917f00..57f727f7968 100644 --- a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp @@ -19,7 +19,7 @@ namespace { struct CallVectorFromDoubles { template <typename CT> static TypedCells - call(eval::InterpretedFunction::State &state, size_t numCells) { + invoke(eval::InterpretedFunction::State &state, size_t numCells) { ArrayRef<CT> outputCells = state.stash.create_array<CT>(numCells); for (size_t i = numCells; i-- > 0; ) { outputCells[i] = (CT) state.peek(0).as_double(); @@ -33,7 +33,8 @@ void my_vector_from_doubles_op(eval::InterpretedFunction::State &state, uint64_t const auto *self = (const VectorFromDoublesFunction::Self *)(param); CellType ct = self->resultType.cell_type(); size_t numCells = self->resultSize; - TypedCells cells = dispatch_0<CallVectorFromDoubles>(ct, state, numCells); + using MyTypify = eval::TypifyCellType; + TypedCells cells = typify_invoke<1,MyTypify,CallVectorFromDoubles>(ct, state, numCells); const Value &result = state.stash.create<DenseTensorView>(self->resultType, cells); state.stack.emplace_back(result); } diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp index 493e4af3caf..4d6cfb1c9af 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp @@ -92,7 +92,7 @@ DenseBinaryFormat::serialize(nbostream &stream, const DenseTensorView &tensor) struct CallDecodeCells { template <typename CT> static std::unique_ptr<DenseTensorView> - call(nbostream &stream, size_t numCells, ValueType &&newType) { + invoke(nbostream &stream, size_t numCells, ValueType &&newType) { std::vector<CT> newCells; newCells.reserve(numCells); decodeCells<CT>(stream, numCells, newCells); @@ -106,7 +106,8 @@ DenseBinaryFormat::deserialize(nbostream &stream, CellType cell_type) std::vector<Dimension> dimensions; size_t numCells = decodeDimensions(stream, dimensions); ValueType newType = ValueType::tensor_type(std::move(dimensions), cell_type); - return dispatch_0<CallDecodeCells>(cell_type, stream, numCells, std::move(newType)); + using MyTypify = eval::TypifyCellType; + return typify_invoke<1,MyTypify,CallDecodeCells>(cell_type, stream, numCells, std::move(newType)); } template <typename T> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index a9ded0f5fd2..8ec6174e8ae 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -69,13 +69,6 @@ public class Flags { "Takes effect on next host admin tick.", HOSTNAME); - public static final UnboundBooleanFlag USE_NEW_VESPA_RPMS = defineFeatureFlag( - "use-new-vespa-rpms", false, - "Whether to use the new vespa-rpms YUM repo when upgrading/downgrading. The vespa-version " + - "when fetching the flag value is the wanted version of the host.", - "Takes effect when upgrading or downgrading host admin to a different version.", - HOSTNAME, NODE_TYPE, VESPA_VERSION); - public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag( "disabled-host-admin-tasks", List.of(), String.class, "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask) that should be skipped", @@ -147,9 +140,9 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundStringFlag DOCPROC_LOADBALANCER_TYPE = defineStringFlag( - "docproc-loadbalancer-type", "", - "Selects what kind of load balancer to use for document processing {'adaptive', 'legacy' ''}", + public static final UnboundStringFlag JVM_GC_OPTIONS = defineStringFlag( + "jvm-gc-options", "", + "Sets deafult jvm gc options", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -282,6 +275,13 @@ public class Flags { CONSOLE_USER_EMAIL ); + public static final UnboundBooleanFlag CONFIGSERVER_PROVISION_LB = defineFeatureFlag( + "configserver-provision-lb", false, + "Provision load balancer for config server cluster", + "Takes effect when zone-config-servers application is redeployed", + ZONE_ID + ); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/hosted-tenant-base/OWNERS b/hosted-tenant-base/OWNERS new file mode 100644 index 00000000000..ff9741f2060 --- /dev/null +++ b/hosted-tenant-base/OWNERS @@ -0,0 +1,2 @@ +mortent +bjorncs
\ No newline at end of file diff --git a/hosted-tenant-base/README b/hosted-tenant-base/README new file mode 100644 index 00000000000..fed0672dcec --- /dev/null +++ b/hosted-tenant-base/README @@ -0,0 +1 @@ +Base pom for Vespa cloud application poms diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml new file mode 100644 index 00000000000..93ef008b8f7 --- /dev/null +++ b/hosted-tenant-base/pom.xml @@ -0,0 +1,388 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <!-- <version>7-SNAPSHOT</version>--> + <!-- <url>https://github.com/vespa-engine</url>--> + <!-- <packaging>pom</packaging>--> + + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-tenant-base</artifactId> + <version>7-SNAPSHOT</version> + <name>Base pom for all tenant base poms</name> + <description>Parent POM for all Vespa base poms.</description> + <url>https://github.com/vespa-engine</url> + <packaging>pom</packaging> + + <licenses> + <license> + <name>The Apache License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + </license> + </licenses> + <developers> + <developer> + <name>Vespa</name> + <url>https://github.com/vespa-engine</url> + </developer> + </developers> + <scm> + <connection>scm:git:git@github.com:vespa-engine/vespa.git</connection> + <developerConnection>scm:git:git@github.com:vespa-engine/vespa.git</developerConnection> + <url>git@github.com:vespa-engine/vespa.git</url> + </scm> + + <properties> + <vespaversion>${project.version}</vespaversion> + <test-framework.version>${project.version}</test-framework.version> + <target_jdk_version>11</target_jdk_version> + <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version> + <maven-surefire-plugin.version>2.22.0</maven-surefire-plugin.version> + <junit.version>5.4.2</junit.version> + <endpoint>https://api.vespa-external.aws.oath.cloud:4443</endpoint> + <test.categories>!integration</test.categories> + </properties> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dependency-versions</artifactId> + <version>${vespaversion}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <version>${junit.version}</version> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <version>${junit.version}</version> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container</artifactId> + <version>${vespaversion}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${vespaversion}</version> + <scope>runtime</scope> + <exclusions> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-cd-api</artifactId> + <version>${test-framework.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + + <profiles> + <profile> + <!-- Build *-fat-test.jar file that includes all non-test classes and resources + that are part of the class path during test and and test.jar that includes + all test classes and resources, and put it inside a zip: + 1. application classes and resources + 2. test classes and resources + 3. classes and resources in all dependencies of both (1) and (2) + 4. copy the fat-test-jar and test-jar to application-test/artifacts directory + 5. zip application-test --> + <id>fat-test-application</id> + <build> + <plugins> + <plugin> + <!-- dependencies, see (3) above --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <!-- JAR-like dependencies --> + <id>unpack-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <configuration> + <includeTypes>jar,test-jar</includeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + <!-- WARNING(2018-06-27): bcpkix-jdk15on-1.58.jar and + bcprov-jdk15on-1.58.jar are pulled in via + container-dev and both contains the same set of + bouncycastle signature files in META-INF: + BC1024KE.DSA, BC1024KE.SF, BC2048KE.DSA, and + BC2048KE.SF. By merging any of these two with any + other JAR file like we're doing here, the signatures + are wrong. Worse, what we're doing is WRONG but not + yet fatal. + + The symptom of this happening is that the tester fails + to load the SystemTest class(!?), and subsequently + tries to run all test-like files in the fat test JAR. + + The solution is to exclude such files. This happens + automatically with maven-assembly-plugin. --> + <excludes>META-INF/*.SF,META-INF/*.DSA</excludes> + </configuration> + </execution> + <execution> + <!-- non-JAR-like dependencies --> + <id>non-jar-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <excludeTypes>jar,test-jar</excludeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>copy-resources</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>target/fat-test-classes</outputDirectory> + <resources> + <!-- application classes and resources, see 1. above --> + <resource> + <directory>target/classes</directory> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>fat-test-jar</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <classesDirectory>target/fat-test-classes</classesDirectory> + <classifier>fat-test</classifier> + </configuration> + </execution> + <execution> + <id>test-jar</id> + <phase>package</phase> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>attach-artifact</id> + <phase>package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <tasks> + <!-- copy fat test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-fat-test.jar" + todir="target/application-test/artifacts/" /> + + <!-- copy slim test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-tests.jar" + todir="target/application-test/artifacts/" /> + + <!-- zip application-test, see 5. above --> + <zip destfile="target/application-test.zip" + basedir="target/application-test/" /> + </tasks> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + + <profile> <!-- Alias vespaversion with a more descriptive vespa.compile.version --> + <id>set-vespa-compile-version</id> + <activation> + <property> + <name>vespa.compile.version</name> + </property> + </activation> + <properties> + <vespaversion>${vespa.compile.version}</vespaversion> + </properties> + </profile> + + <profile> <!-- Alias vespaVersion with a more descriptive vespa.runtime.version --> + <id>set-vespa-runtime-version</id> + <activation> + <property> + <name>vespa.runtime.version</name> + </property> + </activation> + <properties> + <vespaVersion>${vespa.runtime.version}</vespaVersion> + </properties> + </profile> + </profiles> + + <build> + <finalName>${project.artifactId}</finalName> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <groups>${test.categories}</groups> + <redirectTestOutputToFile>false</redirectTestOutputToFile> + <trimStackTrace>false</trimStackTrace> + <systemPropertyVariables> + <application>${application}</application> + <tenant>${tenant}</tenant> + <instance>${instance}</instance> + <environment>${environment}</environment> + <region>${region}</region> + <endpoint>${endpoint}</endpoint> + <apiKeyFile>${apiKeyFile}</apiKeyFile> + <apiCertificateFile>${apiCertificateFile}</apiCertificateFile> + <dataPlaneKeyFile>${dataPlaneKeyFile}</dataPlaneKeyFile> + <dataPlaneCertificateFile>${dataPlaneCertificateFile}</dataPlaneCertificateFile> + </systemPropertyVariables> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + </configuration> + </plugin> + </plugins> + </pluginManagement> + + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>3.0.0-M2</version> + <executions> + <execution> + <id>enforce-java</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireJavaVersion> + <version>[11, )</version> + </requireJavaVersion> + <requireMavenVersion> + <version>[3.5, )</version> + </requireMavenVersion> + </rules> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>${maven-compiler-plugin.version}</version> + <configuration> + <source>${target_jdk_version}</source> + <target>${target_jdk_version}</target> + <showWarnings>true</showWarnings> + <showDeprecation>true</showDeprecation> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-maven-plugin</artifactId> + <version>${vespaversion}</version> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-application-maven-plugin</artifactId> + <version>${vespaversion}</version> + <executions> + <execution> + <goals> + <goal>packageApplication</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${vespaversion}</version> + <extensions>true</extensions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/hosted-zone-api/CMakeLists.txt b/hosted-zone-api/CMakeLists.txt new file mode 100644 index 00000000000..cc6b2953759 --- /dev/null +++ b/hosted-zone-api/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(hosted-zone-api) diff --git a/hosted-zone-api/OWNERS b/hosted-zone-api/OWNERS new file mode 100644 index 00000000000..569bf1cc3a1 --- /dev/null +++ b/hosted-zone-api/OWNERS @@ -0,0 +1 @@ +bjorncs diff --git a/hosted-zone-api/README.md b/hosted-zone-api/README.md new file mode 100644 index 00000000000..8f44632dde1 --- /dev/null +++ b/hosted-zone-api/README.md @@ -0,0 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +# hosted-zone-api + +Contains hosted Zone API for user facing Vespa Java APIs diff --git a/hosted-zone-api/abi-spec.json b/hosted-zone-api/abi-spec.json new file mode 100644 index 00000000000..e5d1db476c2 --- /dev/null +++ b/hosted-zone-api/abi-spec.json @@ -0,0 +1,51 @@ +{ + "ai.vespa.cloud.Environment": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static ai.vespa.cloud.Environment[] values()", + "public static ai.vespa.cloud.Environment valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum ai.vespa.cloud.Environment dev", + "public static final enum ai.vespa.cloud.Environment perf", + "public static final enum ai.vespa.cloud.Environment test", + "public static final enum ai.vespa.cloud.Environment staging", + "public static final enum ai.vespa.cloud.Environment prod" + ] + }, + "ai.vespa.cloud.SystemInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(ai.vespa.cloud.Zone)", + "public ai.vespa.cloud.Zone zone()" + ], + "fields": [] + }, + "ai.vespa.cloud.Zone": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(ai.vespa.cloud.Environment, java.lang.String)", + "public ai.vespa.cloud.Environment environment()", + "public java.lang.String region()", + "public java.lang.String toString()", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public static ai.vespa.cloud.Zone from(java.lang.String)" + ], + "fields": [] + } +}
\ No newline at end of file diff --git a/hosted-zone-api/pom.xml b/hosted-zone-api/pom.xml new file mode 100644 index 00000000000..05a291c52bf --- /dev/null +++ b/hosted-zone-api/pom.xml @@ -0,0 +1,55 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>hosted-zone-api</artifactId> + <packaging>container-plugin</packaging> + <version>7-SNAPSHOT</version> + + <dependencies> + <!-- provided --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <!-- required for bundle-plugin to generate import-package statements for Java's standard library --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <!-- test --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>abi-check-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/container-core/src/main/java/ai/vespa/cloud/Environment.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/Environment.java index 8f1d9fc962a..8f1d9fc962a 100644 --- a/container-core/src/main/java/ai/vespa/cloud/Environment.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/Environment.java diff --git a/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java index 0524ae072cd..0ac93861275 100644 --- a/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java @@ -1,9 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.cloud; -import com.google.inject.Inject; -import com.yahoo.cloud.config.ConfigserverConfig; - /** * Provides information about the system in which this container is running. * This is available and can be injected when running in a cloud environment. @@ -14,13 +11,6 @@ public class SystemInfo { private final Zone zone; - /** Do not use */ - @Inject - public SystemInfo(ConfigserverConfig config) { - this.zone = new Zone(Environment.valueOf(config.environment()), config.region()); - } - - /** Create an instance for testing */ public SystemInfo(Zone zone) { this.zone = zone; } diff --git a/container-core/src/main/java/ai/vespa/cloud/Zone.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/Zone.java index 48293aa7908..48293aa7908 100644 --- a/container-core/src/main/java/ai/vespa/cloud/Zone.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/Zone.java diff --git a/container-core/src/main/java/ai/vespa/cloud/package-info.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/package-info.java index 259a2bda258..259a2bda258 100644 --- a/container-core/src/main/java/ai/vespa/cloud/package-info.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/package-info.java diff --git a/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java b/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java index 6bc8b395e00..6bc8b395e00 100644 --- a/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java +++ b/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java index 808a63eb130..ad67e13e044 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java @@ -22,6 +22,7 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg private boolean wantFrozen = false; private boolean isFrozen = true; private boolean pendingInterrupt = false; + private boolean isWaitingForNextContext = false; public NodeAgentContextManager(Clock clock, NodeAgentContext context) { this.clock = clock; @@ -62,6 +63,8 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg public NodeAgentContext nextContext() throws InterruptedException { synchronized (monitor) { nextContext = null; // Reset any previous context and wait for the next one + isWaitingForNextContext = true; + monitor.notify(); Duration untilNextContext = Duration.ZERO; while (setAndGetIsFrozen(wantFrozen) || nextContext == null || @@ -76,6 +79,7 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg } catch (InterruptedException ignored) { } } + isWaitingForNextContext = false; currentContext = nextContext; return currentContext; } @@ -105,4 +109,15 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg return this.isFrozen; } } + + /** FOR TESTING ONLY */ + void waitUntilWaitingForNextContext() { + synchronized (monitor) { + while (!isWaitingForNextContext) { + try { + monitor.wait(); + } catch (InterruptedException ignored) { } + } + } + } }
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java index 99111366b8e..a082addd775 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java @@ -7,6 +7,8 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -25,13 +27,13 @@ public class NodeAgentContextManagerTest { private final NodeAgentContextManager manager = new NodeAgentContextManager(clock, initialContext); @Test(timeout = TIMEOUT) - public void context_is_ignored_unless_scheduled_while_waiting() throws InterruptedException { + public void context_is_ignored_unless_scheduled_while_waiting() { NodeAgentContext context1 = generateContext(); manager.scheduleTickWith(context1, clock.instant()); assertSame(initialContext, manager.currentContext()); AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - Thread.sleep(20); + manager.waitUntilWaitingForNextContext(); assertFalse(async.isCompleted()); NodeAgentContext context2 = generateContext(); @@ -42,9 +44,9 @@ public class NodeAgentContextManagerTest { } @Test(timeout = TIMEOUT) - public void returns_no_earlier_than_at_given_time() throws InterruptedException { + public void returns_no_earlier_than_at_given_time() { AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - Thread.sleep(20); + manager.waitUntilWaitingForNextContext(); NodeAgentContext context1 = generateContext(); Instant returnAt = clock.instant().plusMillis(500); @@ -57,10 +59,9 @@ public class NodeAgentContextManagerTest { } @Test(timeout = TIMEOUT) - public void blocks_in_nextContext_until_one_is_scheduled() throws InterruptedException { + public void blocks_in_nextContext_until_one_is_scheduled() { AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - assertFalse(async.isCompleted()); - Thread.sleep(10); + manager.waitUntilWaitingForNextContext(); assertFalse(async.isCompleted()); NodeAgentContext context1 = generateContext(); @@ -72,10 +73,9 @@ public class NodeAgentContextManagerTest { } @Test(timeout = TIMEOUT) - public void blocks_in_nextContext_until_interrupt() throws InterruptedException { + public void blocks_in_nextContext_until_interrupt() { AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - assertFalse(async.isCompleted()); - Thread.sleep(10); + manager.waitUntilWaitingForNextContext(); assertFalse(async.isCompleted()); manager.interrupt(); @@ -86,13 +86,13 @@ public class NodeAgentContextManagerTest { } @Test(timeout = TIMEOUT) - public void setFrozen_does_not_block_with_no_timeout() throws InterruptedException { + public void setFrozen_does_not_block_with_no_timeout() { assertFalse(manager.setFrozen(false, Duration.ZERO)); // Generate new context and get it from the supplier, this completes the unfreeze NodeAgentContext context1 = generateContext(); AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - Thread.sleep(20); + manager.waitUntilWaitingForNextContext(); manager.scheduleTickWith(context1, clock.instant()); assertSame(context1, async.awaitResult().response.get()); @@ -116,7 +116,7 @@ public class NodeAgentContextManagerTest { Thread.sleep(200); // Simulate running NodeAgent::converge return context; }); - Thread.sleep(20); + manager.waitUntilWaitingForNextContext(); NodeAgentContext context1 = generateContext(); manager.scheduleTickWith(context1, clock.instant()); @@ -130,9 +130,9 @@ public class NodeAgentContextManagerTest { assertFalse(asyncScheduler.isCompleted()); // Still waiting for consumer to converge to frozen AsyncExecutor<NodeAgentContext> asyncConsumer2 = new AsyncExecutor<>(manager::nextContext); - Thread.sleep(20); + manager.waitUntilWaitingForNextContext(); assertFalse(asyncConsumer2.isCompleted()); // Waiting for next context - assertTrue(asyncScheduler.isCompleted()); // While consumer is waiting, it has converged to frozen + asyncScheduler.awaitResult(); // We should be able to converge to frozen now // Interrupt manager to end asyncConsumer2 manager.interrupt(); @@ -146,46 +146,30 @@ public class NodeAgentContextManagerTest { } private static class AsyncExecutor<T> { - private final Object monitor = new Object(); - private final Thread thread; + private final CountDownLatch latch = new CountDownLatch(1); private volatile Optional<T> response = Optional.empty(); private volatile Optional<Exception> exception = Optional.empty(); - private boolean completed = false; - private AsyncExecutor(ThrowingSupplier<T> supplier) { - this.thread = new Thread(() -> { + private AsyncExecutor(Callable<T> supplier) { + new Thread(() -> { try { - response = Optional.of(supplier.get()); + response = Optional.of(supplier.call()); } catch (Exception e) { exception = Optional.of(e); } - synchronized (monitor) { - completed = true; - monitor.notifyAll(); - } - }); - this.thread.start(); + latch.countDown(); + }).start(); } private AsyncExecutor<T> awaitResult() { - synchronized (monitor) { - while (!completed) { - try { - monitor.wait(); - } catch (InterruptedException ignored) { } - } - } + try { + latch.await(); + } catch (InterruptedException ignored) { } return this; } private boolean isCompleted() { - synchronized (monitor) { - return completed; - } + return latch.getCount() == 0; } } - - private interface ThrowingSupplier<T> { - T get() throws Exception; - } }
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index 09723d83e3e..edf2932ad6e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -3,9 +3,7 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.provision.NodeRepository; - -import java.util.Set; +import com.yahoo.config.provision.NodeType; /** * A managed load balance service. @@ -15,17 +13,14 @@ import java.util.Set; public interface LoadBalancerService { /** - * Create a load balancer for given application cluster. Implementations are expected to be idempotent + * Create a load balancer from the given specification. Implementations are expected to be idempotent * - * @param application Application owning the LB - * @param cluster Target cluster of the LB - * @param reals Reals that should be configured on the LB + * @param spec Load balancer specification * @param force Whether reconfiguration should be forced (e.g. allow configuring an empty set of reals on a * pre-existing load balancer). * @return The provisioned load balancer instance */ - LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository); + LoadBalancerInstance create(LoadBalancerSpec spec, boolean force); /** Permanently remove load balancer for given application cluster */ void remove(ApplicationId application, ClusterSpec.Id cluster); @@ -33,6 +28,12 @@ public interface LoadBalancerService { /** Returns the protocol supported by this load balancer service */ Protocol protocol(); + /** Returns whether load balancers created by this service can forward traffic to given node and cluster type */ + default boolean canForwardTo(NodeType nodeType, ClusterSpec.Type clusterType) { + return (nodeType == NodeType.tenant && clusterType.isContainer()) || + (nodeType == NodeType.config && clusterType == ClusterSpec.Type.admin); + } + /** Load balancer protocols */ enum Protocol { ipv4, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index 9bd1189420a..f4d689056c3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -5,13 +5,11 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Set; /** * @author mpolden @@ -30,19 +28,18 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository) { - var id = new LoadBalancerId(application, cluster); + public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { + var id = new LoadBalancerId(spec.application(), spec.cluster()); var oldInstance = instances.get(id); - if (!force && oldInstance != null && !oldInstance.reals().isEmpty() && reals.isEmpty()) { + if (!force && oldInstance != null && !oldInstance.reals().isEmpty() && spec.reals().isEmpty()) { throw new IllegalArgumentException("Refusing to remove all reals from load balancer " + id); } var instance = new LoadBalancerInstance( - HostName.from("lb-" + application.toShortString() + "-" + cluster.value()), + HostName.from("lb-" + spec.application().toShortString() + "-" + spec.cluster().value()), Optional.of(new DnsZone("zone-id-1")), Collections.singleton(4443), ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), - reals); + spec.reals()); instances.put(id, instance); return instance; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java new file mode 100644 index 00000000000..b76f5ba2bcf --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java @@ -0,0 +1,43 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.lb; + +import com.google.common.collect.ImmutableSortedSet; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; + +import java.util.Objects; +import java.util.Set; + +/** + * A specification for a load balancer. + * + * @author mpolden + */ +public class LoadBalancerSpec { + + private final ApplicationId application; + private final ClusterSpec.Id cluster; + private final Set<Real> reals; + + public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals) { + this.application = Objects.requireNonNull(application); + this.cluster = Objects.requireNonNull(cluster); + this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals)); + } + + /** Owner of the load balancer */ + public ApplicationId application() { + return application; + } + + /** The target cluster of this load balancer */ + public ClusterSpec.Id cluster() { + return cluster; + } + + /** Real servers to attach to this load balancer */ + public Set<Real> reals() { + return reals; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java index 07074bc45af..7667672e470 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.Comparator; import java.util.Optional; @@ -18,11 +17,10 @@ import java.util.Set; public class PassthroughLoadBalancerService implements LoadBalancerService { @Override - public LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository) { - var real = reals.stream() - .min(Comparator.naturalOrder()) - .orElseThrow(() -> new IllegalArgumentException("No reals given")); + public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { + var real = spec.reals().stream() + .min(Comparator.naturalOrder()) + .orElseThrow(() -> new IllegalArgumentException("No reals given")); return new LoadBalancerInstance(real.hostname(), Optional.empty(), Set.of(real.port()), Set.of(real.ipAddress() + "/32"), Set.of()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index a8faafc0bad..bc4381573c6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; -import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; @@ -27,13 +26,14 @@ public class SharedLoadBalancerService implements LoadBalancerService { private static final Comparator<Node> hostnameComparator = Comparator.comparing(Node::hostname); - @Inject - public SharedLoadBalancerService() { + private final NodeRepository nodeRepository; + + public SharedLoadBalancerService(NodeRepository nodeRepository) { + this.nodeRepository = Objects.requireNonNull(nodeRepository); } @Override - public LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository) { + public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { var proxyNodes = new ArrayList<>(nodeRepository.getNodes(NodeType.proxy)); proxyNodes.sort(hostnameComparator); @@ -52,7 +52,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { Optional.empty(), Set.of(4080, 4443), networkNames, - reals + spec.reals() ); } @@ -66,6 +66,12 @@ public class SharedLoadBalancerService implements LoadBalancerService { return Protocol.dualstack; } + @Override + public boolean canForwardTo(NodeType nodeType, ClusterSpec.Type clusterType) { + // Shared routing layer only supports routing to tenant nodes + return nodeType == NodeType.tenant && clusterType.isContainer(); + } + private static String withPrefixLength(String address) { if (IP.isV6(address)) { return address + "/128"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 483b4dc8f84..6edd57de1c1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -1,13 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer.State; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import java.time.Duration; @@ -17,6 +17,7 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.logging.Level; import java.util.stream.Collectors; /** @@ -99,7 +100,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { // Remove any real no longer allocated to this application reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value())); try { - service.create(lb.id().application(), lb.id().cluster(), reals, true, nodeRepository()); + service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals), true); db.writeLoadBalancer(lb.with(lb.instance().withReals(reals))); } catch (Exception e) { failed.add(lb.id()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 7875afeb28c..4323622df8b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -84,7 +84,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { nodeRebooter = new NodeRebooter(nodeRepository, clock, flagSource); metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval, clock); infrastructureProvisioner = new InfrastructureProvisioner(nodeRepository, infraDeployer, defaults.infrastructureProvisionInterval); - loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService().map(lbService -> + loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService(nodeRepository).map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService)); dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner -> new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 7ad69d673e7..7158ccc57e3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -111,10 +111,10 @@ class Activator { /** Activate load balancers */ private void activateLoadBalancers(ApplicationId application, Collection<HostSpec> hosts, NestedTransaction transaction, @SuppressWarnings("unused") Mutex applicationLock) { - loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(application, clustersOf(hosts), applicationLock, transaction)); + loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(application, allClustersOf(hosts), applicationLock, transaction)); } - private static Set<ClusterSpec> clustersOf(Collection<HostSpec> hosts) { + private static Set<ClusterSpec> allClustersOf(Collection<HostSpec> hosts) { return hosts.stream() .map(HostSpec::membership) .flatMap(Optional::stream) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index 004c74b5f70..38dd9f29873 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -17,7 +17,7 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { private final HostResourcesCalculator hostResourcesCalculator = new IdentityHostResourcesCalculator(); @Override - public Optional<LoadBalancerService> getLoadBalancerService() { + public Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository) { return Optional.empty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index c6945e1779b..460e1e71e65 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -8,6 +8,9 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -15,6 +18,7 @@ import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec; import com.yahoo.vespa.hosted.provision.lb.Real; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; @@ -23,6 +27,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.logging.Level; @@ -45,11 +50,13 @@ public class LoadBalancerProvisioner { private final NodeRepository nodeRepository; private final CuratorDatabaseClient db; private final LoadBalancerService service; + private final BooleanFlag provisionConfigServerLoadBalancer; - public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service) { + public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service, FlagSource flagSource) { this.nodeRepository = nodeRepository; this.db = nodeRepository.database(); this.service = service; + this.provisionConfigServerLoadBalancer = Flags.CONFIGSERVER_PROVISION_LB.bindTo(flagSource); // Read and write all load balancers to make sure they are stored in the latest version of the serialization format for (var id : db.readLoadBalancerIds()) { try (var lock = db.lock(id.application())) { @@ -70,11 +77,12 @@ public class LoadBalancerProvisioner { * Calling this for irrelevant node or cluster types is a no-op. */ public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { - if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type - if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type + if (!canForwardTo(requestedNodes.type(), cluster)) return; // Nothing to provision for this node and cluster type if (application.instance().isTester()) return; // Do not provision for tester instances try (var lock = db.lock(application)) { - provision(application, effectiveId(cluster), false, lock); + ClusterSpec.Id clusterId = effectiveId(cluster); + List<Node> nodes = nodesOf(clusterId, application); + provision(application, clusterId, nodes,false, lock); } } @@ -91,13 +99,14 @@ public class LoadBalancerProvisioner { public void activate(ApplicationId application, Set<ClusterSpec> clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { try (var lock = db.lock(application)) { - var containerClusters = containerClustersOf(clusters); - for (var clusterId : containerClusters) { + for (var cluster : loadBalancedClustersOf(application).entrySet()) { // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(application, clusterId, true, lock); + provision(application, cluster.getKey(), cluster.getValue(), true, lock); } // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); + var surplusLoadBalancers = surplusLoadBalancersOf(application, clusters.stream() + .map(LoadBalancerProvisioner::effectiveId) + .collect(Collectors.toSet())); deactivate(surplusLoadBalancers, transaction); } } @@ -138,9 +147,17 @@ public class LoadBalancerProvisioner { db.writeLoadBalancers(deactivatedLoadBalancers, transaction); } + // TODO(mpolden): Inline when feature flag is removed + private boolean canForwardTo(NodeType type, ClusterSpec cluster) { + boolean canForwardTo = service.canForwardTo(type, cluster.type()); + if (canForwardTo && type == NodeType.config) { + return provisionConfigServerLoadBalancer.value(); + } + return canForwardTo; + } /** Idempotently provision a load balancer for given application and cluster */ - private void provision(ApplicationId application, ClusterSpec.Id clusterId, boolean activate, + private void provision(ApplicationId application, ClusterSpec.Id clusterId, List<Node> nodes, boolean activate, @SuppressWarnings("unused") Mutex loadBalancersLock) { var id = new LoadBalancerId(application, clusterId); var now = nodeRepository.clock().instant(); @@ -148,7 +165,7 @@ public class LoadBalancerProvisioner { if (loadBalancer.isEmpty() && activate) return; // Nothing to activate as this load balancer was never prepared var force = loadBalancer.isPresent() && loadBalancer.get().state() != LoadBalancer.State.active; - var instance = create(application, clusterId, allocatedContainers(application, clusterId), force); + var instance = provisionInstance(application, clusterId, nodes, force); LoadBalancer newLoadBalancer; if (loadBalancer.isEmpty()) { newLoadBalancer = new LoadBalancer(id, instance, LoadBalancer.State.reserved, now); @@ -159,7 +176,8 @@ public class LoadBalancerProvisioner { db.writeLoadBalancer(newLoadBalancer); } - private LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, List<Node> nodes, boolean force) { + private LoadBalancerInstance provisionInstance(ApplicationId application, ClusterSpec.Id cluster, List<Node> nodes, + boolean force) { var reals = new LinkedHashSet<Real>(); for (var node : nodes) { for (var ip : reachableIpAddresses(node)) { @@ -169,7 +187,7 @@ public class LoadBalancerProvisioner { log.log(Level.FINE, "Creating load balancer for " + cluster + " in " + application.toShortString() + ", targeting: " + reals); try { - return service.create(application, cluster, reals, force, nodeRepository); + return service.create(new LoadBalancerSpec(application, cluster, reals), force); } catch (Exception e) { throw new LoadBalancerServiceException("Failed to (re)configure load balancer for " + cluster + " in " + application + ", targeting: " + reals + ". The operation will be " + @@ -177,14 +195,21 @@ public class LoadBalancerProvisioner { } } - /** Returns a list of active and reserved nodes of type container in given cluster */ - private List<Node> allocatedContainers(ApplicationId application, ClusterSpec.Id clusterId) { - return NodeList.copyOf(nodeRepository.getNodes(NodeType.tenant, Node.State.reserved, Node.State.active)) - .owner(application) - .matching(node -> node.state().isAllocated()) - .container() - .matching(node -> effectiveId(node.allocation().get().membership().cluster()).equals(clusterId)) - .asList(); + /** Returns the nodes allocated to the given load balanced cluster */ + private List<Node> nodesOf(ClusterSpec.Id loadBalancedCluster, ApplicationId application) { + return loadBalancedClustersOf(application).getOrDefault(loadBalancedCluster, List.of()); + } + + /** Returns the load balanced clusters of given application and their nodes */ + private Map<ClusterSpec.Id, List<Node>> loadBalancedClustersOf(ApplicationId application) { + NodeList nodes = NodeList.copyOf(nodeRepository.getNodes(Node.State.reserved, Node.State.active)) + .owner(application); + if (nodes.stream().anyMatch(node -> node.type() == NodeType.config)) { + nodes = nodes.nodeType(NodeType.config).type(ClusterSpec.Type.admin); + } else { + nodes = nodes.nodeType(NodeType.tenant).container(); + } + return nodes.stream().collect(Collectors.groupingBy(node -> effectiveId(node.allocation().get().membership().cluster()))); } /** Find IP addresses reachable by the load balancer service */ @@ -202,14 +227,6 @@ public class LoadBalancerProvisioner { return reachable; } - /** Returns the container cluster IDs of the given clusters */ - private static Set<ClusterSpec.Id> containerClustersOf(Set<ClusterSpec> clusters) { - return clusters.stream() - .filter(c -> c.type().isContainer()) - .map(LoadBalancerProvisioner::effectiveId) - .collect(Collectors.toUnmodifiableSet()); - } - private static ClusterSpec.Id effectiveId(ClusterSpec cluster) { return cluster.combinedId().orElse(cluster.id()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 59fca955a68..f9d8213072e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; @@ -15,7 +14,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; -import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.FlagSource; @@ -36,6 +34,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -66,7 +65,8 @@ public class NodeRepositoryProvisioner implements Provisioner { this.allocationOptimizer = new AllocationOptimizer(nodeRepository); this.capacityPolicies = new CapacityPolicies(nodeRepository); this.zone = zone; - this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService)); + this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService(nodeRepository) + .map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService, flagSource)); this.nodeResourceLimits = new NodeResourceLimits(nodeRepository); this.preparer = new Preparer(nodeRepository, zone.environment() == Environment.prod ? SPARE_CAPACITY_PROD : SPARE_CAPACITY_NONPROD, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index a86bd581516..563fc50e697 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import java.util.Optional; @@ -12,7 +13,7 @@ import java.util.Optional; */ public interface ProvisionServiceProvider { - Optional<LoadBalancerService> getLoadBalancerService(); + Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository); Optional<HostProvisioner> getHostProvisioner(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java index 9a02f65daf9..20538732c7a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.google.inject.Inject; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; @@ -37,7 +38,7 @@ public class MockProvisionServiceProvider implements ProvisionServiceProvider { } @Override - public Optional<LoadBalancerService> getLoadBalancerService() { + public Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository) { return loadBalancerService; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java index e70fc184b87..997aec8a156 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java @@ -20,8 +20,8 @@ public class PassthroughLoadBalancerServiceTest { var lbService = new PassthroughLoadBalancerService(); var real = new Real(HostName.from("host1.example.com"), "192.0.2.10"); var reals = Set.of(real, new Real(HostName.from("host2.example.com"), "192.0.2.11")); - var instance = lbService.create(ApplicationId.from("tenant1", "app1", "default"), - ClusterSpec.Id.from("c1"), reals, false, null); + var instance = lbService.create(new LoadBalancerSpec(ApplicationId.from("tenant1", "app1", "default"), + ClusterSpec.Id.from("c1"), reals), false); assertEquals(real.hostname(), instance.hostname()); assertEquals(Set.of(real.port()), instance.ports()); assertEquals(Set.of(real.ipAddress() + "/32"), instance.networks()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index 64d189b9111..06f18d94c5f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; public class SharedLoadBalancerServiceTest { private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); - private final SharedLoadBalancerService loadBalancerService = new SharedLoadBalancerService(); + private final SharedLoadBalancerService loadBalancerService = new SharedLoadBalancerService(tester.nodeRepository()); private final ApplicationId applicationId = ApplicationId.from("tenant1", "application1", "default"); private final ClusterSpec.Id clusterId = ClusterSpec.Id.from("qrs1"); private final Set<Real> reals = Set.of( @@ -30,7 +30,7 @@ public class SharedLoadBalancerServiceTest { @Test public void test_create_lb() { tester.makeReadyNodes(2, "default", NodeType.proxy); - var lb = loadBalancerService.create(applicationId, clusterId, reals, false, tester.nodeRepository()); + var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals), false); assertEquals(HostName.from("host-1.yahoo.com"), lb.hostname()); assertEquals(Optional.empty(), lb.dnsZone()); @@ -40,7 +40,7 @@ public class SharedLoadBalancerServiceTest { @Test(expected = IllegalStateException.class) public void test_exception_on_missing_proxies() { - loadBalancerService.create(applicationId, clusterId, reals, false, tester.nodeRepository()); + loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals), false); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 26039c29ae8..f48127f650d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -11,6 +11,8 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; @@ -43,7 +45,8 @@ public class LoadBalancerProvisionerTest { private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default"); private final ApplicationId infraApp1 = ApplicationId.from("vespa", "tenant-host", "default"); - private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private final ProvisioningTester tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); @Test public void provision_load_balancer() { @@ -212,6 +215,21 @@ public class LoadBalancerProvisionerTest { assertEquals(combinedId, lbs.get().get(0).id().cluster()); } + @Test + public void provision_load_balancer_config_server_cluster() { + flagSource.withBooleanFlag(Flags.CONFIGSERVER_PROVISION_LB.id(), true); + ApplicationId configServerApp = ApplicationId.from("hosted-vespa", "zone-config-servers", "default"); + Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(configServerApp).asList(); + var cluster = ClusterSpec.Id.from("zone-config-servers"); + var nodes = prepare(configServerApp, Capacity.fromRequiredNodeType(NodeType.config), false, + clusterRequest(ClusterSpec.Type.admin, cluster)); + assertEquals(1, lbs.get().size()); + assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().reals().size()); + tester.activate(configServerApp, nodes); + assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); + assertEquals(cluster, lbs.get().get(0).id().cluster()); + } + private void dirtyNodesOf(ApplicationId application) { tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, this.getClass().getSimpleName()); } diff --git a/parent/pom.xml b/parent/pom.xml index f3f6896ed5b..ce40eb464fc 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -745,7 +745,7 @@ <dependency> <groupId>xerces</groupId> <artifactId>xercesImpl</artifactId> - <version>2.11.0</version> + <version>2.12.0</version> </dependency> </dependencies> </dependencyManagement> @@ -31,6 +31,8 @@ <module>bundle-plugin-test</module> <module>chain</module> <module>client</module> + <module>cloud-tenant-base</module> + <module>cloud-tenant-cd</module> <module>clustercontroller-apps</module> <module>clustercontroller-apputil</module> <module>clustercontroller-core</module> @@ -80,6 +82,9 @@ <module>filedistribution</module> <module>flags</module> <module>fsa</module> + <module>hosted-api</module> + <module>hosted-tenant-base</module> + <module>hosted-zone-api</module> <module>http-utils</module> <module>indexinglanguage</module> <module>jaxrs_client_utils</module> @@ -123,7 +128,7 @@ <module>storage</module> <module>tenant-auth</module> <module>tenant-base</module> - <module>tenant-cd</module> + <module>tenant-cd-api</module> <module>testutil</module> <module>vdslib</module> <module>vespaclient-core</module> @@ -144,7 +149,6 @@ <module>zkfacade</module> <module>zookeeper-command-line-client</module> <module>zookeeper-server</module> - <module>hosted-api</module> </modules> </project> diff --git a/python/vespa/README.md b/python/vespa/README.md index c316564f3c1..00d8cc2e769 100644 --- a/python/vespa/README.md +++ b/python/vespa/README.md @@ -4,7 +4,7 @@ ## Install -`pip install vespa` +`pip install pyvespa` ## Connect to a Vespa app @@ -52,16 +52,9 @@ query_result = app.query( ``` ``` -query_result["root"]["fields"] +query_result.number_documents_retrieved ``` - - - - {'totalCount': 1077} - - - ## Labelled data > How to structure labelled data @@ -97,346 +90,6 @@ training_data_batch = app.collect_training_data( training_data_batch ``` - - - -<div> -<style scoped> - .dataframe tbody tr th:only-of-type { - vertical-align: middle; - } - - .dataframe tbody tr th { - vertical-align: top; - } - - .dataframe thead th { - text-align: right; - } -</style> -<table border="1" class="dataframe"> - <thead> - <tr style="text-align: right;"> - <th></th> - <th>attributeMatch(authors.first)</th> - <th>attributeMatch(authors.first).averageWeight</th> - <th>attributeMatch(authors.first).completeness</th> - <th>attributeMatch(authors.first).fieldCompleteness</th> - <th>attributeMatch(authors.first).importance</th> - <th>attributeMatch(authors.first).matches</th> - <th>attributeMatch(authors.first).maxWeight</th> - <th>attributeMatch(authors.first).normalizedWeight</th> - <th>attributeMatch(authors.first).normalizedWeightedWeight</th> - <th>attributeMatch(authors.first).queryCompleteness</th> - <th>...</th> - <th>textSimilarity(results).queryCoverage</th> - <th>textSimilarity(results).score</th> - <th>textSimilarity(title).fieldCoverage</th> - <th>textSimilarity(title).order</th> - <th>textSimilarity(title).proximity</th> - <th>textSimilarity(title).queryCoverage</th> - <th>textSimilarity(title).score</th> - <th>document_id</th> - <th>query_id</th> - <th>relevant</th> - </tr> - </thead> - <tbody> - <tr> - <th>0</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.000000</td> - <td>0.000000</td> - <td>0</td> - <td>0</td> - <td>1</td> - </tr> - <tr> - <th>1</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>1.000000</td> - <td>1.0</td> - <td>1.000000</td> - <td>1.000000</td> - <td>1.000000</td> - <td>56212</td> - <td>0</td> - <td>0</td> - </tr> - <tr> - <th>2</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.187500</td> - <td>0.5</td> - <td>0.617188</td> - <td>0.428571</td> - <td>0.457087</td> - <td>34026</td> - <td>0</td> - <td>0</td> - </tr> - <tr> - <th>3</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.000000</td> - <td>0.000000</td> - <td>3</td> - <td>0</td> - <td>1</td> - </tr> - <tr> - <th>4</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>1.000000</td> - <td>1.0</td> - <td>1.000000</td> - <td>1.000000</td> - <td>1.000000</td> - <td>56212</td> - <td>0</td> - <td>0</td> - </tr> - <tr> - <th>5</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.187500</td> - <td>0.5</td> - <td>0.617188</td> - <td>0.428571</td> - <td>0.457087</td> - <td>34026</td> - <td>0</td> - <td>0</td> - </tr> - <tr> - <th>6</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.071429</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.083333</td> - <td>0.039286</td> - <td>1</td> - <td>1</td> - <td>1</td> - </tr> - <tr> - <th>7</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>1.000000</td> - <td>1.0</td> - <td>1.000000</td> - <td>1.000000</td> - <td>1.000000</td> - <td>29774</td> - <td>1</td> - <td>0</td> - </tr> - <tr> - <th>8</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.500000</td> - <td>1.0</td> - <td>1.000000</td> - <td>0.333333</td> - <td>0.700000</td> - <td>22787</td> - <td>1</td> - <td>0</td> - </tr> - <tr> - <th>9</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.058824</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.083333</td> - <td>0.036765</td> - <td>5</td> - <td>1</td> - <td>1</td> - </tr> - <tr> - <th>10</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>1.000000</td> - <td>1.0</td> - <td>1.000000</td> - <td>1.000000</td> - <td>1.000000</td> - <td>29774</td> - <td>1</td> - <td>0</td> - </tr> - <tr> - <th>11</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.500000</td> - <td>1.0</td> - <td>1.000000</td> - <td>0.333333</td> - <td>0.700000</td> - <td>22787</td> - <td>1</td> - <td>0</td> - </tr> - </tbody> -</table> -<p>12 rows × 984 columns</p> -</div> - - - ## Evaluating a query model > Define metrics and evaluate query models. See the [evaluation page](/vespa/evaluation) for more examples. @@ -463,57 +116,3 @@ evaluation = app.evaluate( ) evaluation ``` - - - - -<div> -<style scoped> - .dataframe tbody tr th:only-of-type { - vertical-align: middle; - } - - .dataframe tbody tr th { - vertical-align: top; - } - - .dataframe thead th { - text-align: right; - } -</style> -<table border="1" class="dataframe"> - <thead> - <tr style="text-align: right;"> - <th></th> - <th>query_id</th> - <th>match_ratio_retrieved_docs</th> - <th>match_ratio_docs_available</th> - <th>match_ratio_value</th> - <th>recall_10_value</th> - <th>reciprocal_rank_10_value</th> - </tr> - </thead> - <tbody> - <tr> - <th>0</th> - <td>0</td> - <td>1267</td> - <td>62529</td> - <td>0.020263</td> - <td>0</td> - <td>0</td> - </tr> - <tr> - <th>1</th> - <td>1</td> - <td>887</td> - <td>62529</td> - <td>0.014185</td> - <td>0</td> - <td>0</td> - </tr> - </tbody> -</table> -</div> - - diff --git a/python/vespa/docs/_config.yml b/python/vespa/docs/_config.yml index 6851be04428..1704c669c00 100644 --- a/python/vespa/docs/_config.yml +++ b/python/vespa/docs/_config.yml @@ -61,4 +61,4 @@ sidebars: permalink: pretty theme: jekyll-theme-cayman -baseurl: /vespa/
\ No newline at end of file +baseurl: /pyvespa/
\ No newline at end of file diff --git a/python/vespa/docs/_data/sidebars/home_sidebar.yml b/python/vespa/docs/_data/sidebars/home_sidebar.yml index faeb34eee1f..03a90e47d89 100644 --- a/python/vespa/docs/_data/sidebars/home_sidebar.yml +++ b/python/vespa/docs/_data/sidebars/home_sidebar.yml @@ -19,6 +19,6 @@ entries: title: Query API url: /query output: web - title: vespa + title: pyvespa output: web title: Sidebar diff --git a/python/vespa/docs/index.html b/python/vespa/docs/index.html index 054c74f3bec..7c55143b923 100644 --- a/python/vespa/docs/index.html +++ b/python/vespa/docs/index.html @@ -35,7 +35,7 @@ description: "Provide data analysis support for Vespa applications" </div> <div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> <div class="text_cell_render border-box-sizing rendered_html"> -<p><code>pip install vespa</code></p> +<p><code>pip install pyvespa</code></p> </div> </div> @@ -55,7 +55,7 @@ description: "Provide data analysis support for Vespa applications" <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.application</span> <span class="k">import</span> <span class="n">Vespa</span> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.application</span> <span class="kn">import</span> <span class="n">Vespa</span> <span class="n">app</span> <span class="o">=</span> <span class="n">Vespa</span><span class="p">(</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"https://api.cord19.vespa.ai"</span><span class="p">)</span> </pre></div> @@ -82,8 +82,8 @@ description: "Provide data analysis support for Vespa applications" <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.query</span> <span class="k">import</span> <span class="n">Query</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">WeakAnd</span><span class="p">,</span> <span class="n">ANN</span><span class="p">,</span> <span class="n">RankProfile</span> -<span class="kn">from</span> <span class="nn">random</span> <span class="k">import</span> <span class="n">random</span> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.query</span> <span class="kn">import</span> <span class="n">Query</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">WeakAnd</span><span class="p">,</span> <span class="n">ANN</span><span class="p">,</span> <span class="n">RankProfile</span> +<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span> <span class="n">match_phase</span> <span class="o">=</span> <span class="n">Union</span><span class="p">(</span> <span class="n">WeakAnd</span><span class="p">(</span><span class="n">hits</span> <span class="o">=</span> <span class="mi">10</span><span class="p">),</span> @@ -143,29 +143,13 @@ description: "Provide data analysis support for Vespa applications" <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span></span><span class="n">query_result</span><span class="p">[</span><span class="s2">"root"</span><span class="p">][</span><span class="s2">"fields"</span><span class="p">]</span> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">query_result</span><span class="o">.</span><span class="n">number_documents_retrieved</span> </pre></div> </div> </div> </div> -<div class="output_wrapper"> -<div class="output"> - -<div class="output_area"> - - - -<div class="output_text output_subarea output_execute_result"> -<pre>{'totalCount': 1077}</pre> -</div> - -</div> - -</div> -</div> - </div> {% endraw %} @@ -240,354 +224,6 @@ description: "Provide data analysis support for Vespa applications" </div> </div> -<div class="output_wrapper"> -<div class="output"> - -<div class="output_area"> - - -<div class="output_html rendered_html output_subarea output_execute_result"> -<div> -<style scoped> - .dataframe tbody tr th:only-of-type { - vertical-align: middle; - } - - .dataframe tbody tr th { - vertical-align: top; - } - - .dataframe thead th { - text-align: right; - } -</style> -<table border="1" class="dataframe"> - <thead> - <tr style="text-align: right;"> - <th></th> - <th>attributeMatch(authors.first)</th> - <th>attributeMatch(authors.first).averageWeight</th> - <th>attributeMatch(authors.first).completeness</th> - <th>attributeMatch(authors.first).fieldCompleteness</th> - <th>attributeMatch(authors.first).importance</th> - <th>attributeMatch(authors.first).matches</th> - <th>attributeMatch(authors.first).maxWeight</th> - <th>attributeMatch(authors.first).normalizedWeight</th> - <th>attributeMatch(authors.first).normalizedWeightedWeight</th> - <th>attributeMatch(authors.first).queryCompleteness</th> - <th>...</th> - <th>textSimilarity(results).queryCoverage</th> - <th>textSimilarity(results).score</th> - <th>textSimilarity(title).fieldCoverage</th> - <th>textSimilarity(title).order</th> - <th>textSimilarity(title).proximity</th> - <th>textSimilarity(title).queryCoverage</th> - <th>textSimilarity(title).score</th> - <th>document_id</th> - <th>query_id</th> - <th>relevant</th> - </tr> - </thead> - <tbody> - <tr> - <th>0</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.000000</td> - <td>0.000000</td> - <td>0</td> - <td>0</td> - <td>1</td> - </tr> - <tr> - <th>1</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>1.000000</td> - <td>1.0</td> - <td>1.000000</td> - <td>1.000000</td> - <td>1.000000</td> - <td>56212</td> - <td>0</td> - <td>0</td> - </tr> - <tr> - <th>2</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.187500</td> - <td>0.5</td> - <td>0.617188</td> - <td>0.428571</td> - <td>0.457087</td> - <td>34026</td> - <td>0</td> - <td>0</td> - </tr> - <tr> - <th>3</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.000000</td> - <td>0.000000</td> - <td>3</td> - <td>0</td> - <td>1</td> - </tr> - <tr> - <th>4</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>1.000000</td> - <td>1.0</td> - <td>1.000000</td> - <td>1.000000</td> - <td>1.000000</td> - <td>56212</td> - <td>0</td> - <td>0</td> - </tr> - <tr> - <th>5</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.187500</td> - <td>0.5</td> - <td>0.617188</td> - <td>0.428571</td> - <td>0.457087</td> - <td>34026</td> - <td>0</td> - <td>0</td> - </tr> - <tr> - <th>6</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.071429</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.083333</td> - <td>0.039286</td> - <td>1</td> - <td>1</td> - <td>1</td> - </tr> - <tr> - <th>7</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>1.000000</td> - <td>1.0</td> - <td>1.000000</td> - <td>1.000000</td> - <td>1.000000</td> - <td>29774</td> - <td>1</td> - <td>0</td> - </tr> - <tr> - <th>8</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.500000</td> - <td>1.0</td> - <td>1.000000</td> - <td>0.333333</td> - <td>0.700000</td> - <td>22787</td> - <td>1</td> - <td>0</td> - </tr> - <tr> - <th>9</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.058824</td> - <td>0.0</td> - <td>0.000000</td> - <td>0.083333</td> - <td>0.036765</td> - <td>5</td> - <td>1</td> - <td>1</td> - </tr> - <tr> - <th>10</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>1.000000</td> - <td>1.0</td> - <td>1.000000</td> - <td>1.000000</td> - <td>1.000000</td> - <td>29774</td> - <td>1</td> - <td>0</td> - </tr> - <tr> - <th>11</th> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>0.0</td> - <td>...</td> - <td>0.0</td> - <td>0.0</td> - <td>0.500000</td> - <td>1.0</td> - <td>1.000000</td> - <td>0.333333</td> - <td>0.700000</td> - <td>22787</td> - <td>1</td> - <td>0</td> - </tr> - </tbody> -</table> -<p>12 rows × 984 columns</p> -</div> -</div> - -</div> - -</div> -</div> - </div> {% endraw %} @@ -618,7 +254,7 @@ description: "Provide data analysis support for Vespa applications" <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.evaluation</span> <span class="k">import</span> <span class="n">MatchRatio</span><span class="p">,</span> <span class="n">Recall</span><span class="p">,</span> <span class="n">ReciprocalRank</span> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.evaluation</span> <span class="kn">import</span> <span class="n">MatchRatio</span><span class="p">,</span> <span class="n">Recall</span><span class="p">,</span> <span class="n">ReciprocalRank</span> <span class="n">eval_metrics</span> <span class="o">=</span> <span class="p">[</span><span class="n">MatchRatio</span><span class="p">(),</span> <span class="n">Recall</span><span class="p">(</span><span class="n">at</span><span class="o">=</span><span class="mi">10</span><span class="p">),</span> <span class="n">ReciprocalRank</span><span class="p">(</span><span class="n">at</span><span class="o">=</span><span class="mi">10</span><span class="p">)]</span> </pre></div> @@ -657,68 +293,6 @@ description: "Provide data analysis support for Vespa applications" </div> </div> -<div class="output_wrapper"> -<div class="output"> - -<div class="output_area"> - - -<div class="output_html rendered_html output_subarea output_execute_result"> -<div> -<style scoped> - .dataframe tbody tr th:only-of-type { - vertical-align: middle; - } - - .dataframe tbody tr th { - vertical-align: top; - } - - .dataframe thead th { - text-align: right; - } -</style> -<table border="1" class="dataframe"> - <thead> - <tr style="text-align: right;"> - <th></th> - <th>query_id</th> - <th>match_ratio_retrieved_docs</th> - <th>match_ratio_docs_available</th> - <th>match_ratio_value</th> - <th>recall_10_value</th> - <th>reciprocal_rank_10_value</th> - </tr> - </thead> - <tbody> - <tr> - <th>0</th> - <td>0</td> - <td>1267</td> - <td>62529</td> - <td>0.020263</td> - <td>0</td> - <td>0</td> - </tr> - <tr> - <th>1</th> - <td>1</td> - <td>887</td> - <td>62529</td> - <td>0.014185</td> - <td>0</td> - <td>0</td> - </tr> - </tbody> -</table> -</div> -</div> - -</div> - -</div> -</div> - </div> {% endraw %} diff --git a/python/vespa/docs/sidebar.json b/python/vespa/docs/sidebar.json index ef6e029124f..2acf7014eff 100644 --- a/python/vespa/docs/sidebar.json +++ b/python/vespa/docs/sidebar.json @@ -1,5 +1,5 @@ { - "vespa": { + "pyvespa": { "Overview": "/", "Vespa - collect training data": "/collect_training_data", "Vespa - Evaluate query models": "/evaluation", diff --git a/python/vespa/notebooks/index.ipynb b/python/vespa/notebooks/index.ipynb index 076bb935c02..e6ffdc538e2 100644 --- a/python/vespa/notebooks/index.ipynb +++ b/python/vespa/notebooks/index.ipynb @@ -31,7 +31,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "`pip install vespa`" + "`pip install pyvespa`" ] }, { @@ -113,20 +113,9 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'totalCount': 1077}" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "query_result[\"root\"][\"fields\"]" + "query_result.number_documents_retrieved" ] }, { @@ -178,550 +167,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "<div>\n", - "<style scoped>\n", - " .dataframe tbody tr th:only-of-type {\n", - " vertical-align: middle;\n", - " }\n", - "\n", - " .dataframe tbody tr th {\n", - " vertical-align: top;\n", - " }\n", - "\n", - " .dataframe thead th {\n", - " text-align: right;\n", - " }\n", - "</style>\n", - "<table border=\"1\" class=\"dataframe\">\n", - " <thead>\n", - " <tr style=\"text-align: right;\">\n", - " <th></th>\n", - " <th>attributeMatch(authors.first)</th>\n", - " <th>attributeMatch(authors.first).averageWeight</th>\n", - " <th>attributeMatch(authors.first).completeness</th>\n", - " <th>attributeMatch(authors.first).fieldCompleteness</th>\n", - " <th>attributeMatch(authors.first).importance</th>\n", - " <th>attributeMatch(authors.first).matches</th>\n", - " <th>attributeMatch(authors.first).maxWeight</th>\n", - " <th>attributeMatch(authors.first).normalizedWeight</th>\n", - " <th>attributeMatch(authors.first).normalizedWeightedWeight</th>\n", - " <th>attributeMatch(authors.first).queryCompleteness</th>\n", - " <th>...</th>\n", - " <th>textSimilarity(results).queryCoverage</th>\n", - " <th>textSimilarity(results).score</th>\n", - " <th>textSimilarity(title).fieldCoverage</th>\n", - " <th>textSimilarity(title).order</th>\n", - " <th>textSimilarity(title).proximity</th>\n", - " <th>textSimilarity(title).queryCoverage</th>\n", - " <th>textSimilarity(title).score</th>\n", - " <th>document_id</th>\n", - " <th>query_id</th>\n", - " <th>relevant</th>\n", - " </tr>\n", - " </thead>\n", - " <tbody>\n", - " <tr>\n", - " <th>0</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.000000</td>\n", - " <td>0.0</td>\n", - " <td>0.000000</td>\n", - " <td>0.000000</td>\n", - " <td>0.000000</td>\n", - " <td>0</td>\n", - " <td>0</td>\n", - " <td>1</td>\n", - " </tr>\n", - " <tr>\n", - " <th>1</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>1.000000</td>\n", - " <td>1.0</td>\n", - " <td>1.000000</td>\n", - " <td>1.000000</td>\n", - " <td>1.000000</td>\n", - " <td>56212</td>\n", - " <td>0</td>\n", - " <td>0</td>\n", - " </tr>\n", - " <tr>\n", - " <th>2</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.187500</td>\n", - " <td>0.5</td>\n", - " <td>0.617188</td>\n", - " <td>0.428571</td>\n", - " <td>0.457087</td>\n", - " <td>34026</td>\n", - " <td>0</td>\n", - " <td>0</td>\n", - " </tr>\n", - " <tr>\n", - " <th>3</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.000000</td>\n", - " <td>0.0</td>\n", - " <td>0.000000</td>\n", - " <td>0.000000</td>\n", - " <td>0.000000</td>\n", - " <td>3</td>\n", - " <td>0</td>\n", - " <td>1</td>\n", - " </tr>\n", - " <tr>\n", - " <th>4</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>1.000000</td>\n", - " <td>1.0</td>\n", - " <td>1.000000</td>\n", - " <td>1.000000</td>\n", - " <td>1.000000</td>\n", - " <td>56212</td>\n", - " <td>0</td>\n", - " <td>0</td>\n", - " </tr>\n", - " <tr>\n", - " <th>5</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.187500</td>\n", - " <td>0.5</td>\n", - " <td>0.617188</td>\n", - " <td>0.428571</td>\n", - " <td>0.457087</td>\n", - " <td>34026</td>\n", - " <td>0</td>\n", - " <td>0</td>\n", - " </tr>\n", - " <tr>\n", - " <th>6</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.071429</td>\n", - " <td>0.0</td>\n", - " <td>0.000000</td>\n", - " <td>0.083333</td>\n", - " <td>0.039286</td>\n", - " <td>1</td>\n", - " <td>1</td>\n", - " <td>1</td>\n", - " </tr>\n", - " <tr>\n", - " <th>7</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>1.000000</td>\n", - " <td>1.0</td>\n", - " <td>1.000000</td>\n", - " <td>1.000000</td>\n", - " <td>1.000000</td>\n", - " <td>29774</td>\n", - " <td>1</td>\n", - " <td>0</td>\n", - " </tr>\n", - " <tr>\n", - " <th>8</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.500000</td>\n", - " <td>1.0</td>\n", - " <td>1.000000</td>\n", - " <td>0.333333</td>\n", - " <td>0.700000</td>\n", - " <td>22787</td>\n", - " <td>1</td>\n", - " <td>0</td>\n", - " </tr>\n", - " <tr>\n", - " <th>9</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.058824</td>\n", - " <td>0.0</td>\n", - " <td>0.000000</td>\n", - " <td>0.083333</td>\n", - " <td>0.036765</td>\n", - " <td>5</td>\n", - " <td>1</td>\n", - " <td>1</td>\n", - " </tr>\n", - " <tr>\n", - " <th>10</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>1.000000</td>\n", - " <td>1.0</td>\n", - " <td>1.000000</td>\n", - " <td>1.000000</td>\n", - " <td>1.000000</td>\n", - " <td>29774</td>\n", - " <td>1</td>\n", - " <td>0</td>\n", - " </tr>\n", - " <tr>\n", - " <th>11</th>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>...</td>\n", - " <td>0.0</td>\n", - " <td>0.0</td>\n", - " <td>0.500000</td>\n", - " <td>1.0</td>\n", - " <td>1.000000</td>\n", - " <td>0.333333</td>\n", - " <td>0.700000</td>\n", - " <td>22787</td>\n", - " <td>1</td>\n", - " <td>0</td>\n", - " </tr>\n", - " </tbody>\n", - "</table>\n", - "<p>12 rows × 984 columns</p>\n", - "</div>" - ], - "text/plain": [ - " attributeMatch(authors.first) \\\n", - "0 0.0 \n", - "1 0.0 \n", - "2 0.0 \n", - "3 0.0 \n", - "4 0.0 \n", - "5 0.0 \n", - "6 0.0 \n", - "7 0.0 \n", - "8 0.0 \n", - "9 0.0 \n", - "10 0.0 \n", - "11 0.0 \n", - "\n", - " attributeMatch(authors.first).averageWeight \\\n", - "0 0.0 \n", - "1 0.0 \n", - "2 0.0 \n", - "3 0.0 \n", - "4 0.0 \n", - "5 0.0 \n", - "6 0.0 \n", - "7 0.0 \n", - "8 0.0 \n", - "9 0.0 \n", - "10 0.0 \n", - "11 0.0 \n", - "\n", - " attributeMatch(authors.first).completeness \\\n", - "0 0.0 \n", - "1 0.0 \n", - "2 0.0 \n", - "3 0.0 \n", - "4 0.0 \n", - "5 0.0 \n", - "6 0.0 \n", - "7 0.0 \n", - "8 0.0 \n", - "9 0.0 \n", - "10 0.0 \n", - "11 0.0 \n", - "\n", - " attributeMatch(authors.first).fieldCompleteness \\\n", - "0 0.0 \n", - "1 0.0 \n", - "2 0.0 \n", - "3 0.0 \n", - "4 0.0 \n", - "5 0.0 \n", - "6 0.0 \n", - "7 0.0 \n", - "8 0.0 \n", - "9 0.0 \n", - "10 0.0 \n", - "11 0.0 \n", - "\n", - " attributeMatch(authors.first).importance \\\n", - "0 0.0 \n", - "1 0.0 \n", - "2 0.0 \n", - "3 0.0 \n", - "4 0.0 \n", - "5 0.0 \n", - "6 0.0 \n", - "7 0.0 \n", - "8 0.0 \n", - "9 0.0 \n", - "10 0.0 \n", - "11 0.0 \n", - "\n", - " attributeMatch(authors.first).matches \\\n", - "0 0.0 \n", - "1 0.0 \n", - "2 0.0 \n", - "3 0.0 \n", - "4 0.0 \n", - "5 0.0 \n", - "6 0.0 \n", - "7 0.0 \n", - "8 0.0 \n", - "9 0.0 \n", - "10 0.0 \n", - "11 0.0 \n", - "\n", - " attributeMatch(authors.first).maxWeight \\\n", - "0 0.0 \n", - "1 0.0 \n", - "2 0.0 \n", - "3 0.0 \n", - "4 0.0 \n", - "5 0.0 \n", - "6 0.0 \n", - "7 0.0 \n", - "8 0.0 \n", - "9 0.0 \n", - "10 0.0 \n", - "11 0.0 \n", - "\n", - " attributeMatch(authors.first).normalizedWeight \\\n", - "0 0.0 \n", - "1 0.0 \n", - "2 0.0 \n", - "3 0.0 \n", - "4 0.0 \n", - "5 0.0 \n", - "6 0.0 \n", - "7 0.0 \n", - "8 0.0 \n", - "9 0.0 \n", - "10 0.0 \n", - "11 0.0 \n", - "\n", - " attributeMatch(authors.first).normalizedWeightedWeight \\\n", - "0 0.0 \n", - "1 0.0 \n", - "2 0.0 \n", - "3 0.0 \n", - "4 0.0 \n", - "5 0.0 \n", - "6 0.0 \n", - "7 0.0 \n", - "8 0.0 \n", - "9 0.0 \n", - "10 0.0 \n", - "11 0.0 \n", - "\n", - " attributeMatch(authors.first).queryCompleteness ... \\\n", - "0 0.0 ... \n", - "1 0.0 ... \n", - "2 0.0 ... \n", - "3 0.0 ... \n", - "4 0.0 ... \n", - "5 0.0 ... \n", - "6 0.0 ... \n", - "7 0.0 ... \n", - "8 0.0 ... \n", - "9 0.0 ... \n", - "10 0.0 ... \n", - "11 0.0 ... \n", - "\n", - " textSimilarity(results).queryCoverage textSimilarity(results).score \\\n", - "0 0.0 0.0 \n", - "1 0.0 0.0 \n", - "2 0.0 0.0 \n", - "3 0.0 0.0 \n", - "4 0.0 0.0 \n", - "5 0.0 0.0 \n", - "6 0.0 0.0 \n", - "7 0.0 0.0 \n", - "8 0.0 0.0 \n", - "9 0.0 0.0 \n", - "10 0.0 0.0 \n", - "11 0.0 0.0 \n", - "\n", - " textSimilarity(title).fieldCoverage textSimilarity(title).order \\\n", - "0 0.000000 0.0 \n", - "1 1.000000 1.0 \n", - "2 0.187500 0.5 \n", - "3 0.000000 0.0 \n", - "4 1.000000 1.0 \n", - "5 0.187500 0.5 \n", - "6 0.071429 0.0 \n", - "7 1.000000 1.0 \n", - "8 0.500000 1.0 \n", - "9 0.058824 0.0 \n", - "10 1.000000 1.0 \n", - "11 0.500000 1.0 \n", - "\n", - " textSimilarity(title).proximity textSimilarity(title).queryCoverage \\\n", - "0 0.000000 0.000000 \n", - "1 1.000000 1.000000 \n", - "2 0.617188 0.428571 \n", - "3 0.000000 0.000000 \n", - "4 1.000000 1.000000 \n", - "5 0.617188 0.428571 \n", - "6 0.000000 0.083333 \n", - "7 1.000000 1.000000 \n", - "8 1.000000 0.333333 \n", - "9 0.000000 0.083333 \n", - "10 1.000000 1.000000 \n", - "11 1.000000 0.333333 \n", - "\n", - " textSimilarity(title).score document_id query_id relevant \n", - "0 0.000000 0 0 1 \n", - "1 1.000000 56212 0 0 \n", - "2 0.457087 34026 0 0 \n", - "3 0.000000 3 0 1 \n", - "4 1.000000 56212 0 0 \n", - "5 0.457087 34026 0 0 \n", - "6 0.039286 1 1 1 \n", - "7 1.000000 29774 1 0 \n", - "8 0.700000 22787 1 0 \n", - "9 0.036765 5 1 1 \n", - "10 1.000000 29774 1 0 \n", - "11 0.700000 22787 1 0 \n", - "\n", - "[12 rows x 984 columns]" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "training_data_batch = app.collect_training_data(\n", " labelled_data = labelled_data,\n", @@ -773,74 +219,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "<div>\n", - "<style scoped>\n", - " .dataframe tbody tr th:only-of-type {\n", - " vertical-align: middle;\n", - " }\n", - "\n", - " .dataframe tbody tr th {\n", - " vertical-align: top;\n", - " }\n", - "\n", - " .dataframe thead th {\n", - " text-align: right;\n", - " }\n", - "</style>\n", - "<table border=\"1\" class=\"dataframe\">\n", - " <thead>\n", - " <tr style=\"text-align: right;\">\n", - " <th></th>\n", - " <th>query_id</th>\n", - " <th>match_ratio_retrieved_docs</th>\n", - " <th>match_ratio_docs_available</th>\n", - " <th>match_ratio_value</th>\n", - " <th>recall_10_value</th>\n", - " <th>reciprocal_rank_10_value</th>\n", - " </tr>\n", - " </thead>\n", - " <tbody>\n", - " <tr>\n", - " <th>0</th>\n", - " <td>0</td>\n", - " <td>1267</td>\n", - " <td>62529</td>\n", - " <td>0.020263</td>\n", - " <td>0</td>\n", - " <td>0</td>\n", - " </tr>\n", - " <tr>\n", - " <th>1</th>\n", - " <td>1</td>\n", - " <td>887</td>\n", - " <td>62529</td>\n", - " <td>0.014185</td>\n", - " <td>0</td>\n", - " <td>0</td>\n", - " </tr>\n", - " </tbody>\n", - "</table>\n", - "</div>" - ], - "text/plain": [ - " query_id match_ratio_retrieved_docs match_ratio_docs_available \\\n", - "0 0 1267 62529 \n", - "1 1 887 62529 \n", - "\n", - " match_ratio_value recall_10_value reciprocal_rank_10_value \n", - "0 0.020263 0 0 \n", - "1 0.014185 0 0 " - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "evaluation = app.evaluate(\n", " labelled_data = labelled_data,\n", diff --git a/python/vespa/notebooks/query.ipynb b/python/vespa/notebooks/query.ipynb index 2ecb4b9419f..27053527b52 100644 --- a/python/vespa/notebooks/query.ipynb +++ b/python/vespa/notebooks/query.ipynb @@ -79,7 +79,7 @@ { "data": { "text/plain": [ - "{'totalCount': 52387}" + "108882" ] }, "execution_count": null, @@ -88,7 +88,7 @@ } ], "source": [ - "results[\"root\"][\"fields\"]" + "results.number_documents_retrieved" ] }, { @@ -130,7 +130,7 @@ { "data": { "text/plain": [ - "{'totalCount': 52387}" + "108882" ] }, "execution_count": null, @@ -139,7 +139,7 @@ } ], "source": [ - "results[\"root\"][\"fields\"]" + "results.number_documents_retrieved" ] }, { @@ -190,7 +190,7 @@ { "data": { "text/plain": [ - "{'totalCount': 1084}" + "947" ] }, "execution_count": null, @@ -199,7 +199,7 @@ } ], "source": [ - "results[\"root\"][\"fields\"]" + "results.number_documents_retrieved" ] }, { @@ -224,7 +224,7 @@ { "data": { "text/plain": [ - "[40215, 18456, 33692]" + "[117166, 60125, 28903]" ] }, "execution_count": null, @@ -233,7 +233,7 @@ } ], "source": [ - "top_ids = [hit[\"fields\"][\"id\"] for hit in results[\"root\"][\"children\"][0:3]]\n", + "top_ids = [hit[\"fields\"][\"id\"] for hit in results.hits[0:3]]\n", "top_ids" ] }, @@ -270,7 +270,7 @@ { "data": { "text/plain": [ - "[18456, 33692]" + "[60125, 28903]" ] }, "execution_count": null, @@ -279,7 +279,7 @@ } ], "source": [ - "id_recalled = [hit[\"fields\"][\"id\"] for hit in results_with_recall[\"root\"][\"children\"]]\n", + "id_recalled = [hit[\"fields\"][\"id\"] for hit in results_with_recall.hits]\n", "id_recalled" ] }, diff --git a/python/vespa/settings.ini b/python/vespa/settings.ini index 2a3ad1128db..60394dcb425 100644 --- a/python/vespa/settings.ini +++ b/python/vespa/settings.ini @@ -1,7 +1,7 @@ [DEFAULT] # All sections below are required unless otherwise specified host = github -lib_name = vespa +lib_name = pyvespa user = vespa-engine description = Vespa python API keywords = vespa, search engine, data science diff --git a/python/vespa/setup.py b/python/vespa/setup.py index de97dee06e9..2dea754602e 100644 --- a/python/vespa/setup.py +++ b/python/vespa/setup.py @@ -1,47 +1,77 @@ +import os from pkg_resources import parse_version from configparser import ConfigParser import setuptools -assert parse_version(setuptools.__version__)>=parse_version('36.2') + +assert parse_version(setuptools.__version__) >= parse_version("36.2") # note: all settings are in settings.ini; edit there, not here -config = ConfigParser(delimiters=['=']) -config.read('settings.ini') -cfg = config['DEFAULT'] +config = ConfigParser(delimiters=["="]) +config.read("settings.ini") +cfg = config["DEFAULT"] -cfg_keys = 'version description keywords author author_email'.split() -expected = cfg_keys + "lib_name user branch license status min_python audience language".split() -for o in expected: assert o in cfg, "missing expected setting: {}".format(o) -setup_cfg = {o:cfg[o] for o in cfg_keys} +cfg_keys = "description keywords author author_email".split() +expected = ( + cfg_keys + + "lib_name user branch license status min_python audience language".split() +) +for o in expected: + assert o in cfg, "missing expected setting: {}".format(o) +setup_cfg = {o: cfg[o] for o in cfg_keys} licenses = { - 'apache2': ('Apache Software License 2.0','OSI Approved :: Apache Software License'), + "apache2": ( + "Apache Software License 2.0", + "OSI Approved :: Apache Software License", + ), } -statuses = [ '1 - Planning', '2 - Pre-Alpha', '3 - Alpha', - '4 - Beta', '5 - Production/Stable', '6 - Mature', '7 - Inactive' ] -py_versions = '2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8'.split() +statuses = [ + "1 - Planning", + "2 - Pre-Alpha", + "3 - Alpha", + "4 - Beta", + "5 - Production/Stable", + "6 - Mature", + "7 - Inactive", +] +py_versions = ( + "2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8".split() +) -requirements = cfg.get('requirements','').split() -lic = licenses[cfg['license']] -min_python = cfg['min_python'] +requirements = cfg.get("requirements", "").split() +lic = licenses[cfg["license"]] +min_python = cfg["min_python"] -setuptools.setup( - name = cfg['lib_name'], - license = lic[0], - classifiers = [ - 'Development Status :: ' + statuses[int(cfg['status'])], - 'Intended Audience :: ' + cfg['audience'].title(), - 'License :: ' + lic[1], - 'Natural Language :: ' + cfg['language'].title(), - ] + ['Programming Language :: Python :: '+o for o in py_versions[py_versions.index(min_python):]], - url = cfg['git_url'], - packages = setuptools.find_packages(), - include_package_data = True, - install_requires = requirements, - dependency_links = cfg.get('dep_links','').split(), - python_requires = '>=' + cfg['min_python'], - long_description = open('README.md').read(), - long_description_content_type = 'text/markdown', - zip_safe = False, - entry_points = { 'console_scripts': cfg.get('console_scripts','').split() }, - **setup_cfg) +def get_target_version(): + build_nr = os.environ.get("GITHUB_RUN_NUMBER", "0+dev") + version = "0.1" + return "{}.{}".format(version, build_nr) + + +setuptools.setup( + name=cfg["lib_name"], + version=get_target_version(), + license=lic[0], + classifiers=[ + "Development Status :: " + statuses[int(cfg["status"])], + "Intended Audience :: " + cfg["audience"].title(), + "License :: " + lic[1], + "Natural Language :: " + cfg["language"].title(), + ] + + [ + "Programming Language :: Python :: " + o + for o in py_versions[py_versions.index(min_python) :] + ], + url=cfg["git_url"], + packages=setuptools.find_packages(), + include_package_data=True, + install_requires=requirements, + dependency_links=cfg.get("dep_links", "").split(), + python_requires=">=" + cfg["min_python"], + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + zip_safe=False, + entry_points={"console_scripts": cfg.get("console_scripts", "").split()}, + **setup_cfg +) diff --git a/python/vespa/vespa/application.py b/python/vespa/vespa/application.py index d875998f4d0..3646cf87bf2 100644 --- a/python/vespa/vespa/application.py +++ b/python/vespa/vespa/application.py @@ -4,7 +4,7 @@ from typing import Optional, Dict, Tuple, List from requests import post from pandas import DataFrame -from vespa.query import Query +from vespa.query import Query, VespaResult from vespa.evaluation import EvalMetric @@ -37,7 +37,7 @@ class Vespa(object): debug_request: bool = False, recall: Optional[Tuple] = None, **kwargs - ) -> Dict: + ) -> VespaResult: """ Send a query request to the Vespa application. @@ -71,10 +71,10 @@ class Vespa(object): body.update(kwargs) if debug_request: - return body + return VespaResult(vespa_result={}, request_body=body) else: r = post(self.search_end_point, json=body) - return r.json() + return VespaResult(vespa_result=r.json()) def collect_training_data_point( self, @@ -114,7 +114,7 @@ class Vespa(object): recall=(id_field, [relevant_id]), **kwargs ) - hits = get_hits(vespa_result=relevant_id_result) + hits = relevant_id_result.hits features = [] if len(hits) == 1 and hits[0]["fields"][id_field] == relevant_id: random_hits_result = self.query( @@ -123,7 +123,7 @@ class Vespa(object): hits=number_additional_docs, **kwargs ) - hits.extend(get_hits(random_hits_result)) + hits.extend(random_hits_result.hits) features = annotate_data( hits=hits, @@ -248,14 +248,6 @@ class Vespa(object): return evaluation -# todo: create a VespaResult class to store vespa results -def get_hits(vespa_result): - hits = [] - if "children" in vespa_result["root"]: - hits = vespa_result["root"]["children"] - return hits - - # todo: a better pattern for labelled data would be (query_id, query, doc_id, score) with the possibility od # assigning a specific default value for those docs not mentioned def annotate_data(hits, query_id, id_field, relevant_id, relevant_score, default_score): diff --git a/python/vespa/vespa/evaluation.py b/python/vespa/vespa/evaluation.py index 98365640fb3..4ca7a1d136b 100644 --- a/python/vespa/vespa/evaluation.py +++ b/python/vespa/vespa/evaluation.py @@ -1,8 +1,7 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. from typing import Dict, List - -# todo: When creating a VespaResult class use getters with appropriate defaults to avoid the try clauses here. +from vespa.query import VespaResult class EvalMetric(object): @@ -25,7 +24,7 @@ class MatchRatio(EvalMetric): def evaluate_query( self, - query_results: Dict, + query_results: VespaResult, relevant_docs: List[Dict], id_field: str, default_score: int, @@ -40,16 +39,11 @@ class MatchRatio(EvalMetric): :return: Dict containing the number of retrieved docs (_retrieved_docs), the number of docs available in the corpus (_docs_available) and the match ratio (_value). """ - try: - retrieved_docs = query_results["root"]["fields"]["totalCount"] - except KeyError: - retrieved_docs = 0 - try: - docs_available = query_results["root"]["coverage"]["documents"] + retrieved_docs = query_results.number_documents_retrieved + docs_available = query_results.number_documents_indexed + value = 0 + if docs_available > 0: value = retrieved_docs / docs_available - except KeyError: - docs_available = 0 - value = 0 return { str(self.name) + "_retrieved_docs": retrieved_docs, str(self.name) + "_docs_available": docs_available, @@ -70,7 +64,7 @@ class Recall(EvalMetric): def evaluate_query( self, - query_results: Dict, + query_results: VespaResult, relevant_docs: List[Dict], id_field: str, default_score: int, @@ -88,8 +82,7 @@ class Recall(EvalMetric): relevant_ids = {str(doc["id"]) for doc in relevant_docs} try: retrieved_ids = { - str(hit["fields"][id_field]) - for hit in query_results["root"]["children"][: self.at] + str(hit["fields"][id_field]) for hit in query_results.hits[: self.at] } except KeyError: retrieved_ids = set() @@ -113,7 +106,7 @@ class ReciprocalRank(EvalMetric): def evaluate_query( self, - query_results: Dict, + query_results: VespaResult, relevant_docs: List[Dict], id_field: str, default_score: int, @@ -130,10 +123,7 @@ class ReciprocalRank(EvalMetric): relevant_ids = {str(doc["id"]) for doc in relevant_docs} rr = 0 - try: - hits = query_results["root"]["children"][: self.at] - except KeyError: - hits = [] + hits = query_results.hits[: self.at] for index, hit in enumerate(hits): if hit["fields"][id_field] in relevant_ids: rr = 1 / (index + 1) diff --git a/python/vespa/vespa/query.py b/python/vespa/vespa/query.py index f78c8a23380..ed67819b821 100644 --- a/python/vespa/vespa/query.py +++ b/python/vespa/vespa/query.py @@ -204,13 +204,26 @@ class Query(object): class VespaResult(object): - def __init__(self, vespa_result): - self.vespa_result = vespa_result + def __init__(self, vespa_result, request_body=None): + self._vespa_result = vespa_result + self._request_body = request_body + + @property + def request_body(self) -> Optional[Dict]: + return self._request_body @property def json(self) -> Dict: - return self.vespa_result + return self._vespa_result @property def hits(self) -> List: - return self.vespa_result.get("root", {}).get("children", []) + return self._vespa_result.get("root", {}).get("children", []) + + @property + def number_documents_retrieved(self) -> int: + return self._vespa_result.get("root", {}).get("fields", {}).get("totalCount", 0) + + @property + def number_documents_indexed(self) -> int: + return self._vespa_result.get("root", {}).get("coverage", {}).get("documents", 0) diff --git a/python/vespa/vespa/test_application.py b/python/vespa/vespa/test_application.py index 57d7d784bde..84bd1c0a6ad 100644 --- a/python/vespa/vespa/test_application.py +++ b/python/vespa/vespa/test_application.py @@ -6,7 +6,7 @@ from pandas import DataFrame from pandas.testing import assert_frame_equal from vespa.application import Vespa -from vespa.query import Query, OR, RankProfile +from vespa.query import Query, OR, RankProfile, VespaResult class TestVespa(unittest.TestCase): @@ -27,7 +27,9 @@ class TestVespaQuery(unittest.TestCase): app = Vespa(url="http://localhost", port=8080) body = {"yql": "select * from sources * where test"} - self.assertDictEqual(app.query(body=body, debug_request=True), body) + self.assertDictEqual( + app.query(body=body, debug_request=True).request_body, body + ) self.assertDictEqual( app.query( @@ -35,7 +37,7 @@ class TestVespaQuery(unittest.TestCase): query_model=Query(match_phase=OR(), rank_profile=RankProfile()), debug_request=True, hits=10, - ), + ).request_body, { "yql": 'select * from sources * where ([{"grammar": "any"}]userInput("this is a test"));', "ranking": {"profile": "default", "listFeatures": "false"}, @@ -50,7 +52,7 @@ class TestVespaQuery(unittest.TestCase): debug_request=True, hits=10, recall=("id", [1, 5]), - ), + ).request_body, { "yql": 'select * from sources * where ([{"grammar": "any"}]userInput("this is a test"));', "ranking": {"profile": "default", "listFeatures": "false"}, @@ -149,7 +151,10 @@ class TestVespaCollectData(unittest.TestCase): def test_collect_training_data_point(self): self.app.query = Mock( - side_effect=[self.raw_vespa_result_recall, self.raw_vespa_result_additional] + side_effect=[ + VespaResult(self.raw_vespa_result_recall), + VespaResult(self.raw_vespa_result_additional), + ] ) query_model = Query(rank_profile=RankProfile(list_features=True)) data = self.app.collect_training_data_point( @@ -204,7 +209,10 @@ class TestVespaCollectData(unittest.TestCase): } } self.app.query = Mock( - side_effect=[self.raw_vespa_result_recall, self.raw_vespa_result_additional] + side_effect=[ + VespaResult(self.raw_vespa_result_recall), + VespaResult(self.raw_vespa_result_additional), + ] ) query_model = Query(rank_profile=RankProfile(list_features=True)) data = self.app.collect_training_data_point( diff --git a/python/vespa/vespa/test_evaluation.py b/python/vespa/vespa/test_evaluation.py index 5fa29eb3907..b6941985d94 100644 --- a/python/vespa/vespa/test_evaluation.py +++ b/python/vespa/vespa/test_evaluation.py @@ -2,6 +2,7 @@ import unittest +from vespa.query import VespaResult from vespa.evaluation import MatchRatio, Recall, ReciprocalRank @@ -61,7 +62,7 @@ class TestEvalMetric(unittest.TestCase): metric = MatchRatio() evaluation = metric.evaluate_query( - query_results=self.query_results, + query_results=VespaResult(self.query_results), relevant_docs=self.labelled_data[0]["relevant_docs"], id_field="vespa_id_field", default_score=0, @@ -77,20 +78,22 @@ class TestEvalMetric(unittest.TestCase): ) evaluation = metric.evaluate_query( - query_results={ - "root": { - "id": "toplevel", - "relevance": 1.0, - "coverage": { - "coverage": 100, - "documents": 62529, - "full": True, - "nodes": 2, - "results": 1, - "resultsFull": 1, - }, + query_results=VespaResult( + { + "root": { + "id": "toplevel", + "relevance": 1.0, + "coverage": { + "coverage": 100, + "documents": 62529, + "full": True, + "nodes": 2, + "results": 1, + "resultsFull": 1, + }, + } } - }, + ), relevant_docs=self.labelled_data[0]["relevant_docs"], id_field="vespa_id_field", default_score=0, @@ -106,20 +109,22 @@ class TestEvalMetric(unittest.TestCase): ) evaluation = metric.evaluate_query( - query_results={ - "root": { - "id": "toplevel", - "relevance": 1.0, - "fields": {"totalCount": 1083}, - "coverage": { - "coverage": 100, - "full": True, - "nodes": 2, - "results": 1, - "resultsFull": 1, - }, + query_results=VespaResult( + { + "root": { + "id": "toplevel", + "relevance": 1.0, + "fields": {"totalCount": 1083}, + "coverage": { + "coverage": 100, + "full": True, + "nodes": 2, + "results": 1, + "resultsFull": 1, + }, + } } - }, + ), relevant_docs=self.labelled_data[0]["relevant_docs"], id_field="vespa_id_field", default_score=0, @@ -137,57 +142,45 @@ class TestEvalMetric(unittest.TestCase): def test_recall(self): metric = Recall(at=2) evaluation = metric.evaluate_query( - query_results=self.query_results, + query_results=VespaResult(self.query_results), relevant_docs=self.labelled_data[0]["relevant_docs"], id_field="vespa_id_field", default_score=0, ) self.assertDictEqual( - evaluation, - { - "recall_2_value": 0.5, - }, + evaluation, {"recall_2_value": 0.5,}, ) metric = Recall(at=1) evaluation = metric.evaluate_query( - query_results=self.query_results, + query_results=VespaResult(self.query_results), relevant_docs=self.labelled_data[0]["relevant_docs"], id_field="vespa_id_field", default_score=0, ) self.assertDictEqual( - evaluation, - { - "recall_1_value": 0.0, - }, + evaluation, {"recall_1_value": 0.0,}, ) def test_reciprocal_rank(self): metric = ReciprocalRank(at=2) evaluation = metric.evaluate_query( - query_results=self.query_results, + query_results=VespaResult(self.query_results), relevant_docs=self.labelled_data[0]["relevant_docs"], id_field="vespa_id_field", default_score=0, ) self.assertDictEqual( - evaluation, - { - "reciprocal_rank_2_value": 0.5, - }, + evaluation, {"reciprocal_rank_2_value": 0.5,}, ) metric = ReciprocalRank(at=1) evaluation = metric.evaluate_query( - query_results=self.query_results, + query_results=VespaResult(self.query_results), relevant_docs=self.labelled_data[0]["relevant_docs"], id_field="vespa_id_field", default_score=0, ) self.assertDictEqual( - evaluation, - { - "reciprocal_rank_1_value": 0.0, - }, + evaluation, {"reciprocal_rank_1_value": 0.0,}, ) diff --git a/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h index 3e3683ce60f..c8b196023d6 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h +++ b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h @@ -16,24 +16,29 @@ private: uint32_t _neighbors_to_explore_at_insert; // This is always the same as in the attribute config, and is duplicated here to simplify usage. DistanceMetric _distance_metric; + bool _allow_multi_threaded_indexing; public: HnswIndexParams(uint32_t max_links_per_node_in, uint32_t neighbors_to_explore_at_insert_in, - DistanceMetric distance_metric_in) + DistanceMetric distance_metric_in, + bool allow_multi_threaded_indexing_in = false) : _max_links_per_node(max_links_per_node_in), _neighbors_to_explore_at_insert(neighbors_to_explore_at_insert_in), - _distance_metric(distance_metric_in) + _distance_metric(distance_metric_in), + _allow_multi_threaded_indexing(allow_multi_threaded_indexing_in) {} uint32_t max_links_per_node() const { return _max_links_per_node; } uint32_t neighbors_to_explore_at_insert() const { return _neighbors_to_explore_at_insert; } DistanceMetric distance_metric() const { return _distance_metric; } + bool allow_multi_threaded_indexing() const { return _allow_multi_threaded_indexing; } bool operator==(const HnswIndexParams& rhs) const { return (_max_links_per_node == rhs._max_links_per_node && _neighbors_to_explore_at_insert == rhs._neighbors_to_explore_at_insert && - _distance_metric == rhs._distance_metric); + _distance_metric == rhs._distance_metric && + _allow_multi_threaded_indexing == rhs._allow_multi_threaded_indexing); } }; diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.cpp b/searchcommon/src/vespa/searchcommon/attribute/status.cpp index da13548ec2e..f2bb49c348a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/status.cpp @@ -20,6 +20,42 @@ Status::Status() { } +Status::Status(const Status& rhs) + : _numDocs(rhs._numDocs), + _numValues(rhs._numValues), + _numUniqueValues(rhs._numUniqueValues), + _allocated(rhs._allocated), + _used(rhs._used), + _dead(rhs._dead), + _unused(rhs._unused), + _onHold(rhs._onHold), + _onHoldMax(rhs._onHoldMax), + _lastSyncToken(rhs.getLastSyncToken()), + _updates(rhs._updates), + _nonIdempotentUpdates(rhs._nonIdempotentUpdates), + _bitVectors(rhs._bitVectors) +{ +} + +Status& +Status::operator=(const Status& rhs) +{ + _numDocs = rhs._numDocs; + _numValues = rhs._numValues; + _numUniqueValues = rhs._numUniqueValues; + _allocated = rhs._allocated; + _used = rhs._used; + _dead = rhs._dead; + _unused = rhs._unused; + _onHold = rhs._onHold; + _onHoldMax = rhs._onHoldMax; + setLastSyncToken(rhs.getLastSyncToken()); + _updates = rhs._updates; + _nonIdempotentUpdates = rhs._nonIdempotentUpdates; + _bitVectors = rhs._bitVectors; + return *this; +} + vespalib::string Status::createName(vespalib::stringref index, vespalib::stringref attr) { diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.h b/searchcommon/src/vespa/searchcommon/attribute/status.h index 888355b3f58..a624309da65 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.h +++ b/searchcommon/src/vespa/searchcommon/attribute/status.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/vespalib/stllike/string.h> +#include <atomic> namespace search::attribute { @@ -10,6 +11,8 @@ class Status { public: Status(); + Status(const Status& rhs); + Status& operator=(const Status& rhs); void updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, uint64_t used, uint64_t dead, uint64_t onHold); @@ -22,14 +25,15 @@ public: uint64_t getDead() const { return _dead; } uint64_t getOnHold() const { return _onHold; } uint64_t getOnHoldMax() const { return _onHoldMax; } - uint64_t getLastSyncToken() const { return _lastSyncToken; } + // This might be accessed from other threads than the writer thread. + uint64_t getLastSyncToken() const { return _lastSyncToken.load(std::memory_order_relaxed); } uint64_t getUpdateCount() const { return _updates; } uint64_t getNonIdempotentUpdateCount() const { return _nonIdempotentUpdates; } uint32_t getBitVectors() const { return _bitVectors; } void setNumDocs(uint64_t v) { _numDocs = v; } void incNumDocs() { ++_numDocs; } - void setLastSyncToken(uint64_t v) { _lastSyncToken = v; } + void setLastSyncToken(uint64_t v) { _lastSyncToken.store(v, std::memory_order_relaxed); } void incUpdates(uint64_t v=1) { _updates += v; } void incNonIdempotentUpdates(uint64_t v = 1) { _nonIdempotentUpdates += v; } void incBitVectors() { ++_bitVectors; } @@ -47,7 +51,7 @@ private: uint64_t _unused; uint64_t _onHold; uint64_t _onHoldMax; - uint64_t _lastSyncToken; + std::atomic<uint64_t> _lastSyncToken; uint64_t _updates; uint64_t _nonIdempotentUpdates; uint32_t _bitVectors; diff --git a/searchcommon/src/vespa/searchcommon/common/schema.cpp b/searchcommon/src/vespa/searchcommon/common/schema.cpp index a21cc43572e..c59edbef22f 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.cpp +++ b/searchcommon/src/vespa/searchcommon/common/schema.cpp @@ -70,16 +70,20 @@ namespace index { const uint32_t Schema::UNKNOWN_FIELD_ID(std::numeric_limits<uint32_t>::max()); Schema::Field::Field(vespalib::stringref n, DataType dt) - : _name(n), - _dataType(dt), - _collectionType(schema::CollectionType::SINGLE) + : Field(n, dt, schema::CollectionType::SINGLE, "") { } Schema::Field::Field(vespalib::stringref n, DataType dt, CollectionType ct) + : Field(n, dt, ct, "") +{ +} + +Schema::Field::Field(vespalib::stringref n, DataType dt, CollectionType ct, vespalib::stringref tensor_spec) : _name(n), _dataType(dt), - _collectionType(ct) + _collectionType(ct), + _tensor_spec(tensor_spec) { } @@ -111,15 +115,14 @@ Schema::Field::operator==(const Field &rhs) const { return _name == rhs._name && _dataType == rhs._dataType && - _collectionType == rhs._collectionType; + _collectionType == rhs._collectionType && + _tensor_spec == rhs._tensor_spec; } bool Schema::Field::operator!=(const Field &rhs) const { - return _name != rhs._name || - _dataType != rhs._dataType || - _collectionType != rhs._collectionType; + return !((*this) == rhs); } Schema::IndexField::IndexField(vespalib::stringref name, DataType dt) diff --git a/searchcommon/src/vespa/searchcommon/common/schema.h b/searchcommon/src/vespa/searchcommon/common/schema.h index e17d219d7e8..9003578adaf 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.h +++ b/searchcommon/src/vespa/searchcommon/common/schema.h @@ -35,10 +35,12 @@ public: vespalib::string _name; DataType _dataType; CollectionType _collectionType; + vespalib::string _tensor_spec; public: Field(vespalib::stringref n, DataType dt); Field(vespalib::stringref n, DataType dt, CollectionType ct); + Field(vespalib::stringref n, DataType dt, CollectionType ct, vespalib::stringref tensor_spec); /** * Create this field based on the given config lines. @@ -58,6 +60,7 @@ public: const vespalib::string &getName() const { return _name; } DataType getDataType() const { return _dataType; } CollectionType getCollectionType() const { return _collectionType; } + const vespalib::string& get_tensor_spec() const { return _tensor_spec; } bool matchingTypes(const Field &rhs) const { return getDataType() == rhs.getDataType() && diff --git a/searchcore/src/tests/proton/attribute/CMakeLists.txt b/searchcore/src/tests/proton/attribute/CMakeLists.txt index 79f81f3daa1..c23d97c6e88 100644 --- a/searchcore/src/tests/proton/attribute/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/CMakeLists.txt @@ -7,9 +7,11 @@ vespa_add_executable(searchcore_attribute_test_app TEST searchcore_attribute searchcore_flushengine searchcore_pcommon + searchlib_test + gtest ) -vespa_add_test(NAME searchcore_attribute_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/attribute_test.sh - DEPENDS searchcore_attribute_test_app) +vespa_add_test(NAME searchcore_attribute_test_app COMMAND searchcore_attribute_test_app) + vespa_add_executable(searchcore_attributeflush_test_app TEST SOURCES attributeflush_test.cpp diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 839ef14fcb0..91f580cd221 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -7,41 +7,43 @@ #include <vespa/document/update/arithmeticvalueupdate.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/tensor.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> -#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/test/attribute_utils.h> +#include <vespa/searchcore/proton/test/mock_attribute_manager.h> #include <vespa/searchcorespi/flush/iflushtarget.h> -#include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> +#include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/bitvector_search_cache.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/attribute/imported_attribute_vector_factory.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/predicate_attribute.h> -#include <vespa/vespalib/util/foregroundtaskexecutor.h> +#include <vespa/searchlib/attribute/singlenumericattribute.hpp> #include <vespa/searchlib/common/idestructorcallback.h> -#include <vespa/vespalib/util/sequencedtaskexecutorobserver.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/predicate/predicate_hash.h> #include <vespa/searchlib/predicate/predicate_index.h> +#include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/vespalib/btree/btreeroot.hpp> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchcommon/attribute/iattributevector.h> -#include <vespa/vespalib/btree/btreeroot.hpp> -#include <vespa/searchlib/attribute/singlenumericattribute.hpp> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/foregroundtaskexecutor.h> +#include <vespa/vespalib/util/sequencedtaskexecutorobserver.h> #include <vespa/log/log.h> LOG_SETUP("attribute_test"); @@ -57,8 +59,11 @@ using namespace vespa::config::search; using proton::ImportedAttributesRepo; using proton::test::AttributeUtils; +using proton::test::MockAttributeManager; using search::TuneFileAttributes; using search::attribute::BitVectorSearchCache; +using search::attribute::DistanceMetric; +using search::attribute::HnswIndexParams; using search::attribute::IAttributeVector; using search::attribute::ImportedAttributeVector; using search::attribute::ImportedAttributeVectorFactory; @@ -67,24 +72,25 @@ using search::index::DummyFileHeaderContext; using search::index::schema::CollectionType; using search::predicate::PredicateHash; using search::predicate::PredicateIndex; +using search::tensor::DenseTensorAttribute; +using search::tensor::PrepareResult; using search::tensor::TensorAttribute; using search::test::DirectoryHandler; using std::string; -using vespalib::eval::ValueType; -using vespalib::eval::TensorSpec; -using vespalib::tensor::Tensor; -using vespalib::tensor::DefaultTensorEngine; using vespalib::ForegroundTaskExecutor; using vespalib::SequencedTaskExecutorObserver; +using vespalib::eval::TensorSpec; +using vespalib::eval::ValueType; +using vespalib::tensor::DefaultTensorEngine; +using vespalib::tensor::Tensor; -using AVConfig = search::attribute::Config; using AVBasicType = search::attribute::BasicType; using AVCollectionType = search::attribute::CollectionType; +using AVConfig = search::attribute::Config; using Int32AttributeVector = SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> >; using LidVector = LidVectorContext::LidVector; -namespace -{ +namespace { const uint64_t createSerialNum = 42u; @@ -116,45 +122,48 @@ fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64 const std::shared_ptr<IDestructorCallback> emptyCallback; -struct Fixture -{ +class AttributeWriterTest : public ::testing::Test { +public: DirectoryHandler _dirHandler; - DummyFileHeaderContext _fileHeaderContext; - ForegroundTaskExecutor _attributeFieldWriterReal; - SequencedTaskExecutorObserver _attributeFieldWriter; - HwInfo _hwInfo; - proton::AttributeManager::SP _m; + std::unique_ptr<ForegroundTaskExecutor> _attributeFieldWriterReal; + std::unique_ptr<SequencedTaskExecutorObserver> _attributeFieldWriter; + std::shared_ptr<MockAttributeManager> _mgr; std::unique_ptr<AttributeWriter> _aw; - Fixture(uint32_t threads) + AttributeWriterTest() : _dirHandler(test_dir), - _fileHeaderContext(), - _attributeFieldWriterReal(threads), - _attributeFieldWriter(_attributeFieldWriterReal), - _hwInfo(), - _m(std::make_shared<proton::AttributeManager> - (test_dir, "test.subdb", TuneFileAttributes(), - _fileHeaderContext, _attributeFieldWriter, _hwInfo)), + _attributeFieldWriterReal(), + _attributeFieldWriter(), + _mgr(), _aw() { - allocAttributeWriter(); + setup(1); } - Fixture() - : Fixture(1) - { + ~AttributeWriterTest(); + void setup(uint32_t threads) { + _aw.reset(); + _attributeFieldWriterReal = std::make_unique<ForegroundTaskExecutor>(threads); + _attributeFieldWriter = std::make_unique<SequencedTaskExecutorObserver>(*_attributeFieldWriterReal); + _mgr = std::make_shared<MockAttributeManager>(); + _mgr->set_writer(*_attributeFieldWriter); + allocAttributeWriter(); } - ~Fixture(); void allocAttributeWriter() { - _aw = std::make_unique<AttributeWriter>(_m); + _aw = std::make_unique<AttributeWriter>(_mgr); } AttributeVector::SP addAttribute(const vespalib::string &name) { - return addAttribute({name, AVConfig(AVBasicType::INT32)}, createSerialNum); + return addAttribute({name, AVConfig(AVBasicType::INT32)}); } - AttributeVector::SP addAttribute(const AttributeSpec &spec, SerialNum serialNum) { - auto ret = _m->addAttribute(spec, serialNum); + AttributeVector::SP addAttribute(const AttributeSpec &spec) { + auto ret = _mgr->addAttribute(spec.getName(), + AttributeFactory::createAttribute(spec.getName(), spec.getConfig())); allocAttributeWriter(); return ret; } + void add_attribute(AttributeVector::SP attr) { + _mgr->addAttribute(attr->getName(), std::move(attr)); + allocAttributeWriter(); + } void put(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit = true) { _aw->put(serialNum, doc, lid, immediateCommit, emptyCallback); @@ -177,13 +186,13 @@ struct Fixture _aw->forceCommit(serialNum, emptyCallback); } void assertExecuteHistory(std::vector<uint32_t> expExecuteHistory) { - EXPECT_EQUAL(expExecuteHistory, _attributeFieldWriter.getExecuteHistory()); + EXPECT_EQ(expExecuteHistory, _attributeFieldWriter->getExecuteHistory()); } }; -Fixture::~Fixture() = default; +AttributeWriterTest::~AttributeWriterTest() = default; -TEST_F("require that attribute writer handles put", Fixture) +TEST_F(AttributeWriterTest, handles_put) { Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::INT32, CollectionType::SINGLE)); @@ -193,108 +202,108 @@ TEST_F("require that attribute writer handles put", Fixture) DocBuilder idb(s); - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute({"a2", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); - AttributeVector::SP a3 = f.addAttribute({"a3", AVConfig(AVBasicType::FLOAT)}, createSerialNum); - AttributeVector::SP a4 = f.addAttribute({"a4", AVConfig(AVBasicType::STRING)}, createSerialNum); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute({"a2", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); + auto a3 = addAttribute({"a3", AVConfig(AVBasicType::FLOAT)}); + auto a4 = addAttribute({"a4", AVConfig(AVBasicType::STRING)}); attribute::IntegerContent ibuf; attribute::FloatContent fbuf; attribute::ConstCharContent sbuf; { // empty document should give default values - EXPECT_EQUAL(1u, a1->getNumDocs()); - f.put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - EXPECT_EQUAL(2u, a2->getNumDocs()); - EXPECT_EQUAL(2u, a3->getNumDocs()); - EXPECT_EQUAL(2u, a4->getNumDocs()); - EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, a2->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, a3->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, a4->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a1->getNumDocs()); + put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); + EXPECT_EQ(2u, a1->getNumDocs()); + EXPECT_EQ(2u, a2->getNumDocs()); + EXPECT_EQ(2u, a3->getNumDocs()); + EXPECT_EQ(2u, a4->getNumDocs()); + EXPECT_EQ(1u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a2->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a3->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a4->getStatus().getLastSyncToken()); ibuf.fill(*a1, 1); - EXPECT_EQUAL(1u, ibuf.size()); + EXPECT_EQ(1u, ibuf.size()); EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ibuf[0])); ibuf.fill(*a2, 1); - EXPECT_EQUAL(0u, ibuf.size()); + EXPECT_EQ(0u, ibuf.size()); fbuf.fill(*a3, 1); - EXPECT_EQUAL(1u, fbuf.size()); + EXPECT_EQ(1u, fbuf.size()); EXPECT_TRUE(search::attribute::isUndefined<float>(fbuf[0])); sbuf.fill(*a4, 1); - EXPECT_EQUAL(1u, sbuf.size()); - EXPECT_EQUAL(strcmp("", sbuf[0]), 0); + EXPECT_EQ(1u, sbuf.size()); + EXPECT_EQ(strcmp("", sbuf[0]), 0); } { // document with single value & multi value attribute - Document::UP doc = idb.startDocument("id:ns:searchdocument::2"). + auto doc = idb.startDocument("id:ns:searchdocument::2"). startAttributeField("a1").addInt(10).endField(). startAttributeField("a2").startElement().addInt(20).endElement(). startElement().addInt(30).endElement().endField().endDocument(); - f.put(2, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a2->getNumDocs()); - EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(2u, a2->getStatus().getLastSyncToken()); - EXPECT_EQUAL(2u, a3->getStatus().getLastSyncToken()); - EXPECT_EQUAL(2u, a4->getStatus().getLastSyncToken()); + put(2, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a2->getNumDocs()); + EXPECT_EQ(2u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(2u, a2->getStatus().getLastSyncToken()); + EXPECT_EQ(2u, a3->getStatus().getLastSyncToken()); + EXPECT_EQ(2u, a4->getStatus().getLastSyncToken()); ibuf.fill(*a1, 2); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(10u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(10u, ibuf[0]); ibuf.fill(*a2, 2); - EXPECT_EQUAL(2u, ibuf.size()); - EXPECT_EQUAL(20u, ibuf[0]); - EXPECT_EQUAL(30u, ibuf[1]); + EXPECT_EQ(2u, ibuf.size()); + EXPECT_EQ(20u, ibuf[0]); + EXPECT_EQ(30u, ibuf[1]); } { // replace existing document - Document::UP doc = idb.startDocument("id:ns:searchdocument::2"). + auto doc = idb.startDocument("id:ns:searchdocument::2"). startAttributeField("a1").addInt(100).endField(). startAttributeField("a2").startElement().addInt(200).endElement(). startElement().addInt(300).endElement(). startElement().addInt(400).endElement().endField().endDocument(); - f.put(3, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a2->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(3u, a2->getStatus().getLastSyncToken()); - EXPECT_EQUAL(3u, a3->getStatus().getLastSyncToken()); - EXPECT_EQUAL(3u, a4->getStatus().getLastSyncToken()); + put(3, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a2->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a2->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a3->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a4->getStatus().getLastSyncToken()); ibuf.fill(*a1, 2); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(100u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(100u, ibuf[0]); ibuf.fill(*a2, 2); - EXPECT_EQUAL(3u, ibuf.size()); - EXPECT_EQUAL(200u, ibuf[0]); - EXPECT_EQUAL(300u, ibuf[1]); - EXPECT_EQUAL(400u, ibuf[2]); + EXPECT_EQ(3u, ibuf.size()); + EXPECT_EQ(200u, ibuf[0]); + EXPECT_EQ(300u, ibuf[1]); + EXPECT_EQ(400u, ibuf[2]); } } -TEST_F("require that attribute writer handles predicate put", Fixture) +TEST_F(AttributeWriterTest, handles_predicate_put) { Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE)); DocBuilder idb(s); - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); // empty document should give default values - EXPECT_EQUAL(1u, a1->getNumDocs()); - f.put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); + EXPECT_EQ(1u, a1->getNumDocs()); + put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); + EXPECT_EQ(2u, a1->getNumDocs()); + EXPECT_EQ(1u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(0u, index.getZeroConstraintDocs().size()); // document with single value attribute PredicateSlimeBuilder builder; - Document::UP doc = + auto doc = idb.startDocument("id:ns:searchdocument::2").startAttributeField("a1") .addPredicate(builder.true_predicate().build()) .endField().endDocument(); - f.put(2, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); + put(2, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(2u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, index.getZeroConstraintDocs().size()); auto it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")); EXPECT_FALSE(it.valid()); @@ -303,9 +312,9 @@ TEST_F("require that attribute writer handles predicate put", Fixture) doc = idb.startDocument("id:ns:searchdocument::2").startAttributeField("a1") .addPredicate(builder.feature("foo").value("bar").build()) .endField().endDocument(); - f.put(3, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + put(3, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")); EXPECT_TRUE(it.valid()); @@ -317,21 +326,21 @@ assertUndefined(const IAttributeVector &attr, uint32_t docId) EXPECT_TRUE(search::attribute::isUndefined<int32_t>(attr.getInt(docId))); } -TEST_F("require that attribute writer handles remove", Fixture) +TEST_F(AttributeWriterTest, handles_remove) { - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute("a2"); fillAttribute(a1, 1, 10, 1); fillAttribute(a2, 1, 20, 1); - f.remove(2, 0); + remove(2, 0); - TEST_DO(assertUndefined(*a1, 0)); - TEST_DO(assertUndefined(*a2, 0)); + assertUndefined(*a1, 0); + assertUndefined(*a2, 0); - f.remove(2, 0); // same sync token as previous + remove(2, 0); // same sync token as previous try { - f.remove(1, 0); // lower sync token than previous + remove(1, 0); // lower sync token than previous EXPECT_TRUE(true); // update is ignored } catch (vespalib::IllegalStateException & e) { LOG(info, "Got expected exception: '%s'", e.getMessage().c_str()); @@ -339,62 +348,63 @@ TEST_F("require that attribute writer handles remove", Fixture) } } -TEST_F("require that attribute writer handles batch remove", Fixture) +TEST_F(AttributeWriterTest, handles_batch_remove) { - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute("a2"); fillAttribute(a1, 4, 22, 1); fillAttribute(a2, 4, 33, 1); LidVector lidsToRemove = {1,3}; - f.remove(lidsToRemove, 2); - - TEST_DO(assertUndefined(*a1, 1)); - EXPECT_EQUAL(22, a1->getInt(2)); - TEST_DO(assertUndefined(*a1, 3)); - TEST_DO(assertUndefined(*a2, 1)); - EXPECT_EQUAL(33, a2->getInt(2)); - TEST_DO(assertUndefined(*a2, 3)); + remove(lidsToRemove, 2); + + assertUndefined(*a1, 1); + EXPECT_EQ(22, a1->getInt(2)); + assertUndefined(*a1, 3); + assertUndefined(*a2, 1); + EXPECT_EQ(33, a2->getInt(2)); + assertUndefined(*a2, 3); } -void verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::stringref expected) +void +verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::stringref expected) { attribute::ConstCharContent sbuf; sbuf.fill(v, lid); - EXPECT_EQUAL(1u, sbuf.size()); - EXPECT_EQUAL(expected, sbuf[0]); + EXPECT_EQ(1u, sbuf.size()); + EXPECT_EQ(expected, sbuf[0]); } -TEST_F("require that visibilitydelay is honoured", Fixture) +TEST_F(AttributeWriterTest, visibility_delay_is_honoured) { - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::STRING)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::STRING)}); Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::STRING, CollectionType::SINGLE)); DocBuilder idb(s); - EXPECT_EQUAL(1u, a1->getNumDocs()); - EXPECT_EQUAL(0u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a1->getNumDocs()); + EXPECT_EQ(0u, a1->getStatus().getLastSyncToken()); Document::UP doc = idb.startDocument("id:ns:searchdocument::1") .startAttributeField("a1").addStr("10").endField() .endDocument(); - f.put(3, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); - AttributeWriter awDelayed(f._m); + put(3, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); + AttributeWriter awDelayed(_mgr); awDelayed.put(4, *doc, 2, false, emptyCallback); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); awDelayed.put(5, *doc, 4, false, emptyCallback); - EXPECT_EQUAL(5u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(5u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); awDelayed.forceCommit(6, emptyCallback); - EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(6u, a1->getStatus().getLastSyncToken()); - AttributeWriter awDelayedShort(f._m); + AttributeWriter awDelayedShort(_mgr); awDelayedShort.put(7, *doc, 2, false, emptyCallback); - EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(6u, a1->getStatus().getLastSyncToken()); awDelayedShort.put(8, *doc, 2, false, emptyCallback); awDelayedShort.forceCommit(8, emptyCallback); - EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(8u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "10"); awDelayed.put(9, *idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1").addStr("11").endField().endDocument(), @@ -403,40 +413,39 @@ TEST_F("require that visibilitydelay is honoured", Fixture) 2, false, emptyCallback); awDelayed.put(11, *idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1").addStr("30").endField().endDocument(), 2, false, emptyCallback); - EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(8u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "10"); awDelayed.forceCommit(12, emptyCallback); - EXPECT_EQUAL(12u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(12u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "30"); - } -TEST_F("require that attribute writer handles predicate remove", Fixture) +TEST_F(AttributeWriterTest, handles_predicate_remove) { - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); Schema s; s.addAttributeField( Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE)); DocBuilder idb(s); PredicateSlimeBuilder builder; - Document::UP doc = + auto doc = idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1") .addPredicate(builder.true_predicate().build()) .endField().endDocument(); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); - EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); - f.remove(2, 1); - EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); + EXPECT_EQ(1u, index.getZeroConstraintDocs().size()); + remove(2, 1); + EXPECT_EQ(0u, index.getZeroConstraintDocs().size()); } -TEST_F("require that attribute writer handles update", Fixture) +TEST_F(AttributeWriterTest, handles_update) { - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute("a2"); fillAttribute(a1, 1, 10, 1); fillAttribute(a2, 1, 20, 1); @@ -454,19 +463,19 @@ TEST_F("require that attribute writer handles update", Fixture) DummyFieldUpdateCallback onUpdate; bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit, onUpdate); + update(2, upd, 1, immediateCommit, onUpdate); attribute::IntegerContent ibuf; ibuf.fill(*a1, 1); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(15u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(15u, ibuf[0]); ibuf.fill(*a2, 1); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(30u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(30u, ibuf[0]); - f.update(2, upd, 1, immediateCommit, onUpdate); // same sync token as previous + update(2, upd, 1, immediateCommit, onUpdate); // same sync token as previous try { - f.update(1, upd, 1, immediateCommit, onUpdate); // lower sync token than previous + update(1, upd, 1, immediateCommit, onUpdate); // lower sync token than previous EXPECT_TRUE(true); // update is ignored } catch (vespalib::IllegalStateException & e) { LOG(info, "Got expected exception: '%s'", e.getMessage().c_str()); @@ -474,20 +483,20 @@ TEST_F("require that attribute writer handles update", Fixture) } } -TEST_F("require that attribute writer handles predicate update", Fixture) +TEST_F(AttributeWriterTest, handles_predicate_update) { - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); Schema schema; schema.addAttributeField(Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE)); DocBuilder idb(schema); PredicateSlimeBuilder builder; - Document::UP doc = + auto doc = idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1") .addPredicate(builder.true_predicate().build()) .endField().endDocument(); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); const document::DocumentType &dt(idb.getDocumentType()); DocumentUpdate upd(*idb.getDocumentTypeRepo(), dt, DocumentId("id:ns:searchdocument::1")); @@ -496,20 +505,20 @@ TEST_F("require that attribute writer handles predicate update", Fixture) .addUpdate(AssignValueUpdate(new_value))); PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); - EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); + EXPECT_EQ(1u, index.getZeroConstraintDocs().size()); EXPECT_FALSE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); bool immediateCommit = true; DummyFieldUpdateCallback onUpdate; - f.update(2, upd, 1, immediateCommit, onUpdate); - EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); + update(2, upd, 1, immediateCommit, onUpdate); + EXPECT_EQ(0u, index.getZeroConstraintDocs().size()); EXPECT_TRUE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); } -struct AttributeCollectionSpecFixture -{ +class AttributeCollectionSpecTest : public ::testing::Test { +public: AttributesConfigBuilder _builder; AttributeCollectionSpecFactory _factory; - AttributeCollectionSpecFixture(bool fastAccessOnly) + AttributeCollectionSpecTest(bool fastAccessOnly) : _builder(), _factory(search::GrowStrategy(), 100, fastAccessOnly) { @@ -528,49 +537,47 @@ struct AttributeCollectionSpecFixture } }; -struct NormalAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture -{ - NormalAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(false) {} +class NormalAttributeCollectionSpecTest : public AttributeCollectionSpecTest { +public: + NormalAttributeCollectionSpecTest() : AttributeCollectionSpecTest(false) {} }; -struct FastAccessAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture +struct FastAccessAttributeCollectionSpecTest : public AttributeCollectionSpecTest { - FastAccessAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(true) {} + FastAccessAttributeCollectionSpecTest() : AttributeCollectionSpecTest(true) {} }; -TEST_F("require that normal attribute collection spec can be created", - NormalAttributeCollectionSpecFixture) +TEST_F(NormalAttributeCollectionSpecTest, spec_can_be_created) { - AttributeCollectionSpec::UP spec = f.create(10, 20); - EXPECT_EQUAL(2u, spec->getAttributes().size()); - EXPECT_EQUAL("a1", spec->getAttributes()[0].getName()); - EXPECT_EQUAL("a2", spec->getAttributes()[1].getName()); - EXPECT_EQUAL(10u, spec->getDocIdLimit()); - EXPECT_EQUAL(20u, spec->getCurrentSerialNum()); + AttributeCollectionSpec::UP spec = create(10, 20); + EXPECT_EQ(2u, spec->getAttributes().size()); + EXPECT_EQ("a1", spec->getAttributes()[0].getName()); + EXPECT_EQ("a2", spec->getAttributes()[1].getName()); + EXPECT_EQ(10u, spec->getDocIdLimit()); + EXPECT_EQ(20u, spec->getCurrentSerialNum()); } -TEST_F("require that fast access attribute collection spec can be created", - FastAccessAttributeCollectionSpecFixture) +TEST_F(FastAccessAttributeCollectionSpecTest, spec_can_be_created) { - AttributeCollectionSpec::UP spec = f.create(10, 20); - EXPECT_EQUAL(1u, spec->getAttributes().size()); - EXPECT_EQUAL("a2", spec->getAttributes()[0].getName()); - EXPECT_EQUAL(10u, spec->getDocIdLimit()); - EXPECT_EQUAL(20u, spec->getCurrentSerialNum()); + AttributeCollectionSpec::UP spec = create(10, 20); + EXPECT_EQ(1u, spec->getAttributes().size()); + EXPECT_EQ("a2", spec->getAttributes()[0].getName()); + EXPECT_EQ(10u, spec->getDocIdLimit()); + EXPECT_EQ(20u, spec->getCurrentSerialNum()); } const FilterAttributeManager::AttributeSet ACCEPTED_ATTRIBUTES = {"a2"}; -struct FilterFixture -{ +class FilterAttributeManagerTest : public ::testing::Test { +public: DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; HwInfo _hwInfo; - proton::AttributeManager::SP _baseMgr; FilterAttributeManager _filterMgr; - FilterFixture() + + FilterAttributeManagerTest() : _dirHandler(test_dir), _fileHeaderContext(), _attributeFieldWriter(), @@ -587,34 +594,34 @@ struct FilterFixture } }; -TEST_F("require that filter attribute manager can filter attributes", FilterFixture) +TEST_F(FilterAttributeManagerTest, filter_attributes) { - EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == nullptr); - EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != nullptr); + EXPECT_TRUE(_filterMgr.getAttribute("a1").get() == nullptr); + EXPECT_TRUE(_filterMgr.getAttribute("a2").get() != nullptr); std::vector<AttributeGuard> attrs; - f._filterMgr.getAttributeList(attrs); - EXPECT_EQUAL(1u, attrs.size()); - EXPECT_EQUAL("a2", attrs[0]->getName()); - searchcorespi::IFlushTarget::List targets = f._filterMgr.getFlushTargets(); - EXPECT_EQUAL(2u, targets.size()); - EXPECT_EQUAL("attribute.flush.a2", targets[0]->getName()); - EXPECT_EQUAL("attribute.shrink.a2", targets[1]->getName()); + _filterMgr.getAttributeList(attrs); + EXPECT_EQ(1u, attrs.size()); + EXPECT_EQ("a2", attrs[0]->getName()); + searchcorespi::IFlushTarget::List targets = _filterMgr.getFlushTargets(); + EXPECT_EQ(2u, targets.size()); + EXPECT_EQ("attribute.flush.a2", targets[0]->getName()); + EXPECT_EQ("attribute.shrink.a2", targets[1]->getName()); } -TEST_F("require that filter attribute manager can return flushed serial number", FilterFixture) +TEST_F(FilterAttributeManagerTest, returns_flushed_serial_number) { - f._baseMgr->flushAll(100); - EXPECT_EQUAL(0u, f._filterMgr.getFlushedSerialNum("a1")); - EXPECT_EQUAL(100u, f._filterMgr.getFlushedSerialNum("a2")); + _baseMgr->flushAll(100); + EXPECT_EQ(0u, _filterMgr.getFlushedSerialNum("a1")); + EXPECT_EQ(100u, _filterMgr.getFlushedSerialNum("a2")); } -TEST_F("readable_attribute_vector filters attributes", FilterFixture) +TEST_F(FilterAttributeManagerTest, readable_attribute_vector_filters_attributes) { - auto av = f._filterMgr.readable_attribute_vector("a2"); + auto av = _filterMgr.readable_attribute_vector("a2"); ASSERT_TRUE(av); - EXPECT_EQUAL("a2", av->makeReadGuard(false)->attribute()->getName()); + EXPECT_EQ("a2", av->makeReadGuard(false)->attribute()->getName()); - av = f._filterMgr.readable_attribute_vector("a1"); + av = _filterMgr.readable_attribute_vector("a1"); EXPECT_FALSE(av); } @@ -625,18 +632,20 @@ Tensor::UP make_tensor(const TensorSpec &spec) { return Tensor::UP(dynamic_cast<Tensor*>(tensor.release())); } +const vespalib::string sparse_tensor = "tensor(x{},y{})"; + AttributeVector::SP -createTensorAttribute(Fixture &f) { +createTensorAttribute(AttributeWriterTest &t) { AVConfig cfg(AVBasicType::TENSOR); - cfg.setTensorType(ValueType::from_spec("tensor(x{},y{})")); - auto ret = f.addAttribute({"a1", cfg}, createSerialNum); + cfg.setTensorType(ValueType::from_spec(sparse_tensor)); + auto ret = t.addAttribute({"a1", cfg}); return ret; } Schema -createTensorSchema() { +createTensorSchema(const vespalib::string& tensor_spec = sparse_tensor) { Schema schema; - schema.addAttributeField(Schema::AttributeField("a1", schema::DataType::TENSOR, CollectionType::SINGLE)); + schema.addAttributeField(Schema::AttributeField("a1", schema::DataType::TENSOR, CollectionType::SINGLE, tensor_spec)); return schema; } @@ -649,38 +658,34 @@ createTensorPutDoc(DocBuilder &builder, const Tensor &tensor) { } - -TEST_F("Test that we can use attribute writer to write to tensor attribute", - Fixture) +TEST_F(AttributeWriterTest, can_write_to_tensor_attribute) { - AttributeVector::SP a1 = createTensorAttribute(f); + auto a1 = createTensorAttribute(*this); Schema s = createTensorSchema(); DocBuilder builder(s); - auto tensor = make_tensor(TensorSpec("tensor(x{},y{})") + auto tensor = make_tensor(TensorSpec(sparse_tensor) .add({{"x", "4"}, {"y", "5"}}, 7)); Document::UP doc = createTensorPutDoc(builder, *tensor); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - TensorAttribute *tensorAttribute = - dynamic_cast<TensorAttribute *>(a1.get()); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); + auto *tensorAttribute = dynamic_cast<TensorAttribute *>(a1.get()); EXPECT_TRUE(tensorAttribute != nullptr); auto tensor2 = tensorAttribute->getTensor(1); EXPECT_TRUE(static_cast<bool>(tensor2)); EXPECT_TRUE(tensor->equals(*tensor2)); } -TEST_F("require that attribute writer handles tensor assign update", Fixture) +TEST_F(AttributeWriterTest, handles_tensor_assign_update) { - AttributeVector::SP a1 = createTensorAttribute(f); + auto a1 = createTensorAttribute(*this); Schema s = createTensorSchema(); DocBuilder builder(s); - auto tensor = make_tensor(TensorSpec("tensor(x{},y{})") + auto tensor = make_tensor(TensorSpec(sparse_tensor) .add({{"x", "6"}, {"y", "7"}}, 9)); - Document::UP doc = createTensorPutDoc(builder, *tensor); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - TensorAttribute *tensorAttribute = - dynamic_cast<TensorAttribute *>(a1.get()); + auto doc = createTensorPutDoc(builder, *tensor); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); + auto *tensorAttribute = dynamic_cast<TensorAttribute *>(a1.get()); EXPECT_TRUE(tensorAttribute != nullptr); auto tensor2 = tensorAttribute->getTensor(1); EXPECT_TRUE(static_cast<bool>(tensor2)); @@ -688,23 +693,22 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture) const document::DocumentType &dt(builder.getDocumentType()); DocumentUpdate upd(*builder.getDocumentTypeRepo(), dt, DocumentId("id:ns:searchdocument::1")); - auto new_tensor = make_tensor(TensorSpec("tensor(x{},y{})") + auto new_tensor = make_tensor(TensorSpec(sparse_tensor) .add({{"x", "8"}, {"y", "9"}}, 11)); - TensorDataType xySparseTensorDataType(vespalib::eval::ValueType::from_spec("tensor(x{},y{})")); + TensorDataType xySparseTensorDataType(vespalib::eval::ValueType::from_spec(sparse_tensor)); TensorFieldValue new_value(xySparseTensorDataType); new_value = new_tensor->clone(); upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(AssignValueUpdate(new_value))); bool immediateCommit = true; DummyFieldUpdateCallback onUpdate; - f.update(2, upd, 1, immediateCommit, onUpdate); - EXPECT_EQUAL(2u, a1->getNumDocs()); + update(2, upd, 1, immediateCommit, onUpdate); + EXPECT_EQ(2u, a1->getNumDocs()); EXPECT_TRUE(tensorAttribute != nullptr); tensor2 = tensorAttribute->getTensor(1); EXPECT_TRUE(static_cast<bool>(tensor2)); EXPECT_TRUE(!tensor->equals(*tensor2)); EXPECT_TRUE(new_tensor->equals(*tensor2)); - } namespace { @@ -712,16 +716,16 @@ namespace { void assertPutDone(AttributeVector &attr, int32_t expVal) { - EXPECT_EQUAL(2u, attr.getNumDocs()); - EXPECT_EQUAL(1u, attr.getStatus().getLastSyncToken()); + EXPECT_EQ(2u, attr.getNumDocs()); + EXPECT_EQ(1u, attr.getStatus().getLastSyncToken()); attribute::IntegerContent ibuf; ibuf.fill(attr, 1); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(expVal, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(expVal, ibuf[0]); } void -putAttributes(Fixture &f, std::vector<uint32_t> expExecuteHistory) +putAttributes(AttributeWriterTest &t, std::vector<uint32_t> expExecuteHistory) { Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::INT32, CollectionType::SINGLE)); @@ -730,41 +734,205 @@ putAttributes(Fixture &f, std::vector<uint32_t> expExecuteHistory) DocBuilder idb(s); - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); - AttributeVector::SP a3 = f.addAttribute("a3"); + auto a1 = t.addAttribute("a1"); + auto a2 = t.addAttribute("a2"); + auto a3 = t.addAttribute("a3"); - EXPECT_EQUAL(1u, a1->getNumDocs()); - EXPECT_EQUAL(1u, a2->getNumDocs()); - EXPECT_EQUAL(1u, a3->getNumDocs()); - f.put(1, *idb.startDocument("id:ns:searchdocument::1"). + EXPECT_EQ(1u, a1->getNumDocs()); + EXPECT_EQ(1u, a2->getNumDocs()); + EXPECT_EQ(1u, a3->getNumDocs()); + t.put(1, *idb.startDocument("id:ns:searchdocument::1"). startAttributeField("a1").addInt(10).endField(). startAttributeField("a2").addInt(15).endField(). startAttributeField("a3").addInt(20).endField(). endDocument(), 1); - TEST_DO(assertPutDone(*a1, 10)); - TEST_DO(assertPutDone(*a2, 15)); - TEST_DO(assertPutDone(*a3, 20)); - TEST_DO(f.assertExecuteHistory(expExecuteHistory)); + assertPutDone(*a1, 10); + assertPutDone(*a2, 15); + assertPutDone(*a3, 20); + t.assertExecuteHistory(expExecuteHistory); +} + +} + +TEST_F(AttributeWriterTest, spreads_write_over_1_write_context) +{ + putAttributes(*this, {0}); } +TEST_F(AttributeWriterTest, spreads_write_over_2_write_contexts) +{ + setup(2); + putAttributes(*this, {0, 1}); } -TEST_F("require that attribute writer spreads write over 1 write context", Fixture(1)) +TEST_F(AttributeWriterTest, spreads_write_over_3_write_contexts) { - TEST_DO(putAttributes(f, {0})); + setup(8); + putAttributes(*this, {0, 1, 2}); } -TEST_F("require that attribute writer spreads write over 2 write contexts", Fixture(2)) +struct MockPrepareResult : public PrepareResult { + uint32_t docid; + const Tensor& tensor; + MockPrepareResult(uint32_t docid_in, const Tensor& tensor_in) : docid(docid_in), tensor(tensor_in) {} +}; + +class MockDenseTensorAttribute : public DenseTensorAttribute { +public: + mutable size_t prepare_set_tensor_cnt; + mutable size_t complete_set_tensor_cnt; + size_t clear_doc_cnt; + + MockDenseTensorAttribute(vespalib::stringref name, const AVConfig& cfg) + : DenseTensorAttribute(name, cfg), + prepare_set_tensor_cnt(0), + complete_set_tensor_cnt(0), + clear_doc_cnt(0) + {} + uint32_t clearDoc(DocId docid) override { + ++clear_doc_cnt; + return DenseTensorAttribute::clearDoc(docid); + } + std::unique_ptr<PrepareResult> prepare_set_tensor(uint32_t docid, const Tensor& tensor) const override { + ++prepare_set_tensor_cnt; + return std::make_unique<MockPrepareResult>(docid, tensor); + } + + virtual void complete_set_tensor(DocId docid, const Tensor& tensor, std::unique_ptr<PrepareResult> prepare_result) override { + ++complete_set_tensor_cnt; + assert(prepare_result); + auto* mock_result = dynamic_cast<MockPrepareResult*>(prepare_result.get()); + assert(mock_result); + EXPECT_EQ(docid, mock_result->docid); + EXPECT_EQ(tensor, mock_result->tensor); + } +}; + +const vespalib::string dense_tensor = "tensor(x[2])"; + +AVConfig +get_tensor_config(bool allow_multi_threaded_indexing) { - TEST_DO(putAttributes(f, {0, 1})); + AVConfig cfg(AVBasicType::TENSOR); + cfg.setTensorType(ValueType::from_spec(dense_tensor)); + cfg.set_hnsw_index_params(HnswIndexParams(4, 4, DistanceMetric::Euclidean, allow_multi_threaded_indexing)); + return cfg; } -TEST_F("require that attribute writer spreads write over 3 write contexts", Fixture(8)) +std::shared_ptr<MockDenseTensorAttribute> +make_mock_tensor_attribute(const vespalib::string& name, bool allow_multi_threaded_indexing) { - TEST_DO(putAttributes(f, {0, 1, 2})); + auto cfg = get_tensor_config(allow_multi_threaded_indexing); + return std::make_shared<MockDenseTensorAttribute>(name, cfg); } +TEST_F(AttributeWriterTest, tensor_attributes_using_two_phase_put_are_in_separate_write_contexts) +{ + addAttribute("a1"); + addAttribute({"t1", get_tensor_config(true)}); + addAttribute({"t2", get_tensor_config(true)}); + addAttribute({"t3", get_tensor_config(false)}); + allocAttributeWriter(); + + const auto& ctx = _aw->get_write_contexts(); + EXPECT_EQ(3, ctx.size()); + EXPECT_FALSE(ctx[0].use_two_phase_put()); + EXPECT_EQ(2, ctx[0].getFields().size()); + + EXPECT_TRUE(ctx[1].use_two_phase_put()); + EXPECT_EQ(1, ctx[1].getFields().size()); + EXPECT_EQ("t1", ctx[1].getFields()[0].getAttribute().getName()); + + EXPECT_TRUE(ctx[2].use_two_phase_put()); + EXPECT_EQ(1, ctx[2].getFields().size()); + EXPECT_EQ("t2", ctx[2].getFields()[0].getAttribute().getName()); +} + +class TwoPhasePutTest : public AttributeWriterTest { +public: + Schema schema; + DocBuilder builder; + std::shared_ptr<MockDenseTensorAttribute> attr; + std::unique_ptr<Tensor> tensor; + + TwoPhasePutTest() + : AttributeWriterTest(), + schema(createTensorSchema(dense_tensor)), + builder(schema), + attr() + { + setup(2); + attr = make_mock_tensor_attribute("a1", true); + add_attribute(attr); + AttributeManager::padAttribute(*attr, 4); + attr->clear_doc_cnt = 0; + tensor = make_tensor(TensorSpec(dense_tensor) + .add({{"x", 0}}, 3).add({{"x", 1}}, 5)); + } + void expect_tensor_attr_calls(size_t exp_prepare_cnt, + size_t exp_complete_cnt, + size_t exp_clear_doc_cnt = 0) { + EXPECT_EQ(exp_prepare_cnt, attr->prepare_set_tensor_cnt); + EXPECT_EQ(exp_complete_cnt, attr->complete_set_tensor_cnt); + EXPECT_EQ(exp_clear_doc_cnt, attr->clear_doc_cnt); + } + Document::UP make_doc() { + return createTensorPutDoc(builder, *tensor); + } + Document::UP make_no_field_doc() { + return builder.startDocument("id:ns:searchdocument::1").endDocument(); + } + Document::UP make_no_tensor_doc() { + return builder.startDocument("id:ns:searchdocument::1"). + startAttributeField("a1"). + addTensor(std::unique_ptr<vespalib::tensor::Tensor>()).endField().endDocument(); + } +}; + +TEST_F(TwoPhasePutTest, handles_put_in_two_phases_when_specified_for_tensor_attribute) +{ + auto doc = make_doc(); + + put(1, *doc, 1); + expect_tensor_attr_calls(1, 1); + assertExecuteHistory({1, 0}); + + put(2, *doc, 2); + expect_tensor_attr_calls(2, 2); + assertExecuteHistory({1, 0, 0, 0}); + + put(3, *doc, 3); + expect_tensor_attr_calls(3, 3); + // Note that the prepare step is executed round-robin between the 2 threads. + assertExecuteHistory({1, 0, 0, 0, 1, 0}); +} + +TEST_F(TwoPhasePutTest, put_is_ignored_when_serial_number_is_older_or_equal_to_attribute) +{ + auto doc = make_doc(); + attr->commit(7, 7); + put(7, *doc, 1); + expect_tensor_attr_calls(0, 0); + assertExecuteHistory({1, 0}); +} + +TEST_F(TwoPhasePutTest, document_is_cleared_if_field_is_not_set) +{ + auto doc = make_no_field_doc(); + put(1, *doc, 1); + expect_tensor_attr_calls(0, 0, 1); + assertExecuteHistory({1, 0}); +} + +TEST_F(TwoPhasePutTest, document_is_cleared_if_tensor_in_field_is_not_set) +{ + auto doc = make_no_tensor_doc(); + put(1, *doc, 1); + expect_tensor_attr_calls(0, 0, 1); + assertExecuteHistory({1, 0}); +} + + ImportedAttributeVector::SP createImportedAttribute(const vespalib::string &name) { @@ -787,74 +955,66 @@ createImportedAttributesRepo() return result; } -TEST_F("require that AttributeWriter::forceCommit() clears search cache in imported attribute vectors", Fixture) +TEST_F(AttributeWriterTest, forceCommit_clears_search_cache_in_imported_attribute_vectors) { - f._m->setImportedAttributes(createImportedAttributesRepo()); - f.commit(10); - EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_a")->getSearchCache()->size()); - EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); + _mgr->setImportedAttributes(createImportedAttributesRepo()); + commit(10); + EXPECT_EQ(0u, _mgr->getImportedAttributes()->get("imported_a")->getSearchCache()->size()); + EXPECT_EQ(0u, _mgr->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); } -struct StructFixtureBase : public Fixture -{ +class StructWriterTestBase : public AttributeWriterTest { +public: DocumentType _type; const Field _valueField; StructDataType _structFieldType; - StructFixtureBase() - : Fixture(), + StructWriterTestBase() + : AttributeWriterTest(), _type("test"), _valueField("value", 2, *DataType::INT, true), _structFieldType("struct") { - addAttribute({"value", AVConfig(AVBasicType::INT32, AVCollectionType::SINGLE)}, createSerialNum); + addAttribute({"value", AVConfig(AVBasicType::INT32, AVCollectionType::SINGLE)}); _type.addField(_valueField); _structFieldType.addField(_valueField); } - ~StructFixtureBase(); + ~StructWriterTestBase(); - std::unique_ptr<StructFieldValue> - makeStruct() - { + std::unique_ptr<StructFieldValue> makeStruct() { return std::make_unique<StructFieldValue>(_structFieldType); } - std::unique_ptr<StructFieldValue> - makeStruct(const int32_t value) - { + std::unique_ptr<StructFieldValue> makeStruct(const int32_t value) { auto ret = makeStruct(); ret->setValue(_valueField, IntFieldValue(value)); return ret; } - std::unique_ptr<Document> - makeDoc() - { + std::unique_ptr<Document> makeDoc() { return std::make_unique<Document>(_type, DocumentId("id::test::1")); } }; -StructFixtureBase::~StructFixtureBase() = default; +StructWriterTestBase::~StructWriterTestBase() = default; -struct StructArrayFixture : public StructFixtureBase -{ - using StructFixtureBase::makeDoc; +class StructArrayWriterTest : public StructWriterTestBase { +public: + using StructWriterTestBase::makeDoc; const ArrayDataType _structArrayFieldType; const Field _structArrayField; - StructArrayFixture() - : StructFixtureBase(), + StructArrayWriterTest() + : StructWriterTestBase(), _structArrayFieldType(_structFieldType), _structArrayField("array", _structArrayFieldType, true) { - addAttribute({"array.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + addAttribute({"array.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); _type.addField(_structArrayField); } - ~StructArrayFixture(); + ~StructArrayWriterTest(); - std::unique_ptr<Document> - makeDoc(int32_t value, const std::vector<int32_t> &arrayValues) - { + std::unique_ptr<Document> makeDoc(int32_t value, const std::vector<int32_t> &arrayValues) { auto doc = makeDoc(); doc->setValue(_valueField, IntFieldValue(value)); ArrayFieldValue s(_structArrayFieldType); @@ -865,49 +1025,47 @@ struct StructArrayFixture : public StructFixtureBase return doc; } void checkAttrs(uint32_t lid, int32_t value, const std::vector<int32_t> &arrayValues) { - auto valueAttr = _m->getAttribute("value")->getSP(); - auto arrayValueAttr = _m->getAttribute("array.value")->getSP(); - EXPECT_EQUAL(value, valueAttr->getInt(lid)); + auto valueAttr = _mgr->getAttribute("value")->getSP(); + auto arrayValueAttr = _mgr->getAttribute("array.value")->getSP(); + EXPECT_EQ(value, valueAttr->getInt(lid)); attribute::IntegerContent ibuf; ibuf.fill(*arrayValueAttr, lid); - EXPECT_EQUAL(arrayValues.size(), ibuf.size()); + EXPECT_EQ(arrayValues.size(), ibuf.size()); for (size_t i = 0; i < arrayValues.size(); ++i) { - EXPECT_EQUAL(arrayValues[i], ibuf[i]); + EXPECT_EQ(arrayValues[i], ibuf[i]); } } }; -StructArrayFixture::~StructArrayFixture() = default; +StructArrayWriterTest::~StructArrayWriterTest() = default; -TEST_F("require that update with doc argument updates struct field attributes (array)", StructArrayFixture) +TEST_F(StructArrayWriterTest, update_with_doc_argument_updates_struct_field_attributes) { - auto doc = f.makeDoc(10, {11, 12}); - f.put(10, *doc, 1); - TEST_DO(f.checkAttrs(1, 10, {11, 12})); - doc = f.makeDoc(20, {21}); - f.update(11, *doc, 1, true); - TEST_DO(f.checkAttrs(1, 10, {21})); + auto doc = makeDoc(10, {11, 12}); + put(10, *doc, 1); + checkAttrs(1, 10, {11, 12}); + doc = makeDoc(20, {21}); + update(11, *doc, 1, true); + checkAttrs(1, 10, {21}); } -struct StructMapFixture : public StructFixtureBase -{ - using StructFixtureBase::makeDoc; +class StructMapWriterTest : public StructWriterTestBase { +public: + using StructWriterTestBase::makeDoc; const MapDataType _structMapFieldType; const Field _structMapField; - StructMapFixture() - : StructFixtureBase(), + StructMapWriterTest() + : StructWriterTestBase(), _structMapFieldType(*DataType::INT, _structFieldType), _structMapField("map", _structMapFieldType, true) { - addAttribute({"map.value.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); - addAttribute({"map.key", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + addAttribute({"map.value.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); + addAttribute({"map.key", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); _type.addField(_structMapField); } - std::unique_ptr<Document> - makeDoc(int32_t value, const std::map<int32_t, int32_t> &mapValues) - { + std::unique_ptr<Document> makeDoc(int32_t value, const std::map<int32_t, int32_t> &mapValues) { auto doc = makeDoc(); doc->setValue(_valueField, IntFieldValue(value)); MapFieldValue s(_structMapFieldType); @@ -917,38 +1075,35 @@ struct StructMapFixture : public StructFixtureBase doc->setValue(_structMapField, s); return doc; } + void checkAttrs(uint32_t lid, int32_t expValue, const std::map<int32_t, int32_t> &expMap) { - auto valueAttr = _m->getAttribute("value")->getSP(); - auto mapKeyAttr = _m->getAttribute("map.key")->getSP(); - auto mapValueAttr = _m->getAttribute("map.value.value")->getSP(); - EXPECT_EQUAL(expValue, valueAttr->getInt(lid)); + auto valueAttr = _mgr->getAttribute("value")->getSP(); + auto mapKeyAttr = _mgr->getAttribute("map.key")->getSP(); + auto mapValueAttr = _mgr->getAttribute("map.value.value")->getSP(); + EXPECT_EQ(expValue, valueAttr->getInt(lid)); attribute::IntegerContent mapKeys; mapKeys.fill(*mapKeyAttr, lid); attribute::IntegerContent mapValues; mapValues.fill(*mapValueAttr, lid); - EXPECT_EQUAL(expMap.size(), mapValues.size()); - EXPECT_EQUAL(expMap.size(), mapKeys.size()); + EXPECT_EQ(expMap.size(), mapValues.size()); + EXPECT_EQ(expMap.size(), mapKeys.size()); size_t i = 0; for (const auto &expMapElem : expMap) { - EXPECT_EQUAL(expMapElem.first, mapKeys[i]); - EXPECT_EQUAL(expMapElem.second, mapValues[i]); + EXPECT_EQ(expMapElem.first, mapKeys[i]); + EXPECT_EQ(expMapElem.second, mapValues[i]); ++i; } } }; -TEST_F("require that update with doc argument updates struct field attributes (map)", StructMapFixture) +TEST_F(StructMapWriterTest, update_with_doc_argument_updates_struct_field_attributes) { - auto doc = f.makeDoc(10, {{1, 11}, {2, 12}}); - f.put(10, *doc, 1); - TEST_DO(f.checkAttrs(1, 10, {{1, 11}, {2, 12}})); - doc = f.makeDoc(20, {{42, 21}}); - f.update(11, *doc, 1, true); - TEST_DO(f.checkAttrs(1, 10, {{42, 21}})); + auto doc = makeDoc(10, {{1, 11}, {2, 12}}); + put(10, *doc, 1); + checkAttrs(1, 10, {{1, 11}, {2, 12}}); + doc = makeDoc(20, {{42, 21}}); + update(11, *doc, 1, true); + checkAttrs(1, 10, {{42, 21}}); } -TEST_MAIN() -{ - vespalib::rmdir(test_dir, true); - TEST_RUN_ALL(); -} +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/attribute/attribute_test.sh b/searchcore/src/tests/proton/attribute/attribute_test.sh deleted file mode 100755 index 26aa6d5f57a..00000000000 --- a/searchcore/src/tests/proton/attribute/attribute_test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -set -e -rm -rf test_output -$VALGRIND ./searchcore_attribute_test_app diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index e1785e1e48d..b6d6d2437d8 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -662,10 +662,11 @@ void addField(Schema & s, const std::string &name, Schema::DataType dtype, - Schema::CollectionType ctype) + Schema::CollectionType ctype, + const std::string& tensor_spec = "") { - s.addSummaryField(Schema::SummaryField(name, dtype, ctype)); - s.addAttributeField(Schema::AttributeField(name, dtype, ctype)); + s.addSummaryField(Schema::SummaryField(name, dtype, ctype, tensor_spec)); + s.addAttributeField(Schema::AttributeField(name, dtype, ctype, tensor_spec)); } @@ -682,7 +683,7 @@ Test::requireThatAttributesAreUsed() addField(s, "bg", schema::DataType::INT32, CollectionType::WEIGHTEDSET); addField(s, "bh", schema::DataType::FLOAT, CollectionType::WEIGHTEDSET); addField(s, "bi", schema::DataType::STRING, CollectionType::WEIGHTEDSET); - addField(s, "bj", schema::DataType::TENSOR, CollectionType::SINGLE); + addField(s, "bj", schema::DataType::TENSOR, CollectionType::SINGLE, "tensor(x{},y{})"); BuildContext bc(s); DBContext dc(bc._repo, getDocTypeName()); diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 18235116d27..d1d08a332e3 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -285,8 +285,8 @@ SchemaContext::SchemaContext() : schema(new Schema()), builder() { - schema->addAttributeField(Schema::AttributeField("tensor", DataType::TENSOR, CollectionType::SINGLE)); - schema->addAttributeField(Schema::AttributeField("tensor2", DataType::TENSOR, CollectionType::SINGLE)); + schema->addAttributeField(Schema::AttributeField("tensor", DataType::TENSOR, CollectionType::SINGLE, "tensor(x{},y{})")); + schema->addAttributeField(Schema::AttributeField("tensor2", DataType::TENSOR, CollectionType::SINGLE, "tensor(x{},y{})")); addField("i1"); } diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index 0f55a4c30de..5f93f97f165 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h> #include <vespa/searchcore/proton/flushengine/flush_target_candidates.h> +#include <vespa/searchcore/proton/flushengine/flush_target_candidate.h> #include <vespa/searchcore/proton/flushengine/tls_stats_map.h> #include <vespa/searchcore/proton/test/dummy_flush_handler.h> #include <vespa/searchcore/proton/test/dummy_flush_target.h> @@ -21,20 +22,16 @@ struct SimpleFlushTarget : public test::DummyFlushTarget { SerialNum flushedSerial; uint64_t approxDiskBytes; - SimpleFlushTarget(const vespalib::string &name, - SerialNum flushedSerial_, - uint64_t approxDiskBytes_) - : test::DummyFlushTarget(name), - flushedSerial(flushedSerial_), - approxDiskBytes(approxDiskBytes_) - {} + double replay_operation_cost; SimpleFlushTarget(const vespalib::string &name, const Type &type, SerialNum flushedSerial_, - uint64_t approxDiskBytes_) + uint64_t approxDiskBytes_, + double replay_operation_cost_) : test::DummyFlushTarget(name, type, Component::OTHER), flushedSerial(flushedSerial_), - approxDiskBytes(approxDiskBytes_) + approxDiskBytes(approxDiskBytes_), + replay_operation_cost(replay_operation_cost_) {} virtual SerialNum getFlushedSerialNum() const override { return flushedSerial; @@ -42,6 +39,9 @@ struct SimpleFlushTarget : public test::DummyFlushTarget virtual uint64_t getApproxBytesToWriteToDisk() const override { return approxDiskBytes; } + double get_replay_operation_cost() const override { + return replay_operation_cost; + } }; class ContextsBuilder @@ -66,30 +66,35 @@ public: const vespalib::string &targetName, IFlushTarget::Type targetType, SerialNum flushedSerial, - uint64_t approxDiskBytes) { + uint64_t approxDiskBytes, + double replay_operation_cost) { IFlushHandler::SP handler = createAndGetHandler(handlerName); IFlushTarget::SP target = std::make_shared<SimpleFlushTarget>(targetName, targetType, flushedSerial, - approxDiskBytes); + approxDiskBytes, + replay_operation_cost); _result.push_back(std::make_shared<FlushContext>(handler, target, 0)); return *this; } ContextsBuilder &add(const vespalib::string &handlerName, const vespalib::string &targetName, SerialNum flushedSerial, - uint64_t approxDiskBytes) { - return add(handlerName, targetName, IFlushTarget::Type::FLUSH, flushedSerial, approxDiskBytes); + uint64_t approxDiskBytes, + double replay_operation_cost = 0.0) { + return add(handlerName, targetName, IFlushTarget::Type::FLUSH, flushedSerial, approxDiskBytes, replay_operation_cost); } ContextsBuilder &add(const vespalib::string &targetName, SerialNum flushedSerial, - uint64_t approxDiskBytes) { - return add("handler1", targetName, IFlushTarget::Type::FLUSH, flushedSerial, approxDiskBytes); + uint64_t approxDiskBytes, + double replay_operation_cost = 0.0) { + return add("handler1", targetName, IFlushTarget::Type::FLUSH, flushedSerial, approxDiskBytes, replay_operation_cost); } ContextsBuilder &addGC(const vespalib::string &targetName, SerialNum flushedSerial, - uint64_t approxDiskBytes) { - return add("handler1", targetName, IFlushTarget::Type::GC, flushedSerial, approxDiskBytes); + uint64_t approxDiskBytes, + double replay_operation_cost = 0.0) { + return add("handler1", targetName, IFlushTarget::Type::GC, flushedSerial, approxDiskBytes, replay_operation_cost); } FlushContext::List build() const { return _result; } }; @@ -99,6 +104,7 @@ class CandidatesBuilder private: const FlushContext::List *_sortedFlushContexts; size_t _numCandidates; + mutable std::vector<FlushTargetCandidate> _candidates; flushengine::TlsStats _tlsStats; Config _cfg; @@ -106,6 +112,7 @@ public: CandidatesBuilder(const FlushContext::List &sortedFlushContexts) : _sortedFlushContexts(&sortedFlushContexts), _numCandidates(sortedFlushContexts.size()), + _candidates(), _tlsStats(1000, 11, 110), _cfg(2.0, 3.0, 4.0) {} @@ -125,8 +132,16 @@ public: replayEndSerial); return *this; } + void setup_candidates() const { + _candidates.clear(); + _candidates.reserve(_sortedFlushContexts->size()); + for (const auto &flush_context : *_sortedFlushContexts) { + _candidates.emplace_back(flush_context, _tlsStats.getLastSerial(), _cfg); + } + } FlushTargetCandidates build() const { - return FlushTargetCandidates(*_sortedFlushContexts, + setup_candidates(); + return FlushTargetCandidates(_candidates, _numCandidates, _tlsStats, _cfg); @@ -196,9 +211,12 @@ struct FlushStrategyFixture { flushengine::TlsStatsMap _tlsStatsMap; PrepareRestartFlushStrategy strategy; - FlushStrategyFixture() + FlushStrategyFixture(const Config &config) : _tlsStatsMap(defaultTransactionLogStats()), - strategy(DEFAULT_CFG) + strategy(config) + {} + FlushStrategyFixture() + : FlushStrategyFixture(DEFAULT_CFG) {} FlushContext::List getFlushTargets(const FlushContext::List &targetList, const flushengine::TlsStatsMap &tlsStatsMap) const { @@ -297,6 +315,12 @@ TEST_F("require that flush targets for different flush handlers are treated inde TEST_DO(assertFlushContexts("[foo,baz,quz]", targets)); } +TEST_F("require that expensive to replay target is flushed", FlushStrategyFixture(Config(2.0, 1.0, 4.0))) +{ + FlushContext::List targets = f.getFlushTargets(ContextsBuilder(). + add("foo", 10, 249).add("bar", 60, 150).add("baz", 60, 150, 12.0).build(), f._tlsStatsMap); + TEST_DO(assertFlushContexts("[foo,baz]", targets)); +} TEST_MAIN() { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 33b9d162163..8f19d5c203b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -12,26 +12,48 @@ #include <vespa/searchcore/proton/common/attribute_updater.h> #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/imported_attribute_vector.h> +#include <vespa/searchlib/tensor/prepare_result.h> #include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <future> #include <vespa/log/log.h> LOG_SETUP(".proton.attribute.attribute_writer"); using namespace document; using namespace search; + +using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId; using search::attribute::ImportedAttributeVector; +using search::tensor::PrepareResult; using vespalib::ISequencedTaskExecutor; -using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId; namespace proton { using LidVector = LidVectorContext::LidVector; +namespace { + +bool +use_two_phase_put_for_attribute(const AttributeVector& attr) +{ + const auto& cfg = attr.getConfig(); + if (cfg.basicType() == search::attribute::BasicType::Type::TENSOR && + cfg.hnsw_index_params().has_value() && + cfg.hnsw_index_params().value().allow_multi_threaded_indexing()) + { + return true; + } + return false; +} + +} + AttributeWriter::WriteField::WriteField(AttributeVector &attribute) : _fieldPath(), _attribute(attribute), - _structFieldAttribute(false) + _structFieldAttribute(false), + _use_two_phase_put(use_two_phase_put_for_attribute(attribute)) { const vespalib::string &name = attribute.getName(); _structFieldAttribute = attribute::isStructFieldAttribute(name); @@ -57,11 +79,11 @@ AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType) AttributeWriter::WriteContext::WriteContext(ExecutorId executorId) : _executorId(executorId), _fields(), - _hasStructFieldAttribute(false) + _hasStructFieldAttribute(false), + _use_two_phase_put(false) { } - AttributeWriter::WriteContext::WriteContext(WriteContext &&rhs) noexcept = default; AttributeWriter::WriteContext::~WriteContext() = default; @@ -75,6 +97,13 @@ AttributeWriter::WriteContext::add(AttributeVector &attr) if (_fields.back().isStructFieldAttribute()) { _hasStructFieldAttribute = true; } + if (_fields.back().use_two_phase_put()) { + // Only support for one field per context when this is true. + assert(_fields.size() == 1); + _use_two_phase_put = true; + } else { + assert(!_use_two_phase_put); + } } void @@ -113,6 +142,27 @@ applyPutToAttribute(SerialNum serialNum, const FieldValue::UP &fieldValue, Docum } void +complete_put_to_attribute(SerialNum serial_num, + uint32_t docid, + AttributeVector& attr, + const FieldValue::SP& field_value, + std::future<std::unique_ptr<PrepareResult>>& result_future, + bool immediate_commit, + AttributeWriter::OnWriteDoneType) +{ + ensureLidSpace(serial_num, docid, attr); + if (field_value.get()) { + auto result = result_future.get(); + AttributeUpdater::complete_set_value(attr, docid, *field_value, std::move(result)); + } else { + attr.clearDoc(docid); + } + if (immediate_commit) { + attr.commit(serial_num, serial_num); + } +} + +void applyRemoveToAttribute(SerialNum serialNum, DocumentIdT lid, bool immediateCommit, AttributeVector &attr, AttributeWriter::OnWriteDoneType) { @@ -148,7 +198,6 @@ applyReplayDone(uint32_t docIdLimit, AttributeVector &attr) attr.shrinkLidSpace(); } - void applyHeartBeat(SerialNum serialNum, AttributeVector &attr) { @@ -166,7 +215,6 @@ applyCommit(SerialNum serialNum, AttributeWriter::OnWriteDoneType , AttributeVec } } - void applyCompactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum, AttributeVector &attr) { @@ -208,7 +256,6 @@ struct BatchUpdateTask : public vespalib::Executor::Task { } } - SerialNum _serialNum; DocumentIdT _lid; bool _immediateCommit; @@ -221,6 +268,7 @@ class FieldContext vespalib::string _name; ExecutorId _executorId; AttributeVector *_attr; + bool _use_two_phase_put; public: FieldContext(ISequencedTaskExecutor &writer, AttributeVector *attr); @@ -228,13 +276,14 @@ public: bool operator<(const FieldContext &rhs) const; ExecutorId getExecutorId() const { return _executorId; } AttributeVector *getAttribute() const { return _attr; } + bool use_two_phase_put() const { return _use_two_phase_put; } }; - FieldContext::FieldContext(ISequencedTaskExecutor &writer, AttributeVector *attr) : _name(attr->getName()), _executorId(writer.getExecutorId(attr->getNamePrefix())), - _attr(attr) + _attr(attr), + _use_two_phase_put(use_two_phase_put_for_attribute(*attr)) { } @@ -303,6 +352,100 @@ PutTask::run() } } + +class PreparePutTask : public vespalib::Executor::Task { +private: + const SerialNum _serial_num; + const uint32_t _docid; + AttributeVector& _attr; + FieldValue::SP _field_value; + std::promise<std::unique_ptr<PrepareResult>> _result_promise; + +public: + PreparePutTask(SerialNum serial_num_in, + uint32_t docid_in, + const AttributeWriter::WriteField& field, + std::shared_ptr<DocumentFieldExtractor> field_extractor); + ~PreparePutTask() override; + void run() override; + SerialNum serial_num() const { return _serial_num; } + uint32_t docid() const { return _docid; } + AttributeVector& attr() { return _attr; } + FieldValue::SP field_value() { return _field_value; } + std::future<std::unique_ptr<PrepareResult>> result_future() { + return _result_promise.get_future(); + } +}; + +PreparePutTask::PreparePutTask(SerialNum serial_num_in, + uint32_t docid_in, + const AttributeWriter::WriteField& field, + std::shared_ptr<DocumentFieldExtractor> field_extractor) + : _serial_num(serial_num_in), + _docid(docid_in), + _attr(field.getAttribute()), + _field_value(), + _result_promise() +{ + // Note: No need to store the field extractor as we are not extracting struct fields. + auto value = field_extractor->getFieldValue(field.getFieldPath()); + _field_value.reset(value.release()); +} + +PreparePutTask::~PreparePutTask() = default; + +void +PreparePutTask::run() +{ + if (_attr.getStatus().getLastSyncToken() < _serial_num) { + if (_field_value.get()) { + _result_promise.set_value(AttributeUpdater::prepare_set_value(_attr, _docid, *_field_value)); + } + } +} + +class CompletePutTask : public vespalib::Executor::Task { +private: + const SerialNum _serial_num; + const uint32_t _docid; + AttributeVector& _attr; + FieldValue::SP _field_value; + std::future<std::unique_ptr<PrepareResult>> _result_future; + const bool _immediate_commit; + std::remove_reference_t<AttributeWriter::OnWriteDoneType> _on_write_done; + +public: + CompletePutTask(PreparePutTask& prepare_task, + bool immediate_commit, + AttributeWriter::OnWriteDoneType on_write_done); + ~CompletePutTask() override; + void run() override; +}; + +CompletePutTask::CompletePutTask(PreparePutTask& prepare_task, + bool immediate_commit, + AttributeWriter::OnWriteDoneType on_write_done) + : _serial_num(prepare_task.serial_num()), + _docid(prepare_task.docid()), + _attr(prepare_task.attr()), + _field_value(prepare_task.field_value()), + _result_future(prepare_task.result_future()), + _immediate_commit(immediate_commit), + _on_write_done(on_write_done) +{ +} + +CompletePutTask::~CompletePutTask() = default; + +void +CompletePutTask::run() +{ + if (_attr.getStatus().getLastSyncToken() < _serial_num) { + complete_put_to_attribute(_serial_num, _docid, _attr, _field_value, _result_future, + _immediate_commit, _on_write_done); + } +} + class RemoveTask : public vespalib::Executor::Task { const AttributeWriter::WriteContext &_wc; @@ -316,7 +459,6 @@ public: void run() override; }; - RemoveTask::RemoveTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, uint32_t lid, bool immediateCommit, AttributeWriter::OnWriteDoneType onWriteDone) : _wc(wc), _serialNum(serialNum), @@ -419,13 +561,22 @@ AttributeWriter::setupWriteContexts() fieldContexts.emplace_back(_attributeFieldWriter, attr); } std::sort(fieldContexts.begin(), fieldContexts.end()); - for (auto &fc : fieldContexts) { + for (const auto& fc : fieldContexts) { + if (fc.use_two_phase_put()) { + continue; + } if (_writeContexts.empty() || (_writeContexts.back().getExecutorId() != fc.getExecutorId())) { _writeContexts.emplace_back(fc.getExecutorId()); } _writeContexts.back().add(*fc.getAttribute()); } + for (const auto& fc : fieldContexts) { + if (fc.use_two_phase_put()) { + _writeContexts.emplace_back(fc.getExecutorId()); + _writeContexts.back().add(*fc.getAttribute()); + } + } for (const auto &wc : _writeContexts) { if (wc.hasStructFieldAttribute()) { _hasStructFieldAttribute = true; @@ -452,9 +603,19 @@ AttributeWriter::internalPut(SerialNum serialNum, const Document &doc, DocumentI } auto extractor = std::make_shared<DocumentFieldExtractor>(doc); for (const auto &wc : _writeContexts) { - if (allAttributes || wc.hasStructFieldAttribute()) { - auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, immediateCommit, allAttributes, onWriteDone); - _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask)); + if (wc.use_two_phase_put()) { + assert(wc.getFields().size() == 1); + auto prepare_task = std::make_unique<PreparePutTask>(serialNum, lid, wc.getFields()[0], extractor); + auto complete_task = std::make_unique<CompletePutTask>(*prepare_task, immediateCommit, onWriteDone); + // We use the local docid to create an executor id to round-robin between the threads. + _attributeFieldWriter.executeTask(_attributeFieldWriter.getExecutorId(lid), std::move(prepare_task)); + _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(complete_task)); + } else { + if (allAttributes || wc.hasStructFieldAttribute()) { + auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, immediateCommit, allAttributes, + onWriteDone); + _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask)); + } } } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index 4a9726dd113..726379220e3 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -19,20 +19,23 @@ namespace proton { class AttributeWriter : public IAttributeWriter { private: - typedef search::AttributeVector AttributeVector; - typedef document::FieldPath FieldPath; - typedef document::DataType DataType; - typedef document::DocumentType DocumentType; - typedef document::FieldValue FieldValue; + using AttributeVector = search::AttributeVector; + using FieldPath = document::FieldPath; + using DataType = document::DataType; + using DocumentType = document::DocumentType; + using FieldValue = document::FieldValue; const IAttributeManager::SP _mgr; vespalib::ISequencedTaskExecutor &_attributeFieldWriter; using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId; public: - class WriteField - { + /** + * Represents an attribute vector for a field and details about how to write to it. + */ + class WriteField { FieldPath _fieldPath; AttributeVector &_attribute; bool _structFieldAttribute; // in array/map of struct + bool _use_two_phase_put; public: WriteField(AttributeVector &attribute); ~WriteField(); @@ -40,12 +43,18 @@ public: const FieldPath &getFieldPath() const { return _fieldPath; } void buildFieldPath(const DocumentType &docType); bool isStructFieldAttribute() const { return _structFieldAttribute; } + bool use_two_phase_put() const { return _use_two_phase_put; } }; - class WriteContext - { + + /** + * Represents a set of fields (as attributes) that are handled by the same write thread. + */ + class WriteContext { ExecutorId _executorId; std::vector<WriteField> _fields; bool _hasStructFieldAttribute; + // When this is true, the context only contains a single field. + bool _use_two_phase_put; public: WriteContext(ExecutorId executorId); WriteContext(WriteContext &&rhs) noexcept; @@ -56,6 +65,7 @@ public: ExecutorId getExecutorId() const { return _executorId; } const std::vector<WriteField> &getFields() const { return _fields; } bool hasStructFieldAttribute() const { return _hasStructFieldAttribute; } + bool use_two_phase_put() const { return _use_two_phase_put; } }; private: using AttrWithId = std::pair<search::AttributeVector *, ExecutorId>; @@ -103,6 +113,11 @@ public: void onReplayDone(uint32_t docIdLimit) override; bool hasStructFieldAttribute() const override; + + // Should only be used for unit testing. + const std::vector<WriteContext>& get_write_contexts() const { + return _writeContexts; + } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index 0a61ec8d882..2b5f4b028dc 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -165,9 +165,15 @@ FlushableAttribute::FlushableAttribute(const AttributeVectorSP attr, _fileHeaderContext(fileHeaderContext), _attributeFieldWriter(attributeFieldWriter), _hwInfo(hwInfo), - _attrDir(attrDir) + _attrDir(attrDir), + _replay_operation_cost(0.0) { _lastStats.setPathElementsToLog(8); + auto &config = attr->getConfig(); + if (config.basicType() == search::attribute::BasicType::Type::TENSOR && + config.tensorType().is_tensor() && config.tensorType().is_dense() && config.hnsw_index_params().has_value()) { + _replay_operation_cost = 100.0; // replaying operations to hnsw index is 100 times more expensive than reading from tls + } } @@ -236,4 +242,10 @@ FlushableAttribute::getApproxBytesToWriteToDisk() const return _attr->getEstimatedSaveByteSize(); } +double +FlushableAttribute::get_replay_operation_cost() const +{ + return _replay_operation_cost; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h index 8d807c153c0..a759bcce26e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h @@ -38,6 +38,7 @@ private: vespalib::ISequencedTaskExecutor &_attributeFieldWriter; HwInfo _hwInfo; std::shared_ptr<AttributeDirectory> _attrDir; + double _replay_operation_cost; Task::UP internalInitFlush(SerialNum currentSerial); @@ -71,6 +72,7 @@ public: virtual Task::UP initFlush(SerialNum currentSerial) override; virtual FlushStats getLastFlushStats() const override { return _lastStats; } virtual uint64_t getApproxBytesToWriteToDisk() const override; + virtual double get_replay_operation_cost() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index 7e1b1fb1e9a..4270006e301 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -27,6 +27,6 @@ vespa_add_library(searchcore_pcommon STATIC ${VESPA_STDCXX_FS_LIB} ) -if(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.2") +if(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.2" OR VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8") set_source_files_properties(hw_info_sampler.cpp PROPERTIES COMPILE_FLAGS -DRHEL_8_2_KLUDGE) endif() diff --git a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp index 8fd47c17acb..d7cf6caff28 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp @@ -31,6 +31,7 @@ LOG_SETUP(".proton.common.attribute_updater"); using namespace document; using vespalib::make_string; +using search::tensor::PrepareResult; using search::tensor::TensorAttribute; using search::attribute::ReferenceAttribute; @@ -471,27 +472,33 @@ AttributeUpdater::updateValue(StringAttribute & vec, uint32_t lid, const FieldVa } } +namespace { + +template <typename ExpFieldValueType> void -AttributeUpdater::updateValue(PredicateAttribute &vec, uint32_t lid, const FieldValue &val) +validate_field_value_type(const FieldValue& val, const vespalib::string& attr_type, const vespalib::string& value_type) { - if (!val.inherits(PredicateFieldValue::classId)) { + if (!val.inherits(ExpFieldValueType::classId)) { throw UpdateException( - make_string("PredicateAttribute must be updated with " - "PredicateFieldValues.")); + make_string("%s must be updated with %s, but was '%s'", + attr_type.c_str(), value_type.c_str(), val.toString(false).c_str())); } +} + +} + +void +AttributeUpdater::updateValue(PredicateAttribute &vec, uint32_t lid, const FieldValue &val) +{ + validate_field_value_type<PredicateFieldValue>(val, "PredicateAttribute", "PredicateFieldValue"); vec.updateValue(lid, static_cast<const PredicateFieldValue &>(val)); } void AttributeUpdater::updateValue(TensorAttribute &vec, uint32_t lid, const FieldValue &val) { - if (!val.inherits(TensorFieldValue::classId)) { - throw UpdateException( - make_string("TensorAttribute must be updated with " - "TensorFieldValues.")); - } - const auto &tensor = static_cast<const TensorFieldValue &>(val). - getAsTensorPtr(); + validate_field_value_type<TensorFieldValue>(val, "TensorAttribute", "TensorFieldValue"); + const auto &tensor = static_cast<const TensorFieldValue &>(val).getAsTensorPtr(); if (tensor) { vec.setTensor(lid, *tensor); } else { @@ -506,7 +513,7 @@ AttributeUpdater::updateValue(ReferenceAttribute &vec, uint32_t lid, const Field vec.clearDoc(lid); throw UpdateException( make_string("ReferenceAttribute must be updated with " - "ReferenceFieldValues.")); + "ReferenceFieldValue, but was '%s'", val.toString(false).c_str())); } const auto &reffv = static_cast<const ReferenceFieldValue &>(val); if (reffv.hasValidDocumentId()) { @@ -516,4 +523,57 @@ AttributeUpdater::updateValue(ReferenceAttribute &vec, uint32_t lid, const Field } } +namespace { + +void +validate_tensor_attribute_type(AttributeVector& attr) +{ + const auto& info = attr.getClass(); + if (!info.inherits(TensorAttribute::classId)) { + throw UpdateException( + make_string("Expected attribute vector '%s' to be a TensorAttribute, but was '%s'", + attr.getName().c_str(), info.name())); + } +} + +std::unique_ptr<PrepareResult> +prepare_set_tensor(TensorAttribute& attr, uint32_t docid, const FieldValue& val) +{ + validate_field_value_type<TensorFieldValue>(val, "TensorAttribute", "TensorFieldValue"); + const auto& tensor = static_cast<const TensorFieldValue&>(val).getAsTensorPtr(); + if (tensor) { + return attr.prepare_set_tensor(docid, *tensor); + } + return std::unique_ptr<PrepareResult>(); +} + +void +complete_set_tensor(TensorAttribute& attr, uint32_t docid, const FieldValue& val, std::unique_ptr<PrepareResult> prepare_result) +{ + validate_field_value_type<TensorFieldValue>(val, "TensorAttribute", "TensorFieldValue"); + const auto& tensor = static_cast<const TensorFieldValue&>(val).getAsTensorPtr(); + if (tensor) { + attr.complete_set_tensor(docid, *tensor, std::move(prepare_result)); + } else { + attr.clearDoc(docid); + } +} + +} + +std::unique_ptr<PrepareResult> +AttributeUpdater::prepare_set_value(AttributeVector& attr, uint32_t docid, const FieldValue& val) +{ + validate_tensor_attribute_type(attr); + return prepare_set_tensor(static_cast<TensorAttribute&>(attr), docid, val); +} + +void +AttributeUpdater::complete_set_value(AttributeVector& attr, uint32_t docid, const FieldValue& val, + std::unique_ptr<PrepareResult> prepare_result) +{ + validate_tensor_attribute_type(attr); + complete_set_tensor(static_cast<TensorAttribute&>(attr), docid, val, std::move(prepare_result)); +} + } // namespace search diff --git a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h index 01be6299692..32d14f6dd5a 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h +++ b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h @@ -10,7 +10,10 @@ namespace search { class PredicateAttribute; -namespace tensor { class TensorAttribute; } +namespace tensor { + class PrepareResult; + class TensorAttribute; +} namespace attribute {class ReferenceAttribute; } VESPA_DEFINE_EXCEPTION(UpdateException, vespalib::Exception); @@ -20,14 +23,18 @@ VESPA_DEFINE_EXCEPTION(UpdateException, vespalib::Exception); */ class AttributeUpdater { using Field = document::Field; - using FieldValue = document::FieldValue; using FieldUpdate = document::FieldUpdate; + using FieldValue = document::FieldValue; using ValueUpdate = document::ValueUpdate; public: static void handleUpdate(AttributeVector & vec, uint32_t lid, const FieldUpdate & upd); static void handleValue(AttributeVector & vec, uint32_t lid, const FieldValue & val); + static std::unique_ptr<tensor::PrepareResult> prepare_set_value(AttributeVector& attr, uint32_t docid, const FieldValue& val); + static void complete_set_value(AttributeVector& attr, uint32_t docid, const FieldValue& val, + std::unique_ptr<tensor::PrepareResult> prepare_result); + private: template <typename V> static void handleUpdate(V & vec, uint32_t lid, const ValueUpdate & upd); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt index ecd90d8a992..340007f4513 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_library(searchcore_flushengine STATIC flushcontext.cpp flushengine.cpp flush_engine_explorer.cpp + flush_target_candidate.cpp flush_target_candidates.cpp flushtargetproxy.cpp flushtask.cpp diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.cpp new file mode 100644 index 00000000000..6be6d31372a --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.cpp @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "flush_target_candidate.h" +#include "flushcontext.h" + +namespace proton { + +FlushTargetCandidate::FlushTargetCandidate(std::shared_ptr<FlushContext> flush_context, search::SerialNum current_serial, const Config &cfg) + : _flush_context(std::move(flush_context)), + _replay_operation_cost(_flush_context->getTarget()->get_replay_operation_cost() * cfg.tlsReplayOperationCost), + _flushed_serial(_flush_context->getTarget()->getFlushedSerialNum()), + _current_serial(current_serial), + _replay_cost(_replay_operation_cost * (_current_serial - _flushed_serial)), + _approx_bytes_to_write_to_disk(_flush_context->getTarget()->getApproxBytesToWriteToDisk()), + _write_cost(_approx_bytes_to_write_to_disk * cfg.flushTargetWriteCost), + _always_flush(_replay_cost >= _write_cost) +{ +} + +FlushTargetCandidate::~FlushTargetCandidate() = default; + +} diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.h b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.h new file mode 100644 index 00000000000..5920fff6942 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.h @@ -0,0 +1,37 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "prepare_restart_flush_strategy.h" +#include <memory> +#include <vespa/searchlib/common/serialnum.h> + +namespace proton { + +class FlushContext; + +/** + * Class describing a flush target candidate for the prepare restart flush strategy. + */ +class FlushTargetCandidate +{ + std::shared_ptr<FlushContext> _flush_context; + double _replay_operation_cost; + search::SerialNum _flushed_serial; + search::SerialNum _current_serial; + double _replay_cost; + uint64_t _approx_bytes_to_write_to_disk; + double _write_cost; + bool _always_flush; + + using Config = PrepareRestartFlushStrategy::Config; +public: + FlushTargetCandidate(std::shared_ptr<FlushContext> flush_context, search::SerialNum current_serial, const Config &cfg); + ~FlushTargetCandidate(); + const std::shared_ptr<FlushContext> &get_flush_context() const { return _flush_context; } + search::SerialNum get_flushed_serial() const { return _flushed_serial; } + double get_write_cost() const { return _write_cost; } + bool get_always_flush() const { return _always_flush; } +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp index 0051c209ef9..f71da453559 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "flush_target_candidates.h" +#include "flush_target_candidate.h" #include "tls_stats.h" namespace proton { @@ -13,17 +14,17 @@ using TlsReplayCost = FlushTargetCandidates::TlsReplayCost; namespace { SerialNum -calculateReplayStartSerial(const FlushContext::List &sortedFlushContexts, - size_t numCandidates, +calculateReplayStartSerial(vespalib::ConstArrayRef<FlushTargetCandidate> candidates, + size_t num_candidates, const flushengine::TlsStats &tlsStats) { - if (numCandidates == 0) { + if (num_candidates == 0) { return tlsStats.getFirstSerial(); } - if (numCandidates == sortedFlushContexts.size()) { + if (num_candidates == candidates.size()) { return tlsStats.getLastSerial() + 1; } - return sortedFlushContexts[numCandidates]->getTarget()->getFlushedSerialNum() + 1; + return candidates[num_candidates].get_flushed_serial() + 1; } TlsReplayCost @@ -44,43 +45,44 @@ calculateTlsReplayCost(const flushengine::TlsStats &tlsStats, } double -calculateFlushTargetsWriteCost(const FlushContext::List &sortedFlushContexts, - size_t numCandidates, - const Config &cfg) +calculateFlushTargetsWriteCost(vespalib::ConstArrayRef<FlushTargetCandidate> candidates, + size_t num_candidates) { double result = 0; - for (size_t i = 0; i < numCandidates; ++i) { - const auto &flushContext = sortedFlushContexts[i]; - result += (flushContext->getTarget()->getApproxBytesToWriteToDisk() * - cfg.flushTargetWriteCost); + for (size_t i = 0; i < num_candidates; ++i) { + result += candidates[i].get_write_cost(); } return result; } } -FlushTargetCandidates::FlushTargetCandidates(const FlushContext::List &sortedFlushContexts, - size_t numCandidates, +FlushTargetCandidates::FlushTargetCandidates(vespalib::ConstArrayRef<FlushTargetCandidate> candidates, + size_t num_candidates, const flushengine::TlsStats &tlsStats, const Config &cfg) - : _sortedFlushContexts(&sortedFlushContexts), - _numCandidates(numCandidates), + : _candidates(candidates), + _num_candidates(std::min(num_candidates, _candidates.size())), _tlsReplayCost(calculateTlsReplayCost(tlsStats, cfg, - calculateReplayStartSerial(sortedFlushContexts, - numCandidates, + calculateReplayStartSerial(_candidates, + _num_candidates, tlsStats))), - _flushTargetsWriteCost(calculateFlushTargetsWriteCost(sortedFlushContexts, - numCandidates, - cfg)) + _flushTargetsWriteCost(calculateFlushTargetsWriteCost(_candidates, + _num_candidates)) { } FlushContext::List FlushTargetCandidates::getCandidates() const { - FlushContext::List result(_sortedFlushContexts->begin(), - _sortedFlushContexts->begin() + _numCandidates); + FlushContext::List result; + result.reserve(_num_candidates); + for (const auto &candidate : _candidates) { + if (result.size() < _num_candidates || candidate.get_always_flush()) { + result.emplace_back(candidate.get_flush_context()); + } + } return result; } diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h index ea09989de31..2979173331c 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h @@ -2,11 +2,14 @@ #pragma once #include "prepare_restart_flush_strategy.h" +#include <vespa/vespalib/util/arrayref.h> namespace proton { namespace flushengine { class TlsStats; } +class FlushTargetCandidate; + /** * A set of flush targets that are candidates to be flushed. * @@ -27,8 +30,8 @@ public: double totalCost() const { return bytesCost + operationsCost; } }; private: - const FlushContext::List *_sortedFlushContexts; // NOTE: ownership is handled outside - size_t _numCandidates; + vespalib::ConstArrayRef<FlushTargetCandidate> _candidates; // NOTE: ownership is handled outside + size_t _num_candidates; TlsReplayCost _tlsReplayCost; double _flushTargetsWriteCost; @@ -37,8 +40,8 @@ private: public: using UP = std::unique_ptr<FlushTargetCandidates>; - FlushTargetCandidates(const FlushContext::List &sortedFlushContexts, - size_t numCandidates, + FlushTargetCandidates(vespalib::ConstArrayRef<FlushTargetCandidate> candidates, + size_t num_candidates, const flushengine::TlsStats &tlsStats, const Config &cfg); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp index 6cfb8cb6c3d..21f9c8465b0 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp @@ -2,6 +2,7 @@ #include "prepare_restart_flush_strategy.h" #include "flush_target_candidates.h" +#include "flush_target_candidate.h" #include "tls_stats_map.h" #include <sstream> #include <algorithm> @@ -70,17 +71,15 @@ flatten(const FlushContextsMap &flushContextsPerHandler) } void -sortByOldestFlushedSerialNumber(FlushContext::List &flushContexts) +sortByOldestFlushedSerialNumber(std::vector<FlushTargetCandidate>& candidates) { - std::sort(flushContexts.begin(), flushContexts.end(), - [](const auto &lhs, const auto &rhs) { - if (lhs->getTarget()->getFlushedSerialNum() == - rhs->getTarget()->getFlushedSerialNum()) { - return lhs->getName() < rhs->getName(); - } - return lhs->getTarget()->getFlushedSerialNum() < - rhs->getTarget()->getFlushedSerialNum(); - }); + std::sort(candidates.begin(), candidates.end(), + [](const auto &lhs, const auto &rhs) { + if (lhs.get_flushed_serial() == rhs.get_flushed_serial()) { + return lhs.get_flush_context()->getName() < rhs.get_flush_context()->getName(); + } + return lhs.get_flushed_serial() < rhs.get_flushed_serial(); + }); } vespalib::string @@ -103,12 +102,16 @@ findBestTargetsToFlush(const FlushContext::List &unsortedFlushContexts, const flushengine::TlsStats &tlsStats, const Config &cfg) { - FlushContext::List sortedFlushContexts = unsortedFlushContexts; - sortByOldestFlushedSerialNumber(sortedFlushContexts); + std::vector<FlushTargetCandidate> candidates; + candidates.reserve(unsortedFlushContexts.size()); + for (const auto &flush_context : unsortedFlushContexts) { + candidates.emplace_back(flush_context, tlsStats.getLastSerial(), cfg); + } + sortByOldestFlushedSerialNumber(candidates); - FlushTargetCandidates bestSet(sortedFlushContexts, 0, tlsStats, cfg); - for (size_t numCandidates = 1; numCandidates <= sortedFlushContexts.size(); ++numCandidates) { - FlushTargetCandidates nextSet(sortedFlushContexts, numCandidates, tlsStats, cfg); + FlushTargetCandidates bestSet(candidates, 0, tlsStats, cfg); + for (size_t numCandidates = 1; numCandidates <= candidates.size(); ++numCandidates) { + FlushTargetCandidates nextSet(candidates, numCandidates, tlsStats, cfg); LOG(debug, "findBestTargetsToFlush(): Created candidate set: " "flushTargets=[%s], tlsReplayBytesCost=%f, tlsReplayOperationsCost=%f, flushTargetsWriteCost=%f, totalCost=%f", toString(nextSet.getCandidates()).c_str(), diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 8fd686e235d..5213a2b9230 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -213,6 +213,8 @@ Query::handle_global_filters(uint32_t docid_limit) // optimized order may change after accounting for global filter: _blueprint = Blueprint::optimize(std::move(_blueprint)); LOG(debug, "blueprint after handle_global_filters:\n%s\n", _blueprint->asString().c_str()); + // strictness may change if optimized order changed: + fetchPostings(); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 719ba359ccf..962ee65c10d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -302,8 +302,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _warmupExecutor = std::make_unique<vespalib::ThreadStackExecutor>(4, 128*1024, index_warmup_executor); const size_t sharedThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu()); - _sharedExecutor = std::make_unique<vespalib::BlockingThreadStackExecutor>(sharedThreads, 128*1024, sharedThreads*16, proton_shared_executor); - _compile_cache_executor_binding = vespalib::eval::CompileCache::bind(*_sharedExecutor); + _sharedExecutor = std::make_shared<vespalib::BlockingThreadStackExecutor>(sharedThreads, 128*1024, sharedThreads*16, proton_shared_executor); + _compile_cache_executor_binding = vespalib::eval::CompileCache::bind(_sharedExecutor); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(protonConfig.initialize.threads, 128 * 1024, initialize_executor); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index d0b76bd5804..d5c1a8b7b78 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -112,7 +112,7 @@ private: ProtonConfigurer _protonConfigurer; ProtonConfigFetcher _protonConfigFetcher; std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor; - std::unique_ptr<vespalib::ThreadStackExecutorBase> _sharedExecutor; + std::shared_ptr<vespalib::ThreadStackExecutorBase> _sharedExecutor; vespalib::eval::CompileCache::ExecutorBinding::UP _compile_cache_executor_binding; matching::QueryLimiter _queryLimiter; vespalib::Clock _clock; diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index 3e49bb449ff..8e5d3018532 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -11,16 +11,26 @@ namespace proton::test { class MockAttributeManager : public IAttributeManager { private: search::attribute::test::MockAttributeManager _mock; + std::vector<search::AttributeVector*> _writables; std::unique_ptr<ImportedAttributesRepo> _importedAttributes; + vespalib::ISequencedTaskExecutor* _writer; public: MockAttributeManager() : _mock(), - _importedAttributes() + _writables(), + _importedAttributes(), + _writer() {} - void addAttribute(const vespalib::string &name, const search::AttributeVector::SP &attr) { + search::AttributeVector::SP addAttribute(const vespalib::string &name, const search::AttributeVector::SP &attr) { _mock.addAttribute(name, attr); + _writables.push_back(attr.get()); + return attr; + } + + void set_writer(vespalib::ISequencedTaskExecutor& writer) { + _writer = &writer; } search::AttributeGuard::UP getAttribute(const vespalib::string &name) const override { @@ -56,13 +66,18 @@ public: HDR_ABORT("should not be reached"); } vespalib::ISequencedTaskExecutor &getAttributeFieldWriter() const override { - HDR_ABORT("should not be reached"); - } - search::AttributeVector *getWritableAttribute(const vespalib::string &) const override { + assert(_writer != nullptr); + return *_writer; + } + search::AttributeVector *getWritableAttribute(const vespalib::string &name) const override { + auto attr = getAttribute(name); + if (attr) { + return attr->get(); + } return nullptr; } const std::vector<search::AttributeVector *> &getWritableAttributes() const override { - HDR_ABORT("should not be reached"); + return _writables; } void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor>) const override { } diff --git a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h b/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h index 03d9ba8d55c..10ed19a7244 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h +++ b/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h @@ -137,6 +137,11 @@ public: virtual uint64_t getApproxBytesToWriteToDisk() const = 0; /** + * Return cost of replaying a feed operation relative to cost of reading a feed operation from tls. + */ + virtual double get_replay_operation_cost() const { return 0.0; } + + /** * Returns the last serial number for the transaction applied to * target before it was flushed to disk. The transaction log can * not be pruned beyond this. diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index e9e2087e9d1..c5dd468e4fd 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -100,6 +100,7 @@ vespa_define_module( src/tests/attribute/tensorattribute src/tests/bitcompression/expgolomb src/tests/bitvector + src/tests/btree src/tests/bytecomplens src/tests/common/bitvector src/tests/common/location diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 74e67fb7fcd..7a4c6c9e56a 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -158,6 +158,15 @@ public: auto vector = _vectors.get_vector(docid).typify<double>(); _adds.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); } + std::unique_ptr<search::tensor::PrepareResult> prepare_add_document(uint32_t, + vespalib::tensor::TypedCells, + vespalib::GenerationHandler::Guard) const override { + return std::unique_ptr<search::tensor::PrepareResult>(); + } + void complete_add_document(uint32_t docid, + std::unique_ptr<search::tensor::PrepareResult>) override { + add_document(docid); + } void remove_document(uint32_t docid) override { auto vector = _vectors.get_vector(docid).typify<double>(); _removes.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); diff --git a/searchlib/src/tests/btree/.gitignore b/searchlib/src/tests/btree/.gitignore new file mode 100644 index 00000000000..ec4090e3658 --- /dev/null +++ b/searchlib/src/tests/btree/.gitignore @@ -0,0 +1 @@ +searchlib_scanspeed_app diff --git a/searchlib/src/tests/btree/CMakeLists.txt b/searchlib/src/tests/btree/CMakeLists.txt new file mode 100644 index 00000000000..ff396144c52 --- /dev/null +++ b/searchlib/src/tests/btree/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_scanspeed_app + SOURCES + scanspeed.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_scanspeed_app COMMAND vespalib_scanspeed_app BENCHMARK) diff --git a/searchlib/src/tests/btree/scanspeed.cpp b/searchlib/src/tests/btree/scanspeed.cpp new file mode 100644 index 00000000000..1474edd6b0b --- /dev/null +++ b/searchlib/src/tests/btree/scanspeed.cpp @@ -0,0 +1,181 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/btree/btreeroot.h> +#include <vespa/vespalib/btree/btreebuilder.h> +#include <vespa/vespalib/btree/btreenodeallocator.h> +#include <vespa/vespalib/btree/btree.h> +#include <vespa/vespalib/btree/btreestore.h> +#include <vespa/vespalib/btree/btreenodeallocator.hpp> +#include <vespa/vespalib/btree/btreenode.hpp> +#include <vespa/vespalib/btree/btreenodestore.hpp> +#include <vespa/vespalib/btree/btreeiterator.hpp> +#include <vespa/vespalib/btree/btreeroot.hpp> +#include <vespa/vespalib/btree/btreebuilder.hpp> +#include <vespa/vespalib/btree/btree.hpp> +#include <vespa/vespalib/btree/btreestore.hpp> +#include <vespa/vespalib/util/time.h> +#include <vespa/searchlib/common/bitvector.h> + +#include <vespa/fastos/app.h> + +using vespalib::btree::BTree; +using vespalib::btree::BTreeNode; +using vespalib::btree::BTreeTraits; + +enum class ScanMethod +{ + ITERATOR, + FUNCTOR +}; + +class ScanSpeed : public FastOS_Application +{ + template <typename Traits> + void work_loop(ScanMethod scan_method); + int Main() override; +}; + + +namespace { + +const char *scan_method_name(ScanMethod scan_method) +{ + switch (scan_method) { + case ScanMethod::ITERATOR: + return "iterator"; + default: + return "functor"; + } +} + +class ScanOnce { +public: + virtual ~ScanOnce() = default; + virtual void operator()(search::BitVector &bv) = 0; +}; + +template <typename Tree> +class ScanTree : public ScanOnce { +protected: + const Tree &_tree; + int _startval; + int _endval; +public: + ScanTree(const Tree &tree, int startval, int endval) + : _tree(tree), + _startval(startval), + _endval(endval) + { + } + ~ScanTree() override { } +}; + +template <typename Tree> +class ScanWithIterator : public ScanTree<Tree> { +public: + ScanWithIterator(const Tree &tree, int startval, int endval) + : ScanTree<Tree>(tree, startval, endval) + { + } + ~ScanWithIterator() override = default; + void operator()(search::BitVector &bv) override; +}; + +template <typename Tree> +void +ScanWithIterator<Tree>::operator()(search::BitVector &bv) +{ + using ConstIterator = typename Tree::ConstIterator; + ConstIterator itr(BTreeNode::Ref(), this->_tree.getAllocator()); + itr.lower_bound(this->_tree.getRoot(), this->_startval); + while (itr.valid() && itr.getKey() < this->_endval) { + bv.setBit(itr.getKey()); + ++itr; + } +} + +template <typename Tree> +class ScanWithFunctor : public ScanTree<Tree> { + +public: + ScanWithFunctor(const Tree &tree, int startval, int endval) + : ScanTree<Tree>(tree, startval, endval) + { + } + ~ScanWithFunctor() override = default; + void operator()(search::BitVector &bv) override; +}; + +template <typename Tree> +void +ScanWithFunctor<Tree>::operator()(search::BitVector &bv) +{ + using ConstIterator = typename Tree::ConstIterator; + ConstIterator start(BTreeNode::Ref(), this->_tree.getAllocator()); + ConstIterator end(BTreeNode::Ref(), this->_tree.getAllocator()); + start.lower_bound(this->_tree.getRoot(), this->_startval); + end.lower_bound(this->_tree.getRoot(), this->_endval); + start.foreach_key_range(end, [&](int key) { bv.setBit(key); } ); +} + +} + +template <typename Traits> +void +ScanSpeed::work_loop(ScanMethod scan_method) +{ + vespalib::GenerationHandler g; + using Tree = BTree<int, int, vespalib::btree::NoAggregated, std::less<int>, Traits>; + using Builder = typename Tree::Builder; + Tree tree; + Builder builder(tree.getAllocator()); + size_t numEntries = 1000000; + size_t numInnerLoops = 1000; + for (size_t i = 0; i < numEntries; ++i) { + builder.insert(i, 0); + } + tree.assign(builder); + assert(numEntries == tree.size()); + assert(tree.isValid()); + std::unique_ptr<ScanOnce> scan_once; + if (scan_method == ScanMethod::ITERATOR) { + scan_once = std::make_unique<ScanWithIterator<Tree>>(tree, 4, numEntries - 4); + } else { + scan_once = std::make_unique<ScanWithFunctor<Tree>>(tree, 4, numEntries - 4); + } + auto bv = search::BitVector::create(numEntries); + vespalib::Timer timer; + for (size_t innerl = 0; innerl < numInnerLoops; ++innerl) { + (*scan_once)(*bv); + } + double used = vespalib::to_s(timer.elapsed()); + printf("Elapsed time for scanning %ld entries is %8.5f, " + "scanmethod=%s, fanout=%u,%u\n", + numEntries * numInnerLoops, + used, + scan_method_name(scan_method), + static_cast<int>(Traits::LEAF_SLOTS), + static_cast<int>(Traits::INTERNAL_SLOTS)); + fflush(stdout); +} + + +int +ScanSpeed::Main() +{ + using SmallTraits = BTreeTraits<4, 4, 31, false>; + using DefTraits = vespalib::btree::BTreeDefaultTraits; + using LargeTraits = BTreeTraits<32, 16, 10, true>; + using HugeTraits = BTreeTraits<64, 16, 10, true>; + work_loop<SmallTraits>(ScanMethod::ITERATOR); + work_loop<DefTraits>(ScanMethod::ITERATOR); + work_loop<LargeTraits>(ScanMethod::ITERATOR); + work_loop<HugeTraits>(ScanMethod::ITERATOR); + work_loop<SmallTraits>(ScanMethod::FUNCTOR); + work_loop<DefTraits>(ScanMethod::FUNCTOR); + work_loop<LargeTraits>(ScanMethod::FUNCTOR); + work_loop<HugeTraits>(ScanMethod::FUNCTOR); + return 0; +} + +FASTOS_MAIN(ScanSpeed); diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 04f2076121f..33ea0f2df5b 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -521,15 +521,31 @@ TEST_F(HnswIndexTest, shrink_called_heuristic) TEST(LevelGeneratorTest, gives_various_levels) { InvLogLevelGenerator generator(4); - EXPECT_EQ(2u, generator.max_level()); - EXPECT_EQ(1u, generator.max_level()); - EXPECT_EQ(0u, generator.max_level()); - EXPECT_EQ(1u, generator.max_level()); - EXPECT_EQ(0u, generator.max_level()); - EXPECT_EQ(1u, generator.max_level()); - EXPECT_EQ(0u, generator.max_level()); - EXPECT_EQ(0u, generator.max_level()); - EXPECT_EQ(0u, generator.max_level()); + std::vector<uint32_t> got_levels(16); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 2, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0 + })); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + })); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0 + })); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1 + })); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 2 + })); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 0, 1, 1, 0, 3, 1, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0 + })); uint32_t left = 1000000; std::vector<uint32_t> hist; @@ -549,4 +565,60 @@ TEST(LevelGeneratorTest, gives_various_levels) EXPECT_TRUE(hist.size() < 14); } +class TwoPhaseTest : public HnswIndexTest { +public: + TwoPhaseTest() : HnswIndexTest() { + init(true); + vectors.set(4, {1, 3}).set(5, {13, 3}).set(6, {7, 13}) + .set(1, {3, 7}).set(2, {7, 1}).set(3, {11, 7}) + .set(7, {6, 5}).set(8, {5, 5}).set(9, {6, 6}); + } + using UP = std::unique_ptr<PrepareResult>; + UP prepare_add(uint32_t docid, uint32_t max_level = 0) { + level_generator->level = max_level; + vespalib::GenerationHandler::Guard dummy; + auto vector = vectors.get_vector(docid); + return index->prepare_add_document(docid, vector, dummy); + } + void complete_add(uint32_t docid, UP up) { + index->complete_add_document(docid, std::move(up)); + commit(); + } +}; + +TEST_F(TwoPhaseTest, two_phase_add) +{ + add_document(1); + add_document(2); + add_document(3); + expect_entry_point(1, 0); + add_document(4, 1); + add_document(5, 1); + add_document(6, 2); + expect_entry_point(6, 2); + + expect_level_0(1, {2,4,6}); + expect_level_0(2, {1,3,4,5}); + expect_level_0(3, {2,5,6}); + + expect_levels(4, {{1,2}, {5,6}}); + expect_levels(5, {{2,3}, {4,6}}); + expect_levels(6, {{1,3}, {4,5}, {}}); + + auto up = prepare_add(7, 1); + // simulate things happening while 7 is in progress: + add_document(8); // added + remove_document(1); // removed + remove_document(5); + vectors.set(5, {8, 14}); // updated and moved + add_document(5, 2); + add_document(9, 1); // added + complete_add(7, std::move(up)); + + // 1 filtered out because it was removed + // TODO: 5 filtered out because it was updated + expect_levels(7, {{2}, {4,5}}); +} + + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp index 7aa78fbe06b..bcc886fccad 100644 --- a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp @@ -49,7 +49,7 @@ void populate(HnswGraph &graph) { graph.set_link_array(6, 0, V{1, 2, 4}); graph.set_link_array(2, 1, V{4}); graph.set_link_array(4, 1, V{2}); - graph.set_entry_node(2, 1); + graph.set_entry_node({2, 1}); } void modify(HnswGraph &graph) { @@ -63,7 +63,7 @@ void modify(HnswGraph &graph) { graph.set_link_array(4, 1, V{7}); graph.set_link_array(7, 1, V{4}); - graph.set_entry_node(4, 1); + graph.set_entry_node({4, 1}); } @@ -110,8 +110,9 @@ public: void expect_copy_as_populated() const { EXPECT_EQ(copy.size(), 7); - EXPECT_EQ(copy.entry_docid, 2); - EXPECT_EQ(copy.entry_level, 1); + auto entry = copy.get_entry_node(); + EXPECT_EQ(entry.docid, 2); + EXPECT_EQ(entry.level, 1); expect_empty_d(0); expect_empty_d(3); diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp index 17df4628606..d1226a07703 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp @@ -110,12 +110,67 @@ AttributePostingListIteratorT<PL>::doSeek(uint32_t docId) } } +namespace { + +template <typename> struct is_tree_iterator; + +template <typename P> +struct is_tree_iterator<DocIdIterator<P>> { + static constexpr bool value = false; +}; + +template <typename P> +struct is_tree_iterator<DocIdMinMaxIterator<P>> { + static constexpr bool value = false; +}; + +template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT> +struct is_tree_iterator<vespalib::btree::BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>> { + static constexpr bool value = true; +}; + +template <typename PL> +inline constexpr bool is_tree_iterator_v = is_tree_iterator<PL>::value; + +template <typename PL> +void get_hits_helper(BitVector& result, PL& iterator, uint32_t end_id) +{ + auto end_itr = iterator; + if (end_itr.valid() && end_itr.getKey() < end_id) { + end_itr.seek(end_id); + } + iterator.foreach_key_range(end_itr, [&](uint32_t key) { result.setBit(key); }); + iterator = end_itr; +} + +template <typename PL> +void or_hits_helper(BitVector& result, PL& iterator, uint32_t end_id) +{ + auto end_itr = iterator; + if (end_itr.valid() && end_itr.getKey() < end_id) { + end_itr.seek(end_id); + } + iterator.foreach_key_range(end_itr, [&](uint32_t key) + { + if (!result.testBit(key)) { + result.setBit(key); + } + }); + iterator = end_itr; +} + +} + template <typename PL> std::unique_ptr<BitVector> AttributePostingListIteratorT<PL>::get_hits(uint32_t begin_id) { BitVector::UP result(BitVector::create(begin_id, getEndId())); - for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { - result->setBit(_iterator.getKey()); + if constexpr (is_tree_iterator_v<PL>) { + get_hits_helper(*result, _iterator, getEndId()); + } else { + for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { + result->setBit(_iterator.getKey()); + } } result->invalidateCachedCount(); return result; @@ -125,9 +180,13 @@ template <typename PL> void AttributePostingListIteratorT<PL>::or_hits_into(BitVector & result, uint32_t begin_id) { (void) begin_id; - for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { - if ( ! result.testBit(_iterator.getKey()) ) { - result.setBit(_iterator.getKey()); + if constexpr (is_tree_iterator_v<PL>) { + or_hits_helper(result, _iterator, getEndId()); + } else { + for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { + if ( ! result.testBit(_iterator.getKey()) ) { + result.setBit(_iterator.getKey()); + } } } result.invalidateCachedCount(); @@ -143,8 +202,12 @@ template <typename PL> std::unique_ptr<BitVector> FilterAttributePostingListIteratorT<PL>::get_hits(uint32_t begin_id) { BitVector::UP result(BitVector::create(begin_id, getEndId())); - for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { - result->setBit(_iterator.getKey()); + if constexpr (is_tree_iterator_v<PL>) { + get_hits_helper(*result, _iterator, getEndId()); + } else { + for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { + result->setBit(_iterator.getKey()); + } } result->invalidateCachedCount(); return result; @@ -154,9 +217,13 @@ template <typename PL> void FilterAttributePostingListIteratorT<PL>::or_hits_into(BitVector & result, uint32_t begin_id) { (void) begin_id; - for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { - if ( ! result.testBit(_iterator.getKey()) ) { - result.setBit(_iterator.getKey()); + if constexpr (is_tree_iterator_v<PL>) { + or_hits_helper(result, _iterator, getEndId()); + } else { + for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { + if ( ! result.testBit(_iterator.getKey()) ) { + result.setBit(_iterator.getKey()); + } } } result.invalidateCachedCount(); diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index 5f49111f77b..bc0d965bcc1 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -44,7 +44,6 @@ ImportedSearchContext::ImportedSearchContext( _target_search_context(_target_attribute.createSearchContext(std::move(term), params)), _targetLids(_reference_attribute.getTargetLids()), _merger(_reference_attribute.getCommittedDocIdLimit()), - _fetchPostingsDone(false), _params(params) { } @@ -239,15 +238,11 @@ ImportedSearchContext::considerAddSearchCacheEntry() } void ImportedSearchContext::fetchPostings(const queryeval::ExecuteInfo &execInfo) { - assert(!_fetchPostingsDone); - _fetchPostingsDone = true; if (!_searchCacheLookup) { _target_search_context->fetchPostings(execInfo); - if (execInfo.isStrict() - || (_target_attribute.getIsFastSearch() && execInfo.hitRate() > 0.01)) - { - makeMergedPostings(_target_attribute.getIsFilter()); - considerAddSearchCacheEntry(); + if (!_merger.merge_done() && (execInfo.isStrict() || (_target_attribute.getIsFastSearch() && execInfo.hitRate() > 0.01))) { + makeMergedPostings(_target_attribute.getIsFilter()); + considerAddSearchCacheEntry(); } } } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index 1c73ac6c8c2..4c3b6a89a14 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -37,7 +37,6 @@ class ImportedSearchContext : public ISearchContext { std::unique_ptr<ISearchContext> _target_search_context; TargetLids _targetLids; PostingListMerger<int32_t> _merger; - bool _fetchPostingsDone; SearchContextParams _params; uint32_t getTargetLid(uint32_t lid) const { diff --git a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h index 6a10ba73951..1c2e6583ad9 100644 --- a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h +++ b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h @@ -62,6 +62,8 @@ public: { if (__builtin_expect(key < limit, true)) { bv.setBit(key); } }); } + bool merge_done() const { return hasArray() || hasBitVector(); } + // Until diversity handling has been rewritten PostingVector &getWritableArray() { return _array; } StartVector &getWritableStartPos() { return _startPos; } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 69450acd98d..8cd7a7064f6 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -107,7 +107,6 @@ protected: * Synthetic posting lists for range search, in array or bitvector form */ PostingListMerger<DataT> _merger; - bool _fetchPostingsDone; static const long MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION = 100; static const long MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 20; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index 4cd8db9010a..09e5a9da5bc 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -25,8 +25,7 @@ PostingListSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uin uint32_t minBvDocFreq, bool useBitVector, const ISearchContext &searchContext) : PostingListSearchContext(dictionary, docIdLimit, numValues, hasWeight, esb, minBvDocFreq, useBitVector, searchContext), _postingList(postingList), - _merger(docIdLimit), - _fetchPostingsDone(false) + _merger(docIdLimit) { } @@ -116,22 +115,18 @@ template <typename DataT> void PostingListSearchContextT<DataT>::fetchPostings(const queryeval::ExecuteInfo & execInfo) { - if (_fetchPostingsDone) return; - - _fetchPostingsDone = true; - - if (_uniqueValues < 2u) return; - - if (execInfo.isStrict() && !fallbackToFiltering()) { - size_t sum(countHits()); - if (sum < _docIdLimit / 64) { - _merger.reserveArray(_uniqueValues, sum); - fillArray(); - } else { - _merger.allocBitVector(); - fillBitVector(); + if (!_merger.merge_done() && _uniqueValues >= 2u) { + if (execInfo.isStrict() && !fallbackToFiltering()) { + size_t sum(countHits()); + if (sum < _docIdLimit / 64) { + _merger.reserveArray(_uniqueValues, sum); + fillArray(); + } else { + _merger.allocBitVector(); + fillBitVector(); + } + _merger.merge(); } - _merger.merge(); } } @@ -141,12 +136,12 @@ void PostingListSearchContextT<DataT>::diversify(bool forward, size_t wanted_hits, const IAttributeVector &diversity_attr, size_t max_per_group, size_t cutoff_groups, bool cutoff_strict) { - assert(!_fetchPostingsDone); - _fetchPostingsDone = true; - _merger.reserveArray(128, wanted_hits); - diversity::diversify(forward, _lowerDictItr, _upperDictItr, _postingList, wanted_hits, diversity_attr, - max_per_group, cutoff_groups, cutoff_strict, _merger.getWritableArray(), _merger.getWritableStartPos()); - _merger.merge(); + if (!_merger.merge_done()) { + _merger.reserveArray(128, wanted_hits); + diversity::diversify(forward, _lowerDictItr, _upperDictItr, _postingList, wanted_hits, diversity_attr, + max_per_group, cutoff_groups, cutoff_strict, _merger.getWritableArray(), _merger.getWritableStartPos()); + _merger.merge(); + } } @@ -155,7 +150,6 @@ SearchIterator::UP PostingListSearchContextT<DataT>:: createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) { - assert(_fetchPostingsDone); if (_uniqueValues == 0u) { return std::make_unique<EmptySearch>(); } diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index 96234e373dc..0a33e23de72 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -167,7 +167,7 @@ BitVector::countInterval(Range range_in) const ++endw; } if (startw < endw) { - res += IAccelrated::getAccelrator().populationCount(bitValues + startw, endw - startw); + res += IAccelrated::getAccelerator().populationCount(bitValues + startw, endw - startw); } if (partialEnd) { res += Optimized::popCount(bitValues[endw] & ~endBits(last)); @@ -185,13 +185,13 @@ BitVector::orWith(const BitVector & right) if (right.size() > 0) { ssize_t commonBytes = numActiveBytes(getStartIndex(), right.size()) - sizeof(Word); if (commonBytes > 0) { - IAccelrated::getAccelrator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); + IAccelrated::getAccelerator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); } Index last(right.size() - 1); getWordIndex(last)[0] |= (right.getWordIndex(last)[0] & ~endBits(last)); } } else { - IAccelrated::getAccelrator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes()); + IAccelrated::getAccelerator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes()); } repairEnds(); invalidateCachedCount(); @@ -216,7 +216,7 @@ BitVector::andWith(const BitVector & right) verifyInclusiveStart(*this, right); uint32_t commonBytes = std::min(getActiveBytes(), numActiveBytes(getStartIndex(), right.size())); - IAccelrated::getAccelrator().andBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); + IAccelrated::getAccelerator().andBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); if (right.size() < size()) { clearInterval(right.size(), size()); } @@ -235,13 +235,13 @@ BitVector::andNotWith(const BitVector& right) if (right.size() > 0) { ssize_t commonBytes = numActiveBytes(getStartIndex(), right.size()) - sizeof(Word); if (commonBytes > 0) { - IAccelrated::getAccelrator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); + IAccelrated::getAccelerator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); } Index last(right.size() - 1); getWordIndex(last)[0] &= ~(right.getWordIndex(last)[0] & ~endBits(last)); } } else { - IAccelrated::getAccelrator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes()); + IAccelrated::getAccelerator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes()); } repairEnds(); @@ -250,7 +250,7 @@ BitVector::andNotWith(const BitVector& right) void BitVector::notSelf() { - IAccelrated::getAccelrator().notBit(getActiveStart(), getActiveBytes()); + IAccelrated::getAccelerator().notBit(getActiveStart(), getActiveBytes()); setGuardBit(); invalidateCachedCount(); } diff --git a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp index 09024505450..8231d0b4cd7 100644 --- a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp @@ -70,10 +70,12 @@ void DiskTermBlueprint::fetchPostings(const queryeval::ExecuteInfo &execInfo) { (void) execInfo; - _hasEquivParent = areAnyParentsEquiv(getParent()); - _bitVector = _diskIndex.readBitVector(*_lookupRes); - if (!_useBitVector || !_bitVector) { - _postingHandle = _diskIndex.readPostingList(*_lookupRes); + if (!_fetchPostingsDone) { + _hasEquivParent = areAnyParentsEquiv(getParent()); + _bitVector = _diskIndex.readBitVector(*_lookupRes); + if (!_useBitVector || !_bitVector) { + _postingHandle = _diskIndex.readPostingList(*_lookupRes); + } } _fetchPostingsDone = true; } diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index a8737a19eec..37fd98c9f20 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -256,7 +256,7 @@ namespace dotproduct::array { template <typename BaseType> DotProductExecutorBase<BaseType>::DotProductExecutorBase(const V & queryVector) : FeatureExecutor(), - _multiplier(IAccelrated::getAccelrator()), + _multiplier(IAccelrated::getAccelerator()), _queryVector(queryVector) { } diff --git a/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp b/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp index a7ad475d6aa..4656a5e9edd 100644 --- a/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp +++ b/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp @@ -10,38 +10,36 @@ using namespace document; namespace search::index { namespace { -TensorDataType tensorDataType(vespalib::eval::ValueType::from_spec("tensor(x{}, y{})")); - -const DataType *convert(Schema::DataType type) { +DataType::Type convert(Schema::DataType type) { switch (type) { case schema::DataType::BOOL: case schema::DataType::UINT2: case schema::DataType::UINT4: case schema::DataType::INT8: - return DataType::BYTE; + return DataType::T_BYTE; case schema::DataType::INT16: - return DataType::SHORT; + return DataType::T_SHORT; case schema::DataType::INT32: - return DataType::INT; + return DataType::T_INT; case schema::DataType::INT64: - return DataType::LONG; + return DataType::T_LONG; case schema::DataType::FLOAT: - return DataType::FLOAT; + return DataType::T_FLOAT; case schema::DataType::DOUBLE: - return DataType::DOUBLE; + return DataType::T_DOUBLE; case schema::DataType::STRING: - return DataType::STRING; + return DataType::T_STRING; case schema::DataType::RAW: - return DataType::RAW; + return DataType::T_RAW; case schema::DataType::BOOLEANTREE: - return DataType::PREDICATE; + return DataType::T_PREDICATE; case schema::DataType::TENSOR: - return &tensorDataType; + return DataType::T_TENSOR; default: break; } assert(!"Unknown datatype in schema"); - return 0; + return DataType::MAX; } void @@ -142,12 +140,12 @@ document::DocumenttypesConfig DocTypeBuilder::makeConfig() const { if (usf != usedFields.end()) { continue; // taken as index field } - const DataType *primitiveType = convert(field.getDataType()); - if (primitiveType->getId() == DataType::T_TENSOR) { - header_struct.addTensorField(field.getName(), dynamic_cast<const TensorDataType &>(*primitiveType).getTensorType().to_spec()); + auto type_id = convert(field.getDataType()); + if (type_id == DataType::T_TENSOR) { + header_struct.addTensorField(field.getName(), field.get_tensor_spec()); } else { header_struct.addField(field.getName(), type_cache.getType( - primitiveType->getId(), field.getCollectionType())); + type_id, field.getCollectionType())); } usedFields.insert(field.getName()); } @@ -158,12 +156,12 @@ document::DocumenttypesConfig DocTypeBuilder::makeConfig() const { if (usf != usedFields.end()) { continue; // taken as index field or attribute field } - const DataType *primitiveType(convert(field.getDataType())); - if (primitiveType->getId() == DataType::T_TENSOR) { - header_struct.addTensorField(field.getName(), dynamic_cast<const TensorDataType &>(*primitiveType).getTensorType().to_spec()); + auto type_id = convert(field.getDataType()); + if (type_id == DataType::T_TENSOR) { + header_struct.addTensorField(field.getName(), field.get_tensor_spec()); } else { header_struct.addField(field.getName(), type_cache.getType( - primitiveType->getId(), field.getCollectionType())); + type_id, field.getCollectionType())); } usedFields.insert(field.getName()); } diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp index 105d57b22b1..ca8513a3c91 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp @@ -1,19 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/queryeval/multibitvectoriterator.h> -#include <vespa/searchlib/queryeval/andsearch.h> -#include <vespa/searchlib/queryeval/andnotsearch.h> -#include <vespa/searchlib/queryeval/sourceblendersearch.h> -#include <vespa/searchlib/queryeval/orsearch.h> +#include "multibitvectoriterator.h" +#include "andsearch.h" +#include "andnotsearch.h" +#include "sourceblendersearch.h" #include <vespa/searchlib/common/bitvectoriterator.h> -#include <vespa/searchlib/attribute/attributeiterators.h> -#include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/util/optimized.h> +#include <vespa/vespalib/hwaccelrated/iaccelrated.h> namespace search::queryeval { using vespalib::Trinary; +using vespalib::hwaccelrated::IAccelrated; namespace { @@ -21,7 +20,16 @@ template<typename Update> class MultiBitVectorIterator : public MultiBitVectorIteratorBase { public: - MultiBitVectorIterator(Children children) : MultiBitVectorIteratorBase(std::move(children)) { } + explicit MultiBitVectorIterator(Children children) + : MultiBitVectorIteratorBase(std::move(children)), + _update(), + _accel(IAccelrated::getAccelerator()), + _lastWords() + { + static_assert(sizeof(_lastWords) == 64, "Lastwords should have 64 byte size"); + static_assert(NumWordsInBatch == 8, "Batch size should be 8 words."); + memset(_lastWords, 0, sizeof(_lastWords)); + } protected: void updateLastValue(uint32_t docId); void strictSeek(uint32_t docId); @@ -29,33 +37,56 @@ private: void doSeek(uint32_t docId) override; Trinary is_strict() const override { return Trinary::False; } bool acceptExtraFilter() const override { return Update::isAnd(); } - Update _update; + Update _update; + const IAccelrated & _accel; + alignas(64) Word _lastWords[8]; + static constexpr size_t NumWordsInBatch = sizeof(_lastWords) / sizeof(Word); }; template<typename Update> class MultiBitVectorIteratorStrict : public MultiBitVectorIterator<Update> { public: - MultiBitVectorIteratorStrict(MultiSearch::Children children) : MultiBitVectorIterator<Update>(std::move(children)) { } + explicit MultiBitVectorIteratorStrict(MultiSearch::Children children) + : MultiBitVectorIterator<Update>(std::move(children)) + { } private: void doSeek(uint32_t docId) override { this->strictSeek(docId); } Trinary is_strict() const override { return Trinary::True; } }; +struct And { + using Word = BitWord::Word; + void operator () (const IAccelrated & accel, size_t offset, const std::vector<std::pair<const void *, bool>> & src, void *dest) { + accel.and64(offset, src, dest); + } + static bool isAnd() { return true; } +}; + +struct Or { + using Word = BitWord::Word; + void operator () (const IAccelrated & accel, size_t offset, const std::vector<std::pair<const void *, bool>> & src, void *dest) { + accel.or64(offset, src, dest); + } + static bool isAnd() { return false; } +}; + template<typename Update> void MultiBitVectorIterator<Update>::updateLastValue(uint32_t docId) { if (docId >= _lastMaxDocIdLimit) { - if (__builtin_expect(docId < _numDocs, true)) { - const uint32_t index(wordNum(docId)); - _lastValue = _bvs[0][index]; - for(uint32_t i(1); i < _bvs.size(); i++) { - _lastValue = _update(_lastValue, _bvs[i][index]); - } - _lastMaxDocIdLimit = (index + 1) * WordLen; - } else { + if (__builtin_expect(docId >= _numDocs, false)) { setAtEnd(); + return; + } + const uint32_t index(wordNum(docId)); + if (docId >= _lastMaxDocIdLimitRequireFetch) { + uint32_t baseIndex = index & ~(NumWordsInBatch - 1); + _update(_accel, baseIndex*sizeof(Word), _bvs, _lastWords); + _lastMaxDocIdLimitRequireFetch = (baseIndex + NumWordsInBatch) * WordLen; } + _lastValue = _lastWords[index % NumWordsInBatch]; + _lastMaxDocIdLimit = (index + 1) * WordLen; } } @@ -75,7 +106,7 @@ template<typename Update> void MultiBitVectorIterator<Update>::strictSeek(uint32_t docId) { - for (updateLastValue(docId), _lastValue=_lastValue & checkTab(docId); + for (updateLastValue(docId), _lastValue = _lastValue & checkTab(docId); (_lastValue == 0) && __builtin_expect(! isAtEnd(), true); updateLastValue(_lastMaxDocIdLimit)); if (__builtin_expect(!isAtEnd(), true)) { @@ -88,21 +119,6 @@ MultiBitVectorIterator<Update>::strictSeek(uint32_t docId) } } -struct And { - typedef BitWord::Word Word; - Word operator () (const Word a, const Word b) { - return a & b; - } - static bool isAnd() { return true; } -}; - -struct Or { - typedef BitWord::Word Word; - Word operator () (const Word a, const Word b) { - return a | b; - } - static bool isAnd() { return false; } -}; typedef MultiBitVectorIterator<And> AndBVIterator; typedef MultiBitVectorIteratorStrict<And> AndBVIteratorStrict; @@ -136,14 +152,15 @@ bool canOptimize(const MultiSearch & s) { MultiBitVectorIteratorBase::MultiBitVectorIteratorBase(Children children) : MultiSearch(std::move(children)), _numDocs(std::numeric_limits<unsigned int>::max()), - _lastValue(0), _lastMaxDocIdLimit(0), + _lastMaxDocIdLimitRequireFetch(0), + _lastValue(0), _bvs() { _bvs.reserve(getChildren().size()); - for (size_t i(0); i < getChildren().size(); i++) { - const auto * bv = static_cast<const BitVectorIterator *>(getChildren()[i].get()); - _bvs.emplace_back(reinterpret_cast<const Word *>(bv->getBitValues()), bv->isInverted()); + for (const auto & child : getChildren()) { + const auto * bv = static_cast<const BitVectorIterator *>(child.get()); + _bvs.emplace_back(bv->getBitValues(), bv->isInverted()); _numDocs = std::min(_numDocs, bv->getDocIdLimit()); } } @@ -155,6 +172,7 @@ MultiBitVectorIteratorBase::initRange(uint32_t beginId, uint32_t endId) { MultiSearch::initRange(beginId, endId); _lastMaxDocIdLimit = 0; + _lastMaxDocIdLimitRequireFetch = 0; } SearchIterator::UP @@ -163,9 +181,10 @@ MultiBitVectorIteratorBase::andWith(UP filter, uint32_t estimate) (void) estimate; if (filter->isBitVector() && acceptExtraFilter()) { const auto & bv = static_cast<const BitVectorIterator &>(*filter); - _bvs.emplace_back(reinterpret_cast<const Word *>(bv.getBitValues()), bv.isInverted()); + _bvs.emplace_back(bv.getBitValues(), bv.isInverted()); insert(getChildren().size(), std::move(filter)); _lastMaxDocIdLimit = 0; // force reload + _lastMaxDocIdLimitRequireFetch = 0; } return filter; } diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h index cde9ffcbfe5..29e92584ffe 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h @@ -11,7 +11,7 @@ namespace search::queryeval { class MultiBitVectorIteratorBase : public MultiSearch, protected BitWord { public: - ~MultiBitVectorIteratorBase(); + ~MultiBitVectorIteratorBase() override; void initRange(uint32_t beginId, uint32_t endId) override; void addUnpackIndex(size_t index) { _unpackInfo.add(index); } /** @@ -20,26 +20,21 @@ public: */ static SearchIterator::UP optimize(SearchIterator::UP parent); protected: - MultiBitVectorIteratorBase(Children children); - class MetaWord { - public: - MetaWord(const Word * words, bool inverted) : _words(words), _inverted(inverted) { } - Word operator [] (uint32_t index) const { return _inverted ? ~_words[index] : _words[index]; } - private: - const Word * _words; - bool _inverted; - }; + MultiBitVectorIteratorBase(Children hildren); + using MetaWord = std::pair<const void *, bool>; uint32_t _numDocs; - Word _lastValue; // Last value computed uint32_t _lastMaxDocIdLimit; // next documentid requiring recomputation. + uint32_t _lastMaxDocIdLimitRequireFetch; + Word _lastValue; // Last value computed std::vector<MetaWord> _bvs; private: virtual bool acceptExtraFilter() const = 0; UP andWith(UP filter, uint32_t estimate) override; void doUnpack(uint32_t docid) override; - UnpackInfo _unpackInfo; static SearchIterator::UP optimizeMultiSearch(SearchIterator::UP parent); + + UnpackInfo _unpackInfo; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index 55342f91e93..d8b63909142 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -45,7 +45,7 @@ convert_cells<double,double>(std::unique_ptr<DenseTensorView> &, vespalib::eval: struct ConvertCellsSelector { template <typename LCT, typename RCT> - static auto get_fun() { return convert_cells<LCT, RCT>; } + static auto invoke() { return convert_cells<LCT, RCT>; } }; } // namespace <unnamed> @@ -67,7 +67,8 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f { auto lct = _query_tensor->cellsRef().type; auto rct = _attr_tensor.getTensorType().cell_type(); - auto fixup_fun = vespalib::tensor::select_2<ConvertCellsSelector>(lct, rct); + using MyTypify = vespalib::eval::TypifyCellType; + auto fixup_fun = vespalib::typify_invoke<2,MyTypify,ConvertCellsSelector>(lct, rct); fixup_fun(_query_tensor, _attr_tensor.getTensorType()); _fallback_dist_fun = search::tensor::make_distance_function(_attr_tensor.getConfig().distance_metric(), rct); _dist_fun = _fallback_dist_fun.get(); diff --git a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp index 034f57bbb36..97e7044ec0f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp @@ -163,7 +163,8 @@ PredicateBlueprint::PredicateBlueprint(const FieldSpecBase &field, _bounds_btree_iterators(), _bounds_vector_iterators(), _zstar_btree_iterator(), - _zstar_vector_iterator() + _zstar_vector_iterator(), + _fetch_postings_done(false) { const auto &interval_index = _index.getIntervalIndex(); const auto zero_constraints_docs = _index.getZeroConstraintDocs(); @@ -234,36 +235,39 @@ namespace { } void PredicateBlueprint::fetchPostings(const ExecuteInfo &) { - const auto &interval_index = _index.getIntervalIndex(); - const auto &bounds_index = _index.getBoundsIndex(); - lookupPostingLists(_interval_dict_entries, _interval_vector_iterators, - _interval_btree_iterators, interval_index); - lookupPostingLists(_bounds_dict_entries, _bounds_vector_iterators, - _bounds_btree_iterators, bounds_index); - - // Lookup zstar interval iterator - if (_zstar_dict_entry.valid()) { - auto vector_iterator = interval_index.getVectorPostingList(Constants::z_star_compressed_hash); - if (vector_iterator) { - _zstar_vector_iterator.emplace(std::move(*vector_iterator)); - } else { - _zstar_btree_iterator.emplace(interval_index.getBTreePostingList(_zstar_dict_entry)); + if (!_fetch_postings_done) { + const auto &interval_index = _index.getIntervalIndex(); + const auto &bounds_index = _index.getBoundsIndex(); + lookupPostingLists(_interval_dict_entries, _interval_vector_iterators, + _interval_btree_iterators, interval_index); + lookupPostingLists(_bounds_dict_entries, _bounds_vector_iterators, + _bounds_btree_iterators, bounds_index); + + // Lookup zstar interval iterator + if (_zstar_dict_entry.valid()) { + auto vector_iterator = interval_index.getVectorPostingList(Constants::z_star_compressed_hash); + if (vector_iterator) { + _zstar_vector_iterator.emplace(std::move(*vector_iterator)); + } else { + _zstar_btree_iterator.emplace(interval_index.getBTreePostingList(_zstar_dict_entry)); + } } - } - PredicateAttribute::MinFeatureHandle mfh = predicate_attribute().getMinFeatureVector(); - Alloc kv(Alloc::alloc(mfh.second, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE*4)); - _kVBacking.swap(kv); - _kV = BitVectorCache::CountVector(static_cast<uint8_t *>(_kVBacking.get()), mfh.second); - _index.computeCountVector(_cachedFeatures, _kV); - for (const auto & entry : _bounds_dict_entries) { - addBoundsPostingToK(entry.feature); - } - for (const auto & entry : _interval_dict_entries) { - addPostingToK(entry.feature); + PredicateAttribute::MinFeatureHandle mfh = predicate_attribute().getMinFeatureVector(); + Alloc kv(Alloc::alloc(mfh.second, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE*4)); + _kVBacking.swap(kv); + _kV = BitVectorCache::CountVector(static_cast<uint8_t *>(_kVBacking.get()), mfh.second); + _index.computeCountVector(_cachedFeatures, _kV); + for (const auto & entry : _bounds_dict_entries) { + addBoundsPostingToK(entry.feature); + } + for (const auto & entry : _interval_dict_entries) { + addPostingToK(entry.feature); + } + addPostingToK(Constants::z_star_compressed_hash); + addZeroConstraintToK(); + _fetch_postings_done = true; } - addPostingToK(Constants::z_star_compressed_hash); - addZeroConstraintToK(); } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h index c9a19a0f5bb..9609cd4f6c9 100644 --- a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h @@ -87,6 +87,7 @@ private: // The zstar iterator is either a vector or a btree iterator. optional<BTreeIterator> _zstar_btree_iterator; optional<VectorIterator> _zstar_vector_iterator; + bool _fetch_postings_done; }; } diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h index 79f987c740c..d37495e85da 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h @@ -17,7 +17,7 @@ template <typename FloatType> class SquaredEuclideanDistance : public DistanceFunction { public: SquaredEuclideanDistance() - : _computer(vespalib::hwaccelrated::IAccelrated::getAccelrator()) + : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()) {} double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override { auto lhs_vector = lhs.typify<FloatType>(); @@ -60,7 +60,7 @@ template <typename FloatType> class AngularDistance : public DistanceFunction { public: AngularDistance() - : _computer(vespalib::hwaccelrated::IAccelrated::getAccelrator()) + : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()) {} double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override { auto lhs_vector = lhs.typify<FloatType>(); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp index 0f58a72e794..37e3ea1adbd 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp @@ -11,9 +11,11 @@ HnswGraph::HnswGraph() : node_refs(), nodes(HnswIndex::make_default_node_store_config()), links(HnswIndex::make_default_link_store_config()), - entry_docid(0), // Note that docid 0 is reserved and never used - entry_level(-1) -{} + entry_docid_and_level() +{ + EntryNode entry; + set_entry_node(entry); +} HnswGraph::~HnswGraph() {} diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h index d1d308def99..125692af627 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h @@ -38,8 +38,8 @@ struct HnswGraph { NodeRefVector node_refs; NodeStore nodes; LinkStore links; - uint32_t entry_docid; - int32_t entry_level; + + std::atomic<uint64_t> entry_docid_and_level; HnswGraph(); @@ -63,9 +63,32 @@ struct HnswGraph { void set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& new_links); - void set_entry_node(uint32_t docid, int32_t level) { - entry_docid = docid; - entry_level = level; + struct EntryNode { + uint32_t docid; + int32_t level; + EntryNode() + : docid(0), // Note that docid 0 is reserved and never used + level(-1) + {} + EntryNode(uint32_t docid_in, int32_t level_in) + : docid(docid_in), + level(level_in) + {} + }; + + void set_entry_node(EntryNode node) { + uint64_t value = node.level; + value <<= 32; + value |= node.docid; + entry_docid_and_level.store(value, std::memory_order_release); + } + + EntryNode get_entry_node() const { + EntryNode entry; + uint64_t value = entry_docid_and_level.load(std::memory_order_acquire); + entry.docid = (uint32_t)value; + entry.level = (int32_t)(value >> 32); + return entry; } size_t size() const { return node_refs.size(); } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 904754d6d9f..a03614a785e 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -11,6 +11,9 @@ #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/datastore/array_store.hpp> #include <vespa/vespalib/util/rcuvector.hpp> +#include <vespa/log/log.h> + +LOG_SETUP(".searchlib.tensor.hnsw_index"); namespace search::tensor { @@ -272,38 +275,90 @@ HnswIndex::~HnswIndex() = default; void HnswIndex::add_document(uint32_t docid) { - auto input = get_vector(docid); + PreparedAddDoc op = internal_prepare_add(docid, get_vector(docid)); + internal_complete_add(docid, op); +} + +HnswIndex::PreparedAddDoc +HnswIndex::internal_prepare_add(uint32_t docid, TypedCells input_vector) const +{ // TODO: Add capping on num_levels int level = _level_generator->max_level(); - _graph.make_node_for_document(docid, level + 1); - uint32_t entry_docid = get_entry_docid(); - if (entry_docid == 0) { - _graph.set_entry_node(docid, level); - return; - } - - int search_level = get_entry_level(); - double entry_dist = calc_distance(input, entry_docid); - HnswCandidate entry_point(entry_docid, entry_dist); - while (search_level > level) { - entry_point = find_nearest_in_layer(input, entry_point, search_level); + PreparedAddDoc op(docid, level); + auto entry = _graph.get_entry_node(); + if (entry.docid == 0) { + return op; + } + int search_level = entry.level; + double entry_dist = calc_distance(input_vector, entry.docid); + HnswCandidate entry_point(entry.docid, entry_dist); + while (search_level > op.max_level) { + entry_point = find_nearest_in_layer(input_vector, entry_point, search_level); --search_level; } FurthestPriQ best_neighbors; best_neighbors.push(entry_point); - search_level = std::min(level, search_level); + search_level = std::min(op.max_level, search_level); - // Insert the added document in each level it should exist in. + // Find neighbors of the added document in each level it should exist in. while (search_level >= 0) { - // TODO: Rename to search_level? - search_layer(input, _cfg.neighbors_to_explore_at_construction(), best_neighbors, search_level); + search_layer(input_vector, _cfg.neighbors_to_explore_at_construction(), best_neighbors, search_level); auto neighbors = select_neighbors(best_neighbors.peek(), _cfg.max_links_on_inserts()); - connect_new_node(docid, neighbors.used, search_level); + auto use = neighbors.used; + op.connections[search_level].assign(use.begin(), use.end()); --search_level; } - if (level > get_entry_level()) { - _graph.set_entry_node(docid, level); + return op; +} + +HnswIndex::LinkArray +HnswIndex::filter_valid_docids(const LinkArrayRef &docids) +{ + LinkArray valid; + valid.reserve(docids.size()); + for (uint32_t docid : docids) { + auto node_ref = _graph.node_refs[docid].load_acquire(); + if (node_ref.valid()) { + valid.push_back(docid); + } + } + return valid; +} + +void +HnswIndex::internal_complete_add(uint32_t docid, PreparedAddDoc &op) +{ + _graph.make_node_for_document(docid, op.max_level + 1); + for (int level = 0; level <= op.max_level; ++level) { + auto neighbors = filter_valid_docids(op.connections[level]); + connect_new_node(docid, neighbors, level); + } + if (op.max_level > get_entry_level()) { + _graph.set_entry_node({docid, op.max_level}); + } +} + +std::unique_ptr<PrepareResult> +HnswIndex::prepare_add_document(uint32_t docid, + TypedCells vector, + vespalib::GenerationHandler::Guard read_guard) const +{ + PreparedAddDoc op = internal_prepare_add(docid, vector); + (void) read_guard; // must keep guard until this point + return std::make_unique<PreparedAddDoc>(std::move(op)); +} + +void +HnswIndex::complete_add_document(uint32_t docid, std::unique_ptr<PrepareResult> prepare_result) +{ + auto prepared = dynamic_cast<PreparedAddDoc *>(prepare_result.get()); + if (prepared && (prepared->docid == docid)) { + internal_complete_add(docid, *prepared); + } else { + LOG(warning, "complete_add_document called with invalid prepare_result"); + // fallback to normal add + add_document(docid); } } @@ -343,7 +398,7 @@ HnswIndex::remove_document(uint32_t docid) LinkArrayRef my_links = _graph.get_link_array(docid, level); for (uint32_t neighbor_id : my_links) { if (need_new_entrypoint) { - _graph.set_entry_node(neighbor_id, level); + _graph.set_entry_node({neighbor_id, level}); need_new_entrypoint = false; } remove_link_to(neighbor_id, docid, level); @@ -352,7 +407,8 @@ HnswIndex::remove_document(uint32_t docid) _graph.set_link_array(docid, level, empty); } if (need_new_entrypoint) { - _graph.set_entry_node(0, -1); + HnswGraph::EntryNode entry; + _graph.set_entry_node(entry); } _graph.remove_node_for_document(docid); } @@ -407,8 +463,9 @@ HnswIndex::get_state(const vespalib::slime::Inserter& inserter) const uint32_t reachable = count_reachable_nodes(); uint32_t unreachable = valid_nodes - reachable; object.setLong("unreachable_nodes", unreachable); - object.setLong("entry_docid", _graph.entry_docid); - object.setLong("entry_level", _graph.entry_level); + auto entry_node = _graph.get_entry_node(); + object.setLong("entry_docid", entry_node.docid); + object.setLong("entry_level", entry_node.level); auto& cfgObj = object.setObject("cfg"); cfgObj.setLong("max_links_at_level_0", _cfg.max_links_at_level_0()); cfgObj.setLong("max_links_on_inserts", _cfg.max_links_on_inserts()); @@ -472,13 +529,13 @@ FurthestPriQ HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k, const BitVector *filter) const { FurthestPriQ best_neighbors; - if (get_entry_level() < 0) { + auto entry = _graph.get_entry_node(); + if (entry.level < 0) { return best_neighbors; } - uint32_t entry_docid = get_entry_docid(); - int search_level = get_entry_level(); - double entry_dist = calc_distance(vector, entry_docid); - HnswCandidate entry_point(entry_docid, entry_dist); + int search_level = entry.level; + double entry_dist = calc_distance(vector, entry.docid); + HnswCandidate entry_point(entry.docid, entry_dist); while (search_level > 0) { entry_point = find_nearest_in_layer(vector, entry_point, search_level); --search_level; @@ -517,7 +574,7 @@ HnswIndex::set_node(uint32_t docid, const HnswNode &node) } int max_level = num_levels - 1; if (get_entry_level() < max_level) { - _graph.set_entry_node(docid, max_level); + _graph.set_entry_node({docid, max_level}); } } @@ -548,15 +605,15 @@ HnswIndex::check_link_symmetry() const uint32_t HnswIndex::count_reachable_nodes() const { - int search_level = get_entry_level(); + auto entry = _graph.get_entry_node(); + int search_level = entry.level; if (search_level < 0) { return 0; } auto visited = _visited_set_pool.get(_graph.size()); - uint32_t entry_id = get_entry_docid(); LinkArray found_links; - found_links.push_back(entry_id); - visited.mark(entry_id); + found_links.push_back(entry.docid); + visited.mark(entry.docid); while (search_level >= 0) { for (uint32_t idx = 0; idx < found_links.size(); ++idx) { uint32_t docid = found_links[idx]; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 6bae8bef4c5..df7453023cf 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -120,6 +120,19 @@ protected: std::vector<Neighbor> top_k_by_docid(uint32_t k, TypedCells vector, const BitVector *filter, uint32_t explore_k) const; + struct PreparedAddDoc : public PrepareResult { + uint32_t docid; + int32_t max_level; + std::vector<LinkArray> connections; + PreparedAddDoc(uint32_t docid_in, int32_t max_level_in) + : docid(docid_in), max_level(max_level_in), connections(max_level+1) + {} + ~PreparedAddDoc() = default; + PreparedAddDoc(PreparedAddDoc&& other) = default; + }; + PreparedAddDoc internal_prepare_add(uint32_t docid, TypedCells input_vector) const; + LinkArray filter_valid_docids(const LinkArrayRef &docids); + void internal_complete_add(uint32_t docid, PreparedAddDoc &op); public: HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func, RandomLevelGenerator::UP level_generator, const Config& cfg); @@ -129,6 +142,10 @@ public: // Implements NearestNeighborIndex void add_document(uint32_t docid) override; + std::unique_ptr<PrepareResult> prepare_add_document(uint32_t docid, + TypedCells vector, + vespalib::GenerationHandler::Guard read_guard) const override; + void complete_add_document(uint32_t docid, std::unique_ptr<PrepareResult> prepare_result) override; void remove_document(uint32_t docid) override; void transfer_hold_lists(generation_t current_gen) override; void trim_hold_lists(generation_t first_used_gen) override; @@ -145,8 +162,8 @@ public: FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k, const BitVector *filter) const; - uint32_t get_entry_docid() const { return _graph.entry_docid; } - int32_t get_entry_level() const { return _graph.entry_level; } + uint32_t get_entry_docid() const { return _graph.get_entry_node().docid; } + int32_t get_entry_level() const { return _graph.get_entry_node().level; } // Should only be used by unit tests. HnswNode get_node(uint32_t docid) const; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp index f02ead86a8d..9f49c0647c6 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp @@ -39,7 +39,7 @@ HnswIndexLoader::load(const fileutil::LoadedBuffer& buf) } if (_failed) return false; _graph.node_refs.ensure_size(num_nodes); - _graph.set_entry_node(entry_docid, entry_level); + _graph.set_entry_node({entry_docid, entry_level}); return true; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp index 46a988d575e..6593a60d6b5 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp @@ -11,8 +11,9 @@ HnswIndexSaver::~HnswIndexSaver() {} HnswIndexSaver::HnswIndexSaver(const HnswGraph &graph) : _graph_links(graph.links), _meta_data() { - _meta_data.entry_docid = graph.entry_docid; - _meta_data.entry_level = graph.entry_level; + auto entry = graph.get_entry_node(); + _meta_data.entry_docid = entry.docid; + _meta_data.entry_level = entry.level; size_t num_nodes = graph.node_refs.size(); _meta_data.nodes.reserve(num_nodes); for (size_t i = 0; i < num_nodes; ++i) { diff --git a/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h index 2f7f9f4445e..e70830a78f8 100644 --- a/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h +++ b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h @@ -2,6 +2,7 @@ #include "random_level_generator.h" #include <random> +#include <mutex> namespace search::tensor { @@ -16,17 +17,24 @@ namespace search::tensor { class InvLogLevelGenerator : public RandomLevelGenerator { std::mt19937_64 _rng; + std::mutex _mutex; std::uniform_real_distribution<double> _uniform; - double _levelMultiplier; + const double _levelMultiplier; + + double get_uniform() { + std::lock_guard<std::mutex> guard(_mutex); + return _uniform(_rng); + } public: InvLogLevelGenerator(uint32_t m) : _rng(0x1234deadbeef5678uLL), + _mutex(), _uniform(0.0, 1.0), _levelMultiplier(1.0 / log(1.0 * m)) {} uint32_t max_level() override { - double unif = _uniform(_rng); + double unif = get_uniform(); double r = -log(1.0-unif) * _levelMultiplier; return (uint32_t) r; } diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index c2d37f2d59a..b46c19ac88a 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -3,6 +3,7 @@ #pragma once #include "distance_function.h" +#include "prepare_result.h" #include <vespa/eval/tensor/dense/typed_cells.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/util/memoryusage.h> @@ -36,6 +37,26 @@ public: }; virtual ~NearestNeighborIndex() {} virtual void add_document(uint32_t docid) = 0; + + /** + * Performs the prepare step in a two-phase operation to add a document to the index. + * + * This function can be called by any thread. + * The document to add is represented by the given vector as it is _not_ stored in the enclosing tensor attribute at this point in time. + * It should return the result of the costly and non-modifying part of this operation. + * The given read guard must be kept in the result. + */ + virtual std::unique_ptr<PrepareResult> prepare_add_document(uint32_t docid, + vespalib::tensor::TypedCells vector, + vespalib::GenerationHandler::Guard read_guard) const = 0; + /** + * Performs the complete step in a two-phase operation to add a document to the index. + * + * This function is only called by the attribute writer thread. + * It uses the result from the prepare step to do the modifying changes. + */ + virtual void complete_add_document(uint32_t docid, std::unique_ptr<PrepareResult> prepare_result) = 0; + virtual void remove_document(uint32_t docid) = 0; virtual void transfer_hold_lists(generation_t current_gen) = 0; virtual void trim_hold_lists(generation_t first_used_gen) = 0; diff --git a/searchlib/src/vespa/searchlib/tensor/prepare_result.h b/searchlib/src/vespa/searchlib/tensor/prepare_result.h new file mode 100644 index 00000000000..05300684497 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/prepare_result.h @@ -0,0 +1,15 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::tensor { + +/** + * Interface for a class used to keep the result of the prepare step of a two-phase operation. + */ +class PrepareResult { +public: + virtual ~PrepareResult() {} +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 95af9f0471b..6cf4f6d2689 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -20,7 +20,6 @@ using vespalib::tensor::SparseTensor; using vespalib::tensor::Tensor; using vespalib::tensor::TypedDenseTensorBuilder; using vespalib::tensor::WrappedSimpleTensor; -using vespalib::tensor::dispatch_0; using search::StateExplorerUtils; namespace search::tensor { @@ -34,7 +33,7 @@ constexpr size_t DEAD_SLACK = 0x10000u; struct CallMakeEmptyTensor { template <typename CT> - static Tensor::UP call(const ValueType &type) { + static Tensor::UP invoke(const ValueType &type) { TypedDenseTensorBuilder<CT> builder(type); return builder.build(); } @@ -46,7 +45,8 @@ createEmptyTensor(const ValueType &type) if (type.is_sparse()) { return std::make_unique<SparseTensor>(type, SparseTensor::Cells()); } else if (type.is_dense()) { - return dispatch_0<CallMakeEmptyTensor>(type.cell_type(), type); + using MyTypify = vespalib::eval::TypifyCellType; + return vespalib::typify_invoke<1,MyTypify,CallMakeEmptyTensor>(type.cell_type(), type); } else { return std::make_unique<WrappedSimpleTensor>(std::make_unique<SimpleTensor>(type, SimpleTensor::Cells())); } @@ -253,6 +253,23 @@ TensorAttribute::getRefCopy() const return RefCopyVector(&_refVector[0], &_refVector[0] + size); } +std::unique_ptr<PrepareResult> +TensorAttribute::prepare_set_tensor(DocId docid, const Tensor& tensor) const +{ + (void) docid; + (void) tensor; + return std::unique_ptr<PrepareResult>(); +} + +void +TensorAttribute::complete_set_tensor(DocId docid, const Tensor& tensor, + std::unique_ptr<PrepareResult> prepare_result) +{ + (void) docid; + (void) tensor; + (void) prepare_result; +} + IMPLEMENT_IDENTIFIABLE_ABSTRACT(TensorAttribute, AttributeVector); } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index e8efd2170c9..8380e485172 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -3,8 +3,9 @@ #pragma once #include "i_tensor_attribute.h" -#include <vespa/searchlib/attribute/not_implemented_attribute.h> +#include "prepare_result.h" #include "tensor_store.h" +#include <vespa/searchlib/attribute/not_implemented_attribute.h> #include <vespa/vespalib/util/rcuvector.h> namespace search::tensor { @@ -51,6 +52,23 @@ public: uint32_t getVersion() const override; RefCopyVector getRefCopy() const; virtual void setTensor(DocId docId, const Tensor &tensor) = 0; + + /** + * Performs the prepare step in a two-phase operation to set a tensor for a document. + * + * This function can be called by any thread. + * It should return the result of the costly and non-modifying part of such operation. + */ + virtual std::unique_ptr<PrepareResult> prepare_set_tensor(DocId docid, const Tensor& tensor) const; + + /** + * Performs the complete step in a two-phase operation to set a tensor for a document. + * + * This function is only called by the attribute writer thread. + * It uses the result from the prepare step to do the modifying changes. + */ + virtual void complete_set_tensor(DocId docid, const Tensor& tensor, std::unique_ptr<PrepareResult> prepare_result); + virtual void compactWorst() = 0; }; diff --git a/searchsummary/src/vespa/searchsummary/test/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/test/CMakeLists.txt index ae4414bb078..e79a75de5a2 100644 --- a/searchsummary/src/vespa/searchsummary/test/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/test/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchsummary_test OBJECT +vespa_add_library(searchsummary_test SOURCES mock_attribute_manager.cpp AFTER diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index bdd066e8a4a..c9cda047784 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -14,7 +14,7 @@ #include <sstream> #include <vespa/log/log.h> -LOG_SETUP(".visitor.instance"); +LOG_SETUP(".visitor.instance.visitor"); using document::BucketSpace; @@ -367,8 +367,7 @@ Visitor::sendReplyOnce() { assert(_initiatingCmd.get()); if (!_hasSentReply) { - std::shared_ptr<api::StorageReply> reply( - _initiatingCmd->makeReply().release()); + std::shared_ptr<api::StorageReply> reply(_initiatingCmd->makeReply()); _hitCounter->updateVisitorStatistics(_visitorStatistics); static_cast<api::CreateVisitorReply*>(reply.get()) @@ -563,8 +562,7 @@ Visitor::attach(std::shared_ptr<api::StorageCommand> initiatingCmd, _priority = initiatingCmd->getPriority(); _timeToDie = _component.getClock().getTimeInMicros() + timeout.getMicros(); if (_initiatingCmd.get()) { - std::shared_ptr<api::StorageReply> reply( - _initiatingCmd->makeReply().release()); + std::shared_ptr<api::StorageReply> reply(_initiatingCmd->makeReply()); reply->setResult(api::ReturnCode::ABORTED); _messageHandler->send(reply); } @@ -594,7 +592,7 @@ Visitor::attach(std::shared_ptr<api::StorageCommand> initiatingCmd, // In case there was no messages to resend we need to call // continueVisitor to provoke it to resume. - for (uint32_t i=0; i<_visitorOptions._maxParallelOneBucket; ++i) { + for (uint32_t i = 0; i < _visitorOptions._maxParallelOneBucket; ++i) { if (!continueVisitor()) return; } } @@ -669,8 +667,7 @@ Visitor::handleDocumentApiReply(mbus::Reply::UP reply, return; } assert(!meta.message); - meta.message.reset( - static_cast<documentapi::DocumentMessage*>(message.release())); + meta.message.reset(static_cast<documentapi::DocumentMessage*>(message.release())); meta.retryCount++; const size_t retryCount = meta.retryCount; @@ -702,7 +699,7 @@ Visitor::onCreateIteratorReply( const std::shared_ptr<CreateIteratorReply>& reply, VisitorThreadMetrics& /*metrics*/) { - std::list<BucketIterationState*>::reverse_iterator it = _bucketStates.rbegin(); + auto it = _bucketStates.rbegin(); document::Bucket bucket(reply->getBucket()); document::BucketId bucketId(bucket.getBucketId()); @@ -752,7 +749,7 @@ Visitor::onGetIterReply(const std::shared_ptr<GetIterReply>& reply, _id.c_str(), reply->getBucketId().toString().c_str(), reply->getResult().toString().c_str()); - std::list<BucketIterationState*>::reverse_iterator it = _bucketStates.rbegin(); + auto it = _bucketStates.rbegin(); // New requests will be pushed on end of list.. So searching // in reverse order should quickly get correct result. @@ -803,7 +800,7 @@ Visitor::onGetIterReply(const std::shared_ptr<GetIterReply>& reply, LOG(debug, "Visitor %s handling block of %zu documents.", _id.c_str(), reply->getEntries().size()); - try{ + try { framework::MilliSecTimer processingTimer(_component.getClock()); handleDocuments(reply->getBucketId(), reply->getEntries(), @@ -814,18 +811,16 @@ Visitor::onGetIterReply(const std::shared_ptr<GetIterReply>& reply, MBUS_TRACE(reply->getTrace(), 5, "Done processing data block in visitor plugin"); uint64_t size = 0; - for (size_t i = 0; i < reply->getEntries().size(); ++i) { - size += reply->getEntries()[i]->getPersistedDocumentSize(); + for (const auto& entry : reply->getEntries()) { + size += entry->getPersistedDocumentSize(); } _visitorStatistics.setDocumentsVisited( _visitorStatistics.getDocumentsVisited() + reply->getEntries().size()); - _visitorStatistics.setBytesVisited( - _visitorStatistics.getBytesVisited() + size); + _visitorStatistics.setBytesVisited(_visitorStatistics.getBytesVisited() + size); } catch (std::exception& e) { - LOG(warning, "handleDocuments threw exception %s", - e.what()); + LOG(warning, "handleDocuments threw exception %s", e.what()); reportProblem(e.what()); } } @@ -849,8 +844,7 @@ Visitor::sendDueQueuedMessages(framework::MicroSecTime timeNow) while (!_visitorTarget._queuedMessages.empty() && (_visitorTarget._pendingMessages.size() < _visitorOptions._maxPending)) { - VisitorTarget::MessageQueue::iterator it( - _visitorTarget._queuedMessages.begin()); + auto it = _visitorTarget._queuedMessages.begin(); if (it->first < timeNow) { auto& msgMeta = _visitorTarget.metaForMessageId(it->second); _visitorTarget._queuedMessages.erase(it); @@ -1204,13 +1198,10 @@ Visitor::getIterators() if (sentCount == 0) { if (LOG_WOULD_LOG(debug)) { LOG(debug, "Enough iterators being processed. Doing nothing for " - "visitor '%s' bucketStates = %d.", - _id.c_str(), (int)_bucketStates.size()); - for (std::list<BucketIterationState*>::iterator it( - _bucketStates.begin()); - it != _bucketStates.end(); ++it) - { - LOG(debug, "Existing: %s", (*it)->toString().c_str()); + "visitor '%s' bucketStates = %zu.", + _id.c_str(), _bucketStates.size()); + for (const auto& state : _bucketStates) { + LOG(debug, "Existing: %s", state->toString().c_str()); } } } diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index 006af5edf7d..c6e75735690 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -140,8 +140,7 @@ VisitorThread::shutdown() .getType() != PropagateVisitorConfig::ID)) { std::shared_ptr<api::StorageReply> reply( - static_cast<api::StorageCommand&>(*it->_message) - .makeReply().release()); + static_cast<api::StorageCommand&>(*it->_message).makeReply()); reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, "Shutting down storage node.")); _messageSender.send(reply); @@ -225,9 +224,9 @@ VisitorThread::run(framework::ThreadHandle& thread) if (entry._message.get()) { // If visitor doesn't exist, log failure only if it wasn't // recently deleted - if (_currentlyRunningVisitor == _visitors.end() && - entry._message->getType() != api::MessageType::VISITOR_CREATE && - entry._message->getType() != api::MessageType::INTERNAL) + if ((_currentlyRunningVisitor == _visitors.end()) && + (entry._message->getType() != api::MessageType::VISITOR_CREATE) && + (entry._message->getType() != api::MessageType::INTERNAL)) { handleNonExistingVisitorCall(entry, result); } else { @@ -264,10 +263,8 @@ VisitorThread::run(framework::ThreadHandle& thread) if (!handled && entry._message.get() && !entry._message->getType().isReply()) { - api::StorageCommand& cmd( - dynamic_cast<api::StorageCommand&>(*entry._message)); - std::shared_ptr<api::StorageReply> reply( - cmd.makeReply().release()); + auto& cmd = dynamic_cast<api::StorageCommand&>(*entry._message); + std::shared_ptr<api::StorageReply> reply(cmd.makeReply()); reply->setResult(result); _messageSender.send(reply); } @@ -278,10 +275,8 @@ void VisitorThread::tick() { // Give all visitors an event - for (VisitorMap::iterator it = _visitors.begin(); it != _visitors.end();) - { - LOG(spam, "Giving tick to visitor %s.", - it->second->getVisitorName().c_str()); + for (auto it = _visitors.begin(); it != _visitors.end();) { + LOG(spam, "Giving tick to visitor %s.", it->second->getVisitorName().c_str()); it->second->continueVisitor(); if (it->second->isCompleted()) { LOG(debug, "Closing visitor %s. Visitor marked as completed", @@ -312,11 +307,9 @@ VisitorThread::close() } else { _metrics.completedVisitors[loadType].inc(1); } - framework::SecondTime currentTime( - _component.getClock().getTimeInSeconds()); + framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); trimRecentlyCompletedList(currentTime); - _recentlyCompleted.push_back(std::make_pair( - _currentlyRunningVisitor->first, currentTime)); + _recentlyCompleted.emplace_back(_currentlyRunningVisitor->first, currentTime); _visitors.erase(_currentlyRunningVisitor); _currentlyRunningVisitor = _visitors.end(); } @@ -324,8 +317,7 @@ VisitorThread::close() void VisitorThread::trimRecentlyCompletedList(framework::SecondTime currentTime) { - framework::SecondTime recentLimit( - currentTime - framework::SecondTime(30)); + framework::SecondTime recentLimit(currentTime - framework::SecondTime(30)); // Dump all elements that aren't recent anymore while (!_recentlyCompleted.empty() && _recentlyCompleted.front().second < recentLimit) @@ -339,16 +331,12 @@ VisitorThread::handleNonExistingVisitorCall(const Event& entry, ReturnCode& code) { // Get current time. Set the time that is the oldest still recent. - framework::SecondTime currentTime( - _component.getClock().getTimeInSeconds());; + framework::SecondTime currentTime(_component.getClock().getTimeInSeconds());; trimRecentlyCompletedList(currentTime); // Go through all recent visitors. Ignore request if recent - for (std::deque<std::pair<api::VisitorId, framework::SecondTime> > - ::iterator it = _recentlyCompleted.begin(); - it != _recentlyCompleted.end(); ++it) - { - if (it->first == entry._visitorId) { + for (const auto& e : _recentlyCompleted) { + if (e.first == entry._visitorId) { code = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, "Visitor recently completed/failed/aborted."); return; @@ -371,13 +359,13 @@ VisitorThread::createVisitor(vespalib::stringref libName, vespalib::string str = libName; std::transform(str.begin(), str.end(), str.begin(), tolower); - VisitorFactory::Map::iterator it(_visitorFactories.find(str)); + auto it = _visitorFactories.find(str); if (it == _visitorFactories.end()) { error << "Visitor library " << str << " not found."; return std::shared_ptr<Visitor>(); } - LibMap::iterator libIter = _libs.find(str); + auto libIter = _libs.find(str); if (libIter == _libs.end()) { _libs[str] = std::shared_ptr<VisitorEnvironment>( it->second->makeVisitorEnvironment(_component).release()); @@ -402,17 +390,15 @@ namespace { std::unique_ptr<api::StorageMessageAddress> getDataAddress(const api::CreateVisitorCommand& cmd) { - return std::unique_ptr<api::StorageMessageAddress>( - new api::StorageMessageAddress( - mbus::Route::parse(cmd.getDataDestination()))); + return std::make_unique<api::StorageMessageAddress>( + mbus::Route::parse(cmd.getDataDestination())); } std::unique_ptr<api::StorageMessageAddress> getControlAddress(const api::CreateVisitorCommand& cmd) { - return std::unique_ptr<api::StorageMessageAddress>( - new api::StorageMessageAddress( - mbus::Route::parse(cmd.getControlDestination()))); + return std::make_unique<api::StorageMessageAddress>( + mbus::Route::parse(cmd.getControlDestination())); } void @@ -447,28 +433,27 @@ VisitorThread::onCreateVisitor( std::unique_ptr<api::StorageMessageAddress> dataAddress; std::shared_ptr<Visitor> visitor; do { - // If no buckets are specified, fail command - if (cmd->getBuckets().size() == 0) { + // If no buckets are specified, fail command + if (cmd->getBuckets().empty()) { result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, "No buckets specified"); LOG(warning, "CreateVisitor(%s): No buckets specified. Aborting.", cmd->getInstanceId().c_str()); break; } - // Get the source address + // Get the source address controlAddress = getControlAddress(*cmd); dataAddress = getDataAddress(*cmd); - // Attempt to load library containing visitor + // Attempt to load library containing visitor vespalib::asciistream errors; - visitor = createVisitor(cmd->getLibraryName(), cmd->getParameters(), - errors); - if (visitor.get() == 0) { + visitor = createVisitor(cmd->getLibraryName(), cmd->getParameters(), errors); + if (!visitor) { result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, errors.str()); LOG(warning, "CreateVisitor(%s): Failed to create visitor: %s", cmd->getInstanceId().c_str(), errors.str().data()); break; } - // Set visitor parameters + // Set visitor parameters if (cmd->getMaximumPendingReplyCount() != 0) { visitor->setMaxPending(cmd->getMaximumPendingReplyCount()); } else { @@ -494,11 +479,9 @@ VisitorThread::onCreateVisitor( // Parse document selection try{ - if (cmd->getDocumentSelection() != "") { - std::shared_ptr<const document::DocumentTypeRepo> repo( - _component.getTypeRepo()); - const document::BucketIdFactory& idFactory( - _component.getBucketIdFactory()); + if (!cmd->getDocumentSelection().empty()) { + std::shared_ptr<const document::DocumentTypeRepo> repo(_component.getTypeRepo()); + const document::BucketIdFactory& idFactory(_component.getBucketIdFactory()); document::select::Parser parser(*repo, idFactory); docSelection = parser.parse(cmd->getDocumentSelection()); validateDocumentSelection(*repo, *docSelection); @@ -522,11 +505,11 @@ VisitorThread::onCreateVisitor( } LOG(debug, "CreateVisitor(%s): Successfully created visitor", cmd->getInstanceId().c_str()); - // Insert visitor prior to creating successful reply. + // Insert visitor prior to creating successful reply. } while (false); - // Start the visitor last, as to ensure client will receive - // visitor create reply first, and that all errors we could detect - // resulted in proper error code in reply.. + // Start the visitor last, as to ensure client will receive + // visitor create reply first, and that all errors we could detect + // resulted in proper error code in reply.. if (result.success()) { _visitors[cmd->getVisitorId()] = visitor; try{ @@ -548,18 +531,17 @@ VisitorThread::onCreateVisitor( visitor->attach(cmd, *controlAddress, *dataAddress, framework::MilliSecTime(vespalib::count_ms(cmd->getTimeout()))); } catch (std::exception& e) { - // We don't handle exceptions from this code, as we've - // added visitor to internal structs we'll end up calling - // close() twice. + // We don't handle exceptions from this code, as we've + // added visitor to internal structs we'll end up calling + // close() twice. LOG(error, "Got exception we can't handle: %s", e.what()); assert(false); } _metrics.createdVisitors[visitor->getLoadType()].inc(1); visitorTimer.stop(_metrics.averageVisitorCreationTime[visitor->getLoadType()]); } else { - // Send reply - std::shared_ptr<api::CreateVisitorReply> reply( - new api::CreateVisitorReply(*cmd)); + // Send reply + auto reply = std::make_shared<api::CreateVisitorReply>(*cmd); reply->setResult(result); _messageSender.closed(cmd->getVisitorId()); _messageSender.send(reply); @@ -572,7 +554,7 @@ VisitorThread::handleMessageBusReply(mbus::Reply::UP reply, Visitor& visitor) { vespalib::MonitorGuard sync(_queueMonitor); - _queue.push_back(Event(visitor.getVisitorId(), std::move(reply))); + _queue.emplace_back(visitor.getVisitorId(), std::move(reply)); sync.broadcast(); } @@ -582,8 +564,7 @@ VisitorThread::onInternal(const std::shared_ptr<api::InternalCommand>& cmd) switch (cmd->getType()) { case PropagateVisitorConfig::ID: { - PropagateVisitorConfig& pcmd( - dynamic_cast<PropagateVisitorConfig&>(*cmd)); + auto& pcmd = dynamic_cast<PropagateVisitorConfig&>(*cmd); const vespa::config::content::core::StorVisitorConfig& config(pcmd.getConfig()); if (_defaultDocBlockSize != 0) { // Live update LOG(config, "Updating visitor thread configuration in visitor " @@ -655,12 +636,10 @@ VisitorThread::onInternal(const std::shared_ptr<api::InternalCommand>& cmd) case RequestStatusPage::ID: { LOG(spam, "Got RequestStatusPage request"); - RequestStatusPage& rsp(dynamic_cast<RequestStatusPage&>(*cmd)); + auto& rsp = dynamic_cast<RequestStatusPage&>(*cmd); vespalib::asciistream ost; getStatus(ost, rsp.getPath()); - std::shared_ptr<RequestStatusPageReply> reply( - new RequestStatusPageReply(rsp, ost.str())); - _messageSender.send(reply); + _messageSender.send(std::make_shared<RequestStatusPageReply>(rsp, ost.str())); break; } default: @@ -679,11 +658,9 @@ VisitorThread::onInternalReply(const std::shared_ptr<api::InternalReply>& r) switch (r->getType()) { case GetIterReply::ID: { - std::shared_ptr<GetIterReply> reply( - std::dynamic_pointer_cast<GetIterReply>(r)); + auto reply = std::dynamic_pointer_cast<GetIterReply>(r); assert(reply.get()); - _currentlyRunningVisitor->second->onGetIterReply( - reply, _metrics); + _currentlyRunningVisitor->second->onGetIterReply(reply, _metrics); if (_currentlyRunningVisitor->second->isCompleted()) { LOG(debug, "onGetIterReply(%s): Visitor completed.", _currentlyRunningVisitor->second->getVisitorName().c_str()); @@ -693,11 +670,9 @@ VisitorThread::onInternalReply(const std::shared_ptr<api::InternalReply>& r) } case CreateIteratorReply::ID: { - std::shared_ptr<CreateIteratorReply> reply( - std::dynamic_pointer_cast<CreateIteratorReply>(r)); + auto reply = std::dynamic_pointer_cast<CreateIteratorReply>(r); assert(reply.get()); - _currentlyRunningVisitor->second->onCreateIteratorReply( - reply, _metrics); + _currentlyRunningVisitor->second->onCreateIteratorReply(reply, _metrics); break; } default: @@ -721,25 +696,21 @@ VisitorThread::getStatus(vespalib::asciistream& out, if (status && verbose) { out << "<h3>Visitor libraries loaded</h3>\n<ul>\n"; - if (_libs.size() == 0) { + if (_libs.empty()) { out << "None\n"; } - for (LibMap::const_iterator it = _libs.begin(); it != _libs.end(); ++it) - { - out << "<li>" << it->first << "\n"; + for (const auto& lib : _libs) { + out << "<li>" << lib.first << "\n"; } out << "</ul>\n"; out << "<h3>Recently completed/failed/aborted visitors</h3>\n<ul>\n"; - if (_recentlyCompleted.size() == 0) { + if (_recentlyCompleted.empty()) { out << "None\n"; } - for (std::deque<std::pair<api::VisitorId, framework::SecondTime> > - ::const_iterator it = _recentlyCompleted.begin(); - it != _recentlyCompleted.end(); ++it) - { - out << "<li> Visitor " << it->first << " done at " - << it->second.getTime() << "\n"; + for (const auto& cv : _recentlyCompleted) { + out << "<li> Visitor " << cv.first << " done at " + << cv.second.getTime() << "\n"; } out << "</ul>\n"; out << "<h3>Current queue size: " << _queue.size() << "</h3>\n"; @@ -764,17 +735,15 @@ VisitorThread::getStatus(vespalib::asciistream& out, << "</table>\n"; } if (showAll) { - for (VisitorMap::const_iterator it = _visitors.begin(); - it != _visitors.end(); ++it) - { - out << "<h3>Visitor " << it->first << "</h3>\n"; + for (const auto& v : _visitors) { + out << "<h3>Visitor " << v.first << "</h3>\n"; std::ostringstream tmp; - it->second->getStatus(tmp, verbose); + v.second->getStatus(tmp, verbose); out << tmp.str(); } } else if (path.hasAttribute("visitor")) { out << "<h3>Visitor " << visitor << "</h3>\n"; - VisitorMap::const_iterator it = _visitors.find(visitor); + auto it = _visitors.find(visitor); if (it == _visitors.end()) { out << "Not found\n"; } else { @@ -784,7 +753,7 @@ VisitorThread::getStatus(vespalib::asciistream& out, } } else { // List visitors out << "<h3>Active visitors</h3>\n"; - if (_visitors.size() == 0) { + if (_visitors.empty()) { out << "None\n"; } for (VisitorMap::const_iterator it = _visitors.begin(); diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index a3fac22a93d..767119d2a02 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -93,23 +93,29 @@ <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-cd</artifactId> + <artifactId>tenant-cd-api</artifactId> <version>${test-framework.version}</version> <scope>test</scope> - <exclusions> - <exclusion> - <groupId>net.java.dev.jna</groupId> - <artifactId>jna</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </exclusion> - <exclusion> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </exclusion> - </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>cloud-tenant-cd</artifactId> + <version>${test-framework.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> </dependency> </dependencies> diff --git a/tenant-cd/OWNERS b/tenant-cd-api/OWNERS index d0a102ecbf4..d0a102ecbf4 100644 --- a/tenant-cd/OWNERS +++ b/tenant-cd-api/OWNERS diff --git a/tenant-cd/README b/tenant-cd-api/README index a3803b81d53..a3803b81d53 100644 --- a/tenant-cd/README +++ b/tenant-cd-api/README diff --git a/tenant-cd-api/abi-spec.json b/tenant-cd-api/abi-spec.json new file mode 100644 index 00000000000..677a18c74e4 --- /dev/null +++ b/tenant-cd-api/abi-spec.json @@ -0,0 +1,117 @@ +{ + "ai.vespa.hosted.cd.Deployment": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract ai.vespa.hosted.cd.Endpoint endpoint(java.lang.String)" + ], + "fields": [] + }, + "ai.vespa.hosted.cd.Endpoint": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract java.net.URI uri()", + "public abstract java.net.http.HttpResponse send(java.net.http.HttpRequest$Builder, java.net.http.HttpResponse$BodyHandler)", + "public java.net.http.HttpResponse send(java.net.http.HttpRequest$Builder)", + "public abstract java.net.http.HttpRequest$Builder request(java.lang.String, java.util.Map)", + "public java.net.http.HttpRequest$Builder request(java.lang.String)" + ], + "fields": [] + }, + "ai.vespa.hosted.cd.IntegrationTest": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [], + "fields": [] + }, + "ai.vespa.hosted.cd.ProductionTest": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [], + "fields": [] + }, + "ai.vespa.hosted.cd.StagingSetup": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [], + "fields": [] + }, + "ai.vespa.hosted.cd.StagingTest": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [], + "fields": [] + }, + "ai.vespa.hosted.cd.SystemTest": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [], + "fields": [] + }, + "ai.vespa.hosted.cd.TestRuntime": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public static ai.vespa.hosted.cd.TestRuntime get()", + "public abstract ai.vespa.hosted.cd.Deployment deploymentToTest()", + "public abstract ai.vespa.cloud.Zone zone()" + ], + "fields": [] + } +}
\ No newline at end of file diff --git a/tenant-cd-api/pom.xml b/tenant-cd-api/pom.xml new file mode 100644 index 00000000000..b19d42d094f --- /dev/null +++ b/tenant-cd-api/pom.xml @@ -0,0 +1,76 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-cd-api</artifactId> + <name>Hosted Vespa tenant CD API</name> + <description>Test API library for hosted Vespa applications.</description> + <url>https://github.com/vespa-engine</url> + <packaging>container-plugin</packaging> + + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent</relativePath> + </parent> + + <dependencies> + <!-- provided --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-zone-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <!-- required for bundle-plugin to generate import-package statements for Java's standard library --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <!-- compile --> + <dependency> <!-- TODO(bjorncs): share junit version number with test-runner implementation --> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <version>5.6.2</version> <!-- NOTE: This version must match the string in all ExportPackage annotations --> + <scope>compile</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <attachBundleArtifact>true</attachBundleArtifact> + <bundleClassifierName>deploy</bundleClassifierName> + <useCommonAssemblyIds>false</useCommonAssemblyIds> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>abi-check-plugin</artifactId> + </plugin> + </plugins> + </build> + +</project> diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Deployment.java index 7d7b2f74981..7d7b2f74981 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Deployment.java diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Endpoint.java index bd6f30767f2..afc6aa1b519 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Endpoint.java @@ -1,19 +1,12 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; -import ai.vespa.hosted.api.EndpointAuthenticator; - import java.net.URI; -import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; -import java.util.stream.Collectors; - -import static java.net.URLEncoder.encode; -import static java.nio.charset.StandardCharsets.UTF_8; /** * An endpoint in a Vespa application {@link Deployment}, which allows document retrieval. @@ -25,7 +18,7 @@ public interface Endpoint { /** Returns the URI of the endpoint, with scheme, host and port. */ URI uri(); - /** Sends the given request with required authentication. See {@link EndpointAuthenticator#authenticated} and {@link HttpClient#send}. */ + /** Sends the given request with required authentication. */ <T> HttpResponse<T> send(HttpRequest.Builder request, HttpResponse.BodyHandler<T> handler); /** Sends the given request with required authentication. */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java index f9dc15df32e..f9dc15df32e 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java index 53e7311fefc..b9054689b00 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java @@ -2,7 +2,6 @@ package ai.vespa.hosted.cd; import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; import java.lang.annotation.Retention; import java.lang.annotation.Target; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingSetup.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java index bef3eabcef6..bef3eabcef6 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingSetup.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java index 59360b2753c..59360b2753c 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java index f01f2ca6c90..f01f2ca6c90 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/TestRuntime.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/TestRuntime.java new file mode 100644 index 00000000000..08cc0467b71 --- /dev/null +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/TestRuntime.java @@ -0,0 +1,24 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.hosted.cd; + +import ai.vespa.cloud.Zone; + +import java.util.ServiceLoader; + +/** + * The place to obtain environment-dependent configuration for test of a Vespa deployment. + * + * @author jvenstad + * @author mortent + */ +public interface TestRuntime { + static TestRuntime get() { + ServiceLoader<TestRuntime> serviceLoader = ServiceLoader.load(TestRuntime.class); + return serviceLoader.findFirst().orElseThrow(() -> new RuntimeException("No TestRuntime implementation found")); + } + + Deployment deploymentToTest(); + + Zone zone(); + +} diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/package-info.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/package-info.java new file mode 100644 index 00000000000..fc10fb82c5c --- /dev/null +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/package-info.java @@ -0,0 +1,10 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +@PublicApi +package ai.vespa.hosted.cd; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/condition/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/condition/package-info.java new file mode 100644 index 00000000000..bc0684f0c76 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/condition/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api.condition; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/package-info.java new file mode 100644 index 00000000000..bcb46dbe671 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api.extension; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/function/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/function/package-info.java new file mode 100644 index 00000000000..8d62bceeae7 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/function/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api.function; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/io/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/io/package-info.java new file mode 100644 index 00000000000..7fc2e15c716 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/io/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api.io; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/package-info.java new file mode 100644 index 00000000000..dd82f705bd3 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/parallel/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/parallel/package-info.java new file mode 100644 index 00000000000..dc88b0d33bf --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/parallel/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api.parallel; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd/pom.xml b/tenant-cd/pom.xml deleted file mode 100644 index 829b1de457b..00000000000 --- a/tenant-cd/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - - <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-cd</artifactId> - <name>Hosted Vespa tenant CD</name> - <description>Test library for hosted Vespa applications.</description> - <url>https://github.com/vespa-engine</url> - <packaging>jar</packaging> - - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent</relativePath> - </parent> - - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>security-utils</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-provisioning</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-auth</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>hosted-api</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-engine</artifactId> - </dependency> - </dependencies> - -</project> diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java deleted file mode 100644 index c479bab6e13..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd; - -import ai.vespa.hosted.api.ControllerHttpClient; -import ai.vespa.hosted.api.EndpointAuthenticator; -import ai.vespa.hosted.api.Properties; -import ai.vespa.hosted.api.TestConfig; -import ai.vespa.hosted.cd.http.HttpDeployment; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.zone.ZoneId; - -import java.nio.file.Files; -import java.nio.file.Paths; - -/** - * The place to obtain environment-dependent configuration for test of a Vespa deployment. - * - * @author jvenstad - */ -public class TestRuntime { - - private static TestRuntime theRuntime; - - private final TestConfig config; - private final Deployment deploymentToTest; - - private TestRuntime(TestConfig config, EndpointAuthenticator authenticator) { - this.config = config; - this.deploymentToTest = new HttpDeployment(config.deployments().get(config.zone()), authenticator); - } - - /** - * Returns the config and authenticator to use when running integration tests. - * - * If the system property {@code "vespa.test.config"} is set (to a file path), a file at that location - * is attempted read, and config parsed from it. - * Otherwise, config is fetched over HTTP from the hosted Vespa API, assuming the deployment indicated - * by the optional {@code "environment"} and {@code "region"} system properties exists. - * When environment is not specified, it defaults to {@link Environment#dev}, - * while region must be set unless the environment is {@link Environment#dev} or {@link Environment#perf}. - */ - public static synchronized TestRuntime get() { - if (theRuntime == null) { - String configPath = System.getProperty("vespa.test.config"); - TestConfig config = configPath != null ? fromFile(configPath) : fromController(); - theRuntime = new TestRuntime(config, - new ai.vespa.hosted.auth.EndpointAuthenticator(config.system())); - } - return theRuntime; - } - - /** Returns a copy of this runtime, with the given endpoint authenticator. */ - public TestRuntime with(EndpointAuthenticator authenticator) { - return new TestRuntime(config, authenticator); - } - - /** Returns the full id of the application this is testing. */ - public ApplicationId application() { return config.application(); } - - /** Returns the zone of the deployment this is testing. */ - public ZoneId zone() { return config.zone(); } - - /** Returns the deployment this is testing. */ - public Deployment deploymentToTest() { return deploymentToTest; } - - private static TestConfig fromFile(String path) { - try { - return TestConfig.fromJson(Files.readAllBytes(Paths.get(path))); - } - catch (Exception e) { - throw new IllegalArgumentException("Failed reading config from '" + path + "'!", e); - } - } - - private static TestConfig fromController() { - ControllerHttpClient controller = new ai.vespa.hosted.auth.ApiAuthenticator().controller(); - ApplicationId id = Properties.application(); - Environment environment = Properties.environment().orElse(Environment.dev); - ZoneId zone = Properties.region().map(region -> ZoneId.from(environment, region)) - .orElseGet(() -> controller.defaultZone(environment)); - return controller.testConfig(id, zone); - } - -} diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 95b6528bd77..3faf47ccfa9 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -441,7 +441,6 @@ public class DocumentGenMojo extends AbstractMojo { ind(1)+"/** The doc type of this.*/\n" + ind(1)+"public static final com.yahoo.document.DocumentType type = getDocumentType();\n\n"+ ind(1)+"/** Struct type view of the type of the body of this.*/\n" + - ind(1)+"private static final com.yahoo.document.StructDataType bodyStructType = getBodyStructType();\n\n" + ind(1)+"/** Struct type view of the type of the header of this.*/\n" + ind(1)+"private static final com.yahoo.document.StructDataType headerStructType = getHeaderStructType();\n\n"); @@ -460,9 +459,7 @@ public class DocumentGenMojo extends AbstractMojo { // Mimic header and body to make serialization work. // This can be improved by generating a method to serialize the document _here_, and use that in serialization. exportOverriddenStructGetter(docType.allHeader().getFields(), out, 1, "getHeader", className+".headerStructType"); - exportOverriddenStructGetter(docType.allBody().getFields(), out, 1, "getBody", className+".bodyStructType"); exportStructTypeGetter(docType.getName()+".header", docType.allHeader().getFields(), out, 1, "getHeaderStructType", "com.yahoo.document.StructDataType"); - exportStructTypeGetter(docType.getName()+".body", docType.allBody().getFields(), out, 1, "getBodyStructType", "com.yahoo.document.StructDataType"); Collection<Field> allUniqueFields = getAllUniqueFields(multiExtends, docType.getAllFields()); exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(), diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java index 0b3ee6d0792..eb6bb609970 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java @@ -33,7 +33,7 @@ public class MockedOperationHandler implements OperationHandler { @SuppressWarnings("deprecation") public void put(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException { log.append("PUT: " + data.getDocument().getId()); - log.append(data.getDocument().getBody().toString()); + log.append(data.getDocument().getHeader().toString()); } @Override diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 2675bc16bf2..1ca9816a921 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -24,8 +24,8 @@ vespa_define_module( src/tests/assert src/tests/barrier src/tests/benchmark_timer - src/tests/btree src/tests/box + src/tests/btree src/tests/closure src/tests/component src/tests/compress @@ -128,6 +128,7 @@ vespa_define_module( src/tests/tutorial/minimal src/tests/tutorial/simple src/tests/tutorial/threads + src/tests/typify src/tests/util/generationhandler src/tests/util/generationhandler_stress src/tests/util/md5 diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index 848c8a37125..63afd8b770f 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -36,6 +36,54 @@ toStr(const T & v) return ss.str(); } +class SequenceValidator +{ + int _wanted_count; + int _prev_key; + int _count; + bool _failed; + +public: + SequenceValidator(int start, int wanted_count) + : _wanted_count(wanted_count), + _prev_key(start - 1), + _count(0), + _failed(false) + { + } + + bool failed() const { + return _failed || _wanted_count != _count; + } + + void operator()(int key) { + if (key != _prev_key + 1) { + _failed = true; + } + _prev_key = key; + ++_count; + } +}; + +class ForeachKeyValidator +{ + SequenceValidator & _validator; +public: + ForeachKeyValidator(SequenceValidator &validator) + : _validator(validator) + { + } + void operator()(int key) { + _validator(key); + } +}; + +template <typename Iterator> +void validate_subrange(Iterator &start, Iterator &end, SequenceValidator &validator) { + start.foreach_key_range(end, ForeachKeyValidator(validator)); + EXPECT_FALSE(validator.failed()); +} + } typedef BTreeTraits<4, 4, 31, false> MyTraits; @@ -210,6 +258,8 @@ private: void requireThatIteratorDistanceWorks(); + + void requireThatForeachKeyWorks(); public: int Main() override; }; @@ -1489,6 +1539,32 @@ Test::requireThatIteratorDistanceWorks() requireThatIteratorDistanceWorks(400); } +void +Test::requireThatForeachKeyWorks() +{ + using Tree = BTree<int, int, btree::NoAggregated, MyComp, MyTraits>; + using Iterator = typename Tree::ConstIterator; + Tree t; + populateTree(t, 256, 1); + + { + // Whole range + SequenceValidator validator(1, 256); + t.foreach_key(ForeachKeyValidator(validator)); + EXPECT_FALSE(validator.failed()); + } + { + // Subranges + for (int startval = 1; startval < 259; ++startval) { + for (int endval = 1; endval < 259; ++endval) { + SequenceValidator validator(startval, std::max(0, std::min(endval,257) - std::min(startval, 257))); + Iterator start = t.lowerBound(startval); + Iterator end = t.lowerBound(endval); + validate_subrange(start, end, validator); + } + } + } +}; int Test::Main() @@ -1515,6 +1591,7 @@ Test::Main() requireThatSmallNodesWorks(); requireThatApplyWorks(); requireThatIteratorDistanceWorks(); + requireThatForeachKeyWorks(); TEST_DONE(); } diff --git a/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp b/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp index d6e1aef9394..e95e8a5c58b 100644 --- a/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp +++ b/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp @@ -60,7 +60,7 @@ template <typename T> FullBenchmark<T>::FullBenchmark(size_t numDocs, size_t numValues) : _values(numDocs*numValues), _query(numValues), - _dp(IAccelrated::getAccelrator()) + _dp(IAccelrated::getAccelerator()) { for (size_t i(0); i < numDocs; i++) { for (size_t j(0); j < numValues; j++) { diff --git a/vespalib/src/tests/traits/traits_test.cpp b/vespalib/src/tests/traits/traits_test.cpp index 0a29721df1d..7751327df75 100644 --- a/vespalib/src/tests/traits/traits_test.cpp +++ b/vespalib/src/tests/traits/traits_test.cpp @@ -42,4 +42,14 @@ TEST("require that can_skip_destruction works") { EXPECT_EQUAL(can_skip_destruction<Child2>::value, true); } +struct NoType {}; +struct TypeType { using type = NoType; }; +struct NoTypeType { static constexpr int type = 3; }; + +TEST("require that type type member can be detected") { + EXPECT_FALSE(has_type_type_v<NoType>); + EXPECT_TRUE(has_type_type_v<TypeType>); + EXPECT_FALSE(has_type_type_v<NoTypeType>); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/typify/CMakeLists.txt b/vespalib/src/tests/typify/CMakeLists.txt new file mode 100644 index 00000000000..29e95af1988 --- /dev/null +++ b/vespalib/src/tests/typify/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_typify_test_app TEST + SOURCES + typify_test.cpp + DEPENDS + vespalib + gtest +) +vespa_add_test(NAME vespalib_typify_test_app COMMAND vespalib_typify_test_app) diff --git a/vespalib/src/tests/typify/typify_test.cpp b/vespalib/src/tests/typify/typify_test.cpp new file mode 100644 index 00000000000..4c3f1c512ca --- /dev/null +++ b/vespalib/src/tests/typify/typify_test.cpp @@ -0,0 +1,124 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/typify.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; + +struct A { static constexpr int value_from_type = 1; }; +struct B { static constexpr int value_from_type = 2; }; + +struct MyIntA { int value; }; +struct MyIntB { int value; }; +struct MyIntC { int value; }; // no typifier for this type + +// MyIntA -> A or B +struct TypifyMyIntA { + template <typename T> using Result = TypifyResultType<T>; + template <typename F> static decltype(auto) resolve(MyIntA value, F &&f) { + if (value.value == 1) { + return f(Result<A>()); + } else if (value.value == 2) { + return f(Result<B>()); + } + abort(); + } +}; + +// MyIntB -> TypifyResultValue<int,1> or TypifyResultValue<int,2> +struct TypifyMyIntB { + template <int VALUE> using Result = TypifyResultValue<int,VALUE>; + template <typename F> static decltype(auto) resolve(MyIntB value, F &&f) { + if (value.value == 1) { + return f(Result<1>()); + } else if (value.value == 2) { + return f(Result<2>()); + } + abort(); + } +}; + +using TX = TypifyValue<TypifyBool, TypifyMyIntA, TypifyMyIntB>; + +//----------------------------------------------------------------------------- + +struct GetFromType { + template <typename T> static int invoke() { return T::value_from_type; } +}; + +TEST(TypifyTest, simple_type_typification_works) { + auto res1 = typify_invoke<1,TX,GetFromType>(MyIntA{1}); + auto res2 = typify_invoke<1,TX,GetFromType>(MyIntA{2}); + EXPECT_EQ(res1, 1); + EXPECT_EQ(res2, 2); +} + +struct GetFromValue { + template <typename R> static int invoke() { return R::value; } +}; + +TEST(TypifyTest, simple_value_typification_works) { + auto res1 = typify_invoke<1,TX,GetFromValue>(MyIntB{1}); + auto res2 = typify_invoke<1,TX,GetFromValue>(MyIntB{2}); + EXPECT_EQ(res1, 1); + EXPECT_EQ(res2, 2); +} + +struct MaybeSum { + template <typename F1, typename V1, typename F2, typename V2> static int invoke(MyIntC v3) { + int res = 0; + if (F1::value) { + res += V1::value_from_type; + } + if (F2::value) { + res += V2::value; + } + res += v3.value; + return res; + } +}; + +TEST(TypifyTest, complex_typification_works) { + auto res1 = typify_invoke<4,TX,MaybeSum>(false, MyIntA{2}, false, MyIntB{1}, MyIntC{4}); + auto res2 = typify_invoke<4,TX,MaybeSum>(false, MyIntA{2}, true, MyIntB{1}, MyIntC{4}); + auto res3 = typify_invoke<4,TX,MaybeSum>(true, MyIntA{2}, false, MyIntB{1}, MyIntC{4}); + auto res4 = typify_invoke<4,TX,MaybeSum>(true, MyIntA{2}, true, MyIntB{1}, MyIntC{4}); + EXPECT_EQ(res1, 4); + EXPECT_EQ(res2, 5); + EXPECT_EQ(res3, 6); + EXPECT_EQ(res4, 7); +} + +struct Singleton { + virtual int get() const = 0; + virtual ~Singleton() {} +}; + +template <int A, int B> +struct MySingleton : Singleton { + MySingleton() = default; + MySingleton(const MySingleton &) = delete; + MySingleton &operator=(const MySingleton &) = delete; + int get() const override { return A + B; } +}; + +struct GetSingleton { + template <typename A, typename B> + static const Singleton &invoke() { + static MySingleton<A::value, B::value> obj; + return obj; + } +}; + +TEST(TypifyTest, typify_invoke_can_return_object_reference) { + const Singleton &s1 = typify_invoke<2,TX,GetSingleton>(MyIntB{1}, MyIntB{1}); + const Singleton &s2 = typify_invoke<2,TX,GetSingleton>(MyIntB{2}, MyIntB{2}); + const Singleton &s3 = typify_invoke<2,TX,GetSingleton>(MyIntB{2}, MyIntB{2}); + EXPECT_EQ(s1.get(), 2); + EXPECT_EQ(s2.get(), 4); + EXPECT_EQ(s3.get(), 4); + EXPECT_NE(&s1, &s2); + EXPECT_EQ(&s2, &s3); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.h b/vespalib/src/vespa/vespalib/btree/btreeiterator.h index 55ab37759ad..6933fc1c2d0 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.h +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.h @@ -303,6 +303,47 @@ protected: * @param pathSize New tree height (number of levels of internal nodes) */ VESPA_DLL_LOCAL void clearPath(uint32_t pathSize); + + /** + * Call func with leaf entry key value as argument for all leaf entries in subtree + * from this iterator position to end of subtree. + */ + template <typename FunctionType> + void + foreach_key_range_start(uint32_t level, FunctionType func) const + { + if (level > 0u) { + --level; + foreach_key_range_start(level, func); + auto &store = _allocator->getNodeStore(); + auto node = _path[level].getNode(); + uint32_t idx = _path[level].getIdx(); + node->foreach_key_range(store, idx + 1, node->validSlots(), func); + } else { + _leaf.getNode()->foreach_key_range(_leaf.getIdx(), _leaf.getNode()->validSlots(), func); + } + } + + /** + * Call func with leaf entry key value as argument for all leaf entries in subtree + * from start of subtree until this iterator position is reached (i.e. entries in + * subtree before this iterator position). + */ + template <typename FunctionType> + void + foreach_key_range_end(uint32_t level, FunctionType func) const + { + if (level > 0u) { + --level; + auto &store = _allocator->getNodeStore(); + auto node = _path[level].getNode(); + uint32_t eidx = _path[level].getIdx(); + node->foreach_key_range(store, 0, eidx, func); + foreach_key_range_end(level, func); + } else { + _leaf.getNode()->foreach_key_range(0, _leaf.getIdx(), func); + } + } public: bool @@ -451,6 +492,68 @@ public: _leafRoot->foreach_key(func); } } + + /** + * Call func with leaf entry key value as argument for all leaf entries in tree from + * this iterator position until end_itr position is reached (i.e. entries in + * range [this iterator, end_itr)). + */ + template <typename FunctionType> + void + foreach_key_range(const BTreeIteratorBase &end_itr, FunctionType func) const + { + if (!valid()) { + return; + } + if (!end_itr.valid()) { + foreach_key_range_start(_pathSize, func); + return; + } + assert(_pathSize == end_itr._pathSize); + assert(_allocator == end_itr._allocator); + uint32_t level = _pathSize; + if (level > 0u) { + /** + * Tree has intermediate nodes. Detect lowest shared tree node for this + * iterator and end_itr. + */ + uint32_t idx; + uint32_t eidx; + do { + --level; + assert(_path[level].getNode() == end_itr._path[level].getNode()); + idx = _path[level].getIdx(); + eidx = end_itr._path[level].getIdx(); + if (idx > eidx) { + return; + } + if (idx != eidx) { + ++level; + break; + } + } while (level != 0); + if (level > 0u) { + // Lowest shared node is an intermediate node. + // Left subtree for child [idx], from this iterator position to end of subtree. + foreach_key_range_start(level - 1, func); + auto &store = _allocator->getNodeStore(); + auto node = _path[level - 1].getNode(); + // Any intermediate subtrees for children [idx + 1, eidx). + node->foreach_key_range(store, idx + 1, eidx, func); + // Right subtree for child [eidx], from start of subtree to end_itr position. + end_itr.foreach_key_range_end(level - 1, func); + return; + } else { + // Lowest shared node is a leaf node. + assert(_leaf.getNode() == end_itr._leaf.getNode()); + } + } + uint32_t idx = _leaf.getIdx(); + uint32_t eidx = end_itr._leaf.getIdx(); + if (idx < eidx) { + _leaf.getNode()->foreach_key_range(idx, eidx, func); + } + } }; diff --git a/vespalib/src/vespa/vespalib/btree/btreenode.h b/vespalib/src/vespa/vespalib/btree/btreenode.h index b34be33ccf5..0c70e70bc6a 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenode.h +++ b/vespalib/src/vespa/vespalib/btree/btreenode.h @@ -370,6 +370,26 @@ public: } } + /** + * Call func with leaf entry key value as argument for all leaf entries in subtrees + * for children [start_idx, end_idx). + */ + template <typename NodeStoreType, typename FunctionType> + void foreach_key_range(NodeStoreType &store, uint32_t start_idx, uint32_t end_idx, FunctionType func) const { + const BTreeNode::Ref *it = this->_data; + const BTreeNode::Ref *ite = it + end_idx; + it += start_idx; + if (this->getLevel() > 1u) { + for (; it != ite; ++it) { + store.mapInternalRef(*it)->foreach_key(store, func); + } + } else { + for (; it != ite; ++it) { + store.mapLeafRef(*it)->foreach_key(func); + } + } + } + template <typename NodeStoreType, typename FunctionType> void foreach(NodeStoreType &store, FunctionType func) const { const BTreeNode::Ref *it = this->_data; @@ -459,6 +479,19 @@ public: } } + /** + * Call func with leaf entry key value as argument for leaf entries [start_idx, end_idx). + */ + template <typename FunctionType> + void foreach_key_range(uint32_t start_idx, uint32_t end_idx, FunctionType func) const { + const KeyT *it = _keys; + const KeyT *ite = it + end_idx; + it += start_idx; + for (; it != ite; ++it) { + func(*it); + } + } + template <typename FunctionType> void foreach(FunctionType func) const { const KeyT *it = _keys; diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp index 7ff393c87f8..8588a5510f7 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp @@ -20,4 +20,14 @@ Avx2Accelrator::squaredEuclideanDistance(const double * a, const double * b, siz return avx::euclideanDistanceSelectAlignment<double, 32>(a, b, sz); } +void +Avx2Accelrator::and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::andChunks<32u, 2u>(offset, src, dest); +} + +void +Avx2Accelrator::or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::orChunks<32u, 2u>(offset, src, dest); +} + } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h index 3e0dbb28110..b6f3d299748 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h @@ -15,6 +15,8 @@ public: size_t populationCount(const uint64_t *a, size_t sz) const override; double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override; double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override; + void and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; + void or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp index 0941e6d6ad8..4dade08e77a 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp @@ -32,4 +32,14 @@ Avx512Accelrator::squaredEuclideanDistance(const double * a, const double * b, s return avx::euclideanDistanceSelectAlignment<double, 64>(a, b, sz); } +void +Avx512Accelrator::and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::andChunks<64, 1>(offset, src, dest); +} + +void +Avx512Accelrator::or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::orChunks<64, 1>(offset, src, dest); +} + } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h index 209ec06c857..a54d57407b2 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h @@ -17,6 +17,8 @@ public: size_t populationCount(const uint64_t *a, size_t sz) const override; double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override; double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override; + void and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; + void or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp index f9684e88c63..f9dfaacf626 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp @@ -165,4 +165,14 @@ GenericAccelrator::squaredEuclideanDistance(const double * a, const double * b, return euclideanDistanceT<double, 4>(a, b, sz); } +void +GenericAccelrator::and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::andChunks<16, 4>(offset, src, dest); +} + +void +GenericAccelrator::or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::orChunks<16,4>(offset, src, dest); +} + } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h index 50a3d59d49d..2335b40fe85 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h @@ -25,6 +25,8 @@ public: size_t populationCount(const uint64_t *a, size_t sz) const override; double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override; double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override; + void and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; + void or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp index bb132165e53..de917c5f065 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp @@ -46,7 +46,8 @@ std::vector<T> createAndFill(size_t sz) { } template<typename T> -void verifyDotproduct(const IAccelrated & accel) +void +verifyDotproduct(const IAccelrated & accel) { const size_t testLength(255); srand(1); @@ -66,7 +67,8 @@ void verifyDotproduct(const IAccelrated & accel) } template<typename T> -void verifyEuclideanDistance(const IAccelrated & accel) { +void +verifyEuclideanDistance(const IAccelrated & accel) { const size_t testLength(255); srand(1); std::vector<T> a = createAndFill<T>(testLength); @@ -84,7 +86,8 @@ void verifyEuclideanDistance(const IAccelrated & accel) { } } -void verifyPopulationCount(const IAccelrated & accel) +void +verifyPopulationCount(const IAccelrated & accel) { const uint64_t words[7] = {0x123456789abcdef0L, // 32 0x0000000000000000L, // 0 @@ -101,6 +104,118 @@ void verifyPopulationCount(const IAccelrated & accel) } } +void +fill(std::vector<uint64_t> & v, size_t n) { + v.reserve(n); + for (size_t i(0); i < n; i++) { + v.emplace_back(random()); + } +} + +void +simpleAndWith(std::vector<uint64_t> & dest, const std::vector<uint64_t> & src) { + for (size_t i(0); i < dest.size(); i++) { + dest[i] &= src[i]; + } +} + +void +simpleOrWith(std::vector<uint64_t> & dest, const std::vector<uint64_t> & src) { + for (size_t i(0); i < dest.size(); i++) { + dest[i] |= src[i]; + } +} + +std::vector<uint64_t> +simpleInvert(const std::vector<uint64_t> & src) { + std::vector<uint64_t> inverted; + inverted.reserve(src.size()); + for (size_t i(0); i < src.size(); i++) { + inverted.push_back(~src[i]); + } + return inverted; +} + +std::vector<uint64_t> +optionallyInvert(bool invert, std::vector<uint64_t> v) { + return invert ? simpleInvert(std::move(v)) : std::move(v); +} + +bool shouldInvert(bool invertSome) { + return invertSome ? (random() & 1) : false; +} + +void +verifyOr64(const IAccelrated & accel, const std::vector<std::vector<uint64_t>> & vectors, + size_t offset, size_t num_vectors, bool invertSome) +{ + std::vector<std::pair<const void *, bool>> vRefs; + for (size_t j(0); j < num_vectors; j++) { + vRefs.emplace_back(&vectors[j][0], shouldInvert(invertSome)); + } + + std::vector<uint64_t> expected = optionallyInvert(vRefs[0].second, vectors[0]); + for (size_t j = 1; j < num_vectors; j++) { + simpleOrWith(expected, optionallyInvert(vRefs[j].second, vectors[j])); + } + + uint64_t dest[8] __attribute((aligned(64))); + accel.or64(offset*sizeof(uint64_t), vRefs, dest); + int diff = memcmp(&expected[offset], dest, sizeof(dest)); + if (diff != 0) { + LOG_ABORT("Accelerator fails to compute correct 64 bytes OR"); + } +} + +void +verifyAnd64(const IAccelrated & accel, const std::vector<std::vector<uint64_t>> & vectors, + size_t offset, size_t num_vectors, bool invertSome) +{ + std::vector<std::pair<const void *, bool>> vRefs; + for (size_t j(0); j < num_vectors; j++) { + vRefs.emplace_back(&vectors[j][0], shouldInvert(invertSome)); + } + std::vector<uint64_t> expected = optionallyInvert(vRefs[0].second, vectors[0]); + for (size_t j = 1; j < num_vectors; j++) { + simpleAndWith(expected, optionallyInvert(vRefs[j].second, vectors[j])); + } + + uint64_t dest[8] __attribute((aligned(64))); + accel.and64(offset*sizeof(uint64_t), vRefs, dest); + int diff = memcmp(&expected[offset], dest, sizeof(dest)); + if (diff != 0) { + LOG_ABORT("Accelerator fails to compute correct 64 bytes AND"); + } +} + +void +verifyOr64(const IAccelrated & accel) { + std::vector<std::vector<uint64_t>> vectors(3) ; + for (auto & v : vectors) { + fill(v, 16); + } + for (size_t offset = 0; offset < 8; offset++) { + for (size_t i = 1; i < vectors.size(); i++) { + verifyOr64(accel, vectors, offset, i, false); + verifyOr64(accel, vectors, offset, i, true); + } + } +} + +void +verifyAnd64(const IAccelrated & accel) { + std::vector<std::vector<uint64_t>> vectors(3); + for (auto & v : vectors) { + fill(v, 16); + } + for (size_t offset = 0; offset < 8; offset++) { + for (size_t i = 1; i < vectors.size(); i++) { + verifyAnd64(accel, vectors, offset, i, false); + verifyAnd64(accel, vectors, offset, i, true); + } + } +} + class RuntimeVerificator { public: @@ -114,6 +229,8 @@ private: verifyEuclideanDistance<float>(accelrated); verifyEuclideanDistance<double>(accelrated); verifyPopulationCount(accelrated); + verifyAnd64(accelrated); + verifyOr64(accelrated); } }; @@ -122,7 +239,7 @@ RuntimeVerificator::RuntimeVerificator() GenericAccelrator generic; verify(generic); - const IAccelrated & thisCpu(IAccelrated::getAccelrator()); + const IAccelrated & thisCpu(IAccelrated::getAccelerator()); verify(thisCpu); } @@ -155,7 +272,7 @@ static Selector _G_selector; RuntimeVerificator _G_verifyAccelrator; const IAccelrated & -IAccelrated::getAccelrator() +IAccelrated::getAccelerator() { static IAccelrated::UP accelrator = _G_selector.create(); return *accelrator; diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h index 0292ad14643..2594a48dd33 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h @@ -4,6 +4,7 @@ #include <memory> #include <cstdint> +#include <vector> namespace vespalib::hwaccelrated { @@ -29,8 +30,12 @@ public: virtual size_t populationCount(const uint64_t *a, size_t sz) const = 0; virtual double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const = 0; virtual double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const = 0; + // AND 64 bytes from multiple, optionally inverted sources + virtual void and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const = 0; + // OR 64 bytes from multiple, optionally inverted sources + virtual void or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const = 0; - static const IAccelrated & getAccelrator() __attribute__((noinline)); + static const IAccelrated & getAccelerator() __attribute__((noinline)); }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp index f5daf2b9081..65b4c717681 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp @@ -24,5 +24,55 @@ populationCount(const uint64_t *a, size_t sz) { return count; } +template<typename T> +T get(const void * base, bool invert) { + T v; + memcpy(&v, base, sizeof(T)); + return __builtin_expect(invert, false) ? ~v : v; +} + +template <typename T> +const T * cast(const void * ptr, size_t offsetBytes) { + return static_cast<const T *>(static_cast<const void *>(static_cast<const char *>(ptr) + offsetBytes)); +} + +template<unsigned ChunkSize, unsigned Chunks> +void +andChunks(size_t offset, const std::vector<std::pair<const void *, bool>> & src, void * dest) { + typedef uint64_t Chunk __attribute__ ((vector_size (ChunkSize))); + static_assert(sizeof(Chunk) == ChunkSize, "sizeof(Chunk) == ChunkSize"); + static_assert(ChunkSize*Chunks == 64, "ChunkSize*Chunks == 64"); + Chunk * chunk = static_cast<Chunk *>(dest); + const Chunk * tmp = cast<Chunk>(src[0].first, offset); + for (size_t n=0; n < Chunks; n++) { + chunk[n] = get<Chunk>(tmp+n, src[0].second); + } + for (size_t i(1); i < src.size(); i++) { + tmp = cast<Chunk>(src[i].first, offset); + for (size_t n=0; n < Chunks; n++) { + chunk[n] &= get<Chunk>(tmp+n, src[i].second); + } + } +} + +template<unsigned ChunkSize, unsigned Chunks> +void +orChunks(size_t offset, const std::vector<std::pair<const void *, bool>> & src, void * dest) { + typedef uint64_t Chunk __attribute__ ((vector_size (ChunkSize))); + static_assert(sizeof(Chunk) == ChunkSize, "sizeof(Chunk) == ChunkSize"); + static_assert(ChunkSize*Chunks == 64, "ChunkSize*Chunks == 64"); + Chunk * chunk = static_cast<Chunk *>(dest); + const Chunk * tmp = cast<Chunk>(src[0].first, offset); + for (size_t n=0; n < Chunks; n++) { + chunk[n] = get<Chunk>(tmp+n, src[0].second); + } + for (size_t i(1); i < src.size(); i++) { + tmp = cast<Chunk>(src[i].first, offset); + for (size_t n=0; n < Chunks; n++) { + chunk[n] |= get<Chunk>(tmp+n, src[i].second); + } + } +} + } } diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp index efb1dbf4054..ad5d78d5ab6 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp @@ -87,6 +87,7 @@ ThreadStackExecutorBase::obtainTask(Worker &worker) if (!worker.idle) { assert(_taskCount != 0); --_taskCount; + wakeup(monitor); _barrier.completeEvent(worker.task.token); worker.idle = true; } @@ -96,7 +97,6 @@ ThreadStackExecutorBase::obtainTask(Worker &worker) worker.task = std::move(_tasks.front()); worker.idle = false; _tasks.pop(); - wakeup(monitor); return true; } if (_closed) { diff --git a/vespalib/src/vespa/vespalib/util/traits.h b/vespalib/src/vespa/vespalib/util/traits.h index eb0385abc72..7f8945954a8 100644 --- a/vespalib/src/vespa/vespalib/util/traits.h +++ b/vespalib/src/vespa/vespalib/util/traits.h @@ -39,4 +39,10 @@ struct can_skip_destruction : std::is_trivially_destructible<T> {}; //----------------------------------------------------------------------------- +template <typename, typename = std::void_t<>> struct has_type_type : std::false_type {}; +template <typename T> struct has_type_type<T, std::void_t<typename T::type>> : std::true_type {}; +template <typename T> constexpr bool has_type_type_v = has_type_type<T>::value; + +//----------------------------------------------------------------------------- + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/typify.h b/vespalib/src/vespa/vespalib/util/typify.h new file mode 100644 index 00000000000..0d84d1756a5 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/typify.h @@ -0,0 +1,110 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "traits.h" +#include <stddef.h> +#include <utility> + +namespace vespalib { + +//----------------------------------------------------------------------------- + +/** + * Typification result for values resolving into actual types. Using + * this exact template is not required, but the result type must have + * the name 'type' for the auto-unwrapping performed by typify_invoke + * to work. + **/ +template <typename T> struct TypifyResultType { + using type = T; +}; + +/** + * Typification result for values resolving into non-types. Using this + * exact template is not required, but is supplied for + * convenience. The resolved compile-time value should be called + * 'value' for consistency across typifiers. + **/ +template <typename T, T VALUE> struct TypifyResultValue { + static constexpr T value = VALUE; +}; + +/** + * Typification result for values resolving into simple templates + * (templated on one type). Using this exact template is not required, + * but is supplied for convenience and as example. The resolved + * template should be called 'templ' for consistency across typifiers. + **/ +template <template<typename> typename TT> struct TypifyResultSimpleTemplate { + template <typename T> using templ = TT<T>; +}; + +/** + * A Typifier is able to take a run-time value and resolve it into a + * type, non-type constant value or a template. The resolve result is + * passed to the specified function in the form of a thin result + * wrapper. + **/ +struct TypifyBool { + template <bool VALUE> using Result = TypifyResultValue<bool, VALUE>; + template <typename F> static decltype(auto) resolve(bool value, F &&f) { + if (value) { + return f(Result<true>()); + } else { + return f(Result<false>()); + } + } +}; + +//----------------------------------------------------------------------------- + +/** + * Template used to combine individual typifiers into a typifier able + * to resolve multiple types. + **/ +template <typename ...Ts> struct TypifyValue : Ts... { using Ts::resolve...; }; + +//----------------------------------------------------------------------------- + +template <size_t N, typename Typifier, typename Target, typename ...Rs> struct TypifyInvokeImpl { + static decltype(auto) select() { + static_assert(sizeof...(Rs) == N); + return Target::template invoke<Rs...>(); + } + template <typename T, typename ...Args> static decltype(auto) select(T &&value, Args &&...args) { + if constexpr (N == sizeof...(Rs)) { + return Target::template invoke<Rs...>(std::forward<T>(value), std::forward<Args>(args)...); + } else { + return Typifier::resolve(value, [&](auto t)->decltype(auto) + { + using X = decltype(t); + if constexpr (has_type_type_v<X>) { + return TypifyInvokeImpl<N, Typifier, Target, Rs..., typename X::type>::select(std::forward<Args>(args)...); + } else { + return TypifyInvokeImpl<N, Typifier, Target, Rs..., X>::select(std::forward<Args>(args)...); + } + }); + } + } +}; + +/** + * Typify the N first parameters using 'Typifier' (typically an + * instantiation of the TypifyValue template) and forward the + * remaining parameters to the Target::invoke template function with + * the typification results from the N first parameters as template + * parameters. Note that typification results that are types are + * unwrapped before being used as template parameters while + * typification results that are non-types or templates are kept in + * their wrappers when passed as template parameters. Please refer to + * the unit test for examples. + **/ +template <size_t N, typename Typifier, typename Target, typename ...Args> decltype(auto) typify_invoke(Args && ...args) { + static_assert(N > 0); + return TypifyInvokeImpl<N,Typifier,Target>::select(std::forward<Args>(args)...); +} + +//----------------------------------------------------------------------------- + +} diff --git a/vespamalloc/src/tests/overwrite/.gitignore b/vespamalloc/src/tests/overwrite/.gitignore index 5a8760f913d..537a606f87f 100644 --- a/vespamalloc/src/tests/overwrite/.gitignore +++ b/vespamalloc/src/tests/overwrite/.gitignore @@ -1,8 +1,5 @@ .depend Makefile -expectsignal -overwrite_test -overwrite_testd -/expectsignal-overwrite vespamalloc_overwrite_test_app vespamalloc_expectsignal-overwrite_app +vespamalloc_overwrite_test_with_vespamallocd_app diff --git a/vespamalloc/src/tests/overwrite/CMakeLists.txt b/vespamalloc/src/tests/overwrite/CMakeLists.txt index 29b6ac46eb4..9f8274ea2ce 100644 --- a/vespamalloc/src/tests/overwrite/CMakeLists.txt +++ b/vespamalloc/src/tests/overwrite/CMakeLists.txt @@ -13,3 +13,11 @@ vespa_add_executable(vespamalloc_expectsignal-overwrite_app vespa_add_test(NAME vespamalloc_overwrite_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/overwrite_test.sh DEPENDS vespamalloc_overwrite_test_app vespamalloc_expectsignal-overwrite_app vespamalloc vespamallocd) + +vespa_add_executable(vespamalloc_overwrite_test_with_vespamallocd_app TEST + SOURCES + overwrite.cpp + DEPENDS + vespamallocd +) +vespa_add_test(NAME vespamalloc_overwrite_test_with_vespamallocd_app NO_VALGRIND COMMAND vespamalloc_overwrite_test_with_vespamallocd_app testmemoryfill) diff --git a/vespamalloc/src/tests/overwrite/overwrite.cpp b/vespamalloc/src/tests/overwrite/overwrite.cpp index 84f96fbbb3e..1919a75ab00 100644 --- a/vespamalloc/src/tests/overwrite/overwrite.cpp +++ b/vespamalloc/src/tests/overwrite/overwrite.cpp @@ -29,9 +29,7 @@ private: void verifyWriteAfterFreeDetection(); // Should abort }; -Test::~Test() -{ -} +Test::~Test() = default; void Test::testFillValue(char *a) { diff --git a/vespamalloc/src/tests/test1/.gitignore b/vespamalloc/src/tests/test1/.gitignore index b7fab5d205c..45356cc94e7 100644 --- a/vespamalloc/src/tests/test1/.gitignore +++ b/vespamalloc/src/tests/test1/.gitignore @@ -2,3 +2,5 @@ Makefile testatomic vespamalloc_testatomic_app +vespamalloc_new_test_with_vespamalloc_app +vespamalloc_new_test_with_vespamallocd_app diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index cade2e092b4..15680f22595 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -6,3 +6,25 @@ vespa_add_executable(vespamalloc_testatomic_app TEST ${VESPA_ATOMIC_LIB} ) vespa_add_test(NAME vespamalloc_testatomic_app NO_VALGRIND COMMAND vespamalloc_testatomic_app) + +vespa_add_executable(vespamalloc_new_test_app TEST + SOURCES + new_test.cpp +) +vespa_add_test(NAME vespamalloc_new_test_app NO_VALGRIND COMMAND vespamalloc_new_test_app) + +vespa_add_executable(vespamalloc_new_test_with_vespamalloc_app TEST + SOURCES + new_test.cpp + DEPENDS + vespamalloc +) +vespa_add_test(NAME vespamalloc_new_test_with_vespamalloc_app NO_VALGRIND COMMAND vespamalloc_new_test_with_vespamalloc_app) + +vespa_add_executable(vespamalloc_new_test_with_vespamallocd_app TEST + SOURCES + new_test.cpp + DEPENDS + vespamallocd +) +vespa_add_test(NAME vespamalloc_new_test_with_vespamallocd_app NO_VALGRIND COMMAND vespamalloc_new_test_with_vespamallocd_app) diff --git a/vespamalloc/src/tests/test1/new_test.cpp b/vespamalloc/src/tests/test1/new_test.cpp new file mode 100644 index 00000000000..2400e41a1d9 --- /dev/null +++ b/vespamalloc/src/tests/test1/new_test.cpp @@ -0,0 +1,109 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> + +LOG_SETUP("new_test"); + +void cmp(const void *a, const void *b) { + EXPECT_EQUAL(a, b); +} +void cmp(const void *base, size_t offset, const void *p) { + cmp((static_cast<const char *>(base) + offset), p); +} + +template <typename S> +void veryfy_aligned(S * p) { + EXPECT_TRUE((uintptr_t(p) % alignof(S)) == 0); + memset(p, 0, sizeof(S)); +} + +TEST("verify new with normal alignment") { + struct S { + int a; + long b; + int c; + }; + static_assert(sizeof(S) == 24); + static_assert(alignof(S) == 8); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + cmp(s.get(), 8, &s->b); + cmp(s.get(), 16, &s->c); + LOG(info, "&s=%p &s.b=%p &s.c=%p", s.get(), &s->b, &s->c); +} + +TEST("verify new with alignment = 16") { + struct S { + int a; + alignas(16) long b; + int c; + }; + static_assert(sizeof(S) == 32); + static_assert(alignof(S) == 16); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + cmp(s.get(), 16, &s->b); + cmp(s.get(), 24, &s->c); + LOG(info, "&s=%p &s.b=%p &s.c=%p", s.get(), &s->b, &s->c); +} + +TEST("verify new with alignment = 32") { + struct S { + int a; + alignas(32) long b; + int c; + }; + static_assert(sizeof(S) == 64); + static_assert(alignof(S) == 32); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + cmp(s.get(), 32, &s->b); + cmp(s.get(), 40, &s->c); + LOG(info, "&s=%p &s.b=%p &s.c=%p", s.get(), &s->b, &s->c); +} + +TEST("verify new with alignment = 64") { + struct S { + int a; + alignas(64) long b; + int c; + }; + static_assert(sizeof(S) == 128); + static_assert(alignof(S) == 64); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + cmp(s.get(), 64, &s->b); + cmp(s.get(), 72, &s->c); + LOG(info, "&s=%p &s.b=%p &s.c=%p", s.get(), &s->b, &s->c); +} + +TEST("verify new with alignment = 64 with single element") { + struct S { + alignas(64) long a; + }; + static_assert(sizeof(S) == 64); + static_assert(alignof(S) == 64); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + LOG(info, "&s=%p", s.get()); +} + +TEST("verify new with alignment = 64 with single element") { + struct alignas(64) S { + long a; + }; + static_assert(sizeof(S) == 64); + static_assert(alignof(S) == 64); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + LOG(info, "&s=%p", s.get()); +} + + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespamalloc/src/vespamalloc/malloc/common.h b/vespamalloc/src/vespamalloc/malloc/common.h index b21a2f63ed5..36f1bd0521d 100644 --- a/vespamalloc/src/vespamalloc/malloc/common.h +++ b/vespamalloc/src/vespamalloc/malloc/common.h @@ -66,6 +66,7 @@ template <size_t MinClassSizeC> class CommonT { public: + static constexpr size_t MAX_ALIGN = 0x200000ul; enum {MinClassSize = MinClassSizeC}; static inline SizeClassT sizeClass(size_t sz) { SizeClassT tmp(msbIdx(sz - 1) - (MinClassSizeC - 1)); diff --git a/vespamalloc/src/vespamalloc/malloc/malloc.h b/vespamalloc/src/vespamalloc/malloc/malloc.h index b3131ac5cbb..df40197bbbd 100644 --- a/vespamalloc/src/vespamalloc/malloc/malloc.h +++ b/vespamalloc/src/vespamalloc/malloc/malloc.h @@ -16,7 +16,7 @@ class MemoryManager : public IAllocator { public: MemoryManager(size_t logLimitAtStart); - ~MemoryManager(); + ~MemoryManager() override; bool initThisThread() override; bool quitThisThread() override; void enableThreadSupport() override; @@ -26,9 +26,17 @@ public: size_t getMaxNumThreads() const override { return _threadList.getMaxNumThreads(); } void *malloc(size_t sz); + void *malloc(size_t sz, std::align_val_t); void *realloc(void *oldPtr, size_t sz); - void free(void *ptr) { freeSC(ptr, _segment.sizeClass(ptr)); } - void free(void *ptr, size_t sz) { freeSC(ptr, MemBlockPtrT::sizeClass(sz)); } + void free(void *ptr) { + freeSC(ptr, _segment.sizeClass(ptr)); + } + void free(void *ptr, size_t sz) { + freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz))); + } + void free(void *ptr, size_t sz, std::align_val_t alignment) { + freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz, alignment))); + } size_t getMinSizeForAlignment(size_t align, size_t sz) const { return MemBlockPtrT::getMinSizeForAlignment(align, sz); } size_t sizeClass(const void *ptr) const { return _segment.sizeClass(ptr); } @@ -88,9 +96,7 @@ MemoryManager<MemBlockPtrT, ThreadListT>::MemoryManager(size_t logLimitAtStart) } template <typename MemBlockPtrT, typename ThreadListT> -MemoryManager<MemBlockPtrT, ThreadListT>::~MemoryManager() -{ -} +MemoryManager<MemBlockPtrT, ThreadListT>::~MemoryManager() = default; template <typename MemBlockPtrT, typename ThreadListT> bool MemoryManager<MemBlockPtrT, ThreadListT>::initThisThread() @@ -159,13 +165,27 @@ void * MemoryManager<MemBlockPtrT, ThreadListT>::malloc(size_t sz) fprintf(stderr, "Memory %p(%ld) has been tampered with after free.\n", mem.ptr(), mem.size()); crash(); } - PARANOID_CHECK2(if (!mem.validFree() && mem.ptr()) { crash(); } ); mem.setExact(sz); mem.alloc(_prAllocLimit<=mem.adjustSize(sz)); return mem.ptr(); } template <typename MemBlockPtrT, typename ThreadListT> +void * MemoryManager<MemBlockPtrT, ThreadListT>::malloc(size_t sz, std::align_val_t alignment) +{ + MemBlockPtrT mem; + ThreadPool & tp = _threadList.getCurrent(); + tp.malloc(mem.adjustSize(sz, alignment), mem); + if (!mem.validFree()) { + fprintf(stderr, "Memory %p(%ld) has been tampered with after free.\n", mem.ptr(), mem.size()); + crash(); + } + mem.setExact(sz, alignment); + mem.alloc(_prAllocLimit<=mem.adjustSize(sz, alignment)); + return mem.ptr(); +} + +template <typename MemBlockPtrT, typename ThreadListT> void MemoryManager<MemBlockPtrT, ThreadListT>::freeSC(void *ptr, SizeClassT sc) { if (MemBlockPtrT::verifySizeClass(sc)) { diff --git a/vespamalloc/src/vespamalloc/malloc/mallocd.cpp b/vespamalloc/src/vespamalloc/malloc/mallocd.cpp index 8e8bb642efc..47c12b4f186 100644 --- a/vespamalloc/src/vespamalloc/malloc/mallocd.cpp +++ b/vespamalloc/src/vespamalloc/malloc/mallocd.cpp @@ -8,11 +8,11 @@ typedef ThreadListT<MemBlockBoundsCheck, Stat> ThreadList; typedef MemoryWatcher<MemBlockBoundsCheck, ThreadList> Allocator; static char _Gmem[sizeof(Allocator)]; -static Allocator *_GmemP = NULL; +static Allocator *_GmemP = nullptr; static Allocator * createAllocator() { - if (_GmemP == NULL) { + if (_GmemP == nullptr) { _GmemP = new (_Gmem) Allocator(-1, 0x7fffffffffffffffl); } return _GmemP; diff --git a/vespamalloc/src/vespamalloc/malloc/memblock.h b/vespamalloc/src/vespamalloc/malloc/memblock.h index 118fb0e046c..e8d8e274678 100644 --- a/vespamalloc/src/vespamalloc/malloc/memblock.h +++ b/vespamalloc/src/vespamalloc/malloc/memblock.h @@ -10,14 +10,14 @@ namespace vespamalloc { template <size_t MinSizeClassC, size_t MaxSizeClassMultiAllocC> class MemBlockT : public CommonT<MinSizeClassC> { - static const size_t MAX_ALIGN= 0x200000ul; public: - typedef StackEntry<StackReturnEntry> Stack; + using Parent = CommonT<MinSizeClassC>; + using Stack = StackEntry<StackReturnEntry>; enum { MaxSizeClassMultiAlloc = MaxSizeClassMultiAllocC, SizeClassSpan = (MaxSizeClassMultiAllocC-MinSizeClassC) }; - MemBlockT() : _ptr(NULL) { } + MemBlockT() : _ptr(nullptr) { } MemBlockT(void * p) : _ptr(p) { } MemBlockT(void * p, size_t /*sz*/) : _ptr(p) { } MemBlockT(void * p, size_t, bool) : _ptr(p) { } @@ -28,7 +28,8 @@ public: const void *ptr() const { return _ptr; } bool validAlloc() const { return true; } bool validFree() const { return true; } - void setExact(size_t ) { } + void setExact(size_t) { } + void setExact(size_t, std::align_val_t ) { } void alloc(bool ) { } void setThreadId(int ) { } void free() { } @@ -36,12 +37,13 @@ public: bool allocated() const { return false; } int threadId() const { return 0; } void info(FILE *, unsigned level=0) const { (void) level; } - Stack * callStack() { return NULL; } + Stack * callStack() { return nullptr; } size_t callStackLen() const { return 0; } void fillMemory(size_t) { } void logBigBlock(size_t exact, size_t adjusted, size_t gross) const __attribute__((noinline)); static size_t adjustSize(size_t sz) { return sz; } + static size_t adjustSize(size_t sz, std::align_val_t) { return sz; } static size_t unAdjustSize(size_t sz) { return sz; } static void dumpInfo(size_t level); static void dumpFile(FILE * fp) { _logFile = fp; } @@ -49,9 +51,9 @@ public: static void setFill(uint8_t ) { } static bool verifySizeClass(int sc) { (void) sc; return true; } static size_t getMinSizeForAlignment(size_t align, size_t sz) { - return (sz < MAX_ALIGN) + return (sz < Parent::MAX_ALIGN) ? std::max(sz, align) - : (align < MAX_ALIGN) ? sz : sz + align; + : (align < Parent::MAX_ALIGN) ? sz : sz + align; } private: void * _ptr; @@ -64,4 +66,3 @@ template <> void MemBlock::dumpInfo(size_t level); extern template class MemBlockT<5, 20>; } - diff --git a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.cpp b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.cpp index 0c608fed5d5..d147bd5ba41 100644 --- a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.cpp +++ b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.cpp @@ -15,7 +15,8 @@ void MemBlockBoundsCheckBaseTBase::verifyFill() const const uint8_t *c(static_cast<const uint8_t *>(ptr())), *e(c+size()); for(;(c < e) && (*c == _fillValue); c++) { } if (c != e) { - fprintf(_logFile, "Incorrect fillvalue (%2x) instead of (%2x) at position %ld of %ld\n", *c, _fillValue, c - static_cast<const uint8_t *>(ptr()), size()); + fprintf(_logFile, "Incorrect fillvalue (%2x) instead of (%2x) at position %ld(%p) of %ld(%p - %p)\n", + *c, _fillValue, c - static_cast<const uint8_t *>(ptr()), c, size(), ptr(), e); abort(); } } diff --git a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h index 21e9d74c0d2..1860f2f36d3 100644 --- a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h +++ b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h @@ -11,15 +11,22 @@ class MemBlockBoundsCheckBaseTBase : public CommonT<5> public: typedef StackEntry<StackReturnEntry> Stack; void * rawPtr() { return _ptr; } - void *ptr() { unsigned *p((unsigned*)_ptr); return p ? (p+4) : NULL; } - const void *ptr() const { unsigned *p((unsigned*)_ptr); return p ? (p+4) : NULL; } + void *ptr() { + char *p((char*)_ptr); + return p ? (p+alignment()) : nullptr; + } + const void *ptr() const { + const char *p((const char*)_ptr); + return p ? (p+alignment()) : nullptr; + } void setThreadId(int th) { if (_ptr) { static_cast<uint32_t*>(_ptr)[2] = th; } } - bool allocated() const { return (static_cast<unsigned*>(_ptr)[3] == ALLOC_MAGIC); } - size_t size() const { return static_cast<const uint64_t *>(_ptr)[0]; } + bool allocated() const { return (static_cast<uint32_t*>(_ptr)[3] == ALLOC_MAGIC); } + size_t size() const { return static_cast<const uint32_t *>(_ptr)[0]; } + size_t alignment() const { return static_cast<const uint32_t *>(_ptr)[1]; } int threadId() const { return static_cast<int*>(_ptr)[2]; } - Stack * callStack() { return reinterpret_cast<Stack *>((char *)_ptr + size() + 4*sizeof(unsigned)); } - const Stack * callStack() const { return reinterpret_cast<const Stack *>((const char *)_ptr + size() + 4*sizeof(unsigned)); } + Stack * callStack() { return reinterpret_cast<Stack *>((char *)_ptr + size() + alignment()); } + const Stack * callStack() const { return reinterpret_cast<const Stack *>((const char *)_ptr + size() + alignment()); } void fillMemory(size_t sz) { if (_fillValue != NO_FILL) { memset(ptr(), _fillValue, sz); @@ -44,7 +51,17 @@ protected: MemBlockBoundsCheckBaseTBase(void * p) : _ptr(p) { } void verifyFill() const __attribute__((noinline)); - void setSize(size_t sz) { static_cast<uint64_t *>(_ptr)[0] = sz; } + void setSize(size_t sz) { + assert(sz < 0x100000000ul); + static_cast<uint32_t *>(_ptr)[0] = sz; + } + void setAlignment(size_t alignment) { static_cast<uint32_t *>(_ptr)[1] = alignment; } + static constexpr size_t preambleOverhead(std::align_val_t alignment) { + return std::max(preambleOverhead(), size_t(alignment)); + } + static constexpr size_t preambleOverhead() { + return 4*sizeof(unsigned); + } enum { ALLOC_MAGIC = 0xF1E2D3C4, @@ -69,15 +86,22 @@ public: MaxSizeClassMultiAlloc = MaxSizeClassMultiAllocC, SizeClassSpan = (MaxSizeClassMultiAllocC-5) }; - MemBlockBoundsCheckBaseT() : MemBlockBoundsCheckBaseTBase(NULL) { } - MemBlockBoundsCheckBaseT(void * p) : MemBlockBoundsCheckBaseTBase(p ? (unsigned *)p-4 : NULL) { } - MemBlockBoundsCheckBaseT(void * p, size_t sz) : MemBlockBoundsCheckBaseTBase(p) { setSize(sz); } + MemBlockBoundsCheckBaseT() : MemBlockBoundsCheckBaseTBase(nullptr) { } + MemBlockBoundsCheckBaseT(void * p) + : MemBlockBoundsCheckBaseTBase(p ? static_cast<char *>(p) - preambleOverhead() : nullptr) + { } + MemBlockBoundsCheckBaseT(void * p, size_t sz) + : MemBlockBoundsCheckBaseTBase(p) + { + setSize(sz); + setAlignment(preambleOverhead()); + } MemBlockBoundsCheckBaseT(void * p, size_t, bool) : MemBlockBoundsCheckBaseTBase(p) { } bool validCommon() const { const unsigned *p(reinterpret_cast<const unsigned*>(_ptr)); return p && ((p[3] == ALLOC_MAGIC) || (p[3] == FREE_MAGIC)) - && *(reinterpret_cast<const unsigned *> ((const char*)_ptr + size() + 4*sizeof(unsigned) + StackTraceLen*sizeof(void *))) == TAIL_MAGIC; + && *(reinterpret_cast<const unsigned *> ((const char*)_ptr + size() + alignment() + StackTraceLen*sizeof(void *))) == TAIL_MAGIC; } bool validAlloc1() const { unsigned *p((unsigned*)_ptr); @@ -110,7 +134,12 @@ public: fillMemory(size()); setTailMagic(); } - void setExact(size_t sz) { init(sz); } + void setExact(size_t sz) { + init(sz, preambleOverhead()); + } + void setExact(size_t sz, std::align_val_t alignment) { + init(sz, preambleOverhead(alignment)); + } size_t callStackLen() const { const Stack * stack = callStack(); // Use int to avoid compiler warning about always true. @@ -121,17 +150,30 @@ public: } return StackTraceLen; } - static size_t adjustSize(size_t sz) { return sz + ((4+1)*sizeof(unsigned) + StackTraceLen*sizeof(void *)); } - static size_t unAdjustSize(size_t sz) { return sz - ((4+1)*sizeof(unsigned) + StackTraceLen*sizeof(void *)); } + static constexpr size_t adjustSize(size_t sz) { return sz + overhead(); } + static constexpr size_t adjustSize(size_t sz, std::align_val_t alignment) { return sz + overhead(alignment); } + static constexpr size_t unAdjustSize(size_t sz) { return sz - overhead(); } static void dumpInfo(size_t level) __attribute__((noinline)); - static size_t getMinSizeForAlignment(size_t align, size_t sz) { return sz + align; } + static constexpr size_t getMinSizeForAlignment(size_t align, size_t sz) { return sz + align; } void info(FILE * os, unsigned level=0) const __attribute__((noinline)); protected: - void setTailMagic() { *(reinterpret_cast<unsigned *> ((char*)_ptr + size() + 4*sizeof(unsigned) + StackTraceLen*sizeof(void *))) = TAIL_MAGIC; } - void init(size_t sz) { + static constexpr size_t postambleOverhead() { + return sizeof(unsigned) + StackTraceLen*sizeof(void *); + } + static constexpr size_t overhead() { + return preambleOverhead() + postambleOverhead(); + } + static constexpr size_t overhead(std::align_val_t alignment) { + return preambleOverhead(alignment) + postambleOverhead(); + } + void setTailMagic() { + *(reinterpret_cast<unsigned *> ((char*)_ptr + size() + alignment() + StackTraceLen*sizeof(void *))) = TAIL_MAGIC; + } + void init(size_t sz, size_t alignment) { if (_ptr) { setSize(sz); + setAlignment(alignment); setTailMagic(); } } diff --git a/vespamalloc/src/vespamalloc/malloc/overload.h b/vespamalloc/src/vespamalloc/malloc/overload.h index 7883578cc28..56cd8101731 100644 --- a/vespamalloc/src/vespamalloc/malloc/overload.h +++ b/vespamalloc/src/vespamalloc/malloc/overload.h @@ -21,13 +21,6 @@ private: static CreateAllocator _CreateAllocator __attribute__ ((init_priority (543))); -#if 1 // Only until we get on to a new C++14 compiler -void operator delete(void* ptr, std::size_t sz) noexcept __attribute__((visibility ("default"))); -void operator delete[](void* ptr, std::size_t sz) noexcept __attribute__((visibility ("default"))); -void operator delete(void* ptr, std::size_t sz, const std::nothrow_t&) noexcept __attribute__((visibility ("default"))); -void operator delete[](void* ptr, std::size_t sz, const std::nothrow_t&) noexcept __attribute__((visibility ("default"))); -#endif - void* operator new(std::size_t sz) { void * ptr(vespamalloc::createAllocator()->malloc(sz)); @@ -74,6 +67,42 @@ void operator delete[](void* ptr, std::size_t sz, const std::nothrow_t&) noexcep if (ptr) { vespamalloc::_GmemP->free(ptr, sz); } } +/* + * Below are overloads taking alignment into account too. + * Due to allocation being power of 2 up to huge page size (2M) + * alignment will always be satisfied. size will always be larger or equal to alignment. + */ +void* operator new(std::size_t sz, std::align_val_t alignment) { + return vespamalloc::_GmemP->malloc(sz, alignment); +} +void* operator new(std::size_t sz, std::align_val_t alignment, const std::nothrow_t&) noexcept { + return vespamalloc::_GmemP->malloc(sz, alignment); +} +void operator delete(void* ptr , std::align_val_t) noexcept { + return vespamalloc::_GmemP->free(ptr); +} +void operator delete(void* ptr, std::align_val_t, const std::nothrow_t&) noexcept { + return vespamalloc::_GmemP->free(ptr); +} +void* operator new[](std::size_t sz, std::align_val_t alignment) { + return vespamalloc::_GmemP->malloc(sz, alignment); +} +void* operator new[](std::size_t sz, std::align_val_t alignment, const std::nothrow_t&) noexcept { + return vespamalloc::_GmemP->malloc(sz, alignment); +} +void operator delete[](void* ptr, std::align_val_t) noexcept { + return vespamalloc::_GmemP->free(ptr); +} +void operator delete[](void* ptr, std::align_val_t, const std::nothrow_t&) noexcept { + return vespamalloc::_GmemP->free(ptr); +} +void operator delete(void* ptr, std::size_t sz, std::align_val_t alignment) noexcept { + return vespamalloc::_GmemP->free(ptr, sz, alignment); +} +void operator delete[](void* ptr, std::size_t sz, std::align_val_t alignment) noexcept { + return vespamalloc::_GmemP->free(ptr, sz, alignment); +} + extern "C" { void * malloc(size_t sz) { |