diff options
1398 files changed, 48926 insertions, 15491 deletions
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 00000000000..52c655eebfe --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,48 @@ +name: CI +on: [push, pull_request] +jobs: + build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: python/vespa + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.6' + architecture: 'x64' + - name: Install the library + run: | + pip install nbdev fastcore jupyter pytest + pip install -e . + - name: Test with pytest + run: | + pytest + - name: Read all notebooks + run: | + nbdev_read_nbs + - name: Check if all notebooks are cleaned + run: | + echo "Check we are starting with clean git checkout" + if [ -n "$(git status -uno -s)" ]; then echo "git status is not clean"; false; fi + echo "Trying to strip out notebooks" + nbdev_clean_nbs + echo "Check that strip out was unnecessary" + git status -s # display the status to see which nbs need cleaning up + if [ -n "$(git status -uno -s)" ]; then echo -e "!!! Detected unstripped out notebooks\n!!!Remember to run nbdev_install_git_hooks"; false; fi + - name: Check if there is no diff library/notebooks + run: | + if [ -n "$(nbdev_diff_nbs)" ]; then echo -e "!!! Detected difference between the notebooks and the library"; false; fi + - name: Run notebook tests + run: | + nbdev_test_nbs + - name: Build and publish + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.test_pypi_password }} + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + python setup.py sdist bdist_wheel + twine upload --repository testpypi dist/* diff --git a/CMakeLists.txt b/CMakeLists.txt index 906a00ad843..873e4c5fad7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,6 +85,7 @@ add_subdirectory(filedistribution) add_subdirectory(flags) add_subdirectory(fnet) add_subdirectory(fsa) +add_subdirectory(hosted-zone-api) add_subdirectory(jdisc_core) add_subdirectory(jdisc-security-filters) add_subdirectory(jdisc_http_service) @@ -142,8 +143,8 @@ add_subdirectory(vespaclient-java) add_subdirectory(vespajlib) add_subdirectory(vespalib) add_subdirectory(vespalog) -if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") -else() +if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin" AND + NOT DEFINED VESPA_USE_SANITIZER) add_subdirectory(vespamalloc) endif() add_subdirectory(vsm) diff --git a/bootstrap.sh b/bootstrap.sh index b0c3fc12ca6..b48f4b77836 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -42,7 +42,7 @@ echo "Using maven command: ${MAVEN_CMD}" echo "Using maven extra opts: ${MAVEN_EXTRA_OPTS}" mvn_install() { - ${MAVEN_CMD} --no-snapshot-updates clean install ${MAVEN_EXTRA_OPTS} "$@" + ${MAVEN_CMD} --no-snapshot-updates -Dmaven.wagon.http.retryHandler.count=5 clean install ${MAVEN_EXTRA_OPTS} "$@" } # Generate vtag map diff --git a/build_settings.cmake b/build_settings.cmake index c96a626ad0b..d7dd26f5ee7 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -55,7 +55,9 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" ST else() set(CXX_SPECIFIC_WARN_OPTS "-Wsuggest-override -Wnon-virtual-dtor -Wformat-security") if(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8" OR - VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.1") + (VESPA_OS_DISTRO STREQUAL "rhel" AND + VESPA_OS_DISTRO_VERSION VERSION_GREATER_EQUAL "8" AND + VESPA_OS_DISTRO_VERSION VERSION_LESS "9")) set(VESPA_ATOMIC_LIB "") else() set(VESPA_ATOMIC_LIB "atomic") @@ -167,8 +169,11 @@ endif() if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") else() -# Don't allow unresolved symbols in executables or shared libraries -set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") +if(NOT VESPA_USE_SANITIZER OR NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + # Don't allow unresolved symbols in shared libraries + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") +endif() +# Don't allow unresolved symbols in executables set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") # Enable GTest unit tests in shared libraries diff --git a/cloud-tenant-base/OWNERS b/cloud-tenant-base/OWNERS new file mode 100644 index 00000000000..ff9741f2060 --- /dev/null +++ b/cloud-tenant-base/OWNERS @@ -0,0 +1,2 @@ +mortent +bjorncs
\ No newline at end of file diff --git a/cloud-tenant-base/README b/cloud-tenant-base/README new file mode 100644 index 00000000000..3863ef33b12 --- /dev/null +++ b/cloud-tenant-base/README @@ -0,0 +1 @@ +Parent pom for Vespa Cloud applications diff --git a/cloud-tenant-base/pom.xml b/cloud-tenant-base/pom.xml new file mode 100644 index 00000000000..8ffa6be4f9c --- /dev/null +++ b/cloud-tenant-base/pom.xml @@ -0,0 +1,383 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <artifactId>cloud-tenant-base</artifactId> + <name>Vespa Cloud tenant base</name> + <version>7-SNAPSHOT</version> + <description>Parent POM for all Vespa Cloud applications.</description> + <url>https://github.com/vespa-engine</url> + <packaging>pom</packaging> + + <parent> + <artifactId>hosted-tenant-base</artifactId> + <groupId>com.yahoo.vespa</groupId> + <version>7-SNAPSHOT</version> + <relativePath>../hosted-tenant-base/pom.xml</relativePath> + </parent> + + <licenses> + <license> + <name>The Apache License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + </license> + </licenses> + <developers> + <developer> + <name>Vespa</name> + <url>https://github.com/vespa-engine</url> + </developer> + </developers> + <scm> + <connection>scm:git:git@github.com:vespa-engine/vespa.git</connection> + <developerConnection>scm:git:git@github.com:vespa-engine/vespa.git</developerConnection> + <url>git@github.com:vespa-engine/vespa.git</url> + </scm> + + <properties> + <endpoint>https://api.vespa-external.aws.oath.cloud:4443</endpoint> + </properties> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dependency-versions</artifactId> + <version>${vespaversion}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <version>${junit.version}</version> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <version>${junit.version}</version> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container</artifactId> + <version>${vespaversion}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${vespaversion}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>cloud-tenant-cd</artifactId> + <version>${test-framework.version}</version> + <scope>runtime</scope> + </dependency> + </dependencies> + + <profiles> + <profile> + <!-- Build *-fat-test.jar file that includes all non-test classes and resources + that are part of the class path during test and and test.jar that includes + all test classes and resources, and put it inside a zip: + 1. application classes and resources + 2. test classes and resources + 3. classes and resources in all dependencies of both (1) and (2) + 4. copy the fat-test-jar and test-jar to application-test/artifacts directory + 5. zip application-test --> + <id>fat-test-application</id> + <build> + <plugins> + <plugin> + <!-- dependencies, see (3) above --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <!-- JAR-like dependencies --> + <id>unpack-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <configuration> + <includeTypes>jar,test-jar</includeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + <!-- WARNING(2018-06-27): bcpkix-jdk15on-1.58.jar and + bcprov-jdk15on-1.58.jar are pulled in via + container-dev and both contains the same set of + bouncycastle signature files in META-INF: + BC1024KE.DSA, BC1024KE.SF, BC2048KE.DSA, and + BC2048KE.SF. By merging any of these two with any + other JAR file like we're doing here, the signatures + are wrong. Worse, what we're doing is WRONG but not + yet fatal. + + The symptom of this happening is that the tester fails + to load the SystemTest class(!?), and subsequently + tries to run all test-like files in the fat test JAR. + + The solution is to exclude such files. This happens + automatically with maven-assembly-plugin. --> + <excludes>META-INF/*.SF,META-INF/*.DSA</excludes> + </configuration> + </execution> + <execution> + <!-- non-JAR-like dependencies --> + <id>non-jar-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <excludeTypes>jar,test-jar</excludeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>copy-resources</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>target/fat-test-classes</outputDirectory> + <resources> + <!-- application classes and resources, see 1. above --> + <resource> + <directory>target/classes</directory> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>fat-test-jar</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <classesDirectory>target/fat-test-classes</classesDirectory> + <classifier>fat-test</classifier> + </configuration> + </execution> + <execution> + <id>test-jar</id> + <phase>package</phase> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>attach-artifact</id> + <phase>package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <tasks> + <!-- copy fat test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-fat-test.jar" + todir="target/application-test/artifacts/" /> + + <!-- copy slim test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-tests.jar" + todir="target/application-test/artifacts/" /> + + <!-- zip application-test, see 5. above --> + <zip destfile="target/application-test.zip" + basedir="target/application-test/" /> + </tasks> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + + <profile> <!-- Alias vespaversion with a more descriptive vespa.compile.version --> + <id>set-vespa-compile-version</id> + <activation> + <property> + <name>vespa.compile.version</name> + </property> + </activation> + <properties> + <vespaversion>${vespa.compile.version}</vespaversion> + </properties> + </profile> + + <profile> <!-- Alias vespaVersion with a more descriptive vespa.runtime.version --> + <id>set-vespa-runtime-version</id> + <activation> + <property> + <name>vespa.runtime.version</name> + </property> + </activation> + <properties> + <vespaVersion>${vespa.runtime.version}</vespaVersion> + </properties> + </profile> + </profiles> + + <build> + <finalName>${project.artifactId}</finalName> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <groups>${test.categories}</groups> + <redirectTestOutputToFile>false</redirectTestOutputToFile> + <trimStackTrace>false</trimStackTrace> + <systemPropertyVariables> + <application>${application}</application> + <tenant>${tenant}</tenant> + <instance>${instance}</instance> + <environment>${environment}</environment> + <region>${region}</region> + <endpoint>${endpoint}</endpoint> + <apiKeyFile>${apiKeyFile}</apiKeyFile> + <apiCertificateFile>${apiCertificateFile}</apiCertificateFile> + <dataPlaneKeyFile>${dataPlaneKeyFile}</dataPlaneKeyFile> + <dataPlaneCertificateFile>${dataPlaneCertificateFile}</dataPlaneCertificateFile> + </systemPropertyVariables> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + </configuration> + </plugin> + </plugins> + </pluginManagement> + + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>3.0.0-M2</version> + <executions> + <execution> + <id>enforce-java</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireJavaVersion> + <version>[11, )</version> + </requireJavaVersion> + <requireMavenVersion> + <version>[3.5, )</version> + </requireMavenVersion> + </rules> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>${maven-compiler-plugin.version}</version> + <configuration> + <source>${target_jdk_version}</source> + <target>${target_jdk_version}</target> + <showWarnings>true</showWarnings> + <showDeprecation>true</showDeprecation> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-maven-plugin</artifactId> + <version>${vespaversion}</version> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-application-maven-plugin</artifactId> + <version>${vespaversion}</version> + <executions> + <execution> + <goals> + <goal>packageApplication</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${vespaversion}</version> + <extensions>true</extensions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/cloud-tenant-cd/pom.xml b/cloud-tenant-cd/pom.xml new file mode 100644 index 00000000000..ba4b7d02020 --- /dev/null +++ b/cloud-tenant-cd/pom.xml @@ -0,0 +1,86 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <artifactId>cloud-tenant-cd</artifactId> + <name>Vespa Cloud tenant CD implementation</name> + <description>Test library implementation for Vespa Cloud applications.</description> + <url>https://github.com/vespa-engine</url> + <packaging>container-plugin</packaging> + + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent</relativePath> + </parent> + + <dependencies> + <!-- provided scope --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-cd-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-zone-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <!-- compile scope --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-auth</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-api</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <attachBundleArtifact>true</attachBundleArtifact> + <bundleClassifierName>deploy</bundleClassifierName> + <useCommonAssemblyIds>false</useCommonAssemblyIds> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> +</project>
\ No newline at end of file diff --git a/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java new file mode 100644 index 00000000000..75aaaec78ba --- /dev/null +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.hosted.cd; + +import ai.vespa.cloud.Zone; +import ai.vespa.hosted.api.ControllerHttpClient; +import ai.vespa.hosted.api.Properties; +import ai.vespa.hosted.api.TestConfig; +import ai.vespa.hosted.cd.http.HttpDeployment; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.zone.ZoneId; + +import java.nio.file.Files; +import java.nio.file.Paths; + +/** + * @author mortent + */ +public class VespaTestRuntime implements TestRuntime { + private final TestConfig config; + private final Deployment deploymentToTest; + + public VespaTestRuntime() { + String configPath = System.getProperty("vespa.test.config"); + TestConfig config = configPath != null ? fromFile(configPath) : fromController(); + this.config = config; + this.deploymentToTest = new HttpDeployment(config.deployments().get(config.zone()), new ai.vespa.hosted.auth.EndpointAuthenticator(config.system())); + } + + @Override + public Zone zone() { + return new Zone( + ai.vespa.cloud.Environment.valueOf(config.zone().environment().name()), + config.zone().region().value()); } + + /** Returns the deployment this is testing. */ + @Override + public Deployment deploymentToTest() { return deploymentToTest; } + + private static TestConfig fromFile(String path) { + try { + return TestConfig.fromJson(Files.readAllBytes(Paths.get(path))); + } + catch (Exception e) { + throw new IllegalArgumentException("Failed reading config from '" + path + "'!", e); + } + } + + private static TestConfig fromController() { + ControllerHttpClient controller = new ai.vespa.hosted.auth.ApiAuthenticator().controller(); + ApplicationId id = Properties.application(); + Environment environment = Properties.environment().orElse(Environment.dev); + ZoneId zone = Properties.region().map(region -> ZoneId.from(environment, region)) + .orElseGet(() -> controller.defaultZone(environment)); + return controller.testConfig(id, zone); + } +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java index e17589c325d..80d5416ab34 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.http; import ai.vespa.hosted.api.EndpointAuthenticator; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java index 74eb765dd0b..a803fc3e0e2 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.http; import ai.vespa.hosted.api.EndpointAuthenticator; diff --git a/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime b/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime new file mode 100644 index 00000000000..695fe363e4e --- /dev/null +++ b/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime @@ -0,0 +1,2 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +ai.vespa.hosted.cd.VespaTestRuntime
\ No newline at end of file diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationFileTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationFileTest.java index 201066179d1..a2caaafe968 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationFileTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationFileTest.java @@ -11,7 +11,6 @@ import java.io.File; /** * @author Ulf Lilleengen - * @since 5.1 */ public class FilesApplicationFileTest extends ApplicationFileTest { diff --git a/config-lib/abi-spec.json b/config-lib/abi-spec.json index 0a6a673e39e..fa352d8f6bd 100644 --- a/config-lib/abi-spec.json +++ b/config-lib/abi-spec.json @@ -178,8 +178,8 @@ "methods": [ "public void <init>(java.lang.String)", "public java.lang.String value()", - "public int hashCode()", "public boolean equals(java.lang.Object)", + "public int hashCode()", "public java.lang.String toString()", "public static java.util.List toValues(java.util.Collection)", "public static java.util.Map toValueMap(java.util.Map)", diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 1b5a8924fea..d536eaf8bfc 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -422,6 +422,7 @@ "methods": [ "public abstract com.yahoo.config.FileReference addFile(java.lang.String)", "public abstract com.yahoo.config.FileReference addUri(java.lang.String)", + "public com.yahoo.config.FileReference addApplicationPackage()", "public abstract java.lang.String fileSourceHost()", "public abstract java.util.List export()" ], diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java index 7df0e941731..0d4c0ad76eb 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java @@ -11,8 +11,10 @@ import com.yahoo.config.FileReference; */ public interface FileRegistry { + FileReference addFile(String relativePath); FileReference addUri(String uri); + default FileReference addApplicationPackage() { return addFile(""); } /** * Returns the name of the host which is the source of the files diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 68f27a21ce4..48a675fa182 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -45,6 +45,7 @@ import java.util.stream.Stream; */ public class DeploymentSpecXmlReader { + private static final String deploymentTag = "deployment"; private static final String instanceTag = "instance"; private static final String majorVersionTag = "major-version"; private static final String testTag = "test"; @@ -93,6 +94,9 @@ public class DeploymentSpecXmlReader { /** Reads a deployment spec from XML */ public DeploymentSpec read(String xmlForm) { Element root = XML.getDocument(xmlForm).getDocumentElement(); + if ( ! root.getTagName().equals(deploymentTag)) + throw new IllegalArgumentException("The root tag must be <deployment>"); + if (isEmptySpec(root)) return DeploymentSpec.empty; @@ -133,6 +137,13 @@ public class DeploymentSpecXmlReader { Element instanceTag, MutableOptional<String> globalServiceId, Element parentTag) { + if (instanceNameString.isBlank()) + throw new IllegalArgumentException("<instance> attribute 'id' must be specified, and not be blank"); + + // If this is an absolutely empty instance, or the implicit "default" instance but without content, ignore it + if (XML.getChildren(instanceTag).isEmpty() && (instanceTag.getAttributes().getLength() == 0 || instanceTag == parentTag)) + return List.of(); + if (validate) validateTagOrder(instanceTag); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationRoles.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationRoles.java new file mode 100644 index 00000000000..8492603e2e3 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationRoles.java @@ -0,0 +1,51 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +import com.google.common.base.Strings; + +/** + * @author mortent + */ +public class ApplicationRoles { + private final String applicationHostRole; + private final String applicationContainerRole; + + public ApplicationRoles(String applicationHostRole, String applicationContainerRole) { + this.applicationHostRole = applicationHostRole; + this.applicationContainerRole = applicationContainerRole; + } + + /** + * @return an ApplicationRoles instance if both hostRole and containerRole is non-empty, <code>null</code> otherwise + */ + public static ApplicationRoles fromString(String hostRole, String containerRole) { + if(Strings.isNullOrEmpty(hostRole) || Strings.isNullOrEmpty(containerRole)) { + return null; + } + return new ApplicationRoles(hostRole, containerRole); + } + + public String applicationContainerRole() { + return applicationContainerRole; + } + + public String applicationHostRole() { + return applicationHostRole; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ApplicationRoles that = (ApplicationRoles) o; + if (!applicationHostRole.equals(that.applicationHostRole)) return false; + return applicationContainerRole.equals(that.applicationContainerRole); + } + + @Override + public int hashCode() { + int result = applicationHostRole.hashCode(); + result = 31 * result + applicationContainerRole.hashCode(); + return result; + } +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 7641f14b007..6c0c2c1260f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.Zone; import java.io.File; import java.net.URI; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.Set; @@ -61,16 +62,16 @@ public interface ModelContext { // TODO: Only needed for LbServicesProducerTest default boolean useDedicatedNodeForLogserver() { return true; } - // TODO Revisit in May or June 2020 - boolean useAdaptiveDispatch(); + // TODO Remove when 7.225 is last + default boolean useAdaptiveDispatch() { return true; } default Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return Optional.empty(); } // TODO Revisit in May or June 2020 double defaultTermwiseLimit(); - // TODO Revisit in May or June 2020 - int defaultNumResponseThreads(); + // TODO Remove when 7.225 is last + default int defaultNumResponseThreads() { return 1; } // TODO Revisit in May or June 2020 double threadPoolSizeFactor(); @@ -78,11 +79,21 @@ public interface ModelContext { // TODO Revisit in May or June 2020 double queueSizeFactor(); - // TODO Revisit in May or June 2020 - double defaultSoftStartSeconds(); + // TODO Remove when 7.229 is last + default double defaultSoftStartSeconds() { return 0; } - // TODO Revisit in May or June 2020 - double defaultTopKProbability(); + // TODO Remove when 7.226 is last + default double defaultTopKProbability() { + return 0.9999; + } + + // TODO Remove when 7.238 is last + default String docprocLoadBalancerType() { + return "adaptive"; + } + + /// Default setting for the gc-options attribute if not specified explicit by application + String jvmGCOptions(); boolean useDistributorBtreeDb(); @@ -93,6 +104,11 @@ public interface ModelContext { // TODO(mpolden): Remove after May 2020 default boolean useDedicatedNodesWhenUnspecified() { return true; } + + Optional<ApplicationRoles> applicationRoles(); + + // TODO(bjorncs): Temporary feature flag, revisit July 2020 + default Duration jdiscHealthCheckProxyClientTimeout() { return Duration.ofSeconds(1); } } } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/ApplicationFileTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/ApplicationFileTest.java index b7ae6d9dcdf..5b0b11b88b7 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/ApplicationFileTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/ApplicationFileTest.java @@ -6,16 +6,25 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.util.ConfigUtils; import org.junit.Test; -import java.io.*; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.StringReader; import java.util.List; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen */ public abstract class ApplicationFileTest { + protected void writeAppTo(File destFolder) throws IOException { createFiles(destFolder, "vespa-services.xml", "vespa-hosts.xml"); createFolders(destFolder, "searchdefinitions", "components", "files", "templates"); @@ -34,7 +43,7 @@ public abstract class ApplicationFileTest { private void createFiles(File destFolder, String ... names) throws IOException { for (String name : names) { File f = new File(destFolder, name); - f.createNewFile(); + assertTrue(f.createNewFile()); IOUtils.writeFile(f, "foo", false); } } @@ -61,11 +70,11 @@ public abstract class ApplicationFileTest { ApplicationFile f1 = getApplicationFile(p1); ApplicationFile f2 = getApplicationFile(p2); - assertTrue(f1.equals(f1)); - assertFalse(f1.equals(f2)); + assertEquals(f1, f1); + assertNotEquals(f1, f2); - assertFalse(f2.equals(f1)); - assertTrue(f2.equals(f2)); + assertNotEquals(f2, f1); + assertEquals(f2, f2); } @Test @@ -315,4 +324,5 @@ public abstract class ApplicationFileTest { } public abstract ApplicationFile getApplicationFile(Path path) throws Exception; + } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 32d903786dc..a793630c8b9 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -3,6 +3,7 @@ package com.yahoo.config.application.api; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import org.junit.Test; @@ -422,6 +423,23 @@ public class DeploymentSpecTest { } @Test + public void testOnlyAthenzServiceDefinedInInstance() { + StringReader r = new StringReader( + "<deployment athenz-domain='domain'>" + + " <instance id='default' athenz-service='service' />" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + + assertEquals("domain", spec.athenzDomain().get().value()); + assertEquals(1, spec.instances().size()); + + DeploymentInstanceSpec instance = spec.instances().get(0); + assertEquals("default", instance.name().value()); + assertEquals("service", instance.athenzService(Environment.prod, RegionName.defaultName()).get().value()); + } + + @Test public void productionSpecWithParallelDeployments() { StringReader r = new StringReader( "<deployment>" + @@ -652,7 +670,7 @@ public class DeploymentSpecTest { public void deploymentSpecWithIncreasinglyStrictUpgradePoliciesInParallel() { StringReader r = new StringReader( "<deployment version='1.0'>" + - " <instance />" + + " <instance id='instance0'/>" + " <parallel>" + " <instance id='instance1'>" + " <upgrade policy='conservative'/>" + @@ -678,7 +696,7 @@ public class DeploymentSpecTest { " <upgrade policy='canary'/>" + " </instance>" + " </parallel>" + - " <instance />" + + " <instance id='instance3'/>" + "</deployment>" ); DeploymentSpec.fromXml(r); diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java index 4a7ef7b43f7..89972773e1c 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java @@ -3,6 +3,7 @@ package com.yahoo.config.application.api; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import org.junit.Test; @@ -326,6 +327,18 @@ public class DeploymentSpecWithoutInstanceTest { } @Test + public void testOnlyAthenzServiceDefined() { + StringReader r = new StringReader( + "<deployment athenz-domain='domain' athenz-service='service'>" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + + assertEquals("domain", spec.athenzDomain().get().value()); + assertEquals(List.of(), spec.instances()); + } + + @Test public void productionSpecWithParallelDeployments() { StringReader r = new StringReader( "<deployment>\n" + diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java index 761c3645bad..b9e7976ccf7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java @@ -40,7 +40,7 @@ public abstract class ConfigModel { * * @param configModelRepo The ConfigModelRepo of the VespaModel */ - public void initialize(ConfigModelRepo configModelRepo) { return; } + public void initialize(ConfigModelRepo configModelRepo) { } /** * Prepares this model to start serving config requests, possibly using properties of other models. diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 4dbfc8f7a8f..2232e57a06f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -2,6 +2,7 @@ package com.yahoo.config.model.deploy; import com.google.common.collect.ImmutableList; +import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; @@ -35,20 +36,16 @@ public class TestProperties implements ModelContext.Properties { private boolean hostedVespa = false; private Zone zone; private Set<ContainerEndpoint> endpoints = Collections.emptySet(); - private boolean isBootstrap = false; - private boolean isFirstTimeDeployment = false; private boolean useDedicatedNodeForLogserver = false; - private boolean useAdaptiveDispatch = false; - private double topKProbability = 1.0; private boolean useDistributorBtreeDb = false; private boolean useThreePhaseUpdates = false; private double defaultTermwiseLimit = 1.0; - private double softStartSeconds = 0.0; private double threadPoolSizeFactor = 0.0; private double queueSizeFactor = 0.0; - private int defaultNumResponseThreads = 0; + private String jvmGCOptions = null; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private AthenzDomain athenzDomain; + private ApplicationRoles applicationRoles; @Override public boolean multitenant() { return multitenant; } @Override public ApplicationId applicationId() { return applicationId; } @@ -59,45 +56,30 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean hostedVespa() { return hostedVespa; } @Override public Zone zone() { return zone; } @Override public Set<ContainerEndpoint> endpoints() { return endpoints; } + @Override public String jvmGCOptions() { return jvmGCOptions; } - @Override public boolean isBootstrap() { return isBootstrap; } - @Override public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } - @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } + @Override public boolean isBootstrap() { return false; } + @Override public boolean isFirstTimeDeployment() { return false; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } - - @Override - public double threadPoolSizeFactor() { + @Override public double threadPoolSizeFactor() { return threadPoolSizeFactor; } - - @Override - public double queueSizeFactor() { + @Override public double queueSizeFactor() { return queueSizeFactor; } - - @Override - public double defaultSoftStartSeconds() { - return softStartSeconds; - } - - @Override public int defaultNumResponseThreads() { - return defaultNumResponseThreads; - } - - @Override public double defaultTopKProbability() { return topKProbability; } @Override public boolean useDistributorBtreeDb() { return useDistributorBtreeDb; } @Override public boolean useThreePhaseUpdates() { return useThreePhaseUpdates; } @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); } + @Override public Optional<ApplicationRoles> applicationRoles() { return Optional.ofNullable(applicationRoles); } - public TestProperties setDefaultTermwiseLimit(double limit) { - defaultTermwiseLimit = limit; + public TestProperties setJvmGCOptions(String gcOptions) { + jvmGCOptions = gcOptions; return this; } - - public TestProperties setTopKProbability(double probability) { - topKProbability = probability; + public TestProperties setDefaultTermwiseLimit(double limit) { + defaultTermwiseLimit = limit; return this; } @@ -106,20 +88,11 @@ public class TestProperties implements ModelContext.Properties { return this; } - public TestProperties setDefaultNumResponseThreads(int numResponseThreads) { - defaultNumResponseThreads = numResponseThreads; - return this; - } - public TestProperties setUseThreePhaseUpdates(boolean useThreePhaseUpdates) { this.useThreePhaseUpdates = useThreePhaseUpdates; return this; } - public TestProperties setSoftStartSeconds(double softStartSeconds) { - this.softStartSeconds = softStartSeconds; - return this; - } public TestProperties setThreadPoolSizeFactor(double threadPoolSizeFactor) { this.threadPoolSizeFactor = threadPoolSizeFactor; return this; @@ -139,11 +112,6 @@ public class TestProperties implements ModelContext.Properties { return this; } - public TestProperties setUseAdaptiveDispatch(boolean useAdaptiveDispatch) { - this.useAdaptiveDispatch = useAdaptiveDispatch; - return this; - } - public TestProperties setMultitenant(boolean multitenant) { this.multitenant = multitenant; return this; @@ -174,6 +142,11 @@ public class TestProperties implements ModelContext.Properties { return this; } + public TestProperties setApplicationRoles(ApplicationRoles applicationRoles) { + this.applicationRoles = applicationRoles; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java index e55686accca..ab2e0f632e4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.model.container.Container; import java.io.Reader; import java.util.List; +import java.util.Optional; /** * A host provisioner based on a hosts.xml file. @@ -38,7 +39,7 @@ public class HostsXmlProvisioner implements HostProvisioner { } for (Host host : hosts.asCollection()) { if (host.aliases().contains(alias)) { - return new HostSpec(host.hostname(), host.aliases()); + return new HostSpec(host.hostname(), host.aliases(), Optional.empty()); } } throw new IllegalArgumentException("Unable to find host for alias '" + alias + "'"); @@ -50,7 +51,7 @@ public class HostsXmlProvisioner implements HostProvisioner { } private HostSpec host2HostSpec(Host host) { - return new HostSpec(host.hostname(), host.aliases()); + return new HostSpec(host.hostname(), host.aliases(), Optional.empty()); } private Host getFirstHost() { diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 1a1ed000478..62b9cefab78 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ProvisionLogger; @@ -118,8 +117,7 @@ public class InMemoryProvisioner implements HostProvisioner { List<Host> defaultHosts = freeNodes.get(defaultResources); if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts with default resources available"); Host newHost = freeNodes.removeValue(defaultResources, 0); - // Note: Always returns HostSpec with empty dockerImageRepo, which is OK since this method is never used when docker image repo is set - return new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version(), Optional.empty()); + return new HostSpec(newHost.hostname(), List.of(alias), Optional.empty()); } @Override @@ -173,12 +171,12 @@ public class InMemoryProvisioner implements HostProvisioner { private HostSpec retire(HostSpec host) { return new HostSpec(host.hostname(), - host.aliases(), - host.flavor(), - Optional.of(host.membership().get().retire()), + host.realResources(), + host.advertisedResources(), + host.requestedResources().orElse(NodeResources.unspecified()), + host.membership().get().retire(), host.version(), Optional.empty(), - Optional.empty(), host.dockerImageRepo()); } @@ -189,11 +187,11 @@ public class InMemoryProvisioner implements HostProvisioner { // Check if the current allocations are compatible with the new request for (int i = allocation.size() - 1; i >= 0; i--) { - Optional<NodeResources> currentResources = allocation.get(0).flavor().map(Flavor::resources); - if (currentResources.isEmpty() || requestedResources == NodeResources.unspecified) continue; - if (!currentResources.get().compatibleWith(requestedResources)) { + NodeResources currentResources = allocation.get(0).advertisedResources(); + if (currentResources.isUnspecified() || requestedResources.isUnspecified()) continue; + if ( ! currentResources.compatibleWith(requestedResources)) { HostSpec removed = allocation.remove(i); - freeNodes.put(currentResources.get(), new Host(removed.hostname())); // Return the node back to free pool + freeNodes.put(currentResources, new Host(removed.hostname())); // Return the node back to free pool } } @@ -202,11 +200,11 @@ public class InMemoryProvisioner implements HostProvisioner { // Find the smallest host that can fit the requested requested Optional<NodeResources> hostResources = freeNodes.keySet().stream() .sorted(new MemoryDiskCpu()) - .filter(resources -> requestedResources == NodeResources.unspecified || resources.satisfies(requestedResources)) + .filter(resources -> requestedResources.isUnspecified() || resources.satisfies(requestedResources)) .findFirst(); if (hostResources.isEmpty()) { if (canFail) - throw new IllegalArgumentException("Insufficient capacity of for " + requestedResources); + throw new IllegalArgumentException("Insufficient capacity for " + requestedResources); else break; // ¯\_(ツ)_/¯ } @@ -214,10 +212,11 @@ public class InMemoryProvisioner implements HostProvisioner { Host newHost = freeNodes.removeValue(hostResources.get(), 0); if (freeNodes.get(hostResources.get()).isEmpty()) freeNodes.removeAll(hostResources.get()); ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++); - allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(), - hostResources.map(Flavor::new), Optional.of(membership), + allocation.add(new HostSpec(newHost.hostname(), + hostResources.get(), hostResources.get(), requestedResources, + membership, newHost.version(), Optional.empty(), - requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources))); + Optional.empty())); } nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex); diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index 1f0e0755667..b4fd55c6d33 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -1,17 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.provision; +import com.yahoo.component.Version; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.net.HostName; +import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * A host provisioner used when there is no hosts.xml file (using localhost as the only host) @@ -28,12 +32,15 @@ public class SingleNodeProvisioner implements HostProvisioner { public SingleNodeProvisioner() { host = new Host(HostName.getLocalhost()); - this.hostSpec = new HostSpec(host.hostname(), host.aliases()); + this.hostSpec = new HostSpec(host.hostname(), host.aliases(), Optional.empty()); } public SingleNodeProvisioner(Flavor flavor) { host = new Host(HostName.getLocalhost()); - this.hostSpec = new HostSpec(host.hostname(), host.aliases(), flavor); + this.hostSpec = new HostSpec(host.hostname(), + flavor.resources(), flavor.resources(), flavor.resources(), + ClusterMembership.from(ClusterSpec.specification(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).group(ClusterSpec.Group.from(0)).vespaVersion("1").build(), 0), + Optional.empty(), Optional.empty(), Optional.empty()); } @Override @@ -45,7 +52,10 @@ public class SingleNodeProvisioner implements HostProvisioner { public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1 List<HostSpec> hosts = new ArrayList<>(); - hosts.add(new HostSpec(host.hostname(), host.aliases(), ClusterMembership.from(cluster, counter++))); + hosts.add(new HostSpec(host.hostname(), + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + ClusterMembership.from(cluster, counter++), + Optional.empty(), Optional.empty(), Optional.empty())); return hosts; } diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index 41a30c4553d..df9f72b2182 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -66,7 +66,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private final Map<Integer, NewDocumentType> inherits = new LinkedHashMap<>(); private final AnnotationTypeRegistry annotations = new AnnotationTypeRegistry(); private final StructDataType header; - private final StructDataType body; private final Set<FieldSet> fieldSets = new LinkedHashSet<>(); private final Set<Name> documentReferences; // Imported fields are virtual and therefore exist outside of the SD's document field definition @@ -84,7 +83,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp this( name, new StructDataType(name.getName() + ".header"), - new StructDataType(name.getName() + ".body"), new FieldSets(), documentReferences, importedFieldNames); @@ -96,14 +94,12 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp public NewDocumentType(Name name, StructDataType header, - StructDataType body, FieldSets fs, Set<Name> documentReferences, Set<String> importedFieldNames) { super(name.getName()); this.name = name; this.header = header; - this.body = body; if (fs != null) { this.fieldSets.addAll(fs.userFieldSets().values()); for (FieldSet f : fs.builtInFieldSets().values()) { @@ -122,7 +118,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } public DataType getHeader() { return header; } - public DataType getBody() { return body; } public Collection<NewDocumentType> getInherited() { return inherits.values(); } public NewDocumentType getInherited(Name inherited) { return inherits.get(inherited.getId()); } public NewDocumentType removeInherited(Name inherited) { return inherits.remove(inherited.getId()); } @@ -144,23 +139,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp return ret; } - /** - * Data type of the body fields of this and all inherited document types - * @return merged {@link StructDataType} - */ - public StructDataType allBody() { - StructDataType ret = new StructDataType(body.getName()); - for (Field f : body.getFields()) { - ret.addField(f); - } - for (NewDocumentType inherited : getInherited()) { - for (Field f : ((StructDataType) inherited.getBody()).getFields()) { - ret.addField(f); - } - } - return ret; - } - @Override public Class getValueClass() { return Document.class; @@ -219,9 +197,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp public Field getField(String name) { Field field = header.getField(name); if (field == null) { - field = body.getField(name); - } - if (field == null) { for (NewDocumentType inheritedType : inherits.values()) { field = inheritedType.getField(name); if (field != null) { @@ -240,9 +215,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp public Field getField(int id) { Field field = header.getField(id); if (field == null) { - field = body.getField(id); - } - if (field == null) { for (NewDocumentType inheritedType : inherits.values()) { field = inheritedType.getField(id); if (field != null) { @@ -261,14 +233,12 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } collection.addAll(header.getFields()); - collection.addAll(body.getFields()); return Collections.unmodifiableCollection(collection); } public Collection<Field> getFields() { Collection<Field> collection = new LinkedList<>(); collection.addAll(header.getFields()); - collection.addAll(body.getFields()); return Collections.unmodifiableCollection(collection); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index d3a78321106..fed35382b21 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -338,7 +338,6 @@ public class DocumentModelBuilder { Map<StructDataType, String> structInheritance = new HashMap<>(); NewDocumentType dt = new NewDocumentType(new NewDocumentType.Name(sdoc.getName()), sdoc.getDocumentType().contentStruct(), - sdoc.getDocumentType().getBodyType(), sdoc.getFieldSets(), convertDocumentReferencesToNames(sdoc.getDocumentReferences()), convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields())); @@ -391,7 +390,6 @@ public class DocumentModelBuilder { } } handleStruct(dt, sdoc.getDocumentType().contentStruct()); - handleStruct(dt, sdoc.getDocumentType().getBodyType()); extractDataTypesFromFields(dt, sdoc.fieldSet()); return dt; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index aba6cf9a233..577639ead7a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -24,8 +24,6 @@ import java.util.Set; */ public class Index implements Cloneable, Serializable { - public static enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES } - public enum Type { VESPA("vespa"); @@ -65,8 +63,6 @@ public class Index implements Cloneable, Serializable { private Optional<HnswIndexParams> hnswIndexParams = Optional.empty(); - private Optional<DistanceMetric> distanceMetric = Optional.empty(); - /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */ private boolean interleavedFeatures = false; @@ -138,13 +134,12 @@ public class Index implements Cloneable, Serializable { stemming == index.stemming && type == index.type && Objects.equals(boolIndex, index.boolIndex) && - Objects.equals(distanceMetric, index.distanceMetric) && Objects.equals(hnswIndexParams, index.hnswIndexParams); } @Override public int hashCode() { - return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, distanceMetric, hnswIndexParams, interleavedFeatures); + return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, hnswIndexParams, interleavedFeatures); } public String toString() { @@ -192,16 +187,6 @@ public class Index implements Cloneable, Serializable { boolIndex = def; } - public Optional<DistanceMetric> getDistanceMetric() { - return distanceMetric; - } - - public void setDistanceMetric(String value) { - String upper = value.toUpperCase(Locale.ENGLISH); - DistanceMetric dm = DistanceMetric.valueOf(upper); - distanceMetric = Optional.of(dm); - } - public Optional<HnswIndexParams> getHnswIndexParams() { return hnswIndexParams; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index ea126123a25..d309f48d6df 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -91,6 +91,7 @@ public class RankProfile implements Cloneable { private double rankScoreDropLimit = -Double.MAX_VALUE; private Set<ReferenceNode> summaryFeatures; + private String inheritedSummaryFeatures; private Set<ReferenceNode> rankFeatures; @@ -303,6 +304,12 @@ public class RankProfile implements Cloneable { } public void addConstant(String name, Value value) { + if (value instanceof TensorValue) { + TensorType type = ((TensorValue)value).type(); + if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) + throw new IllegalArgumentException("Illegal type of constant " + name + " type " + type + + ": Dense tensor dimensions must have a size"); + } constants.put(name, value.freeze()); } @@ -386,9 +393,15 @@ public class RankProfile implements Cloneable { /** Returns a read-only view of the summary features to use in this profile. This is never null */ public Set<ReferenceNode> getSummaryFeatures() { + if (inheritedSummaryFeatures != null && summaryFeatures != null) { + Set<ReferenceNode> combined = new HashSet<>(); + combined.addAll(getInherited().getSummaryFeatures()); + combined.addAll(summaryFeatures); + return Collections.unmodifiableSet(combined); + } if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); if (getInherited() != null) return getInherited().getSummaryFeatures(); - return Collections.emptySet(); + return Set.of(); } private void addSummaryFeature(ReferenceNode feature) { @@ -397,17 +410,30 @@ public class RankProfile implements Cloneable { summaryFeatures.add(feature); } - /** - * Adds the content of the given feature list to the internal list of summary features. - * - * @param features The features to add. - */ + /** Adds the content of the given feature list to the internal list of summary features. */ public void addSummaryFeatures(FeatureList features) { for (ReferenceNode feature : features) { addSummaryFeature(feature); } } + /** + * Sets the name this should inherit the summary features of. + * Without setting this, this will either have the summary features of the parent, + * or if summary features are set in this, only have the summary features in this. + * With this set the resulting summary features of this will be the superset of those defined in this and + * the final (with inheritance included) summary features of the given parent. + * The profile must be the profile which is directly inherited by this. + * + * @param parentProfile + */ + public void setInheritedSummaryFeatures(String parentProfile) { + if ( ! parentProfile.equals(inheritedName)) + throw new IllegalArgumentException("This can only inherit the summary features of its parent, '" + + inheritedName + ", but attemtping to inherit '" + parentProfile); + this.inheritedSummaryFeatures = parentProfile; + } + /** Returns a read-only view of the rank features to use in this profile. This is never null */ public Set<ReferenceNode> getRankFeatures() { if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java index 7b7265e02ae..b41cf582204 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java @@ -56,7 +56,9 @@ public class RankingConstant { this.pathType = PathType.URI; } - public void setType(TensorType tensorType) { this.tensorType = tensorType; } + public void setType(TensorType type) { + this.tensorType = type; + } /** Initiate sending of this constant to some services over file distribution */ public void sendTo(Collection<? extends AbstractService> services) { @@ -78,6 +80,9 @@ public class RankingConstant { throw new IllegalArgumentException("Ranking constants must have a file or uri."); if (tensorType == null) throw new IllegalArgumentException("Ranking constant '" + name + "' must have a type."); + if (tensorType.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) + throw new IllegalArgumentException("Illegal type in field " + name + " type " + tensorType + + ": Dense tensor dimensions must have a size"); } public String toString() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 8b5f7658475..004e44a7659 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -241,6 +241,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce } aaB.imported(imported); var dma = attribute.distanceMetric(); + aaB.distancemetric(AttributesConfig.Attribute.Distancemetric.Enum.valueOf(dma.toString())); if (attribute.hnswIndexParams().isPresent()) { var ib = new AttributesConfig.Attribute.Index.Builder(); var params = attribute.hnswIndexParams().get(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index 1661a80f238..49d396327bf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -24,7 +24,6 @@ import com.yahoo.document.datatypes.Float16FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.tensor.TensorType; -import static com.yahoo.searchdefinition.Index.DistanceMetric; import java.io.Serializable; import java.util.LinkedHashSet; @@ -40,6 +39,8 @@ import java.util.Set; */ public final class Attribute implements Cloneable, Serializable { + public enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES } + // Remember to change hashCode and equals when you add new fields private String name; @@ -228,7 +229,7 @@ public final class Attribute implements Cloneable, Serializable { public void setUpperBound(long upperBound) { this.upperBound = upperBound; } public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; } public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } - public void setDistanceMetric(Optional<DistanceMetric> dm) { this.distanceMetric = dm; } + public void setDistanceMetric(DistanceMetric metric) { this.distanceMetric = Optional.of(metric); } public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } public String getName() { return name; } @@ -348,7 +349,7 @@ public final class Attribute implements Cloneable, Serializable { public int hashCode() { return Objects.hash( name, type, collectionType, sorting, isPrefetch(), fastAccess, removeIfZero, createIfNonExistent, - isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType, hnswIndexParams); + isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType, distanceMetric, hnswIndexParams); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index b3f98cc6f26..c44bf44f4fe 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -155,7 +155,6 @@ public class SDDocumentType implements Cloneable, Serializable { public SDDocumentType(String name, Search search) { docType = new DocumentType(name); docType.contentStruct().setCompressionConfig(new CompressionConfig()); - docType.getBodyType().setCompressionConfig(new CompressionConfig()); validateId(search); inherit(VESPA_DOCUMENT); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 30ce142d503..e36635ba6b8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -7,12 +7,14 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.StructDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.fieldoperation.FieldOperation; import com.yahoo.searchdefinition.fieldoperation.FieldOperationContainer; +import com.yahoo.tensor.TensorType; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; @@ -116,9 +118,10 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, private boolean isExtraField = false; /** - * Creates a new field. This method is only used to create reserved fields - * @param name The name of the field - * @param dataType The datatype of the field + * Creates a new field. This method is only used to create reserved fields. + * + * @param name the name of the field + * @param dataType the datatype of the field */ protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) { super(name, id, dataType); @@ -129,64 +132,36 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, this(repo, name, id, dataType, true); } - /** - Creates a new field. - - @param name The name of the field - @param dataType The datatype of the field - */ + /** Creates a new field */ public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) { - super(name,dataType); + super(name, dataType); populate(populate, repo, name, dataType); } - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) { - populate(populate,repo, name, dataType, null, 0); - } - - private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) { - if (populate || (dataType instanceof MapDataType)) { - populateWithStructFields(repo, name, dataType, recursion); - populateWithStructMatching(repo, name, dataType, fieldMatching); - } - } - - - /** - * Creates a new field. - * - * @param name The name of the field - * @param dataType The datatype of the field - * @param owner the owning document (used to check for id collisions) - */ + /** Creates a new field */ protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) { super(name, dataType, owner == null ? null : owner.getDocumentType()); - this.ownerDocType=owner; + this.ownerDocType = owner; populate(populate, repo, name, dataType); } /** - * Creates a new field. + * Creates a new field * - * @param name The name of the field - * @param dataType The datatype of the field - * @param owner The owning document (used to check for id collisions) - * @param fieldMatching The matching object to set for the field + * @param name the name of the field + * @param dataType the datatype of the field + * @param owner the owning document (used to check for id collisions) + * @param fieldMatching the matching object to set for the field */ protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, Matching fieldMatching, boolean populate, int recursion) { super(name, dataType, owner == null ? null : owner.getDocumentType()); - this.ownerDocType=owner; + this.ownerDocType = owner; if (fieldMatching != null) this.setMatching(fieldMatching); populate(populate, repo, name, dataType, fieldMatching, recursion); } - /** - * - * @param name The name of the field - * @param dataType The datatype of the field - */ public SDField(SDDocumentType repo, String name, DataType dataType) { this(repo, name,dataType, true); } @@ -194,6 +169,23 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, this(null, name,dataType); } + private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) { + populate(populate, repo, name, dataType, null, 0); + } + + private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching, int recursion) { + if (dataType instanceof TensorDataType) { + TensorType type = ((TensorDataType)dataType).getTensorType(); + if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) + throw new IllegalArgumentException("Illegal type in field " + name + " type " + type + + ": Dense tensor dimensions must have a size"); + } + if (populate || (dataType instanceof MapDataType)) { + populateWithStructFields(repo, name, dataType, recursion); + populateWithStructMatching(repo, name, dataType, fieldMatching); + } + } + public void setIsExtraField(boolean isExtra) { isExtraField = isExtra; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java index 04d11792379..2921253461d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.document; import com.yahoo.document.DataType; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class TemporarySDField extends SDField { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java index 861a9f530d4..b638932a4a8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java @@ -5,6 +5,7 @@ import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.tensor.TensorType; +import java.util.Locale; import java.util.Optional; /** @@ -24,6 +25,7 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain private String alias; private String aliasedName; private Optional<TensorType> tensorType = Optional.empty(); + private Optional<String> distanceMetric = Optional.empty(); public AttributeOperation(String name) { this.name = name; @@ -116,6 +118,10 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain this.tensorType = Optional.of(tensorType); } + public void setDistanceMetric(String value) { + this.distanceMetric = Optional.of(value); + } + public void apply(SDField field) { Attribute attribute = null; if (attributeIsSuffixOfStructField(field.getName())) { @@ -153,6 +159,10 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain if (tensorType.isPresent()) { attribute.setTensorType(tensorType.get()); } + if (distanceMetric.isPresent()) { + String upper = distanceMetric.get().toUpperCase(Locale.ENGLISH); + attribute.setDistanceMetric(Attribute.DistanceMetric.valueOf(upper)); + } } private boolean attributeIsSuffixOfStructField(String fieldName) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java index 0c1f443dee3..7f9da28b9ca 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java @@ -32,7 +32,6 @@ public class IndexOperation implements FieldOperation { private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); private Optional<Boolean> enableBm25 = Optional.empty(); - private Optional<String> distanceMetric = Optional.empty(); private Optional<HnswIndexParams.Builder> hnswIndexParams = Optional.empty(); public String getIndexName() { @@ -95,9 +94,6 @@ public class IndexOperation implements FieldOperation { if (enableBm25.isPresent()) { index.setInterleavedFeatures(enableBm25.get()); } - if (distanceMetric.isPresent()) { - index.setDistanceMetric(distanceMetric.get()); - } if (hnswIndexParams.isPresent()) { index.setHnswIndexParams(hnswIndexParams.get().build()); } @@ -131,10 +127,6 @@ public class IndexOperation implements FieldOperation { enableBm25 = Optional.of(value); } - public void setDistanceMetric(String value) { - this.distanceMetric = Optional.of(value); - } - public void setHnswIndexParams(HnswIndexParams.Builder params) { this.hnswIndexParams = Optional.of(params); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index c97ee2bd935..8010be5043e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -83,7 +83,6 @@ public class TensorFieldProcessor extends Processor { var params = new HnswIndexParams(); if (index != null) { params = params.overrideFrom(index.getHnswIndexParams()); - field.getAttribute().setDistanceMetric(index.getDistanceMetric()); } field.getAttribute().setHnswIndexParams(params); } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index e0d015cc8b2..9b516767c9b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -94,8 +94,7 @@ public class DocumentManager { builder.documenttype(doc); doc. name(dt.getName()). - headerstruct(dt.contentStruct().getId()). - bodystruct(dt.getBodyType().getId()); + headerstruct(dt.contentStruct().getId()); for (DocumentType inherited : dt.getInheritedTypes()) { doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); } @@ -105,8 +104,7 @@ public class DocumentManager { builder.documenttype(doc); doc. name(dt.getName()). - headerstruct(dt.getHeader().getId()). - bodystruct(dt.getBody().getId()); + headerstruct(dt.getHeader().getId()); for (NewDocumentType inherited : dt.getInherited()) { doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index e6bf826dccc..4f075264608 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -42,8 +42,7 @@ public class DocumentTypes { db. id(documentType.getId()). name(documentType.getName()). - headerstruct(documentType.getHeader().getId()). - bodystruct(documentType.getBody().getId()); + headerstruct(documentType.getHeader().getId()); Set<Integer> built = new HashSet<>(); for (NewDocumentType inherited : documentType.getInherited()) { db.inherits(new DocumenttypesConfig.Documenttype.Inherits.Builder().id(inherited.getId())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 9dba6fde9d4..e98c09f8128 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -1,23 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; -import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; /** @@ -44,7 +38,7 @@ public class HostResource implements Comparable<HostResource> { * @param host {@link com.yahoo.vespa.model.Host} object to bind to. */ public HostResource(Host host) { - this(host, new HostSpec(host.getHostname(), Optional.empty())); + this(host, new HostSpec(host.getHostname(), List.of(), Optional.empty())); } public HostResource(Host host, HostSpec spec) { @@ -105,8 +99,16 @@ public class HostResource implements Comparable<HostResource> { } /** Returns the flavor of this resource. Empty for self-hosted Vespa. */ + // TODO: Remove when models older than 7.226 are gone + @Deprecated public Optional<Flavor> getFlavor() { return spec.flavor(); } + /** The real resources available for Vespa processes on this node, after subtracting infrastructure overhead. */ + public NodeResources realResources() { return spec.realResources(); } + + /** The total advertised resources of this node, typically matching what's requested. */ + public NodeResources advertisedResources() { return spec.advertisedResources(); } + @Override public String toString() { return "host '" + host.getHostname() + "'"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 292d4761d8a..3d2918e0ee1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -120,7 +120,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { HostResource host = getExistingHost(spec).orElseGet(() -> addNewHost(spec)); retAllocatedHosts.put(host, spec.membership().orElse(null)); } - retAllocatedHosts.keySet().forEach(host -> log.log(FINE, () -> "Allocated host " + host.getHostname() + " with flavor " + host.getFlavor())); + retAllocatedHosts.keySet().forEach(host -> log.log(FINE, () -> "Allocated host " + host.getHostname() + " with resources " + host.advertisedResources())); return retAllocatedHosts; } @@ -131,7 +131,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { if (hosts.isEmpty()) { return Optional.empty(); } else { - log.log(FINE, () -> "Found existing host resource for " + key.hostname() + " with flavor " + hosts.get(0).getFlavor()); + log.log(FINE, () -> "Found existing host resource for " + key.hostname() + " with resources" + hosts.get(0).advertisedResources()); return Optional.of(hosts.get(0)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 141794256b8..b9fc9643ac3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -396,6 +396,7 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.resource_usage.disk_utilization.average")); metrics.add(new Metric("content.proton.resource_usage.memory.average")); metrics.add(new Metric("content.proton.resource_usage.memory_utilization.average")); + metrics.add(new Metric("content.proton.resource_usage.transient_memory.average")); metrics.add(new Metric("content.proton.resource_usage.memory_mappings.max")); metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.enum_store.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java index c6bf54af760..75e6922ad15 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java @@ -90,7 +90,7 @@ public class ConstantTensorJsonValidator { validateTensorValue(); } } else { - throw new InvalidConstantTensor(parser, "Only \"address\" or \"value\" fields are permitted within a cell object"); + throw new InvalidConstantTensor(parser, "Only 'address' or 'value' fields are permitted within a cell object"); } } @@ -110,55 +110,52 @@ public class ConstantTensorJsonValidator { String dimensionName = parser.getCurrentName(); TensorType.Dimension dimension = tensorDimensions.get(dimensionName); if (dimension == null) { - throw new InvalidConstantTensor(parser, String.format("Tensor dimension \"%s\" does not exist", parser.getCurrentName())); + throw new InvalidConstantTensor(parser, String.format("Tensor dimension '%s' does not exist", parser.getCurrentName())); } if (!cellDimensions.contains(dimensionName)) { - throw new InvalidConstantTensor(parser, String.format("Duplicate tensor dimension \"%s\"", parser.getCurrentName())); + throw new InvalidConstantTensor(parser, String.format("Duplicate tensor dimension '%s'", parser.getCurrentName())); } cellDimensions.remove(dimensionName); - validateTensorCoordinate(dimension); + validateLabel(dimension); } if (!cellDimensions.isEmpty()) { - throw new InvalidConstantTensor(parser, String.format("Tensor address missing dimension(s): %s", Joiner.on(", ").join(cellDimensions))); + throw new InvalidConstantTensor(parser, String.format("Tensor address missing dimension(s) %s", Joiner.on(", ").join(cellDimensions))); } } - /* - * Tensor coordinates are always strings. Coordinates for a mapped dimension can be any string, + /** + * Tensor labels are always strings. Labels for a mapped dimension can be any string, * but those for indexed dimensions needs to be able to be interpreted as integers, and, * additionally, those for indexed bounded dimensions needs to fall within the dimension size. */ - private void validateTensorCoordinate(TensorType.Dimension dimension) throws IOException { + private void validateLabel(TensorType.Dimension dimension) throws IOException { JsonToken token = parser.nextToken(); - if (token != JsonToken.VALUE_STRING) { - throw new InvalidConstantTensor(parser, String.format("Tensor coordinate is not a string (%s)", token.toString())); - } + if (token != JsonToken.VALUE_STRING) + throw new InvalidConstantTensor(parser, String.format("Tensor label is not a string (%s)", token.toString())); if (dimension instanceof TensorType.IndexedBoundDimension) { - validateBoundedCoordinate((TensorType.IndexedBoundDimension) dimension); + validateBoundIndex((TensorType.IndexedBoundDimension) dimension); } else if (dimension instanceof TensorType.IndexedUnboundDimension) { - validateUnboundedCoordinate(dimension); + validateUnboundIndex(dimension); } } - private void validateBoundedCoordinate(TensorType.IndexedBoundDimension dimension) { + private void validateBoundIndex(TensorType.IndexedBoundDimension dimension) { wrapIOException(() -> { try { int value = Integer.parseInt(parser.getValueAsString()); - if (value >= dimension.size().get()) { - throw new InvalidConstantTensor(parser, String.format("Coordinate \"%s\" not within limits of bounded dimension %s", value, dimension.name())); - - } + if (value >= dimension.size().get()) + throw new InvalidConstantTensor(parser, String.format("Index %s not within limits of bound dimension '%s'", value, dimension.name())); } catch (NumberFormatException e) { throwCoordinateIsNotInteger(parser.getValueAsString(), dimension.name()); } }); } - private void validateUnboundedCoordinate(TensorType.Dimension dimension) { + private void validateUnboundIndex(TensorType.Dimension dimension) { wrapIOException(() -> { try { Integer.parseInt(parser.getValueAsString()); @@ -169,7 +166,7 @@ public class ConstantTensorJsonValidator { } private void throwCoordinateIsNotInteger(String value, String dimensionName) { - throw new InvalidConstantTensor(parser, String.format("Coordinate \"%s\" for dimension %s is not an integer", value, dimensionName)); + throw new InvalidConstantTensor(parser, String.format("Index '%s' for dimension '%s' is not an integer", value, dimensionName)); } private void validateTensorValue() throws IOException { @@ -198,11 +195,12 @@ public class ConstantTensorJsonValidator { String actualFieldName = parser.getCurrentName(); if (!actualFieldName.equals(wantedFieldName)) { - throw new InvalidConstantTensor(parser, String.format("Expected field name \"%s\", got \"%s\"", wantedFieldName, actualFieldName)); + throw new InvalidConstantTensor(parser, String.format("Expected field name '%s', got '%s'", wantedFieldName, actualFieldName)); } } static class InvalidConstantTensor extends RuntimeException { + InvalidConstantTensor(JsonParser parser, String message) { super(message + " " + parser.getCurrentLocation().toString()); } @@ -210,6 +208,7 @@ public class ConstantTensorJsonValidator { InvalidConstantTensor(JsonParser parser, Exception base) { super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base); } + } @FunctionalInterface diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java index 9568ea5c27c..cf8a9201668 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -31,7 +31,7 @@ public class RankingConstantsValidator extends Validator { public ExceptionMessageCollector add(Throwable throwable, String rcName, String rcFilename) { exceptionsOccurred = true; - combinedMessage += String.format("\nRanking constant \"%s\" (%s): %s", rcName, rcFilename, throwable.getMessage()); + combinedMessage += String.format("\nRanking constant '%s' (%s): %s", rcName, rcFilename, throwable.getMessage()); return this; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index 5343a322382..24b7b0949f6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -44,8 +44,8 @@ public class ResourcesReductionValidator implements ChangeValidator { ClusterSpec.Id clusterId, ValidationOverrides overrides, Instant now) { - if (current.minResources().nodeResources() == NodeResources.unspecified) return; - if (next.minResources().nodeResources() == NodeResources.unspecified) return; + if (current.minResources().nodeResources().isUnspecified()) return; + if (next.minResources().nodeResources().isUnspecified()) return; List<String> illegalChanges = Stream.of( validateResource("vCPU", diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index ea47e490b12..8f737f02dca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -159,8 +159,8 @@ public class NodesSpecification { * Returns a requirement from <code>count</code> non-dedicated nodes in one group */ public static NodesSpecification nonDedicated(int count, ConfigModelContext context) { - return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified), - new ClusterResources(count, 1, NodeResources.unspecified), + return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified()), + new ClusterResources(count, 1, NodeResources.unspecified()), false, context.getDeployState().getWantedNodeVespaVersion(), false, @@ -172,8 +172,8 @@ public class NodesSpecification { /** Returns a requirement from <code>count</code> dedicated nodes in one group */ public static NodesSpecification dedicated(int count, ConfigModelContext context) { - return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified), - new ClusterResources(count, 1, NodeResources.unspecified), + return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified()), + new ClusterResources(count, 1, NodeResources.unspecified()), true, context.getDeployState().getWantedNodeVespaVersion(), false, @@ -224,7 +224,7 @@ public class NodesSpecification { return new Pair<>(flavorResources, flavorResources); } else { - return new Pair<>(NodeResources.unspecified, NodeResources.unspecified); + return new Pair<>(NodeResources.unspecified(), NodeResources.unspecified()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 29bb578a67b..5207a0163cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.provision.Flavor; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; @@ -45,8 +44,8 @@ public final class ApplicationContainer extends Container implements @Override public void getConfig(QrStartConfig.Builder builder) { if (getHostResource() != null) { - if (getHostResource().getFlavor().isPresent()) { - NodeFlavorTuning flavorTuning = new NodeFlavorTuning(getHostResource().getFlavor().get()); + if ( ! getHostResource().realResources().isUnspecified()) { + NodeResourcesTuning flavorTuning = new NodeResourcesTuning(getHostResource().realResources()); flavorTuning.getConfig(builder); } } @@ -80,13 +79,13 @@ public final class ApplicationContainer extends Container implements @Override public void getConfig(ThreadpoolConfig.Builder builder) { if (! (parent instanceof ContainerCluster)) return; - if ((getHostResource() == null) || getHostResource().getFlavor().isEmpty()) return; + if ((getHostResource() == null) || getHostResource().realResources().isUnspecified()) return; ContainerCluster containerCluster = (ContainerCluster) parent; if (containerCluster.getThreadPoolSizeFactor() <= 0.0) return; - NodeFlavorTuning flavorTuning = new NodeFlavorTuning(getHostResource().getFlavor().get()) + NodeResourcesTuning resourcesTuning = new NodeResourcesTuning(getHostResource().realResources()) .setThreadPoolSizeFactor(containerCluster.getThreadPoolSizeFactor()) .setQueueSizeFactor(containerCluster.getQueueSizeFactor()); - flavorTuning.getConfig(builder); + resourcesTuning.getConfig(builder); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 37548a15835..c8908495c0a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -58,6 +58,10 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public static final String METRICS_V2_HANDLER_BINDING_1 = "http://*" + MetricsV2Handler.V2_PATH; public static final String METRICS_V2_HANDLER_BINDING_2 = METRICS_V2_HANDLER_BINDING_1 + "/*"; + public static final int heapSizePercentageOfTotalNodeMemory = 60; + public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 17; + + private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); private final ConfigProducerGroup<Servlet> servletGroup; @@ -69,7 +73,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private MbusParams mbusParams; private boolean messageBusEnabled = true; - private final double softStartSeconds; private Integer memoryPercentage = null; @@ -84,10 +87,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.DeprecatedSecretStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); - addSimpleComponent("ai.vespa.cloud.SystemInfo"); + addSimpleComponent("com.yahoo.container.jdisc.SystemInfoProvider"); addMetricsV2Handler(); addTestrunnerComponentsIfTester(deployState); - softStartSeconds = deployState.getProperties().defaultSoftStartSeconds(); } @Override @@ -220,13 +222,15 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat super.getConfig(builder); builder.jvm.verbosegc(true) .availableProcessors(0) - .compressedClassSpaceSize(0) //TODO Reduce, next step is 512m + .compressedClassSpaceSize(0) .minHeapsize(1536) .heapsize(1536); if (getMemoryPercentage().isPresent()) { builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getMemoryPercentage().get()); } else if (isHostedVespa()) { - builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getHostClusterId().isPresent() ? 17 : 60); + builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getHostClusterId().isPresent() ? + heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster : + heapSizePercentageOfTotalNodeMemory); } } @@ -254,7 +258,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat @Override public void getConfig(ThreadpoolConfig.Builder builder) { - builder.softStartSeconds(softStartSeconds); } public static class MbusParams { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 31c8724d634..c6de198c06a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -46,7 +46,7 @@ import static com.yahoo.container.QrConfig.Rpc; * @author Einar M R Rosenvinge * @author Tony Vaagenes */ -//qr is restart because it is handled by ConfiguredApplication.start +// qr is restart because it is handled by ConfiguredApplication.start @RestartConfigs({QrStartConfig.class, QrConfig.class}) public abstract class Container extends AbstractService implements QrConfig.Producer, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index b35b7562704..5127616ad5e 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -18,7 +18,6 @@ import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.core.document.ContainerDocumentConfig; -import com.yahoo.container.handler.ThreadPoolProvider; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; @@ -176,7 +175,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addComponent(new StatisticsComponent()); addSimpleComponent(AccessLog.class); - addSimpleComponent(ThreadPoolProvider.class); + addComponent(new ThreadPoolExecutorComponent.Builder("default-pool").build()); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent(SecurityFilterInvoker.class); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java index f9b50d0e641..7eb7a1fb518 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeResourcesTuning.java @@ -1,28 +1,25 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; -import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; /** - * Tuning of qr-start config for a container service based on the node flavor of that node. + * Tuning of qr-start config for a container service based on node resources. * * @author balder */ -public class NodeFlavorTuning implements - QrStartConfig.Producer, - ThreadpoolConfig.Producer -{ +public class NodeResourcesTuning implements QrStartConfig.Producer, ThreadpoolConfig.Producer { - private final Flavor flavor; + private final NodeResources resources; - public NodeFlavorTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) { + public NodeResourcesTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) { this.threadPoolSizeFactor = threadPoolSizeFactor; return this; } - public NodeFlavorTuning setQueueSizeFactor(double queueSizeFactor) { + public NodeResourcesTuning setQueueSizeFactor(double queueSizeFactor) { this.queueSizeFactor = queueSizeFactor; return this; } @@ -30,19 +27,19 @@ public class NodeFlavorTuning implements private double threadPoolSizeFactor = 8.0; private double queueSizeFactor = 8.0; - NodeFlavorTuning(Flavor flavor) { - this.flavor = flavor; + NodeResourcesTuning(NodeResources resources) { + this.resources = resources; } @Override public void getConfig(QrStartConfig.Builder builder) { - builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(flavor.getMinCpuCores()))); + builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(resources.vcpu()))); } @Override public void getConfig(ThreadpoolConfig.Builder builder) { // Controls max number of concurrent requests per container - int workerThreads = Math.max(2, (int)Math.ceil(flavor.getMinCpuCores() * threadPoolSizeFactor)); + int workerThreads = Math.max(2, (int)Math.ceil(resources.vcpu() * threadPoolSizeFactor)); builder.maxthreads(workerThreads); // This controls your burst handling capability. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java new file mode 100644 index 00000000000..2926cb3ee6c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java @@ -0,0 +1,70 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.handler.ThreadPoolProvider; +import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +import java.time.Duration; + +/** + * Component definition for a {@link java.util.concurrent.Executor} using {@link ThreadPoolProvider}. + * + * @author bjorncs + */ +public class ThreadPoolExecutorComponent extends SimpleComponent implements ThreadpoolConfig.Producer { + + private final String name; + private final Integer maxPoolSize; + private final Integer corePoolSize; + private final Duration keepAliveTime; + private final Integer queueSize; + private final Duration maxThreadExecutionTime; + + private ThreadPoolExecutorComponent(Builder builder) { + super(new ComponentModel( + BundleInstantiationSpecification.getFromStrings( + "threadpool-provider@" + builder.name, + ThreadPoolProvider.class.getName(), + null))); + this.name = builder.name; + this.maxPoolSize = builder.maxPoolSize; + this.corePoolSize = builder.corePoolSize; + this.keepAliveTime = builder.keepAliveTime; + this.queueSize = builder.queueSize; + this.maxThreadExecutionTime = builder.maxThreadExecutionTime; + } + + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + builder.name(this.name); + if (maxPoolSize != null) builder.maxthreads(maxPoolSize); + if (corePoolSize != null) builder.corePoolSize(corePoolSize); + if (keepAliveTime != null) builder.keepAliveTime(keepAliveTime.toMillis() / 1000D); + if (queueSize != null) builder.queueSize(queueSize); + if (maxThreadExecutionTime != null) builder.maxThreadExecutionTimeSeconds((int)maxThreadExecutionTime.toMillis() / 1000); + } + + public static class Builder { + + private final String name; + private Integer maxPoolSize; + private Integer corePoolSize; + private Duration keepAliveTime; + private Integer queueSize; + private Duration maxThreadExecutionTime; + + public Builder(String name) { this.name = name; } + + public Builder maxPoolSize(int size) { this.maxPoolSize = size; return this; } + public Builder corePoolSize(int size) { this.corePoolSize = size; return this; } + public Builder keepAliveTime(Duration time) { this.keepAliveTime = time; return this; } + public Builder queueSize(int size) { this.queueSize = size; return this; } + public Builder maxThreadExecutionTime(Duration time) { this.maxThreadExecutionTime = time; return this; } + + public ThreadPoolExecutorComponent build() { return new ThreadPoolExecutorComponent(this); } + + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java index 08b6e321aa3..6d891c55075 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java @@ -26,6 +26,11 @@ public class Component<CHILD extends AbstractConfigProducer<?>, MODEL extends Co this.model = model; } + /** Returns a component that uses its class name as id. */ + public static Component<?,?> fromClassAndBundle(String className, String bundle) { + return new Component<>(new ComponentModel(className, null, bundle)); + } + public ComponentId getGlobalComponentId() { return model.getComponentId(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java index 46ee0d0bd37..0b72971336f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java @@ -6,7 +6,7 @@ import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.config.model.producer.AbstractConfigProducer; /** - * A component that only needs a simple ComponentModel. + * A component that uses the class name as id, and resides in the container-disc bundle. * * @author gjoranv */ @@ -16,7 +16,6 @@ public class SimpleComponent extends Component<AbstractConfigProducer<?>, Compon super(model); } - // For a component that uses the class name as id, and resides in the container-disc bundle. public SimpleComponent(String className) { this(new ComponentModel(BundleInstantiationSpecification.getFromStrings(className, null, null))); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 4c4b1ca7f82..58d12663c86 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -1,13 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; +import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ContainerSubsystem; import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; @@ -21,6 +25,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import static com.yahoo.vespa.model.container.xml.BundleMapper.searchAndDocprocBundle; + /** * @author gjoranv * @author Tony Vaagenes @@ -34,6 +40,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> SemanticRulesConfig.Producer, PageTemplatesConfig.Producer { + public static final String QUERY_PROFILE_REGISTRY_CLASS = CompiledQueryProfileRegistry.class.getName(); + private ApplicationContainerCluster owningCluster; private final List<AbstractSearchCluster> searchClusters = new LinkedList<>(); private final Options options; @@ -46,6 +54,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> super(chains); this.owningCluster = cluster; this.options = options; + + owningCluster.addComponent(Component.fromClassAndBundle(QUERY_PROFILE_REGISTRY_CLASS, searchAndDocprocBundle)); } public void connectSearchClusters(Map<String, AbstractSearchCluster> searchClusters) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java index 0a9618e7b08..079baf6fe7d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java @@ -9,6 +9,7 @@ import com.yahoo.search.query.profile.SubstituteString; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.tensor.TensorType; import java.io.Serializable; import java.util.ArrayList; @@ -56,6 +57,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer Set<String> tensorFields = new HashSet<>(); for (QueryProfileType type : registry.getTypeRegistry().allComponents()) { for (var fieldEntry : type.fields().entrySet()) { + validateTensorField(fieldEntry.getKey(), fieldEntry.getValue().getType().asTensorType()); if (fieldEntry.getValue().getType().asTensorType().rank() > 0) tensorFields.add(fieldEntry.getKey()); } @@ -73,6 +75,12 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer } + private void validateTensorField(String fieldName, TensorType type) { + if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) + throw new IllegalArgumentException("Illegal type in field " + fieldName + " type " + type + + ": Dense tensor dimensions must have a size"); + } + @Override public void getConfig(QueryProfilesConfig.Builder builder) { for (QueryProfile profile : registry.allComponents()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 741e5ebffd1..b83632a58a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -513,17 +513,16 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return (gcAlgorithm.matcher(jvmargs).find() ||cmsArgs.matcher(jvmargs).find()); } - private static String buildJvmGCOptions(Zone zone, String jvmGCOPtions, boolean isHostedVespa) { - if (jvmGCOPtions != null) { - return jvmGCOPtions; - } else if ((zone.system() == SystemName.dev) || isHostedVespa) { - return null; - } else { - return ContainerCluster.G1GC; - } + private static String buildJvmGCOptions(DeployState deployState, String jvmGCOPtions) { + String options = (jvmGCOPtions != null) + ? jvmGCOPtions + : deployState.getProperties().jvmGCOptions(); + return (options == null ||options.isEmpty()) + ? (deployState.isHosted() ? ContainerCluster.CMS : ContainerCluster.G1GC) + : options; } private static String getJvmOptions(ApplicationContainerCluster cluster, Element nodesElement, DeployLogger deployLogger) { - String jvmOptions = ""; + String jvmOptions; if (nodesElement.hasAttribute(VespaDomBuilder.JVM_OPTIONS)) { jvmOptions = nodesElement.getAttribute(VespaDomBuilder.JVM_OPTIONS); if (nodesElement.hasAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME)) { @@ -541,15 +540,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return jvmOptions; } + private static String extractAttribute(Element element, String attrName) { + return element.hasAttribute(attrName) ? element.getAttribute(attrName) : null; + } + void extractJvmFromLegacyNodesTag(List<ApplicationContainer> nodes, ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { applyNodesTagJvmArgs(nodes, getJvmOptions(cluster, nodesElement, context.getDeployLogger())); if (!cluster.getJvmGCOptions().isPresent()) { - String jvmGCOptions = nodesElement.hasAttribute(VespaDomBuilder.JVM_GC_OPTIONS) - ? nodesElement.getAttribute(VespaDomBuilder.JVM_GC_OPTIONS) - : null; - cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState().zone(), jvmGCOptions, context.getDeployState().isHosted())); + String jvmGCOptions = extractAttribute(nodesElement, VespaDomBuilder.JVM_GC_OPTIONS); + cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState(), jvmGCOptions)); } applyMemoryPercentage(cluster, nodesElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); @@ -559,10 +560,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element jvmElement, ConfigModelContext context) { applyNodesTagJvmArgs(nodes, jvmElement.getAttribute(VespaDomBuilder.OPTIONS)); applyMemoryPercentage(cluster, jvmElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME)); - String jvmGCOptions = jvmElement.hasAttribute(VespaDomBuilder.GC_OPTIONS) - ? jvmElement.getAttribute(VespaDomBuilder.GC_OPTIONS) - : null; - cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState().zone(), jvmGCOptions, context.getDeployState().isHosted())); + String jvmGCOptions = extractAttribute(jvmElement, VespaDomBuilder.GC_OPTIONS); + cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState(), jvmGCOptions)); } /** @@ -664,7 +663,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .dockerImageRepository(deployState.getWantedDockerImageRepo()) .build(); int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; - Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified), + Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified()), false, !deployState.getProperties().isBootstrap()); var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java index 4d9ba2f920e..fd0797d6098 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java @@ -22,12 +22,7 @@ public class ContainerServiceBuilder extends VespaDomBuilder.DomConfigProducerBu @Override protected ApplicationContainer doBuild(DeployState deployState, AbstractConfigProducer parent, Element nodeElem) { - return new ApplicationContainer( - parent, - id, - index, - deployState.isHosted() - ); + return new ApplicationContainer(parent, id, index, deployState.isHosted()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index b3f2b81014b..404a0ceb4ea 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -39,7 +39,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.logging.Logger; /** * The config model from a content tag in services. @@ -49,8 +48,6 @@ import java.util.logging.Logger; */ public class Content extends ConfigModel { - private static final Logger log = Logger.getLogger(Content.class.getName()); - private ContentCluster cluster; private Optional<ApplicationContainerCluster> ownedIndexingCluster = Optional.empty(); @@ -207,6 +204,11 @@ public class Content extends ConfigModel { content.cluster = new ContentCluster.Builder(admin).build(content.containers, modelContext, xml); buildIndexingClusters(content, modelContext, (ApplicationConfigProducerRoot)modelContext.getParentProducer()); + markCombinedClusters(); + } + + private void markCombinedClusters() { + } /** Select/creates and initializes the indexing cluster coupled to this */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index fcaba66ef69..6cb45c4559b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -60,32 +60,39 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot private Map<StorageGroup, NodeSpec> groupToSpecMap = new LinkedHashMap<>(); private Optional<ResourceLimits> resourceLimits = Optional.empty(); + /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ + private final boolean combined; + public void prepare() { - List<SearchNode> allBackends = getSearchNodes(); - for (AbstractSearchCluster cluster : clusters.values()) { - cluster.prepareToDistributeFiles(allBackends); - } + clusters.values().forEach(cluster -> cluster.prepareToDistributeFiles(getSearchNodes())); } public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ContentSearchCluster> { private final Map<String, NewDocumentType> documentDefinitions; private final Set<NewDocumentType> globallyDistributedDocuments; + private final boolean combined; public Builder(Map<String, NewDocumentType> documentDefinitions, - Set<NewDocumentType> globallyDistributedDocuments) { + Set<NewDocumentType> globallyDistributedDocuments, + boolean combined) { this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; + this.combined = combined; } @Override protected ContentSearchCluster doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { ModelElement clusterElem = new ModelElement(producerSpec); - String clusterName = ContentCluster.getClusterName(clusterElem); + String clusterName = ContentCluster.getClusterId(clusterElem); Boolean flushOnShutdownElem = clusterElem.childAsBoolean("engine.proton.flush-on-shutdown"); - ContentSearchCluster search = new ContentSearchCluster(ancestor, clusterName, documentDefinitions, globallyDistributedDocuments, - getFlushOnShutdown(flushOnShutdownElem, deployState)); + ContentSearchCluster search = new ContentSearchCluster(ancestor, + clusterName, + documentDefinitions, + globallyDistributedDocuments, + getFlushOnShutdown(flushOnShutdownElem, deployState), + combined); ModelElement tuning = clusterElem.childByPath("engine.proton.tuning"); if (tuning != null) { @@ -171,13 +178,15 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot String clusterName, Map<String, NewDocumentType> documentDefinitions, Set<NewDocumentType> globallyDistributedDocuments, - boolean flushOnShutdown) + boolean flushOnShutdown, + boolean combined) { super(parent, "search"); this.clusterName = clusterName; this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; this.flushOnShutdown = flushOnShutdown; + this.combined = combined; } public void setVisibilityDelay(double delay) { @@ -236,27 +245,27 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot AbstractConfigProducer parent = hasIndexedCluster() ? getIndexed() : this; NodeSpec spec = getNextSearchNodeSpec(parentGroup); - SearchNode snode; + SearchNode searchNode; TransactionLogServer tls; Optional<Tuning> tuning = Optional.ofNullable(this.tuning); if (element == null) { - snode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, - clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted()); - snode.setHostResource(node.getHostResource()); - snode.initService(deployState.getDeployLogger()); + searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, + clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), combined); + searchNode.setHostResource(node.getHostResource()); + searchNode.initService(deployState.getDeployLogger()); - tls = new TransactionLogServer(snode, clusterName); - tls.setHostResource(snode.getHostResource()); + tls = new TransactionLogServer(searchNode, clusterName); + tls.setHostResource(searchNode.getHostResource()); tls.initService(deployState.getDeployLogger()); } else { - snode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits).build(deployState, parent, element.getXml()); - tls = new TransactionLogServer.Builder(clusterName).build(deployState, snode, element.getXml()); + searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, combined).build(deployState, parent, element.getXml()); + tls = new TransactionLogServer.Builder(clusterName).build(deployState, searchNode, element.getXml()); } - snode.setTls(tls); + searchNode.setTls(tls); if (hasIndexedCluster()) { - getIndexed().addSearcher(snode); + getIndexed().addSearcher(searchNode); } else { - nonIndexed.add(snode); + nonIndexed.add(searchNode); } } @@ -315,7 +324,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot @Override public void getConfig(ProtonConfig.Builder builder) { - builder.feeding.concurrency(0.40); // As if specified 0.8 in services.xml + builder.feeding.concurrency(0.50); // As if specified 1.0 in services.xml boolean hasAnyNonIndexedCluster = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); @@ -359,9 +368,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot int numDocumentDbs = builder.documentdb.size(); builder.initialize(new ProtonConfig.Initialize.Builder().threads(numDocumentDbs + 1)); - if (resourceLimits.isPresent()) { - resourceLimits.get().getConfig(builder); - } + resourceLimits.ifPresent(limits -> limits.getConfig(builder)); if (tuning != null) { tuning.getConfig(builder); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java index c8220071373..f41188eccde 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java @@ -86,9 +86,8 @@ public class StorageNode extends ContentNode implements StorServerConfig.Produce @Override public void getConfig(StorFilestorConfig.Builder builder) { - if (getHostResource() != null && getHostResource().getFlavor().isPresent()) { - Flavor nodeFlavor = getHostResource().getFlavor().get(); - builder.num_threads(Math.max(4, (int)nodeFlavor.getMinCpuCores())); + if (getHostResource() != null && ! getHostResource().realResources().isUnspecified()) { + builder.num_threads(Math.max(4, (int)getHostResource().realResources().vcpu())); } cluster.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 6dd3e619ec2..9fb553fffde 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -60,6 +60,7 @@ import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.logging.Level; @@ -125,11 +126,14 @@ public class ContentCluster extends AbstractConfigProducer implements RedundancyBuilder redundancyBuilder = new RedundancyBuilder(contentElement); Set<NewDocumentType> globallyDistributedDocuments = new GlobalDistributionBuilder(documentDefinitions).build(documentsElement); - ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterName(contentElement), documentDefinitions, + ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterId(contentElement), documentDefinitions, globallyDistributedDocuments, routingSelection, deployState.zone(), deployState.isHosted()); - c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterName(contentElement), contentElement).build(deployState, c, contentElement.getXml()); - c.search = new ContentSearchCluster.Builder(documentDefinitions, globallyDistributedDocuments).build(deployState, c, contentElement.getXml()); + c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), contentElement).build(deployState, c, contentElement.getXml()); + c.search = new ContentSearchCluster.Builder(documentDefinitions, + globallyDistributedDocuments, + isCombined(getClusterId(contentElement), containers)) + .build(deployState, c, contentElement.getXml()); c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement); c.distributorNodes = new DistributorCluster.Builder(c).build(deployState, c, w3cContentElement); @@ -237,6 +241,14 @@ public class ContentCluster extends AbstractConfigProducer implements } } + /** Returns whether this hosts one of the given container clusters */ + private boolean isCombined(String clusterId, Collection<ContainerModel> containers) { + return containers.stream() + .map(model -> model.getCluster().getHostClusterId()) + .filter(Optional::isPresent) + .anyMatch(id -> id.get().equals(clusterId)); + } + private void setupExperimental(ContentCluster cluster, ModelElement experimental) { // Put handling of experimental flags here } @@ -496,9 +508,8 @@ public class ContentCluster extends AbstractConfigProducer implements public void prepare(DeployState deployState) { search.prepare(); - if (clusterControllers != null) { + if (clusterControllers != null) clusterControllers.prepare(deployState); - } } /** Returns cluster controllers if this is multitenant, null otherwise */ @@ -509,13 +520,9 @@ public class ContentCluster extends AbstractConfigProducer implements return getPersistence().getDefaultDistributionMode(); } - public static String getClusterName(ModelElement clusterElem) { - String clusterName = clusterElem.stringAttribute("id"); - if (clusterName == null) { - clusterName = "content"; - } - - return clusterName; + public static String getClusterId(ModelElement clusterElem) { + String clusterId = clusterElem.stringAttribute("id"); + return clusterId != null ? clusterId : "content"; } public String getName() { return clusterName; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index 76c05d4028a..eaefa8ea35f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -11,8 +11,8 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster; public class FileStorProducer implements StorFilestorConfig.Producer { public static class Builder { - protected FileStorProducer build(ContentCluster parent, Integer numResponseThreads, ModelElement clusterElem) { - return new FileStorProducer(parent, getThreads(clusterElem), numResponseThreads); + protected FileStorProducer build(ContentCluster parent, ModelElement clusterElem) { + return new FileStorProducer(parent, getThreads(clusterElem)); } private Integer getThreads(ModelElement clusterElem) { @@ -43,12 +43,10 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final Integer numThreads; private final ContentCluster cluster; - private final Integer numResponseThreads; - public FileStorProducer(ContentCluster parent, Integer numThreads, Integer numResponseThreads) { + public FileStorProducer(ContentCluster parent, Integer numThreads) { this.numThreads = numThreads; this.cluster = parent; - this.numResponseThreads = numResponseThreads; } @Override @@ -56,9 +54,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { if (numThreads != null) { builder.num_threads(numThreads); } - if (numResponseThreads != null) { - builder.num_response_threads(numResponseThreads); - } builder.enable_multibit_split_optimalization(cluster.getPersistence().enableMultiLevelSplitting()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 51fc610bf28..36e4554e610 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -14,15 +14,15 @@ public class StorServerProducer implements StorServerConfig.Producer { ModelElement tuning = element.child("tuning"); if (tuning == null) { - return new StorServerProducer(ContentCluster.getClusterName(element), null, null); + return new StorServerProducer(ContentCluster.getClusterId(element), null, null); } ModelElement merges = tuning.child("merges"); if (merges == null) { - return new StorServerProducer(ContentCluster.getClusterName(element), null, null); + return new StorServerProducer(ContentCluster.getClusterId(element), null, null); } - return new StorServerProducer(ContentCluster.getClusterName(element), + return new StorServerProducer(ContentCluster.getClusterId(element), merges.integerAttribute("max-per-node"), merges.integerAttribute("max-queue-size")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index c97b4c50484..2e5594a001d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -35,8 +35,8 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> final ContentCluster cluster = (ContentCluster)ancestor; return new StorageCluster(ancestor, - ContentCluster.getClusterName(clusterElem), - new FileStorProducer.Builder().build(cluster, deployState.getProperties().defaultNumResponseThreads(), clusterElem), + ContentCluster.getClusterId(clusterElem), + new FileStorProducer.Builder().build(cluster, clusterElem), new IntegrityCheckerProducer.Builder().build(cluster, clusterElem), new StorServerProducer.Builder().build(clusterElem), new StorVisitorProducer.Builder().build(clusterElem), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index c3d6f457ce8..f8b48da291d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -448,7 +448,7 @@ public class ConvertedModel { int i = rename.fromDimensions().indexOf(dimension.name()); if (i < 0) { throw new IllegalArgumentException("Rename does not contain dimension '" + - dimension + "' in child expression type: " + childType); + dimension + "' in child expression type: " + childType); } from.add((String)rename.fromDimensions().get(i)); to.add((String)rename.toDimensions().get(i)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index ecfa9a9e53e..c2a85790f89 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -157,14 +157,15 @@ public final class DocumentProtocol implements Protocol, Documentrouteselectorpo } private static void addChainHop(RoutingTableSpec table, String configId, String policy, DocprocChain chain) { - final String selector; + final StringBuilder selector = new StringBuilder(); if (policy != null) { - selector = configId + "/" + policy + "/" + chain.getSessionName(); + selector.append(configId).append("/").append(policy).append("/").append(chain.getSessionName()); } else { - selector = "[LoadBalancer:cluster=" + configId + - ";session=" + chain.getSessionName() + "]"; + selector.append("[LoadBalancer:cluster=").append(configId) + .append(";session=").append(chain.getSessionName()) + .append("]"); } - table.addHop(new HopSpec(chain.getServiceName(), selector)); + table.addHop(new HopSpec(chain.getServiceName(), selector.toString())); } private static String policy(ContainerDocproc docproc) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index fe6c6c52e2d..58d608ec9f9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -75,7 +75,6 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer public abstract int getRowBits(); public final void setClusterIndex(int index) { this.index = index; } public final int getClusterIndex() { return index; } - protected abstract void assureSdConsistent(); @Override public abstract void getConfig(DocumentdbInfoConfig.Builder builder); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 56adc227df4..ef8c236fd94 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -52,8 +52,6 @@ public class IndexedSearchCluster extends SearchCluster private final DispatchGroup rootDispatch; private DispatchSpec dispatchSpec; - private final boolean useAdaptiveDispatch; - private final double defaultTopKProbability; private List<SearchNode> searchNodes = new ArrayList<>(); /** @@ -70,8 +68,6 @@ public class IndexedSearchCluster extends SearchCluster super(parent, clusterName, index); unionCfg = new UnionConfiguration(this, documentDbs); rootDispatch = new DispatchGroup(this); - useAdaptiveDispatch = deployState.getProperties().useAdaptiveDispatch(); - defaultTopKProbability = deployState.getProperties().defaultTopKProbability(); } @Override @@ -256,9 +252,6 @@ public class IndexedSearchCluster extends SearchCluster unionCfg.getConfig(builder); } - @Override - protected void exportSdFiles(File toDir) { } - boolean useFixedRowInDispatch() { for (SearchNode node : getSearchNodes()) { if (node.getNodeSpec().groupIndex() > 0) { @@ -307,12 +300,8 @@ public class IndexedSearchCluster extends SearchCluster nodeBuilder.port(node.getRpcPort()); builder.node(nodeBuilder); } - if (useAdaptiveDispatch) - builder.distributionPolicy(DistributionPolicy.ADAPTIVE); if (tuning.dispatch.getTopkProbability() != null) { builder.topKProbability(tuning.dispatch.getTopkProbability()); - } else { - builder.topKProbability(defaultTopKProbability); } if (tuning.dispatch.getMinActiveDocsCoverage() != null) builder.minActivedocsPercentage(tuning.dispatch.getMinActiveDocsCoverage()); @@ -345,9 +334,6 @@ public class IndexedSearchCluster extends SearchCluster } @Override - protected void assureSdConsistent() { } - - @Override public int getRowBits() { return 8; } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 0998d583d6c..c36d3c1dabb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -1,34 +1,37 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; -import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import static java.lang.Long.min; /** - * Tuning of proton config for a search node based on the node flavor of that node. + * Tuning of proton config for a search node based on the resources on the node. * * @author geirst */ -public class NodeFlavorTuning implements ProtonConfig.Producer { +public class NodeResourcesTuning implements ProtonConfig.Producer { - static long MB = 1024 * 1024; - static long GB = MB * 1024; - private final Flavor nodeFlavor; + final static long MB = 1024 * 1024; + final static long GB = MB * 1024; + private final NodeResources resources; private final int redundancy; private final int searchableCopies; private final int threadsPerSearch; - - - public NodeFlavorTuning(Flavor nodeFlavor, int redundancy, int searchableCopies) { - this(nodeFlavor, redundancy, searchableCopies, 1); - } - public NodeFlavorTuning(Flavor nodeFlavor, int redundancy, int searchableCopies, int threadsPerSearch) { - this.nodeFlavor = nodeFlavor; + private final boolean combined; + + public NodeResourcesTuning(NodeResources resources, + int redundancy, + int searchableCopies, + int threadsPerSearch, + boolean combined) { + this.resources = resources; this.redundancy = redundancy; this.searchableCopies = searchableCopies; this.threadsPerSearch = threadsPerSearch; + this.combined = combined; } @Override @@ -51,31 +54,31 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { private void getConfig(ProtonConfig.Documentdb.Builder builder) { ProtonConfig.Documentdb dbCfg = builder.build(); if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) { - long numDocs = (long)nodeFlavor.getMinMainMemoryAvailableGb()*GB/64L; + long numDocs = (long)usableMemoryGb() * GB / 64L; builder.allocation.initialnumdocs(numDocs/Math.max(searchableCopies, redundancy)); } } private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) { - long memoryLimitBytes = (long) ((nodeFlavor.getMinMainMemoryAvailableGb() * 0.05) * GB); + long memoryLimitBytes = (long) ((usableMemoryGb() * 0.05) * GB); builder.maxbytes(memoryLimitBytes); } private void setHwInfo(ProtonConfig.Builder builder) { - builder.hwinfo.disk.shared(nodeFlavor.getType().equals(Flavor.Type.DOCKER_CONTAINER)); - builder.hwinfo.cpu.cores((int)nodeFlavor.getMinCpuCores()); - builder.hwinfo.memory.size((long)nodeFlavor.resources().memoryGb() * GB); - builder.hwinfo.disk.size((long)nodeFlavor.resources().diskGb() * GB); + builder.hwinfo.disk.shared(true); + builder.hwinfo.cpu.cores((int)resources.vcpu()); + builder.hwinfo.memory.size((long)(usableMemoryGb() * GB)); + builder.hwinfo.disk.size((long)(resources.diskGb() * GB)); } private void tuneDiskWriteSpeed(ProtonConfig.Builder builder) { - if (!nodeFlavor.hasFastDisk()) { + if (resources.diskSpeed() != NodeResources.DiskSpeed.fast) { builder.hwinfo.disk.writespeed(40); } } private void tuneDocumentStoreMaxFileSize(ProtonConfig.Summary.Log.Builder builder) { - double memoryGb = nodeFlavor.getMinMainMemoryAvailableGb(); + double memoryGb = usableMemoryGb(); long fileSizeBytes = 4 * GB; if (memoryGb <= 12.0) { fileSizeBytes = 256 * MB; @@ -88,31 +91,31 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { } private void tuneFlushStrategyMemoryLimits(ProtonConfig.Flush.Memory.Builder builder) { - long memoryLimitBytes = (long) ((nodeFlavor.getMinMainMemoryAvailableGb() / 8) * GB); + long memoryLimitBytes = (long) ((usableMemoryGb() / 8) * GB); builder.maxmemory(memoryLimitBytes); builder.each.maxmemory(memoryLimitBytes); } private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { - long tlsSizeBytes = (long) ((nodeFlavor.getMinDiskAvailableGb() * 0.07) * GB); + long tlsSizeBytes = (long) ((resources.diskGb() * 0.07) * GB); tlsSizeBytes = min(tlsSizeBytes, 100 * GB); builder.maxtlssize(tlsSizeBytes); } private void tuneSummaryReadIo(ProtonConfig.Summary.Read.Builder builder) { - if (nodeFlavor.hasFastDisk()) { + if (resources.diskSpeed() == NodeResources.DiskSpeed.fast) { builder.io(ProtonConfig.Summary.Read.Io.DIRECTIO); } } private void tuneSearchReadIo(ProtonConfig.Search.Mmap.Builder builder) { - if (nodeFlavor.hasFastDisk()) { + if (resources.diskSpeed() == NodeResources.DiskSpeed.fast) { builder.advise(ProtonConfig.Search.Mmap.Advise.RANDOM); } } private void tuneRequestThreads(ProtonConfig.Builder builder) { - int numCores = (int)Math.ceil(nodeFlavor.getMinCpuCores()); + int numCores = (int)Math.ceil(resources.vcpu()); builder.numsearcherthreads(numCores*threadsPerSearch); builder.numsummarythreads(numCores); builder.numthreadspersearch(threadsPerSearch); @@ -122,8 +125,16 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { // "Reserve" 1GB of memory for other processes running on the content node (config-proxy, cluster-controller, metrics-proxy) double reservedMemoryGb = 1; double defaultMemoryLimit = new ProtonConfig.Writefilter(new ProtonConfig.Writefilter.Builder()).memorylimit(); - double scaledMemoryLimit = ((nodeFlavor.getMinMainMemoryAvailableGb() - reservedMemoryGb) * defaultMemoryLimit) / nodeFlavor.getMinMainMemoryAvailableGb(); + double scaledMemoryLimit = ((usableMemoryGb() - reservedMemoryGb) * defaultMemoryLimit) / usableMemoryGb(); builder.memorylimit(scaledMemoryLimit); } + /** Returns the memory we can expect will be available for the content node processes */ + private double usableMemoryGb() { + if ( ! combined ) return resources.memoryGb(); + + double fractionTakenByContainer = (double)ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster / 100; + return resources.memoryGb() * (1 - fractionTakenByContainer); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 0139e949c7a..64dfb3b5597 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -136,10 +136,5 @@ public abstract class SearchCluster extends AbstractSearchCluster public abstract void defaultDocumentsConfig(); public abstract DerivedConfiguration getSdConfig(); - protected abstract void exportSdFiles(File toDir) throws IOException; - protected final void writeSdFiles(File toDir) throws IOException { - assureSdConsistent(); - exportSdFiles(toDir); - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 373c62c2eda..738a4cf66d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search; import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.provision.Flavor; import com.yahoo.metrics.MetricsmanagerConfig; import com.yahoo.searchlib.TranslogserverConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; @@ -51,7 +50,12 @@ public class SearchNode extends AbstractService implements MetricsmanagerConfig.Producer, TranslogserverConfig.Producer { - private static final long serialVersionUID = 1L; + private static final int RPC_PORT = 0; + private static final int UNUSED_1 = 1; + private static final int UNUSED_2 = 2; + private static final int UNUSED_3 = 3; + private static final int HEALTH_PORT = 4; + private final boolean isHostedVespa; private final boolean flushOnShutdown; private NodeSpec nodeSpec; @@ -63,11 +67,9 @@ public class SearchNode extends AbstractService implements private AbstractService serviceLayerService; private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; - private static final int RPC_PORT = 0; - private static final int UNUSED_1 = 1; - private static final int UNUSED_2 = 2; - private static final int UNUSED_3 = 3; - private static final int HEALTH_PORT = 4; + + /** Whether this search node is co-located with a container node on a hosted system */ + private final boolean combined; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<SearchNode> { @@ -78,8 +80,11 @@ public class SearchNode extends AbstractService implements private final boolean flushOnShutdown; private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; + private boolean combined; + public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, - boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits) { + boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, + boolean combined) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -87,38 +92,41 @@ public class SearchNode extends AbstractService implements this.flushOnShutdown = flushOnShutdown; this.tuning = tuning; this.resourceLimits = resourceLimits; + this.combined = combined; } @Override protected SearchNode doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { return new SearchNode(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, - flushOnShutdown, tuning, resourceLimits, deployState.isHosted()); + flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), combined); } + } - /** - * Creates a SearchNode in elastic mode. - */ public static SearchNode create(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, - Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa) { + Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, + boolean combined) { return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, - flushOnShutdown, tuning, resourceLimits, isHostedVespa); + flushOnShutdown, tuning, resourceLimits, isHostedVespa, combined); } private SearchNode(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, - Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa) { - this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa); + Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, + boolean combined) { + this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, combined); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; setPropertiesElastic(clusterName, distributionKey); } private SearchNode(AbstractConfigProducer parent, String name, NodeSpec nodeSpec, String clusterName, - boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa) { + boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, + boolean combined) { super(parent, name); this.isHostedVespa = isHostedVespa; + this.combined = combined; this.nodeSpec = nodeSpec; this.clusterName = clusterName; this.flushOnShutdown = flushOnShutdown; @@ -270,20 +278,16 @@ public class SearchNode extends AbstractService implements // to make sure the node failer has done its work builder.pruneremoveddocumentsage(4 * 24 * 3600 + 3600 + 60); } - if (getHostResource() != null && getHostResource().getFlavor().isPresent()) { - Flavor nodeFlavor = getHostResource().getFlavor().get(); - NodeFlavorTuning nodeFlavorTuning = tuning.isPresent() - ? new NodeFlavorTuning(nodeFlavor, redundancy, searchableCopies, tuning.get().getNumThreadsPerSearch()) - : new NodeFlavorTuning(nodeFlavor, redundancy, searchableCopies); - nodeFlavorTuning.getConfig(builder); - - if (tuning.isPresent()) { - tuning.get().getConfig(builder); - } - - if (resourceLimits.isPresent()) { - resourceLimits.get().getConfig(builder); - } + if (getHostResource() != null && ! getHostResource().realResources().isUnspecified()) { + var nodeResourcesTuning = new NodeResourcesTuning(getHostResource().realResources(), + redundancy, + searchableCopies, + tuning.map(Tuning::threadsPerSearch).orElse(1), + combined); + nodeResourcesTuning.getConfig(builder); + + tuning.ifPresent(t -> t.getConfig(builder)); + resourceLimits.ifPresent(l -> l.getConfig(builder)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 28e7b3eb37a..aa735d5ae4c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -84,13 +84,6 @@ public class StreamingSearchCluster extends SearchCluster implements } @Override - protected void assureSdConsistent() { - if (sdConfig == null) { - throw new IllegalStateException("Search cluster '" + getClusterName() + "' does not have any search definitions"); - } - } - - @Override protected void deriveAllSchemas(List<SchemaSpec> local, DeployState deployState) { if (local.size() == 1) { deriveSingleSearchDefinition(local.get(0).getSearchDefinition().getSearch(), deployState); @@ -112,12 +105,7 @@ public class StreamingSearchCluster extends SearchCluster implements public DerivedConfiguration getSdConfig() { return sdConfig; } - @Override - protected void exportSdFiles(File toDir) throws IOException { - if (sdConfig!=null) { - sdConfig.export(toDir.getCanonicalPath()); - } - } + @Override public void defaultDocumentsConfig() { } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index 5984aad7f16..d8a2e67bf9a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -406,7 +406,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ if (searchNode != null) searchNode.getConfig(builder); } - public int getNumThreadsPerSearch() { + public int threadsPerSearch() { if (searchNode == null) return 1; if (searchNode.threads == null) return 1; if (searchNode.threads.numThreadsPerSearch == null) return 1; diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 3560cf2cd84..75ef3a831e8 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -339,6 +339,7 @@ TOKEN : | < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" > | < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? > | < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > +| < SUMMARYFEATURES_ML_INHERITS: "summary-features inherits " (<IDENTIFIER>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > | < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? > | < RANKFEATURES_ML: "rank-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > | < EXPRESSION_SL: "expression" (" ")* ":" (("{"<BRACE_SL_LEVEL_1>)|<BRACE_SL_CONTENT>)* ("\n")? > @@ -585,9 +586,6 @@ void compression(SDDocumentType document, String name) : if (name == null || name.equals("header")) { document.getDocumentType().contentStruct().setCompressionConfig(cfg); } - if (name == null || name.equals("body")) { - document.getDocumentType().getBodyType().setCompressionConfig(cfg); - } } } @@ -1212,6 +1210,7 @@ Object sortingSetting(SortingOperation sorting, String attributeName) : */ Object attributeSetting(FieldOperationContainer field, AttributeOperation attribute, String attributeName) : { + String str; } { ( @@ -1228,6 +1227,7 @@ Object attributeSetting(FieldOperationContainer field, AttributeOperation attrib attribute.setAliasedName(aliasedName); } | attributeTensorType(attribute) + | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { attribute.setDistanceMetric(str); } ) { return null; } } @@ -1816,7 +1816,6 @@ Object indexBody(IndexOperation index) : | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); } | <ENABLE_BM25> { index.setEnableBm25(true); } - | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { index.setDistanceMetric(str); } | hnswIndex(index) { } ) { return null; } @@ -2143,23 +2142,26 @@ Object secondPhaseItem(RankProfile profile) : Object summaryFeatures(RankProfile profile) : { String features; + String inherited = null; } { ( <SUMMARYFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } | <SUMMARYFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1, - token.image.lastIndexOf("}")).trim(); } ) + token.image.lastIndexOf("}")).trim(); } | + <SUMMARYFEATURES_ML_INHERITS> { + int inheritsIndex = token.image.indexOf("inherits "); + String rest = token.image.substring(inheritsIndex + "inherits ".length()); + profile.setInheritedSummaryFeatures(rest.substring(0, rest.indexOf(" ")).trim()); + features = token.image.substring(token.image.indexOf("{") + 1, token.image.lastIndexOf("}")).trim(); + } + ) { profile.addSummaryFeatures(getFeatureList(features)); return null; } } -/** - * This rule consumes a rank-features block of a rank profile. - * - * @param profile The rank profile to modify. - * @return Null. - */ +/** Consumes a rank-features block of a rank profile */ Object rankFeatures(RankProfile profile) : { String features; diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd index 247df8a0241..6d10c50e80a 100644 --- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd +++ b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd @@ -2,7 +2,7 @@ search test { document test { - field argument type tensor<float>(d0[],d1[784]) { + field argument type tensor<float>(d0[1],d1[784]) { indexing: attribute } } diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd index 126f8d00724..8b782a01946 100644 --- a/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd @@ -4,12 +4,12 @@ search simple { constant constant_tensor_1 { file: tensors/constant_tensor_1.json - type: tensor(x[], y[]) + type: tensor(x[4], y[3]) } constant constant_tensor_2 { file: tensors/constant_tensor_2.json - type: tensor(x[]) + type: tensor(x[6]) } constant constant_tensor_3 { @@ -24,6 +24,6 @@ search simple { constant constant_tensor_5 { file: tensors/constant_tensor_5.json - type: tensor(x[], y[], z[]) + type: tensor(x[33], y[10], z[46]) } } diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd index 126f8d00724..8b782a01946 100644 --- a/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd +++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd @@ -4,12 +4,12 @@ search simple { constant constant_tensor_1 { file: tensors/constant_tensor_1.json - type: tensor(x[], y[]) + type: tensor(x[4], y[3]) } constant constant_tensor_2 { file: tensors/constant_tensor_2.json - type: tensor(x[]) + type: tensor(x[6]) } constant constant_tensor_3 { @@ -24,6 +24,6 @@ search simple { constant constant_tensor_5 { file: tensors/constant_tensor_5.json - type: tensor(x[], y[], z[]) + type: tensor(x[33], y[10], z[46]) } } diff --git a/config-model/src/test/configmodel/types/documentmanager.cfg b/config-model/src/test/configmodel/types/documentmanager.cfg index 631886181d7..f59dbeeb3ca 100644 --- a/config-model/src/test/configmodel/types/documentmanager.cfg +++ b/config-model/src/test/configmodel/types/documentmanager.cfg @@ -214,44 +214,37 @@ datatype[26].structtype[0].field[26].detailedtype "" datatype[26].structtype[0].field[27].name "other" datatype[26].structtype[0].field[27].datatype 4 datatype[26].structtype[0].field[27].detailedtype "" -datatype[27].id 348447225 -datatype[27].structtype[0].name "types.body" -datatype[27].structtype[0].version 0 -datatype[27].structtype[0].compresstype NONE -datatype[27].structtype[0].compresslevel 0 -datatype[27].structtype[0].compressthreshold 95 -datatype[27].structtype[0].compressminsize 800 -datatype[28].id -853072901 -datatype[28].documenttype[0].name "types" -datatype[28].documenttype[0].version 0 -datatype[28].documenttype[0].inherits[0].name "document" -datatype[28].documenttype[0].inherits[0].version 0 -datatype[28].documenttype[0].headerstruct 1328581348 -datatype[28].documenttype[0].bodystruct 348447225 -datatype[28].documenttype[0].fieldsets{[document]}.fields[0] "Folders" -datatype[28].documenttype[0].fieldsets{[document]}.fields[1] "abyte" -datatype[28].documenttype[0].fieldsets{[document]}.fields[2] "album0" -datatype[28].documenttype[0].fieldsets{[document]}.fields[3] "album1" -datatype[28].documenttype[0].fieldsets{[document]}.fields[4] "along" -datatype[28].documenttype[0].fieldsets{[document]}.fields[5] "arrarr" -datatype[28].documenttype[0].fieldsets{[document]}.fields[6] "arrayfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[7] "arraymapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[8] "complexarray" -datatype[28].documenttype[0].fieldsets{[document]}.fields[9] "doublemapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[10] "floatmapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[11] "intmapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[12] "juletre" -datatype[28].documenttype[0].fieldsets{[document]}.fields[13] "longmapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[14] "maparr" -datatype[28].documenttype[0].fieldsets{[document]}.fields[15] "mystructarr" -datatype[28].documenttype[0].fieldsets{[document]}.fields[16] "mystructfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[17] "mystructmap" -datatype[28].documenttype[0].fieldsets{[document]}.fields[18] "pos" -datatype[28].documenttype[0].fieldsets{[document]}.fields[19] "setfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[20] "setfield2" -datatype[28].documenttype[0].fieldsets{[document]}.fields[21] "setfield3" -datatype[28].documenttype[0].fieldsets{[document]}.fields[22] "setfield4" -datatype[28].documenttype[0].fieldsets{[document]}.fields[23] "stringmapfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[24] "structarrayfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[25] "structfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[26] "tagfield" +datatype[27].id -853072901 +datatype[27].documenttype[0].name "types" +datatype[27].documenttype[0].version 0 +datatype[27].documenttype[0].inherits[0].name "document" +datatype[27].documenttype[0].inherits[0].version 0 +datatype[27].documenttype[0].headerstruct 1328581348 +datatype[27].documenttype[0].bodystruct 0 +datatype[27].documenttype[0].fieldsets{[document]}.fields[0] "Folders" +datatype[27].documenttype[0].fieldsets{[document]}.fields[1] "abyte" +datatype[27].documenttype[0].fieldsets{[document]}.fields[2] "album0" +datatype[27].documenttype[0].fieldsets{[document]}.fields[3] "album1" +datatype[27].documenttype[0].fieldsets{[document]}.fields[4] "along" +datatype[27].documenttype[0].fieldsets{[document]}.fields[5] "arrarr" +datatype[27].documenttype[0].fieldsets{[document]}.fields[6] "arrayfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[7] "arraymapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[8] "complexarray" +datatype[27].documenttype[0].fieldsets{[document]}.fields[9] "doublemapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[10] "floatmapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[11] "intmapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[12] "juletre" +datatype[27].documenttype[0].fieldsets{[document]}.fields[13] "longmapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[14] "maparr" +datatype[27].documenttype[0].fieldsets{[document]}.fields[15] "mystructarr" +datatype[27].documenttype[0].fieldsets{[document]}.fields[16] "mystructfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[17] "mystructmap" +datatype[27].documenttype[0].fieldsets{[document]}.fields[18] "pos" +datatype[27].documenttype[0].fieldsets{[document]}.fields[19] "setfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[20] "setfield2" +datatype[27].documenttype[0].fieldsets{[document]}.fields[21] "setfield3" +datatype[27].documenttype[0].fieldsets{[document]}.fields[22] "setfield4" +datatype[27].documenttype[0].fieldsets{[document]}.fields[23] "stringmapfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[24] "structarrayfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[25] "structfield" +datatype[27].documenttype[0].fieldsets{[document]}.fields[26] "tagfield" diff --git a/config-model/src/test/configmodel/types/documenttypes.cfg b/config-model/src/test/configmodel/types/documenttypes.cfg index 8c88ee4f4e0..8f576715a4f 100644 --- a/config-model/src/test/configmodel/types/documenttypes.cfg +++ b/config-model/src/test/configmodel/types/documenttypes.cfg @@ -3,7 +3,7 @@ documenttype[0].id -853072901 documenttype[0].name "types" documenttype[0].version 0 documenttype[0].headerstruct 1328581348 -documenttype[0].bodystruct 348447225 +documenttype[0].bodystruct 0 documenttype[0].inherits[0].id 8 documenttype[0].datatype[0].id -1865479609 documenttype[0].datatype[0].type MAP @@ -547,21 +547,6 @@ documenttype[0].datatype[25].sstruct.field[27].name "other" documenttype[0].datatype[25].sstruct.field[27].id 2443357 documenttype[0].datatype[25].sstruct.field[27].datatype 4 documenttype[0].datatype[25].sstruct.field[27].detailedtype "" -documenttype[0].datatype[26].id 348447225 -documenttype[0].datatype[26].type STRUCT -documenttype[0].datatype[26].array.element.id 0 -documenttype[0].datatype[26].map.key.id 0 -documenttype[0].datatype[26].map.value.id 0 -documenttype[0].datatype[26].wset.key.id 0 -documenttype[0].datatype[26].wset.createifnonexistent false -documenttype[0].datatype[26].wset.removeifzero false -documenttype[0].datatype[26].annotationref.annotation.id 0 -documenttype[0].datatype[26].sstruct.name "types.body" -documenttype[0].datatype[26].sstruct.version 0 -documenttype[0].datatype[26].sstruct.compression.type NONE -documenttype[0].datatype[26].sstruct.compression.level 0 -documenttype[0].datatype[26].sstruct.compression.threshold 95 -documenttype[0].datatype[26].sstruct.compression.minsize 200 documenttype[0].fieldsets{[document]}.fields[0] "Folders" documenttype[0].fieldsets{[document]}.fields[1] "abyte" documenttype[0].fieldsets{[document]}.fields[2] "album0" diff --git a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg index 4f04e901a92..283e5c2fe79 100644 --- a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg +++ b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg @@ -3,7 +3,7 @@ documenttype[0].id -1368624373 documenttype[0].name "other_doc" documenttype[0].version 0 documenttype[0].headerstruct 1631005140 -documenttype[0].bodystruct 549879017 +documenttype[0].bodystruct 0 documenttype[0].inherits[0].id 8 documenttype[0].datatype[0].id 1631005140 documenttype[0].datatype[0].type STRUCT @@ -20,26 +20,11 @@ documenttype[0].datatype[0].sstruct.compression.type NONE documenttype[0].datatype[0].sstruct.compression.level 0 documenttype[0].datatype[0].sstruct.compression.threshold 95 documenttype[0].datatype[0].sstruct.compression.minsize 200 -documenttype[0].datatype[1].id 549879017 -documenttype[0].datatype[1].type STRUCT -documenttype[0].datatype[1].array.element.id 0 -documenttype[0].datatype[1].map.key.id 0 -documenttype[0].datatype[1].map.value.id 0 -documenttype[0].datatype[1].wset.key.id 0 -documenttype[0].datatype[1].wset.createifnonexistent false -documenttype[0].datatype[1].wset.removeifzero false -documenttype[0].datatype[1].annotationref.annotation.id 0 -documenttype[0].datatype[1].sstruct.name "other_doc.body" -documenttype[0].datatype[1].sstruct.version 0 -documenttype[0].datatype[1].sstruct.compression.type NONE -documenttype[0].datatype[1].sstruct.compression.level 0 -documenttype[0].datatype[1].sstruct.compression.threshold 95 -documenttype[0].datatype[1].sstruct.compression.minsize 200 documenttype[1].id -853072901 documenttype[1].name "types" documenttype[1].version 0 documenttype[1].headerstruct 1328581348 -documenttype[1].bodystruct 348447225 +documenttype[1].bodystruct 0 documenttype[1].inherits[0].id 8 documenttype[1].datatype[0].id -1368624373 documenttype[1].datatype[0].type STRUCT @@ -75,19 +60,4 @@ documenttype[1].datatype[1].sstruct.field[0].name "doc_field" documenttype[1].datatype[1].sstruct.field[0].id 819293364 documenttype[1].datatype[1].sstruct.field[0].datatype -1368624373 documenttype[1].datatype[1].sstruct.field[0].detailedtype "" -documenttype[1].datatype[2].id 348447225 -documenttype[1].datatype[2].type STRUCT -documenttype[1].datatype[2].array.element.id 0 -documenttype[1].datatype[2].map.key.id 0 -documenttype[1].datatype[2].map.value.id 0 -documenttype[1].datatype[2].wset.key.id 0 -documenttype[1].datatype[2].wset.createifnonexistent false -documenttype[1].datatype[2].wset.removeifzero false -documenttype[1].datatype[2].annotationref.annotation.id 0 -documenttype[1].datatype[2].sstruct.name "types.body" -documenttype[1].datatype[2].sstruct.version 0 -documenttype[1].datatype[2].sstruct.compression.type NONE -documenttype[1].datatype[2].sstruct.compression.level 0 -documenttype[1].datatype[2].sstruct.compression.threshold 95 -documenttype[1].datatype[2].sstruct.compression.minsize 200 documenttype[1].fieldsets{[document]}.fields[0] "doc_field" diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg index 7b50176625d..7ae73c23685 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -29,76 +29,55 @@ datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].field[1].name "person_ref" datatype[3].structtype[0].field[1].datatype 542332920 datatype[3].structtype[0].field[1].detailedtype "" -datatype[4].id -255288561 -datatype[4].structtype[0].name "ad.body" -datatype[4].structtype[0].version 0 -datatype[4].structtype[0].compresstype NONE -datatype[4].structtype[0].compresslevel 0 -datatype[4].structtype[0].compressthreshold 95 -datatype[4].structtype[0].compressminsize 800 -datatype[5].id 2987301 -datatype[5].documenttype[0].name "ad" -datatype[5].documenttype[0].version 0 -datatype[5].documenttype[0].inherits[0].name "document" -datatype[5].documenttype[0].inherits[0].version 0 -datatype[5].documenttype[0].headerstruct 959075962 -datatype[5].documenttype[0].bodystruct -255288561 -datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" -datatype[5].documenttype[0].importedfield[0].name "my_cool_field" -datatype[5].documenttype[0].importedfield[1].name "my_swag_field" -datatype[5].documenttype[0].importedfield[2].name "my_name" -datatype[6].id -2041471955 -datatype[6].structtype[0].name "campaign.header" -datatype[6].structtype[0].version 0 -datatype[6].structtype[0].compresstype NONE -datatype[6].structtype[0].compresslevel 0 -datatype[6].structtype[0].compressthreshold 95 -datatype[6].structtype[0].compressminsize 800 -datatype[6].structtype[0].field[0].name "cool_field" -datatype[6].structtype[0].field[0].datatype 2 -datatype[6].structtype[0].field[0].detailedtype "" -datatype[6].structtype[0].field[1].name "swag_field" -datatype[6].structtype[0].field[1].datatype 4 -datatype[6].structtype[0].field[1].detailedtype "" -datatype[7].id 1448849794 -datatype[7].structtype[0].name "campaign.body" +datatype[4].id 2987301 +datatype[4].documenttype[0].name "ad" +datatype[4].documenttype[0].version 0 +datatype[4].documenttype[0].inherits[0].name "document" +datatype[4].documenttype[0].inherits[0].version 0 +datatype[4].documenttype[0].headerstruct 959075962 +datatype[4].documenttype[0].bodystruct 0 +datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +datatype[4].documenttype[0].importedfield[0].name "my_cool_field" +datatype[4].documenttype[0].importedfield[1].name "my_swag_field" +datatype[4].documenttype[0].importedfield[2].name "my_name" +datatype[5].id -2041471955 +datatype[5].structtype[0].name "campaign.header" +datatype[5].structtype[0].version 0 +datatype[5].structtype[0].compresstype NONE +datatype[5].structtype[0].compresslevel 0 +datatype[5].structtype[0].compressthreshold 95 +datatype[5].structtype[0].compressminsize 800 +datatype[5].structtype[0].field[0].name "cool_field" +datatype[5].structtype[0].field[0].datatype 2 +datatype[5].structtype[0].field[0].detailedtype "" +datatype[5].structtype[0].field[1].name "swag_field" +datatype[5].structtype[0].field[1].datatype 4 +datatype[5].structtype[0].field[1].detailedtype "" +datatype[6].id -1318255918 +datatype[6].documenttype[0].name "campaign" +datatype[6].documenttype[0].version 0 +datatype[6].documenttype[0].inherits[0].name "document" +datatype[6].documenttype[0].inherits[0].version 0 +datatype[6].documenttype[0].headerstruct -2041471955 +datatype[6].documenttype[0].bodystruct 0 +datatype[6].documenttype[0].fieldsets{[document]}.fields[0] "cool_field" +datatype[6].documenttype[0].fieldsets{[document]}.fields[1] "swag_field" +datatype[7].id 3129224 +datatype[7].structtype[0].name "person.header" datatype[7].structtype[0].version 0 datatype[7].structtype[0].compresstype NONE datatype[7].structtype[0].compresslevel 0 datatype[7].structtype[0].compressthreshold 95 datatype[7].structtype[0].compressminsize 800 -datatype[8].id -1318255918 -datatype[8].documenttype[0].name "campaign" +datatype[7].structtype[0].field[0].name "name" +datatype[7].structtype[0].field[0].datatype 2 +datatype[7].structtype[0].field[0].detailedtype "" +datatype[8].id 443162583 +datatype[8].documenttype[0].name "person" datatype[8].documenttype[0].version 0 datatype[8].documenttype[0].inherits[0].name "document" datatype[8].documenttype[0].inherits[0].version 0 -datatype[8].documenttype[0].headerstruct -2041471955 -datatype[8].documenttype[0].bodystruct 1448849794 -datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "cool_field" -datatype[8].documenttype[0].fieldsets{[document]}.fields[1] "swag_field" -datatype[9].id 3129224 -datatype[9].structtype[0].name "person.header" -datatype[9].structtype[0].version 0 -datatype[9].structtype[0].compresstype NONE -datatype[9].structtype[0].compresslevel 0 -datatype[9].structtype[0].compressthreshold 95 -datatype[9].structtype[0].compressminsize 800 -datatype[9].structtype[0].field[0].name "name" -datatype[9].structtype[0].field[0].datatype 2 -datatype[9].structtype[0].field[0].detailedtype "" -datatype[10].id -2003767395 -datatype[10].structtype[0].name "person.body" -datatype[10].structtype[0].version 0 -datatype[10].structtype[0].compresstype NONE -datatype[10].structtype[0].compresslevel 0 -datatype[10].structtype[0].compressthreshold 95 -datatype[10].structtype[0].compressminsize 800 -datatype[11].id 443162583 -datatype[11].documenttype[0].name "person" -datatype[11].documenttype[0].version 0 -datatype[11].documenttype[0].inherits[0].name "document" -datatype[11].documenttype[0].inherits[0].version 0 -datatype[11].documenttype[0].headerstruct 3129224 -datatype[11].documenttype[0].bodystruct -2003767395 -datatype[11].documenttype[0].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file +datatype[8].documenttype[0].headerstruct 3129224 +datatype[8].documenttype[0].bodystruct 0 +datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "name" diff --git a/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg index e624ffdf7f5..a613c2c034d 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_ref_to_self_type.cfg @@ -24,18 +24,11 @@ datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "self_ref" datatype[].structtype[].field[].datatype -1895788438 datatype[].structtype[].field[].detailedtype "" -datatype[].id -255288561 -datatype[].structtype[].name "ad.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 2987301 datatype[].documenttype[].name "ad" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 959075962 -datatype[].documenttype[].bodystruct -255288561 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "self_ref" diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg index 90ddb8c9f8d..2b6e2e852a3 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg @@ -29,61 +29,40 @@ datatype[3].structtype[0].field[0].detailedtype "" datatype[3].structtype[0].field[1].name "person_ref" datatype[3].structtype[0].field[1].datatype 542332920 datatype[3].structtype[0].field[1].detailedtype "" -datatype[4].id -255288561 -datatype[4].structtype[0].name "ad.body" -datatype[4].structtype[0].version 0 -datatype[4].structtype[0].compresstype NONE -datatype[4].structtype[0].compresslevel 0 -datatype[4].structtype[0].compressthreshold 95 -datatype[4].structtype[0].compressminsize 800 -datatype[5].id 2987301 -datatype[5].documenttype[0].name "ad" -datatype[5].documenttype[0].version 0 -datatype[5].documenttype[0].inherits[0].name "document" -datatype[5].documenttype[0].inherits[0].version 0 -datatype[5].documenttype[0].headerstruct 959075962 -datatype[5].documenttype[0].bodystruct -255288561 -datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" -datatype[6].id -2041471955 -datatype[6].structtype[0].name "campaign.header" -datatype[6].structtype[0].version 0 -datatype[6].structtype[0].compresstype NONE -datatype[6].structtype[0].compresslevel 0 -datatype[6].structtype[0].compressthreshold 95 -datatype[6].structtype[0].compressminsize 800 -datatype[7].id 1448849794 -datatype[7].structtype[0].name "campaign.body" +datatype[4].id 2987301 +datatype[4].documenttype[0].name "ad" +datatype[4].documenttype[0].version 0 +datatype[4].documenttype[0].inherits[0].name "document" +datatype[4].documenttype[0].inherits[0].version 0 +datatype[4].documenttype[0].headerstruct 959075962 +datatype[4].documenttype[0].bodystruct 0 +datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "person_ref" +datatype[5].id -2041471955 +datatype[5].structtype[0].name "campaign.header" +datatype[5].structtype[0].version 0 +datatype[5].structtype[0].compresstype NONE +datatype[5].structtype[0].compresslevel 0 +datatype[5].structtype[0].compressthreshold 95 +datatype[5].structtype[0].compressminsize 800 +datatype[6].id -1318255918 +datatype[6].documenttype[0].name "campaign" +datatype[6].documenttype[0].version 0 +datatype[6].documenttype[0].inherits[0].name "document" +datatype[6].documenttype[0].inherits[0].version 0 +datatype[6].documenttype[0].headerstruct -2041471955 +datatype[6].documenttype[0].bodystruct 0 +datatype[7].id 3129224 +datatype[7].structtype[0].name "person.header" datatype[7].structtype[0].version 0 datatype[7].structtype[0].compresstype NONE datatype[7].structtype[0].compresslevel 0 datatype[7].structtype[0].compressthreshold 95 datatype[7].structtype[0].compressminsize 800 -datatype[8].id -1318255918 -datatype[8].documenttype[0].name "campaign" +datatype[8].id 443162583 +datatype[8].documenttype[0].name "person" datatype[8].documenttype[0].version 0 datatype[8].documenttype[0].inherits[0].name "document" datatype[8].documenttype[0].inherits[0].version 0 -datatype[8].documenttype[0].headerstruct -2041471955 -datatype[8].documenttype[0].bodystruct 1448849794 -datatype[9].id 3129224 -datatype[9].structtype[0].name "person.header" -datatype[9].structtype[0].version 0 -datatype[9].structtype[0].compresstype NONE -datatype[9].structtype[0].compresslevel 0 -datatype[9].structtype[0].compressthreshold 95 -datatype[9].structtype[0].compressminsize 800 -datatype[10].id -2003767395 -datatype[10].structtype[0].name "person.body" -datatype[10].structtype[0].version 0 -datatype[10].structtype[0].compresstype NONE -datatype[10].structtype[0].compresslevel 0 -datatype[10].structtype[0].compressthreshold 95 -datatype[10].structtype[0].compressminsize 800 -datatype[11].id 443162583 -datatype[11].documenttype[0].name "person" -datatype[11].documenttype[0].version 0 -datatype[11].documenttype[0].inherits[0].name "document" -datatype[11].documenttype[0].inherits[0].version 0 -datatype[11].documenttype[0].headerstruct 3129224 -datatype[11].documenttype[0].bodystruct -2003767395 +datatype[8].documenttype[0].headerstruct 3129224 +datatype[8].documenttype[0].bodystruct 0 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg index 1807adeb68d..bab281cca36 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg @@ -27,40 +27,26 @@ datatype[2].structtype[0].field[0].detailedtype "" datatype[2].structtype[0].field[1].name "other_campaign_ref" datatype[2].structtype[0].field[1].datatype 595216861 datatype[2].structtype[0].field[1].detailedtype "" -datatype[3].id -255288561 -datatype[3].structtype[0].name "ad.body" -datatype[3].structtype[0].version 0 -datatype[3].structtype[0].compresstype NONE -datatype[3].structtype[0].compresslevel 0 -datatype[3].structtype[0].compressthreshold 95 -datatype[3].structtype[0].compressminsize 800 -datatype[4].id 2987301 -datatype[4].documenttype[0].name "ad" -datatype[4].documenttype[0].version 0 -datatype[4].documenttype[0].inherits[0].name "document" -datatype[4].documenttype[0].inherits[0].version 0 -datatype[4].documenttype[0].headerstruct 959075962 -datatype[4].documenttype[0].bodystruct -255288561 -datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" -datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "other_campaign_ref" -datatype[5].id -2041471955 -datatype[5].structtype[0].name "campaign.header" -datatype[5].structtype[0].version 0 -datatype[5].structtype[0].compresstype NONE -datatype[5].structtype[0].compresslevel 0 -datatype[5].structtype[0].compressthreshold 95 -datatype[5].structtype[0].compressminsize 800 -datatype[6].id 1448849794 -datatype[6].structtype[0].name "campaign.body" -datatype[6].structtype[0].version 0 -datatype[6].structtype[0].compresstype NONE -datatype[6].structtype[0].compresslevel 0 -datatype[6].structtype[0].compressthreshold 95 -datatype[6].structtype[0].compressminsize 800 -datatype[7].id -1318255918 -datatype[7].documenttype[0].name "campaign" -datatype[7].documenttype[0].version 0 -datatype[7].documenttype[0].inherits[0].name "document" -datatype[7].documenttype[0].inherits[0].version 0 -datatype[7].documenttype[0].headerstruct -2041471955 -datatype[7].documenttype[0].bodystruct 1448849794 +datatype[3].id 2987301 +datatype[3].documenttype[0].name "ad" +datatype[3].documenttype[0].version 0 +datatype[3].documenttype[0].inherits[0].name "document" +datatype[3].documenttype[0].inherits[0].version 0 +datatype[3].documenttype[0].headerstruct 959075962 +datatype[3].documenttype[0].bodystruct 0 +datatype[3].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" +datatype[3].documenttype[0].fieldsets{[document]}.fields[1] "other_campaign_ref" +datatype[4].id -2041471955 +datatype[4].structtype[0].name "campaign.header" +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].compresstype NONE +datatype[4].structtype[0].compresslevel 0 +datatype[4].structtype[0].compressthreshold 95 +datatype[4].structtype[0].compressminsize 800 +datatype[5].id -1318255918 +datatype[5].documenttype[0].name "campaign" +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0].name "document" +datatype[5].documenttype[0].inherits[0].version 0 +datatype[5].documenttype[0].headerstruct -2041471955 +datatype[5].documenttype[0].bodystruct 0 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg index 7859703ffe0..242310b57a4 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -3,7 +3,7 @@ documenttype[0].id 2987301 documenttype[0].name "ad" documenttype[0].version 0 documenttype[0].headerstruct 959075962 -documenttype[0].bodystruct -255288561 +documenttype[0].bodystruct 0 documenttype[0].inherits[0].id 8 documenttype[0].datatype[0].id 959075962 documenttype[0].datatype[0].type STRUCT @@ -28,21 +28,6 @@ documenttype[0].datatype[0].sstruct.field[1].name "person_ref" documenttype[0].datatype[0].sstruct.field[1].id 100779805 documenttype[0].datatype[0].sstruct.field[1].datatype 542332920 documenttype[0].datatype[0].sstruct.field[1].detailedtype "" -documenttype[0].datatype[1].id -255288561 -documenttype[0].datatype[1].type STRUCT -documenttype[0].datatype[1].array.element.id 0 -documenttype[0].datatype[1].map.key.id 0 -documenttype[0].datatype[1].map.value.id 0 -documenttype[0].datatype[1].wset.key.id 0 -documenttype[0].datatype[1].wset.createifnonexistent false -documenttype[0].datatype[1].wset.removeifzero false -documenttype[0].datatype[1].annotationref.annotation.id 0 -documenttype[0].datatype[1].sstruct.name "ad.body" -documenttype[0].datatype[1].sstruct.version 0 -documenttype[0].datatype[1].sstruct.compression.type NONE -documenttype[0].datatype[1].sstruct.compression.level 0 -documenttype[0].datatype[1].sstruct.compression.threshold 95 -documenttype[0].datatype[1].sstruct.compression.minsize 200 documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" documenttype[0].fieldsets{[document]}.fields[1] "person_ref" documenttype[0].referencetype[0].id 595216861 @@ -56,7 +41,7 @@ documenttype[1].id -1318255918 documenttype[1].name "campaign" documenttype[1].version 0 documenttype[1].headerstruct -2041471955 -documenttype[1].bodystruct 1448849794 +documenttype[1].bodystruct 0 documenttype[1].inherits[0].id 8 documenttype[1].datatype[0].id -2041471955 documenttype[1].datatype[0].type STRUCT @@ -81,28 +66,13 @@ documenttype[1].datatype[0].sstruct.field[1].name "swag_field" documenttype[1].datatype[0].sstruct.field[1].id 1691224741 documenttype[1].datatype[0].sstruct.field[1].datatype 4 documenttype[1].datatype[0].sstruct.field[1].detailedtype "" -documenttype[1].datatype[1].id 1448849794 -documenttype[1].datatype[1].type STRUCT -documenttype[1].datatype[1].array.element.id 0 -documenttype[1].datatype[1].map.key.id 0 -documenttype[1].datatype[1].map.value.id 0 -documenttype[1].datatype[1].wset.key.id 0 -documenttype[1].datatype[1].wset.createifnonexistent false -documenttype[1].datatype[1].wset.removeifzero false -documenttype[1].datatype[1].annotationref.annotation.id 0 -documenttype[1].datatype[1].sstruct.name "campaign.body" -documenttype[1].datatype[1].sstruct.version 0 -documenttype[1].datatype[1].sstruct.compression.type NONE -documenttype[1].datatype[1].sstruct.compression.level 0 -documenttype[1].datatype[1].sstruct.compression.threshold 95 -documenttype[1].datatype[1].sstruct.compression.minsize 200 documenttype[1].fieldsets{[document]}.fields[0] "cool_field" documenttype[1].fieldsets{[document]}.fields[1] "swag_field" documenttype[2].id 443162583 documenttype[2].name "person" documenttype[2].version 0 documenttype[2].headerstruct 3129224 -documenttype[2].bodystruct -2003767395 +documenttype[2].bodystruct 0 documenttype[2].inherits[0].id 8 documenttype[2].datatype[0].id 3129224 documenttype[2].datatype[0].type STRUCT @@ -123,19 +93,4 @@ documenttype[2].datatype[0].sstruct.field[0].name "name" documenttype[2].datatype[0].sstruct.field[0].id 1160796772 documenttype[2].datatype[0].sstruct.field[0].datatype 2 documenttype[2].datatype[0].sstruct.field[0].detailedtype "" -documenttype[2].datatype[1].id -2003767395 -documenttype[2].datatype[1].type STRUCT -documenttype[2].datatype[1].array.element.id 0 -documenttype[2].datatype[1].map.key.id 0 -documenttype[2].datatype[1].map.value.id 0 -documenttype[2].datatype[1].wset.key.id 0 -documenttype[2].datatype[1].wset.createifnonexistent false -documenttype[2].datatype[1].wset.removeifzero false -documenttype[2].datatype[1].annotationref.annotation.id 0 -documenttype[2].datatype[1].sstruct.name "person.body" -documenttype[2].datatype[1].sstruct.version 0 -documenttype[2].datatype[1].sstruct.compression.type NONE -documenttype[2].datatype[1].sstruct.compression.level 0 -documenttype[2].datatype[1].sstruct.compression.threshold 95 -documenttype[2].datatype[1].sstruct.compression.minsize 200 -documenttype[2].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file +documenttype[2].fieldsets{[document]}.fields[0] "name" diff --git a/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg index ca568aaa43d..f925ac99a25 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg @@ -3,7 +3,7 @@ documenttype[].id 2987301 documenttype[].name "ad" documenttype[].version 0 documenttype[].headerstruct 959075962 -documenttype[].bodystruct -255288561 +documenttype[].bodystruct 0 documenttype[].inherits[].id 8 documenttype[].datatype[].id 959075962 documenttype[].datatype[].type STRUCT @@ -24,21 +24,6 @@ documenttype[].datatype[].sstruct.field[].name "self_ref" documenttype[].datatype[].sstruct.field[].id 852207313 documenttype[].datatype[].sstruct.field[].datatype -1895788438 documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id -255288561 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "ad.body" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 documenttype[].fieldsets{[document]}.fields[] "self_ref" documenttype[].referencetype[].id -1895788438 documenttype[].referencetype[].target_type_id 2987301 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg index a7fd3577789..c3aba21a498 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg @@ -3,7 +3,7 @@ documenttype[0].id 2987301 documenttype[0].name "ad" documenttype[0].version 0 documenttype[0].headerstruct 959075962 -documenttype[0].bodystruct -255288561 +documenttype[0].bodystruct 0 documenttype[0].inherits[0].id 8 documenttype[0].datatype[0].id 959075962 documenttype[0].datatype[0].type STRUCT @@ -28,21 +28,6 @@ documenttype[0].datatype[0].sstruct.field[1].name "person_ref" documenttype[0].datatype[0].sstruct.field[1].id 100779805 documenttype[0].datatype[0].sstruct.field[1].datatype 542332920 documenttype[0].datatype[0].sstruct.field[1].detailedtype "" -documenttype[0].datatype[1].id -255288561 -documenttype[0].datatype[1].type STRUCT -documenttype[0].datatype[1].array.element.id 0 -documenttype[0].datatype[1].map.key.id 0 -documenttype[0].datatype[1].map.value.id 0 -documenttype[0].datatype[1].wset.key.id 0 -documenttype[0].datatype[1].wset.createifnonexistent false -documenttype[0].datatype[1].wset.removeifzero false -documenttype[0].datatype[1].annotationref.annotation.id 0 -documenttype[0].datatype[1].sstruct.name "ad.body" -documenttype[0].datatype[1].sstruct.version 0 -documenttype[0].datatype[1].sstruct.compression.type NONE -documenttype[0].datatype[1].sstruct.compression.level 0 -documenttype[0].datatype[1].sstruct.compression.threshold 95 -documenttype[0].datatype[1].sstruct.compression.minsize 200 documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" documenttype[0].fieldsets{[document]}.fields[1] "person_ref" documenttype[0].referencetype[0].id 595216861 @@ -53,7 +38,7 @@ documenttype[1].id -1318255918 documenttype[1].name "campaign" documenttype[1].version 0 documenttype[1].headerstruct -2041471955 -documenttype[1].bodystruct 1448849794 +documenttype[1].bodystruct 0 documenttype[1].inherits[0].id 8 documenttype[1].datatype[0].id -2041471955 documenttype[1].datatype[0].type STRUCT @@ -70,26 +55,11 @@ documenttype[1].datatype[0].sstruct.compression.type NONE documenttype[1].datatype[0].sstruct.compression.level 0 documenttype[1].datatype[0].sstruct.compression.threshold 95 documenttype[1].datatype[0].sstruct.compression.minsize 200 -documenttype[1].datatype[1].id 1448849794 -documenttype[1].datatype[1].type STRUCT -documenttype[1].datatype[1].array.element.id 0 -documenttype[1].datatype[1].map.key.id 0 -documenttype[1].datatype[1].map.value.id 0 -documenttype[1].datatype[1].wset.key.id 0 -documenttype[1].datatype[1].wset.createifnonexistent false -documenttype[1].datatype[1].wset.removeifzero false -documenttype[1].datatype[1].annotationref.annotation.id 0 -documenttype[1].datatype[1].sstruct.name "campaign.body" -documenttype[1].datatype[1].sstruct.version 0 -documenttype[1].datatype[1].sstruct.compression.type NONE -documenttype[1].datatype[1].sstruct.compression.level 0 -documenttype[1].datatype[1].sstruct.compression.threshold 95 -documenttype[1].datatype[1].sstruct.compression.minsize 200 documenttype[2].id 443162583 documenttype[2].name "person" documenttype[2].version 0 documenttype[2].headerstruct 3129224 -documenttype[2].bodystruct -2003767395 +documenttype[2].bodystruct 0 documenttype[2].inherits[0].id 8 documenttype[2].datatype[0].id 3129224 documenttype[2].datatype[0].type STRUCT @@ -106,18 +76,3 @@ documenttype[2].datatype[0].sstruct.compression.type NONE documenttype[2].datatype[0].sstruct.compression.level 0 documenttype[2].datatype[0].sstruct.compression.threshold 95 documenttype[2].datatype[0].sstruct.compression.minsize 200 -documenttype[2].datatype[1].id -2003767395 -documenttype[2].datatype[1].type STRUCT -documenttype[2].datatype[1].array.element.id 0 -documenttype[2].datatype[1].map.key.id 0 -documenttype[2].datatype[1].map.value.id 0 -documenttype[2].datatype[1].wset.key.id 0 -documenttype[2].datatype[1].wset.createifnonexistent false -documenttype[2].datatype[1].wset.removeifzero false -documenttype[2].datatype[1].annotationref.annotation.id 0 -documenttype[2].datatype[1].sstruct.name "person.body" -documenttype[2].datatype[1].sstruct.version 0 -documenttype[2].datatype[1].sstruct.compression.type NONE -documenttype[2].datatype[1].sstruct.compression.level 0 -documenttype[2].datatype[1].sstruct.compression.threshold 95 -documenttype[2].datatype[1].sstruct.compression.minsize 200 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg index 46a951ae8ea..c5930449dc1 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg @@ -3,7 +3,7 @@ documenttype[0].id 2987301 documenttype[0].name "ad" documenttype[0].version 0 documenttype[0].headerstruct 959075962 -documenttype[0].bodystruct -255288561 +documenttype[0].bodystruct 0 documenttype[0].inherits[0].id 8 documenttype[0].datatype[0].id 959075962 documenttype[0].datatype[0].type STRUCT @@ -28,21 +28,6 @@ documenttype[0].datatype[0].sstruct.field[1].name "other_campaign_ref" documenttype[0].datatype[0].sstruct.field[1].id 874751172 documenttype[0].datatype[0].sstruct.field[1].datatype 595216861 documenttype[0].datatype[0].sstruct.field[1].detailedtype "" -documenttype[0].datatype[1].id -255288561 -documenttype[0].datatype[1].type STRUCT -documenttype[0].datatype[1].array.element.id 0 -documenttype[0].datatype[1].map.key.id 0 -documenttype[0].datatype[1].map.value.id 0 -documenttype[0].datatype[1].wset.key.id 0 -documenttype[0].datatype[1].wset.createifnonexistent false -documenttype[0].datatype[1].wset.removeifzero false -documenttype[0].datatype[1].annotationref.annotation.id 0 -documenttype[0].datatype[1].sstruct.name "ad.body" -documenttype[0].datatype[1].sstruct.version 0 -documenttype[0].datatype[1].sstruct.compression.type NONE -documenttype[0].datatype[1].sstruct.compression.level 0 -documenttype[0].datatype[1].sstruct.compression.threshold 95 -documenttype[0].datatype[1].sstruct.compression.minsize 200 documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref" documenttype[0].fieldsets{[document]}.fields[1] "other_campaign_ref" documenttype[0].referencetype[0].id 595216861 @@ -51,7 +36,7 @@ documenttype[1].id -1318255918 documenttype[1].name "campaign" documenttype[1].version 0 documenttype[1].headerstruct -2041471955 -documenttype[1].bodystruct 1448849794 +documenttype[1].bodystruct 0 documenttype[1].inherits[0].id 8 documenttype[1].datatype[0].id -2041471955 documenttype[1].datatype[0].type STRUCT @@ -68,18 +53,3 @@ documenttype[1].datatype[0].sstruct.compression.type NONE documenttype[1].datatype[0].sstruct.compression.level 0 documenttype[1].datatype[0].sstruct.compression.threshold 95 documenttype[1].datatype[0].sstruct.compression.minsize 200 -documenttype[1].datatype[1].id 1448849794 -documenttype[1].datatype[1].type STRUCT -documenttype[1].datatype[1].array.element.id 0 -documenttype[1].datatype[1].map.key.id 0 -documenttype[1].datatype[1].map.value.id 0 -documenttype[1].datatype[1].wset.key.id 0 -documenttype[1].datatype[1].wset.createifnonexistent false -documenttype[1].datatype[1].wset.removeifzero false -documenttype[1].datatype[1].annotationref.annotation.id 0 -documenttype[1].datatype[1].sstruct.name "campaign.body" -documenttype[1].datatype[1].sstruct.version 0 -documenttype[1].datatype[1].sstruct.compression.type NONE -documenttype[1].datatype[1].sstruct.compression.level 0 -documenttype[1].datatype[1].sstruct.compression.threshold 95 -documenttype[1].datatype[1].sstruct.compression.minsize 200 diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg index 0a76e44c4ac..0217d957432 100644 --- a/config-model/src/test/derived/advanced/attributes.cfg +++ b/config-model/src/test/derived/advanced/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/advanced/documentmanager.cfg b/config-model/src/test/derived/advanced/documentmanager.cfg index a0a59fbf7ac..4da92d82fb9 100644 --- a/config-model/src/test/derived/advanced/documentmanager.cfg +++ b/config-model/src/test/derived/advanced/documentmanager.cfg @@ -67,20 +67,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "mysummary" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id -704605648 -datatype[].structtype[].name "advanced.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 686681444 datatype[].documenttype[].name "advanced" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1337915045 -datatype[].documenttype[].bodystruct -704605648 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{titleabstract}.fields[] "title" datatype[].documenttype[].fieldsets{default}.fields[] "title" datatype[].documenttype[].fieldsets{[document]}.fields[] "attributes_src" diff --git a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg index fae6bd46ad7..aa74ecebd5b 100644 --- a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg @@ -29,20 +29,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -1503592268 -datatype[].structtype[].name "annotationsimplicitstruct.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -2099544992 datatype[].documenttype[].name "annotationsimplicitstruct" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -364910881 -datatype[].documenttype[].bodystruct -1503592268 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -269517759 annotationtype[].name "banana" annotationtype[].datatype 517946310 diff --git a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg index 21baed26dbf..e103218793d 100644 --- a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg @@ -94,20 +94,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 1181354668 -datatype[].structtype[].name "annotationsinheritance.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -748546200 datatype[].documenttype[].name "annotationsinheritance" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1406250281 -datatype[].documenttype[].bodystruct 1181354668 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -269517759 annotationtype[].name "banana" annotationtype[].datatype 517946310 diff --git a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg index 3ef71148f12..5b5b2ac348f 100644 --- a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg @@ -57,20 +57,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 1375438150 -datatype[].structtype[].name "annotationsinheritance2.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1730091890 datatype[].documenttype[].name "annotationsinheritance2" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 424382193 -datatype[].documenttype[].bodystruct 1375438150 +datatype[].documenttype[].bodystruct 0 annotationtype[].id 1769416289 annotationtype[].name "a" annotationtype[].datatype -1 diff --git a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg index e9ec2cb3715..1f71057f268 100644 --- a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg +++ b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg @@ -31,20 +31,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -570750959 -datatype[].structtype[].name "annotationspolymorphy.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1383624989 datatype[].documenttype[].name "annotationspolymorphy" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1552577796 -datatype[].documenttype[].bodystruct -570750959 +datatype[].documenttype[].bodystruct 0 annotationtype[].id 668095690 annotationtype[].name "super" annotationtype[].datatype -1 diff --git a/config-model/src/test/derived/annotationsreference/documentmanager.cfg b/config-model/src/test/derived/annotationsreference/documentmanager.cfg index 6526f56a906..737bcbf3cac 100644 --- a/config-model/src/test/derived/annotationsreference/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsreference/documentmanager.cfg @@ -65,20 +65,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 1692909067 -datatype[].structtype[].name "annotationsreference.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1448377175 datatype[].documenttype[].name "annotationsreference" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 571255414 -datatype[].documenttype[].bodystruct 1692909067 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -269517759 annotationtype[].name "banana" annotationtype[].datatype 517946310 diff --git a/config-model/src/test/derived/annotationssimple/documentmanager.cfg b/config-model/src/test/derived/annotationssimple/documentmanager.cfg index d32f0addceb..3af65e96558 100644 --- a/config-model/src/test/derived/annotationssimple/documentmanager.cfg +++ b/config-model/src/test/derived/annotationssimple/documentmanager.cfg @@ -19,20 +19,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -682121732 -datatype[].structtype[].name "annotationssimple.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1584092648 datatype[].documenttype[].name "annotationssimple" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1205708249 -datatype[].documenttype[].bodystruct -682121732 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -269517759 annotationtype[].name "banana" annotationtype[].datatype -1 diff --git a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg index c91b5c5e97e..0a1cda99a95 100644 --- a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg @@ -39,20 +39,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -1180029319 -datatype[].structtype[].name "annotationsstruct.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -263977093 datatype[].documenttype[].name "annotationsstruct" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1341437796 -datatype[].documenttype[].bodystruct -1180029319 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -160036815 annotationtype[].name "my_anno" annotationtype[].datatype -1080124700 diff --git a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg index 22b951b1b5d..fca86c58ffa 100644 --- a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg @@ -41,20 +41,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 1616435858 -datatype[].structtype[].name "annotationsstructarray.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1593733058 datatype[].documenttype[].name "annotationsstructarray" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 94945597 -datatype[].documenttype[].bodystruct 1616435858 +datatype[].documenttype[].bodystruct 0 annotationtype[].id -160036815 annotationtype[].name "my_anno" annotationtype[].datatype -1080124700 diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg index 006400c09d4..8391a8a9bdd 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/arrays/documentmanager.cfg b/config-model/src/test/derived/arrays/documentmanager.cfg index a2d8e2e78b4..f542a936574 100644 --- a/config-model/src/test/derived/arrays/documentmanager.cfg +++ b/config-model/src/test/derived/arrays/documentmanager.cfg @@ -42,20 +42,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "c" datatype[].structtype[].field[].datatype 1328286588 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1747896808 -datatype[].structtype[].name "arrays.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1292863364 datatype[].documenttype[].name "arrays" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1081627459 -datatype[].documenttype[].bodystruct -1747896808 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{default}.fields[] "a" datatype[].documenttype[].fieldsets{default}.fields[] "b" datatype[].documenttype[].fieldsets{default}.fields[] "c" diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg index cd91c15700b..448f9a82d36 100644 --- a/config-model/src/test/derived/attributeprefetch/attributes.cfg +++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -94,6 +97,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -119,6 +123,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -144,6 +149,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -169,6 +175,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -194,6 +201,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -219,6 +227,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -244,6 +253,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -269,6 +279,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -294,6 +305,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -319,6 +331,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -344,6 +357,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -369,6 +383,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -394,6 +409,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -419,6 +435,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -444,6 +461,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg index e27c72fbe50..dc208a86913 100644 --- a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg +++ b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg @@ -109,20 +109,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "wsstring" datatype[].structtype[].field[].datatype 1328286588 datatype[].structtype[].field[].detailedtype "" -datatype[].id 932425403 -datatype[].structtype[].name "prefetch.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1458051591 datatype[].documenttype[].name "prefetch" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -109105370 -datatype[].documenttype[].bodystruct 932425403 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "multibyte" datatype[].documenttype[].fieldsets{[document]}.fields[] "multidouble" datatype[].documenttype[].fieldsets{[document]}.fields[] "multifloat" diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg index 29e4c209ef2..d88c6b8b70a 100644 --- a/config-model/src/test/derived/attributes/attributes.cfg +++ b/config-model/src/test/derived/attributes/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -94,6 +97,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -119,6 +123,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -144,6 +149,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -169,6 +175,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -194,6 +201,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -219,6 +227,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -244,6 +253,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -269,6 +279,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -294,6 +305,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -319,6 +331,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -344,6 +357,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -369,6 +383,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -394,6 +409,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -419,6 +435,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -444,6 +461,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg index 5606e5ea730..337b736471d 100644 --- a/config-model/src/test/derived/complex/attributes.cfg +++ b/config-model/src/test/derived/complex/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -94,6 +97,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -119,6 +123,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -144,6 +149,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -169,6 +175,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -194,6 +201,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -219,6 +227,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/complex/documentmanager.cfg b/config-model/src/test/derived/complex/documentmanager.cfg index 42234e52211..50d5dac1ef9 100644 --- a/config-model/src/test/derived/complex/documentmanager.cfg +++ b/config-model/src/test/derived/complex/documentmanager.cfg @@ -98,20 +98,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "exact" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1665926686 -datatype[].structtype[].name "complex.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1402929550 datatype[].documenttype[].name "complex" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1749463923 -datatype[].documenttype[].bodystruct -1665926686 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{default}.fields[] "stringfield" datatype[].documenttype[].fieldsets{default}.fields[] "title" datatype[].documenttype[].fieldsets{special}.fields[] "special1" diff --git a/config-model/src/test/derived/emptydefault/documentmanager.cfg b/config-model/src/test/derived/emptydefault/documentmanager.cfg index b6cb2d06718..e69b2c5d8c3 100644 --- a/config-model/src/test/derived/emptydefault/documentmanager.cfg +++ b/config-model/src/test/derived/emptydefault/documentmanager.cfg @@ -25,19 +25,12 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "two" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id 311791038 -datatype[].structtype[].name "emptydefault.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1663995626 datatype[].documenttype[].name "emptydefault" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 461724009 -datatype[].documenttype[].bodystruct 311791038 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "one" datatype[].documenttype[].fieldsets{[document]}.fields[] "two" diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg index b61fd7e2ee5..087a116f4cd 100644 --- a/config-model/src/test/derived/hnsw_index/attributes.cfg +++ b/config-model/src/test/derived/hnsw_index/attributes.cfg @@ -19,7 +19,34 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor(x[128])" attribute[].imported false +attribute[].distancemetric ANGULAR attribute[].index.hnsw.enabled true attribute[].index.hnsw.maxlinkspernode 32 attribute[].index.hnsw.distancemetric ANGULAR attribute[].index.hnsw.neighborstoexploreatinsert 300 +attribute[].name "t2" +attribute[].datatype TENSOR +attribute[].collectiontype SINGLE +attribute[].removeifzero false +attribute[].createifnonexistent false +attribute[].fastsearch false +attribute[].huge false +attribute[].ismutable false +attribute[].sortascending true +attribute[].sortfunction UCA +attribute[].sortstrength PRIMARY +attribute[].sortlocale "" +attribute[].enablebitvectors false +attribute[].enableonlybitvector false +attribute[].fastaccess false +attribute[].arity 8 +attribute[].lowerbound -9223372036854775808 +attribute[].upperbound 9223372036854775807 +attribute[].densepostinglistthreshold 0.4 +attribute[].tensortype "tensor(x[2])" +attribute[].imported false +attribute[].distancemetric GEODEGREES +attribute[].index.hnsw.enabled false +attribute[].index.hnsw.maxlinkspernode 16 +attribute[].index.hnsw.distancemetric EUCLIDEAN +attribute[].index.hnsw.neighborstoexploreatinsert 200 diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg index e9fc265ca67..0c8266336b1 100644 --- a/config-model/src/test/derived/hnsw_index/ilscripts.cfg +++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg @@ -2,4 +2,6 @@ maxtermoccurrences 100 fieldmatchmaxlength 1000000 ilscript[].doctype "test" ilscript[].docfield[] "t1" +ilscript[].docfield[] "t2" ilscript[].content[] "clear_state | guard { input t1 | attribute t1 | index t1; }" +ilscript[].content[] "clear_state | guard { input t2 | attribute t2; }" diff --git a/config-model/src/test/derived/hnsw_index/test.sd b/config-model/src/test/derived/hnsw_index/test.sd index 207ed764a87..82291be9747 100644 --- a/config-model/src/test/derived/hnsw_index/test.sd +++ b/config-model/src/test/derived/hnsw_index/test.sd @@ -2,13 +2,21 @@ search test { document test { field t1 type tensor(x[128]) { indexing: attribute | index - index { + attribute { distance-metric: angular + } + index { hnsw { max-links-per-node: 32 neighbors-to-explore-at-insert: 300 } } } + field t2 type tensor(x[2]) { + indexing: attribute + attribute { + distance-metric: geodegrees + } + } } } diff --git a/config-model/src/test/derived/id/documentmanager.cfg b/config-model/src/test/derived/id/documentmanager.cfg index 5140abc65fa..8ee82cdd946 100644 --- a/config-model/src/test/derived/id/documentmanager.cfg +++ b/config-model/src/test/derived/id/documentmanager.cfg @@ -22,18 +22,11 @@ datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "uri" datatype[].structtype[].field[].datatype 10 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1830022377 -datatype[].structtype[].name "id.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 3225629 datatype[].documenttype[].name "id" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -531633022 -datatype[].documenttype[].bodystruct -1830022377 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "uri" diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg index 8f68068c8e7..685f708768a 100644 --- a/config-model/src/test/derived/imported_position_field/attributes.cfg +++ b/config-model/src/test/derived/imported_position_field/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg index eb87c6cf53e..e8b7a7b6235 100644 --- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg +++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -94,6 +97,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -119,6 +123,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -144,6 +149,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -169,6 +175,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -194,6 +201,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg index d3a51523e05..6f3514102b4 100644 --- a/config-model/src/test/derived/importedfields/attributes.cfg +++ b/config-model/src/test/derived/importedfields/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -94,6 +97,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -119,6 +123,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -144,6 +149,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -169,6 +175,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -194,6 +201,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported true +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/indexswitches/documentmanager.cfg b/config-model/src/test/derived/indexswitches/documentmanager.cfg index 78dbdb7ae74..ffeaab177ba 100644 --- a/config-model/src/test/derived/indexswitches/documentmanager.cfg +++ b/config-model/src/test/derived/indexswitches/documentmanager.cfg @@ -31,20 +31,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "source" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1892617122 -datatype[].structtype[].name "indexswitches.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -753375626 datatype[].documenttype[].name "indexswitches" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -555640823 -datatype[].documenttype[].bodystruct -1892617122 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{default}.fields[] "descr" datatype[].documenttype[].fieldsets{default}.fields[] "title" datatype[].documenttype[].fieldsets{[document]}.fields[] "descr" diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg index 397d8878792..541092622ff 100644 --- a/config-model/src/test/derived/inheritance/attributes.cfg +++ b/config-model/src/test/derived/inheritance/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/inheritance/documentmanager.cfg b/config-model/src/test/derived/inheritance/documentmanager.cfg index b15ef13ed3f..e054019bd8f 100644 --- a/config-model/src/test/derived/inheritance/documentmanager.cfg +++ b/config-model/src/test/derived/inheritance/documentmanager.cfg @@ -25,20 +25,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "overridden" datatype[].structtype[].field[].datatype 0 datatype[].structtype[].field[].detailedtype "" -datatype[].id 978262812 -datatype[].structtype[].name "grandparent.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -154107656 datatype[].documenttype[].name "grandparent" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 990971719 -datatype[].documenttype[].bodystruct 978262812 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "onlygrandparent" datatype[].documenttype[].fieldsets{[document]}.fields[] "overridden" datatype[].id 1306663898 @@ -54,13 +47,6 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "overridden" datatype[].structtype[].field[].datatype 0 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1989003153 -datatype[].structtype[].name "mother.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -158393403 datatype[].documenttype[].name "mother" datatype[].documenttype[].version 0 @@ -69,7 +55,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1306663898 -datatype[].documenttype[].bodystruct -1989003153 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "onlygrandparent" datatype[].documenttype[].fieldsets{[document]}.fields[] "onlymother" datatype[].documenttype[].fieldsets{[document]}.fields[] "overridden" @@ -86,13 +72,6 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "overridden" datatype[].structtype[].field[].datatype 0 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1742340170 -datatype[].structtype[].name "father.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 986686494 datatype[].documenttype[].name "father" datatype[].documenttype[].version 0 @@ -101,7 +80,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 2126589281 -datatype[].documenttype[].bodystruct -1742340170 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "onlyfather" datatype[].documenttype[].fieldsets{[document]}.fields[] "onlygrandparent" datatype[].documenttype[].fieldsets{[document]}.fields[] "overridden" @@ -118,13 +97,6 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "overridden" datatype[].structtype[].field[].datatype 0 datatype[].structtype[].field[].detailedtype "" -datatype[].id -126593034 -datatype[].structtype[].name "child.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 746267614 datatype[].documenttype[].name "child" datatype[].documenttype[].version 0 @@ -135,7 +107,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "mother" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 81425825 -datatype[].documenttype[].bodystruct -126593034 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "onlychild" datatype[].documenttype[].fieldsets{[document]}.fields[] "onlyfather" datatype[].documenttype[].fieldsets{[document]}.fields[] "onlygrandparent" diff --git a/config-model/src/test/derived/inheritdiamond/documentmanager.cfg b/config-model/src/test/derived/inheritdiamond/documentmanager.cfg index c3ead0d31f8..df3f8908a60 100644 --- a/config-model/src/test/derived/inheritdiamond/documentmanager.cfg +++ b/config-model/src/test/derived/inheritdiamond/documentmanager.cfg @@ -1,11 +1,4 @@ enablecompression false -datatype[].id -126593034 -datatype[].structtype[].name "child.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 336538650 datatype[].structtype[].name "child_struct" datatype[].structtype[].version 0 @@ -40,7 +33,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "father" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 81425825 -datatype[].documenttype[].bodystruct -126593034 +datatype[].documenttype[].bodystruct 0 datatype[].id -1913265190 datatype[].structtype[].name "father_struct" datatype[].structtype[].version 0 @@ -66,27 +59,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -52742073 -datatype[].structtype[].name "father_search.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1464571117 datatype[].documenttype[].name "father_search" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -1962244686 -datatype[].documenttype[].bodystruct -52742073 -datatype[].id -1852215954 -datatype[].structtype[].name "mother_search.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 +datatype[].documenttype[].bodystruct 0 datatype[].id -384824039 datatype[].structtype[].name "mother_search.header" datatype[].structtype[].version 0 @@ -109,7 +88,7 @@ datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -384824039 -datatype[].documenttype[].bodystruct -1852215954 +datatype[].documenttype[].bodystruct 0 datatype[].id 1306663898 datatype[].structtype[].name "mother.header" datatype[].structtype[].version 0 @@ -117,13 +96,6 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -1989003153 -datatype[].structtype[].name "mother.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -158393403 datatype[].documenttype[].name "mother" datatype[].documenttype[].version 0 @@ -132,7 +104,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1306663898 -datatype[].documenttype[].bodystruct -1989003153 +datatype[].documenttype[].bodystruct 0 datatype[].id -205818510 datatype[].structtype[].name "child_search.header" datatype[].structtype[].version 0 @@ -140,20 +112,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -1467672569 -datatype[].structtype[].name "child_search.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -580592339 datatype[].documenttype[].name "child_search" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -205818510 -datatype[].documenttype[].bodystruct -1467672569 +datatype[].documenttype[].bodystruct 0 datatype[].id 111553393 datatype[].structtype[].name "url" datatype[].structtype[].version 0 @@ -186,13 +151,6 @@ datatype[].structtype[].field[].name "x" datatype[].structtype[].field[].datatype 0 datatype[].structtype[].field[].name "y" datatype[].structtype[].field[].datatype 0 -datatype[].id 1845861921 -datatype[].structtype[].name "grandparent_search.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1530060044 datatype[].structtype[].name "grandparent_search.header" datatype[].structtype[].version 0 @@ -206,7 +164,7 @@ datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1530060044 -datatype[].documenttype[].bodystruct 1845861921 +datatype[].documenttype[].bodystruct 0 datatype[].id 990971719 datatype[].structtype[].name "grandparent.header" datatype[].structtype[].version 0 @@ -214,27 +172,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 978262812 -datatype[].structtype[].name "grandparent.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -154107656 datatype[].documenttype[].name "grandparent" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 990971719 -datatype[].documenttype[].bodystruct 978262812 -datatype[].id -1742340170 -datatype[].structtype[].name "father.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 +datatype[].documenttype[].bodystruct 0 datatype[].id 2126589281 datatype[].structtype[].name "father.header" datatype[].structtype[].version 0 @@ -250,4 +194,4 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 2126589281 -datatype[].documenttype[].bodystruct -1742340170 +datatype[].documenttype[].bodystruct 0 diff --git a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg index 8e2ee3bbc4e..25872641741 100644 --- a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg @@ -29,20 +29,13 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id 978262812 -datatype[].structtype[].name "grandparent.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -154107656 datatype[].documenttype[].name "grandparent" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 990971719 -datatype[].documenttype[].bodystruct 978262812 +datatype[].documenttype[].bodystruct 0 datatype[].id 836075987 datatype[].structtype[].name "parent.header" datatype[].structtype[].version 0 @@ -50,13 +43,6 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -389494616 -datatype[].structtype[].name "parent.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1175161836 datatype[].documenttype[].name "parent" datatype[].documenttype[].version 0 @@ -65,7 +51,7 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 836075987 -datatype[].documenttype[].bodystruct -389494616 +datatype[].documenttype[].bodystruct 0 datatype[].id 81425825 datatype[].structtype[].name "child.header" datatype[].structtype[].version 0 @@ -76,13 +62,6 @@ datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "child_field" datatype[].structtype[].field[].datatype 1246084544 datatype[].structtype[].field[].detailedtype "" -datatype[].id -126593034 -datatype[].structtype[].name "child.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 746267614 datatype[].documenttype[].name "child" datatype[].documenttype[].version 0 @@ -91,5 +70,5 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "parent" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 81425825 -datatype[].documenttype[].bodystruct -126593034 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "child_field" diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg index 3b30af10a8f..dda86f765be 100644 --- a/config-model/src/test/derived/inheritfromparent/attributes.cfg +++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg index 7c65a7b72f3..c9cd6fd3042 100644 --- a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg @@ -35,20 +35,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "weight" datatype[].structtype[].field[].datatype 1 datatype[].structtype[].field[].detailedtype "" -datatype[].id -389494616 -datatype[].structtype[].name "parent.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1175161836 datatype[].documenttype[].name "parent" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 836075987 -datatype[].documenttype[].bodystruct -389494616 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[]}.fields[] "weight_src" datatype[].id 81425825 datatype[].structtype[].name "child.header" @@ -60,13 +53,6 @@ datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "child_field" datatype[].structtype[].field[].datatype 1091188812 datatype[].structtype[].field[].detailedtype "" -datatype[].id -126593034 -datatype[].structtype[].name "child.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 746267614 datatype[].documenttype[].name "child" datatype[].documenttype[].version 0 @@ -75,6 +61,6 @@ datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].inherits[].name "parent" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 81425825 -datatype[].documenttype[].bodystruct -126593034 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[]}.fields[] "child_field" datatype[].documenttype[].fieldsets{[]}.fields[] "weight_src" diff --git a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg index f5ec18c4203..faef3f6923b 100644 --- a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg +++ b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg @@ -3,7 +3,7 @@ documenttype[].id 1175161836 documenttype[].name "parent" documenttype[].version 0 documenttype[].headerstruct 836075987 -documenttype[].bodystruct -389494616 +documenttype[].bodystruct 0 documenttype[].inherits[].id 8 documenttype[].datatype[].id 1091188812 documenttype[].datatype[].type STRUCT @@ -47,27 +47,12 @@ documenttype[].datatype[].sstruct.field[].name "weight" documenttype[].datatype[].sstruct.field[].id 1001392207 documenttype[].datatype[].sstruct.field[].datatype 1 documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id -389494616 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "parent.body" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 documenttype[].fieldsets{[]}.fields[] "weight_src" documenttype[].id 746267614 documenttype[].name "child" documenttype[].version 0 documenttype[].headerstruct 81425825 -documenttype[].bodystruct -126593034 +documenttype[].bodystruct 0 documenttype[].inherits[].id 8 documenttype[].inherits[].id 1175161836 documenttype[].datatype[].id 81425825 @@ -89,20 +74,5 @@ documenttype[].datatype[].sstruct.field[].name "child_field" documenttype[].datatype[].sstruct.field[].id 1814271363 documenttype[].datatype[].sstruct.field[].datatype 1091188812 documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id -126593034 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "child.body" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 documenttype[].fieldsets{[]}.fields[] "child_field" documenttype[].fieldsets{[]}.fields[] "weight_src" diff --git a/config-model/src/test/derived/mail/documentmanager.cfg b/config-model/src/test/derived/mail/documentmanager.cfg index 2fa9e5923c9..baf122d0241 100644 --- a/config-model/src/test/derived/mail/documentmanager.cfg +++ b/config-model/src/test/derived/mail/documentmanager.cfg @@ -37,15 +37,6 @@ datatype[].structtype[].field[].name "subject" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].name "snippet" datatype[].structtype[].field[].datatype 2 -datatype[].id -1206550296 -datatype[].arraytype[].datatype 12 -datatype[].id -953584901 -datatype[].structtype[].name "mail.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "body" datatype[].structtype[].field[].datatype 12 datatype[].structtype[].field[].name "attachmentcount" @@ -60,13 +51,15 @@ datatype[].structtype[].field[].name "attachmentcontent" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].name "attachments" datatype[].structtype[].field[].datatype -1206550296 +datatype[].id -1206550296 +datatype[].arraytype[].datatype 12 datatype[].id -1081574983 datatype[].documenttype[].name "mail" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -88808602 -datatype[].documenttype[].bodystruct -953584901 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{sender}.fields[] "from" datatype[].documenttype[].fieldsets{address}.fields[] "cc" datatype[].documenttype[].fieldsets{address}.fields[] "from" diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg index 4de78c52efd..702e065d15a 100644 --- a/config-model/src/test/derived/map_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg index 72e60b857a2..3eca0d5bbf8 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -94,6 +97,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -119,6 +123,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg index 4d19dc69b33..f75b8e06b80 100644 --- a/config-model/src/test/derived/music/attributes.cfg +++ b/config-model/src/test/derived/music/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -94,6 +97,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -119,6 +123,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -144,6 +149,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -169,6 +175,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -194,6 +201,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -219,6 +227,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -244,6 +253,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -269,6 +279,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg index 2aed2288773..303ea35223a 100644 --- a/config-model/src/test/derived/newrank/attributes.cfg +++ b/config-model/src/test/derived/newrank/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -94,6 +97,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -119,6 +123,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -144,6 +149,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -169,6 +175,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -194,6 +201,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -219,6 +227,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -244,6 +253,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg index 3a9daf7af94..d095b68752a 100644 --- a/config-model/src/test/derived/predicate_attribute/attributes.cfg +++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 200 attribute[].densepostinglistthreshold 0.2 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg index 0a8cbd82186..5cae6b784ff 100644 --- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg +++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg index 060510c3578..9ab2da3f686 100644 --- a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg +++ b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg @@ -34,20 +34,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "indexfield2" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id -480519133 -datatype[].structtype[].name "prefixexactattribute.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -1812793455 datatype[].documenttype[].name "prefixexactattribute" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -739138930 -datatype[].documenttype[].bodystruct -480519133 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "attributefield1" datatype[].documenttype[].fieldsets{[document]}.fields[] "attributefield2" datatype[].documenttype[].fieldsets{[document]}.fields[] "indexfield0" diff --git a/config-model/src/test/derived/rankprofileinheritance/child.sd b/config-model/src/test/derived/rankprofileinheritance/child.sd new file mode 100644 index 00000000000..f76aa3a1f10 --- /dev/null +++ b/config-model/src/test/derived/rankprofileinheritance/child.sd @@ -0,0 +1,37 @@ +schema child { + + document child inherits parent1, parent2 { + + field field3 type int { + indexing: attribute + } + + } + + rank-profile profile3 inherits profile1 { + + function function3() { + expression: attribute(field3) + 5 + } + + summary-features { + function3 + attribute(field3) + } + + } + + rank-profile profile4 inherits profile2 { + + function function4() { + expression: attribute(field3) + 5 + } + + summary-features inherits profile2 { + function4 + attribute(field3) + } + + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/rankprofileinheritance/parent1.sd b/config-model/src/test/derived/rankprofileinheritance/parent1.sd new file mode 100644 index 00000000000..ea11ffbc82e --- /dev/null +++ b/config-model/src/test/derived/rankprofileinheritance/parent1.sd @@ -0,0 +1,24 @@ +schema parent1 { + + document parent1 { + + field field1 type int { + indexing: attribute + } + + } + + rank-profile profile1 { + + function function1() { + expression: attribute(field1) + 5 + } + + summary-features { + function1 + attribute(field1) + } + + } + +} diff --git a/config-model/src/test/derived/rankprofileinheritance/parent2.sd b/config-model/src/test/derived/rankprofileinheritance/parent2.sd new file mode 100644 index 00000000000..1246f7264b3 --- /dev/null +++ b/config-model/src/test/derived/rankprofileinheritance/parent2.sd @@ -0,0 +1,24 @@ +schema parent2 { + + document parent2 { + + field field2 type int { + indexing: attribute + } + + } + + rank-profile profile2 { + + function function2() { + expression: attribute(field2) + 5 + } + + summary-features { + function2 + attribute(field2) + } + + } + +} diff --git a/config-model/src/test/derived/rankprofileinheritance/rank-profiles.cfg b/config-model/src/test/derived/rankprofileinheritance/rank-profiles.cfg new file mode 100644 index 00000000000..88788f5a93a --- /dev/null +++ b/config-model/src/test/derived/rankprofileinheritance/rank-profiles.cfg @@ -0,0 +1,32 @@ +rankprofile[].name "default" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].name "profile3" +rankprofile[].fef.property[].name "rankingExpression(function3).rankingScript" +rankprofile[].fef.property[].value "attribute(field3) + 5" +rankprofile[].fef.property[].name "rankingExpression(function1).rankingScript" +rankprofile[].fef.property[].value "attribute(field1) + 5" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "attribute(field3)" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "rankingExpression(function3)" +rankprofile[].name "profile4" +rankprofile[].fef.property[].name "rankingExpression(function2).rankingScript" +rankprofile[].fef.property[].value "attribute(field2) + 5" +rankprofile[].fef.property[].name "rankingExpression(function4).rankingScript" +rankprofile[].fef.property[].value "attribute(field3) + 5" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "attribute(field2)" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "attribute(field3)" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "rankingExpression(function2)" +rankprofile[].fef.property[].name "vespa.summary.feature" +rankprofile[].fef.property[].value "rankingExpression(function4)" diff --git a/config-model/src/test/derived/ranktypes/documentmanager.cfg b/config-model/src/test/derived/ranktypes/documentmanager.cfg index 072a0fff126..a8bb9e904dc 100644 --- a/config-model/src/test/derived/ranktypes/documentmanager.cfg +++ b/config-model/src/test/derived/ranktypes/documentmanager.cfg @@ -34,20 +34,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "identity_literal" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id 1374506021 -datatype[].structtype[].name "ranktypes.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -883421617 datatype[].documenttype[].name "ranktypes" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -471393776 -datatype[].documenttype[].bodystruct 1374506021 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "descr" datatype[].documenttype[].fieldsets{[document]}.fields[] "identity" datatype[].documenttype[].fieldsets{[document]}.fields[] "keywords" diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg index e83b187a0cc..ec3ca030373 100644 --- a/config-model/src/test/derived/reference_fields/attributes.cfg +++ b/config-model/src/test/derived/reference_fields/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg index 83c310ca5ca..6e50bd625ee 100644 --- a/config-model/src/test/derived/sorting/attributes.cfg +++ b/config-model/src/test/derived/sorting/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/streamingstruct/documentmanager.cfg b/config-model/src/test/derived/streamingstruct/documentmanager.cfg index 2cd35c7bdfa..63001ea38ca 100644 --- a/config-model/src/test/derived/streamingstruct/documentmanager.cfg +++ b/config-model/src/test/derived/streamingstruct/documentmanager.cfg @@ -119,20 +119,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "snippet2" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].detailedtype "" -datatype[].id 1858438651 -datatype[].structtype[].name "streamingstruct.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1433175737 datatype[].documenttype[].name "streamingstruct" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 731395686 -datatype[].documenttype[].bodystruct 1858438651 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "a" datatype[].documenttype[].fieldsets{[document]}.fields[] "array1" datatype[].documenttype[].fieldsets{[document]}.fields[] "array2" diff --git a/config-model/src/test/derived/structanyorder/documentmanager.cfg b/config-model/src/test/derived/structanyorder/documentmanager.cfg index c18b1cc11b0..3ffc2f22a9b 100644 --- a/config-model/src/test/derived/structanyorder/documentmanager.cfg +++ b/config-model/src/test/derived/structanyorder/documentmanager.cfg @@ -69,20 +69,13 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "structarrayfield" datatype[].structtype[].field[].datatype -1244829667 datatype[].structtype[].field[].detailedtype "" -datatype[].id -1503592268 -datatype[].structtype[].name "annotationsimplicitstruct.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id -2099544992 datatype[].documenttype[].name "annotationsimplicitstruct" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct -364910881 -datatype[].documenttype[].bodystruct -1503592268 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "structarrayfield" datatype[].documenttype[].fieldsets{[document]}.fields[] "structfield" annotationtype[].id -269517759 diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index 780f47ee3d5..6ed960728c2 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor<float>(x[2],y[1])" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor(x{})" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor(x[10],y[10])" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -94,6 +97,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "tensor<float>(x[10])" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -119,6 +123,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg index 68bd394c9d6..acf5c7ed12f 100644 --- a/config-model/src/test/derived/tensor/documenttypes.cfg +++ b/config-model/src/test/derived/tensor/documenttypes.cfg @@ -3,7 +3,7 @@ documenttype[].id -1290043429 documenttype[].name "tensor" documenttype[].version 0 documenttype[].headerstruct 2125927172 -documenttype[].bodystruct -1903234535 +documenttype[].bodystruct 0 documenttype[].inherits[].id 8 documenttype[].datatype[].id 2125927172 documenttype[].datatype[].type STRUCT @@ -23,7 +23,7 @@ documenttype[].datatype[].sstruct.compression.minsize 200 documenttype[].datatype[].sstruct.field[].name "f1" documenttype[].datatype[].sstruct.field[].id 26661415 documenttype[].datatype[].sstruct.field[].datatype 21 -documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x[])" +documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x[3])" documenttype[].datatype[].sstruct.field[].name "f2" documenttype[].datatype[].sstruct.field[].id 2080644671 documenttype[].datatype[].sstruct.field[].datatype 21 @@ -44,21 +44,6 @@ documenttype[].datatype[].sstruct.field[].name "f6" documenttype[].datatype[].sstruct.field[].id 596352344 documenttype[].datatype[].sstruct.field[].datatype 1 documenttype[].datatype[].sstruct.field[].detailedtype "" -documenttype[].datatype[].id -1903234535 -documenttype[].datatype[].type STRUCT -documenttype[].datatype[].array.element.id 0 -documenttype[].datatype[].map.key.id 0 -documenttype[].datatype[].map.value.id 0 -documenttype[].datatype[].wset.key.id 0 -documenttype[].datatype[].wset.createifnonexistent false -documenttype[].datatype[].wset.removeifzero false -documenttype[].datatype[].annotationref.annotation.id 0 -documenttype[].datatype[].sstruct.name "tensor.body" -documenttype[].datatype[].sstruct.version 0 -documenttype[].datatype[].sstruct.compression.type NONE -documenttype[].datatype[].sstruct.compression.level 0 -documenttype[].datatype[].sstruct.compression.threshold 95 -documenttype[].datatype[].sstruct.compression.minsize 200 documenttype[].fieldsets{[document]}.fields[] "f1" documenttype[].fieldsets{[document]}.fields[] "f2" documenttype[].fieldsets{[document]}.fields[] "f3" diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd index a7248fe3200..44a84a3cbb3 100644 --- a/config-model/src/test/derived/tensor/tensor.sd +++ b/config-model/src/test/derived/tensor/tensor.sd @@ -2,7 +2,7 @@ search tensor { document tensor { - field f1 type tensor(x[]) { + field f1 type tensor(x[3]) { indexing: summary } field f2 type tensor<float>(x[2],y[1]) { diff --git a/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg b/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg index bb5bb001036..19d00483a5a 100644 --- a/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg +++ b/config-model/src/test/derived/twostreamingstructs/documentmanager.cfg @@ -90,20 +90,13 @@ datatype[].structtype[].field[].name "snippet" datatype[].structtype[].field[].datatype 2 datatype[].structtype[].field[].name "snippet2" datatype[].structtype[].field[].datatype 2 -datatype[].id 1858438651 -datatype[].structtype[].name "streamingstruct.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].id 1433175737 datatype[].documenttype[].name "streamingstruct" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 731395686 -datatype[].documenttype[].bodystruct 1858438651 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "a" datatype[].documenttype[].fieldsets{[document]}.fields[] "array1" datatype[].documenttype[].fieldsets{[document]}.fields[] "array2" @@ -139,13 +132,6 @@ datatype[].structtype[].compresstype NONE datatype[].structtype[].compresslevel 0 datatype[].structtype[].compressthreshold 95 datatype[].structtype[].compressminsize 800 -datatype[].id -1417926544 -datatype[].structtype[].name "whatever.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 datatype[].structtype[].field[].name "f1" datatype[].structtype[].field[].datatype -995681764 datatype[].id -778211548 @@ -154,5 +140,5 @@ datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 355471259 -datatype[].documenttype[].bodystruct -1417926544 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "f1" diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index 82535324864..8e37a1f1c63 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -19,6 +19,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -44,6 +45,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -69,6 +71,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -94,6 +97,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -119,6 +123,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -144,6 +149,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -169,6 +175,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -194,6 +201,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -219,6 +227,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -244,6 +253,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -269,6 +279,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -294,6 +305,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN @@ -319,6 +331,7 @@ attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 attribute[].tensortype "" attribute[].imported false +attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.distancemetric EUCLIDEAN diff --git a/config-model/src/test/derived/types/documentmanager.cfg b/config-model/src/test/derived/types/documentmanager.cfg index a4fcd4f49f6..9556f77f6d9 100644 --- a/config-model/src/test/derived/types/documentmanager.cfg +++ b/config-model/src/test/derived/types/documentmanager.cfg @@ -209,28 +209,21 @@ datatype[].structtype[].field[].detailedtype "" datatype[].structtype[].field[].name "other" datatype[].structtype[].field[].datatype 4 datatype[].structtype[].field[].detailedtype "" +datatype[].structtype[].field[].name "complexarray" +datatype[].structtype[].field[].datatype 1416345047 +datatype[].structtype[].field[].detailedtype "" datatype[].id -372512406 datatype[].maptype[].keytype 0 datatype[].maptype[].valtype 1707615575 datatype[].id 1416345047 datatype[].arraytype[].datatype -372512406 -datatype[].id 348447225 -datatype[].structtype[].name "types.body" -datatype[].structtype[].version 0 -datatype[].structtype[].compresstype NONE -datatype[].structtype[].compresslevel 0 -datatype[].structtype[].compressthreshold 95 -datatype[].structtype[].compressminsize 800 -datatype[].structtype[].field[].name "complexarray" -datatype[].structtype[].field[].datatype 1416345047 -datatype[].structtype[].field[].detailedtype "" datatype[].id -853072901 datatype[].documenttype[].name "types" datatype[].documenttype[].version 0 datatype[].documenttype[].inherits[].name "document" datatype[].documenttype[].inherits[].version 0 datatype[].documenttype[].headerstruct 1328581348 -datatype[].documenttype[].bodystruct 348447225 +datatype[].documenttype[].bodystruct 0 datatype[].documenttype[].fieldsets{[document]}.fields[] "Folders" datatype[].documenttype[].fieldsets{[document]}.fields[] "abool" datatype[].documenttype[].fieldsets{[document]}.fields[] "abyte" diff --git a/config-model/src/test/examples/fieldoftypedocument.cfg b/config-model/src/test/examples/fieldoftypedocument.cfg index b7bb444ec93..8074d86b45f 100644 --- a/config-model/src/test/examples/fieldoftypedocument.cfg +++ b/config-model/src/test/examples/fieldoftypedocument.cfg @@ -22,51 +22,37 @@ datatype[1].structtype[0].compressminsize 800 datatype[1].structtype[0].field[0].name "soundtrack" datatype[1].structtype[0].field[0].datatype 1412693671 datatype[1].structtype[0].field[0].detailedtype "" -datatype[2].id -820813431 -datatype[2].structtype[0].name "book.body" -datatype[2].structtype[0].version 0 -datatype[2].structtype[0].compresstype NONE -datatype[2].structtype[0].compresslevel 0 -datatype[2].structtype[0].compressthreshold 95 -datatype[2].structtype[0].compressminsize 800 -datatype[3].id -1383388565 -datatype[3].documenttype[0].name "book" -datatype[3].documenttype[0].version 0 -datatype[3].documenttype[0].inherits[0].name "document" -datatype[3].documenttype[0].inherits[0].version 0 -datatype[3].documenttype[0].headerstruct -1344444812 -datatype[3].documenttype[0].bodystruct -820813431 -datatype[3].documenttype[0].fieldsets{[document]}.fields[0] "soundtrack" -datatype[4].id -1910204744 -datatype[4].structtype[0].name "music.header" -datatype[4].structtype[0].version 0 -datatype[4].structtype[0].compresstype NONE -datatype[4].structtype[0].compresslevel 0 -datatype[4].structtype[0].compressthreshold 95 -datatype[4].structtype[0].compressminsize 800 -datatype[4].structtype[0].field[0].name "intfield" -datatype[4].structtype[0].field[0].datatype 0 -datatype[4].structtype[0].field[0].detailedtype "" -datatype[4].structtype[0].field[1].name "stringfield" -datatype[4].structtype[0].field[1].datatype 2 -datatype[4].structtype[0].field[1].detailedtype "" -datatype[4].structtype[0].field[2].name "longfield" -datatype[4].structtype[0].field[2].datatype 4 -datatype[4].structtype[0].field[2].detailedtype "" -datatype[5].id 993120973 -datatype[5].structtype[0].name "music.body" -datatype[5].structtype[0].version 0 -datatype[5].structtype[0].compresstype NONE -datatype[5].structtype[0].compresslevel 0 -datatype[5].structtype[0].compressthreshold 95 -datatype[5].structtype[0].compressminsize 800 -datatype[6].id 1412693671 -datatype[6].documenttype[0].name "music" -datatype[6].documenttype[0].version 0 -datatype[6].documenttype[0].inherits[0].name "document" -datatype[6].documenttype[0].inherits[0].version 0 -datatype[6].documenttype[0].headerstruct -1910204744 -datatype[6].documenttype[0].bodystruct 993120973 -datatype[6].documenttype[0].fieldsets{[document]}.fields[0] "intfield" -datatype[6].documenttype[0].fieldsets{[document]}.fields[1] "longfield" -datatype[6].documenttype[0].fieldsets{[document]}.fields[2] "stringfield" +datatype[2].id -1383388565 +datatype[2].documenttype[0].name "book" +datatype[2].documenttype[0].version 0 +datatype[2].documenttype[0].inherits[0].name "document" +datatype[2].documenttype[0].inherits[0].version 0 +datatype[2].documenttype[0].headerstruct -1344444812 +datatype[2].documenttype[0].bodystruct 0 +datatype[2].documenttype[0].fieldsets{[document]}.fields[0] "soundtrack" +datatype[3].id -1910204744 +datatype[3].structtype[0].name "music.header" +datatype[3].structtype[0].version 0 +datatype[3].structtype[0].compresstype NONE +datatype[3].structtype[0].compresslevel 0 +datatype[3].structtype[0].compressthreshold 95 +datatype[3].structtype[0].compressminsize 800 +datatype[3].structtype[0].field[0].name "intfield" +datatype[3].structtype[0].field[0].datatype 0 +datatype[3].structtype[0].field[0].detailedtype "" +datatype[3].structtype[0].field[1].name "stringfield" +datatype[3].structtype[0].field[1].datatype 2 +datatype[3].structtype[0].field[1].detailedtype "" +datatype[3].structtype[0].field[2].name "longfield" +datatype[3].structtype[0].field[2].datatype 4 +datatype[3].structtype[0].field[2].detailedtype "" +datatype[4].id 1412693671 +datatype[4].documenttype[0].name "music" +datatype[4].documenttype[0].version 0 +datatype[4].documenttype[0].inherits[0].name "document" +datatype[4].documenttype[0].inherits[0].version 0 +datatype[4].documenttype[0].headerstruct -1910204744 +datatype[4].documenttype[0].bodystruct 0 +datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "intfield" +datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "longfield" +datatype[4].documenttype[0].fieldsets{[document]}.fields[2] "stringfield" diff --git a/config-model/src/test/examples/structresult.cfg b/config-model/src/test/examples/structresult.cfg index eff48f18914..ceaad2e6584 100755 --- a/config-model/src/test/examples/structresult.cfg +++ b/config-model/src/test/examples/structresult.cfg @@ -54,20 +54,13 @@ datatype[4].structtype[0].field[1].detailedtype "" datatype[4].structtype[0].field[2].name "advanced" datatype[4].structtype[0].field[2].datatype 93505813 datatype[4].structtype[0].field[2].detailedtype "" -datatype[5].id 993120973 -datatype[5].structtype[0].name "music.body" -datatype[5].structtype[0].version 0 -datatype[5].structtype[0].compresstype NONE -datatype[5].structtype[0].compresslevel 0 -datatype[5].structtype[0].compressthreshold 95 -datatype[5].structtype[0].compressminsize 800 -datatype[6].id 1412693671 -datatype[6].documenttype[0].name "music" -datatype[6].documenttype[0].version 0 -datatype[6].documenttype[0].inherits[0].name "document" -datatype[6].documenttype[0].inherits[0].version 0 -datatype[6].documenttype[0].headerstruct -1910204744 -datatype[6].documenttype[0].bodystruct 993120973 -datatype[6].documenttype[0].fieldsets{[document]}.fields[0] "advanced" -datatype[6].documenttype[0].fieldsets{[document]}.fields[1] "arraystruct" -datatype[6].documenttype[0].fieldsets{[document]}.fields[2] "mystruct" +datatype[5].id 1412693671 +datatype[5].documenttype[0].name "music" +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0].name "document" +datatype[5].documenttype[0].inherits[0].version 0 +datatype[5].documenttype[0].headerstruct -1910204744 +datatype[5].documenttype[0].bodystruct 0 +datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "advanced" +datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "arraystruct" +datatype[5].documenttype[0].fieldsets{[document]}.fields[2] "mystruct" diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java index 51b039a7532..91f5fdc5f11 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java @@ -6,6 +6,8 @@ import org.junit.Test; import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.Optional; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -17,10 +19,10 @@ public class HostSpecTest { @Test public void testEquals() { - HostSpec h1 = new HostSpec("foo", Collections.<String>emptyList()); - HostSpec h2 = new HostSpec("foo", Collections.<String>emptyList()); - HostSpec h3 = new HostSpec("foo", Arrays.asList("my", "alias")); - HostSpec h4 = new HostSpec("bar", Collections.<String>emptyList()); + HostSpec h1 = new HostSpec("foo", List.of(), Optional.empty()); + HostSpec h2 = new HostSpec("foo", List.of(), Optional.empty()); + HostSpec h3 = new HostSpec("foo", List.of("my", "alias"), Optional.empty()); + HostSpec h4 = new HostSpec("bar", List.of(), Optional.empty()); assertTrue(h1.equals(h1)); assertTrue(h1.equals(h2)); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 7208d8c5fc1..1be58564d1b 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -252,22 +252,59 @@ public class ModelProvisioningTest { " <documents>" + " <document type='type1' mode='index'/>" + " </documents>" + - " <nodes count='2'/>" + + " <nodes count='2'>" + + " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + + " </nodes>" + " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(2); VespaModel model = tester.createModel(xmlWithNodes, true); - assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); assertEquals("Heap size is lowered with combined clusters", 17, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + assertEquals("Memory for proton is lowered to account for the jvm heap", + (long)(3 * (Math.pow(1024, 3)) * (1 - 0.17)), protonMemorySize(model.getContentClusters().get("content1"))); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); } } + /** For comparison with the above */ + @Test + public void testNonCombinedCluster() { + var containerElements = Set.of("jdisc", "container"); + for (var containerElement : containerElements) { + String xmlWithNodes = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <" + containerElement + " version='1.0' id='container1'>" + + " <search/>" + + " <nodes count='2'/>" + + " </" + containerElement + ">" + + " <content version='1.0' id='content1'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'>" + + " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(4); + VespaModel model = tester.createModel(xmlWithNodes, true); + assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); + assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); + assertEquals("Heap size is normal", + 60, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + assertEquals("Memory for proton is normal", + (long)(3 * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1"))); + } + } + @Test public void testCombinedClusterWithJvmOptions() { String xmlWithNodes = @@ -1741,7 +1778,13 @@ public class ModelProvisioningTest { private int physicalMemoryPercentage(ContainerCluster cluster) { QrStartConfig.Builder b = new QrStartConfig.Builder(); cluster.getConfig(b); - return new QrStartConfig(b).jvm().heapSizeAsPercentageOfPhysicalMemory(); + return b.build().jvm().heapSizeAsPercentageOfPhysicalMemory(); + } + + private long protonMemorySize(ContentCluster cluster) { + ProtonConfig.Builder b = new ProtonConfig.Builder(); + cluster.getSearch().getIndexed().getSearchNode(0).getConfig(b); + return b.build().hwinfo().memory().size(); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index e2f2c1fd407..0f83a4db98f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -125,7 +125,7 @@ public class RankProfileTestCase extends SchemaTestCase { " document test { \n" + " field a type tensor(x[10]) { indexing: attribute }\n" + " field b type tensor(y{}) { indexing: attribute }\n" + - " field c type tensor(x[]) { indexing: attribute }\n" + + " field c type tensor(x[5]) { indexing: attribute }\n" + " }\n" + " rank-profile p1 {}\n" + " rank-profile p2 {}\n" + @@ -140,11 +140,28 @@ public class RankProfileTestCase extends SchemaTestCase { assertAttributeTypeSettings(registry.get(search, "p2"), search); } + @Test + public void requireThatDenseDimensionsMustBeBound() throws ParseException { + try { + SearchBuilder builder = new SearchBuilder(new RankProfileRegistry()); + builder.importString("search test {\n" + + " document test { \n" + + " field a type tensor(x[]) { indexing: attribute }\n" + + " }\n" + + "}"); + builder.build(); + } + catch (IllegalArgumentException e) { + assertEquals("Illegal type in field a type tensor(x[]): Dense tensor dimensions must have a size", + e.getMessage()); + } + } + private static void assertAttributeTypeSettings(RankProfile profile, Search search) { RawRankProfile rawProfile = new RawRankProfile(profile, new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(search)); assertEquals("tensor(x[10])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.a").get()); assertEquals("tensor(y{})", findProperty(rawProfile.configProperties(), "vespa.type.attribute.b").get()); - assertEquals("tensor(x[])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.c").get()); + assertEquals("tensor(x[5])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.c").get()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index e4ca83640e9..02d1c3fc3b0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -159,7 +159,7 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { public void testNeuralNetworkSetup() throws ParseException { // Note: the type assigned to query profile and constant tensors here is not the correct type RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfileRegistry queryProfiles = queryProfileWith("query(q)", "tensor(x[])"); + QueryProfileRegistry queryProfiles = queryProfileWith("query(q)", "tensor(x[1])"); SearchBuilder builder = new SearchBuilder(rankProfileRegistry, queryProfiles); builder.importString( "search test {\n" + @@ -184,19 +184,19 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { " }\n" + " }\n" + " constant W_hidden {\n" + - " type: tensor(x[])\n" + + " type: tensor(x[1])\n" + " file: ignored.json\n" + " }\n" + " constant b_input {\n" + - " type: tensor(x[])\n" + + " type: tensor(x[1])\n" + " file: ignored.json\n" + " }\n" + " constant W_final {\n" + - " type: tensor(x[])\n" + + " type: tensor(x[1])\n" + " file: ignored.json\n" + " }\n" + " constant b_final {\n" + - " type: tensor(x[])\n" + + " type: tensor(x[1])\n" + " file: ignored.json\n" + " }\n" + "}\n"); @@ -211,11 +211,11 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { censorBindingHash(testRankProperties.get(0).toString())); assertEquals("(rankingExpression(hidden_layer).rankingScript,rankingExpression(relu@))", censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(hidden_layer).type,tensor(x[]))", + assertEquals("(rankingExpression(hidden_layer).type,tensor(x[1]))", censorBindingHash(testRankProperties.get(2).toString())); assertEquals("(rankingExpression(final_layer).rankingScript,sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))", testRankProperties.get(3).toString()); - assertEquals("(rankingExpression(final_layer).type,tensor(x[]))", + assertEquals("(rankingExpression(final_layer).type,tensor(x[1]))", testRankProperties.get(4).toString()); assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", testRankProperties.get(5).toString()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 3c55aa808b5..71cca20f8fa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -156,4 +156,9 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectDeriving("hnsw_index"); } + @Test + public void testRankProfileInheritance() throws IOException, ParseException { + assertCorrectDeriving("rankprofileinheritance", "child"); + } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java index 02233608eea..1b51fd354f3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java @@ -22,6 +22,7 @@ import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; /** * Tests inheritance @@ -68,8 +69,8 @@ public class InheritanceTestCase extends AbstractExportingTestCase { DocumentmanagerConfig.Builder b = new DocumentmanagerConfig.Builder(); DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), b), outDir.getPath()); DocumentmanagerConfig dc = b.build(); - assertEquals(17, dc.datatype().size()); - assertNotNull(structType("child.body", dc)); + assertEquals(13, dc.datatype().size()); + assertNull(structType("child.body", dc)); DocumentmanagerConfig.Datatype.Structtype childHeader = structType("child.header", dc); assertEquals(childHeader.field(0).name(), "foo"); assertEquals(childHeader.field(1).name(), "bar"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java index 994538deed6..71277f136f8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java @@ -39,7 +39,7 @@ public class FastAccessValidatorTest { " indexing: attribute ", " attribute: fast-access", " }", - " field tensor_attribute type tensor(x[]) {", + " field tensor_attribute type tensor(x[5]) {", " indexing: attribute ", " attribute: fast-access", " }", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java index c679bb7e61f..b7db9134b88 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java @@ -114,7 +114,7 @@ public class ImportedFieldsResolverTestCase { parentSearch.getDocument().addField(createField("attribute_field", DataType.INT, "{ attribute }")); parentSearch.getDocument().addField(createField("attribute_and_index", DataType.INT, "{ attribute | index }")); parentSearch.getDocument().addField(new TemporarySDField("not_attribute", DataType.INT)); - parentSearch.getDocument().addField(createField("tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[])")), "{ attribute }")); + parentSearch.getDocument().addField(createField("tensor_field", new TensorDataType(TensorType.fromSpec("tensor(x[5])")), "{ attribute }")); parentSearch.getDocument().addField(createField("predicate_field", DataType.PREDICATE, "{ attribute }")); addRefField(parentSearch, grandParentSearch, "ref"); addImportedField(parentSearch, "ancient_field", "ref", "ancient_field"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java index a306e0f2c90..96f12a47a2f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java @@ -37,7 +37,7 @@ public class RankingExpressionTypeResolverTestCase { builder.importString(joinLines( "search test {", " document test { ", - " field a type tensor(x[],y[]) {", + " field a type tensor(x[10],y[3]) {", " indexing: attribute", " }", " }", @@ -52,7 +52,7 @@ public class RankingExpressionTypeResolverTestCase { fail("Expected exception"); } catch (IllegalArgumentException expected) { - assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[],y[])", + assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[10],y[3])", Exceptions.toMessageString(expected)); } } @@ -64,7 +64,7 @@ public class RankingExpressionTypeResolverTestCase { builder.importString(joinLines( "search test {", " document test { ", - " field a type tensor(x[],y[]) {", + " field a type tensor(x[10],y[3]) {", " indexing: attribute", " }", " }", @@ -82,7 +82,7 @@ public class RankingExpressionTypeResolverTestCase { fail("Expected exception"); } catch (IllegalArgumentException expected) { - assertEquals("In search definition 'test', rank profile 'my_rank_profile': The second-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[],y[])", + assertEquals("In search definition 'test', rank profile 'my_rank_profile': The second-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[10],y[3])", Exceptions.toMessageString(expected)); } } @@ -94,7 +94,7 @@ public class RankingExpressionTypeResolverTestCase { searchBuilder.importString(joinLines( "search test {", " document test { ", - " field a type tensor(x[],y[]) {", + " field a type tensor(x[10],y[5]) {", " indexing: attribute", " }", " field b type tensor(z[10]) {", @@ -112,7 +112,7 @@ public class RankingExpressionTypeResolverTestCase { fail("Expected exception"); } catch (IllegalArgumentException expected) { - assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression is invalid: An if expression must produce compatible types in both alternatives, but the 'true' type is tensor(x[],y[]) while the 'false' type is tensor(z[10])" + + assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression is invalid: An if expression must produce compatible types in both alternatives, but the 'true' type is tensor(x[10],y[5]) while the 'false' type is tensor(z[10])" + "\n'true' branch: attribute(a)" + "\n'false' branch: attribute(b)", Exceptions.toMessageString(expected)); @@ -126,7 +126,7 @@ public class RankingExpressionTypeResolverTestCase { builder.importString(joinLines( "search test {", " document test { ", - " field a type tensor(x[],y[]) {", + " field a type tensor(x[10],y[3]) {", " indexing: attribute", " }", " field b type tensor(z[10]) {", @@ -147,7 +147,7 @@ public class RankingExpressionTypeResolverTestCase { builder.build(); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSearch(), "my_rank_profile"); - assertEquals(TensorType.fromSpec("tensor(x[],y[])"), + assertEquals(TensorType.fromSpec("tensor(x[10],y[3])"), summaryFeatures(profile).get("macro1(a)").type(profile.typeContext(builder.getQueryProfileRegistry()))); assertEquals(TensorType.fromSpec("tensor(z[10])"), summaryFeatures(profile).get("macro1(b)").type(profile.typeContext(builder.getQueryProfileRegistry()))); @@ -159,7 +159,7 @@ public class RankingExpressionTypeResolverTestCase { builder.importString(joinLines( "search test {", " document test { ", - " field a type tensor(x[],y[]) {", + " field a type tensor(x[10],y[1]) {", " indexing: attribute", " }", " field b type tensor(z[10]) {", @@ -189,7 +189,7 @@ public class RankingExpressionTypeResolverTestCase { builder.build(); RankProfile profile = builder.getRankProfileRegistry().get(builder.getSearch(), "my_rank_profile"); - assertEquals(TensorType.fromSpec("tensor(x[],y[])"), + assertEquals(TensorType.fromSpec("tensor(x[10],y[1])"), summaryFeatures(profile).get("return_a").type(profile.typeContext(builder.getQueryProfileRegistry()))); assertEquals(TensorType.fromSpec("tensor(z[10])"), summaryFeatures(profile).get("return_b").type(profile.typeContext(builder.getQueryProfileRegistry()))); @@ -201,7 +201,7 @@ public class RankingExpressionTypeResolverTestCase { builder.importString(joinLines( "search parent {", " document parent {", - " field a type tensor(x[],y[]) {", + " field a type tensor(x[5],y[1000]) {", " indexing: attribute", " }", " }", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java index 4387d19c474..1fe1ebf2bb3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -29,6 +29,7 @@ public class RankingExpressionWithOnnxTestCase { private final static String name = "mnist_softmax"; private final static String vespaExpression = "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(" + name + "_Variable), f(a,b)(a * b)), sum, d2), constant(" + name + "_Variable_1), f(a,b)(a + b))"; + private final static String vespaExpressionWithBatchReduce = "join(join(reduce(join(reduce(rename(Placeholder, (d0, d1), (d0, d2)), sum, d0), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b)), tensor<float>(d0[1])(1.0), f(a,b)(a * b))"; @After public void removeGeneratedModelFiles() { @@ -93,10 +94,10 @@ public class RankingExpressionWithOnnxTestCase { RankProfileSearchFixture search = fixtureWith("attribute(mytensor)", "onnx('mnist_softmax.onnx')", null, - "field mytensor type tensor<float>(d0[],d1[784]) { indexing: attribute }", + "field mytensor type tensor<float>(d0[1],d1[784]) { indexing: attribute }", "Placeholder", application); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + search.assertFirstPhaseExpression(vespaExpressionWithBatchReduce, "my_profile"); } @@ -105,18 +106,16 @@ public class RankingExpressionWithOnnxTestCase { String queryProfile = "<query-profile id='default' type='root'/>"; String queryProfileType = "<query-profile-type id='root'>" + - " <field name='query(mytensor)' type='tensor<float>(d0[3],d1[784],d2[10])'/>" + + " <field name='query(mytensor)' type='tensor<float>(d0[1],d1[784],d2[10])'/>" + "</query-profile-type>"; - StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, - queryProfile, - queryProfileType); + StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, queryProfile, queryProfileType); RankProfileSearchFixture search = fixtureWith("sum(query(mytensor) * attribute(mytensor) * constant(mytensor),d2)", "onnx('mnist_softmax.onnx')", - "constant mytensor { file: ignored\ntype: tensor<float>(d0[7],d1[784]) }", - "field mytensor type tensor<float>(d0[],d1[784]) { indexing: attribute }", + "constant mytensor { file: ignored\ntype: tensor<float>(d0[1],d1[784]) }", + "field mytensor type tensor<float>(d0[1],d1[784]) { indexing: attribute }", "Placeholder", application); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + search.assertFirstPhaseExpression(vespaExpressionWithBatchReduce, "my_profile"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 680f2dd9659..126a41e14ad 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -44,6 +44,7 @@ public class RankingExpressionWithTensorFlowTestCase { private final String name = "mnist_softmax_saved"; private final String vespaExpression = "join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(" + name + "_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_layer_Variable_1_read), f(a,b)(a + b))"; + private final static String vespaExpressionWithBatchReduce = "join(join(reduce(join(reduce(rename(Placeholder, (d0, d1), (d0, d2)), sum, d0), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; @After public void removeGeneratedModelFiles() { @@ -114,28 +115,28 @@ public class RankingExpressionWithTensorFlowTestCase { RankProfileSearchFixture search = fixtureWith("attribute(mytensor)", "tensorflow('mnist_softmax/saved')", null, - "field mytensor type tensor(d0[],d1[784]) { indexing: attribute }", + "field mytensor type tensor(d0[1],d1[784]) { indexing: attribute }", "Placeholder", application); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + search.assertFirstPhaseExpression(vespaExpressionWithBatchReduce, "my_profile"); } @Test public void testTensorFlowReferenceWithFeatureCombination() { String queryProfile = "<query-profile id='default' type='root'/>"; String queryProfileType = "<query-profile-type id='root'>" + - " <field name='query(mytensor)' type='tensor(d0[3],d1[784],d2[10])'/>" + + " <field name='query(mytensor)' type='tensor(d0[1],d1[784],d2[10])'/>" + "</query-profile-type>"; StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, queryProfile, queryProfileType); RankProfileSearchFixture search = fixtureWith("sum(query(mytensor) * attribute(mytensor) * constant(mytensor),d2)", "tensorflow('mnist_softmax/saved')", - "constant mytensor { file: ignored\ntype: tensor(d0[7],d1[784]) }", - "field mytensor type tensor(d0[],d1[784]) { indexing: attribute }", + "constant mytensor { file: ignored\ntype: tensor(d0[1],d1[784]) }", + "field mytensor type tensor(d0[1],d1[784]) { indexing: attribute }", "Placeholder", application); - search.assertFirstPhaseExpression(vespaExpression, "my_profile"); + search.assertFirstPhaseExpression(vespaExpressionWithBatchReduce, "my_profile"); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index 6aea0593f8a..dff1d338ffe 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -6,8 +6,10 @@ import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; import org.junit.Test; +import java.util.List; import java.util.Optional; import static com.yahoo.config.provision.ClusterSpec.Type.container; @@ -53,7 +55,10 @@ public class HostResourceTest { private static HostResource hostResourceWithMemberships(ClusterMembership membership) { return new HostResource(Host.createHost(null, "hostname"), - new HostSpec("hostname", Optional.of(membership))); + new HostSpec("hostname", + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + membership, + Optional.empty(), Optional.empty(), Optional.empty())); } private static int counter = 0; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java index ac77b821d4a..a9bf8bdcc49 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -17,7 +17,9 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ProvisionLogger; +import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; import org.junit.Before; import org.junit.Test; @@ -103,17 +105,17 @@ public class VespaModelFactoryTest { @Override public HostSpec allocateHost(String alias) { return new HostSpec(hostName, - List.of(), - ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), - 0)); + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), 0), + Optional.empty(), Optional.empty(), Optional.empty()); } @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { return List.of(new HostSpec(hostName, - List.of(), - ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), - 0))); + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(), 0), + Optional.empty(), Optional.empty(), Optional.empty())); } }; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java index 1b61215618a..c4212a877e8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java @@ -16,7 +16,8 @@ public class AwsAccessControlValidatorTest extends AccessControlValidatorTestBas @Before public void setup() { validator = new AwsAccessControlValidator(); - zone = new Zone(Cloud.defaultCloud().withRequireAccessControl(true), SystemName.main, Environment.prod, RegionName.from("foo")); + zone = new Zone(Cloud.builder().requireAccessControl(true).build(), + SystemName.main, Environment.prod, RegionName.from("foo")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java index 1f8dcc2da64..b594a3329d1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java @@ -86,9 +86,9 @@ public class ConstantTensorJsonValidatorTest { } @Test - public void ensure_that_bounded_tensor_outside_limits_is_disallowed() { + public void ensure_that_bound_tensor_outside_limits_is_disallowed() { expectedException.expect(InvalidConstantTensor.class); - expectedException.expectMessage("Coordinate \"5\" not within limits of bounded dimension x"); + expectedException.expectMessage("Index 5 not within limits of bound dimension 'x'"); validateTensorJson( TensorType.fromSpec("tensor(x[5], y[10])"), @@ -119,9 +119,9 @@ public class ConstantTensorJsonValidatorTest { } @Test - public void ensure_that_non_integer_strings_in_address_points_are_disallowed_unbounded() { + public void ensure_that_non_integer_strings_in_address_points_are_disallowed_unbound() { expectedException.expect(InvalidConstantTensor.class); - expectedException.expectMessage("Coordinate \"a\" for dimension x is not an integer"); + expectedException.expectMessage("Index 'a' for dimension 'x' is not an integer"); validateTensorJson( TensorType.fromSpec("tensor(x[])"), @@ -139,7 +139,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_tensor_coordinates_are_strings() { expectedException.expect(InvalidConstantTensor.class); - expectedException.expectMessage("Tensor coordinate is not a string (VALUE_NUMBER_INT)"); + expectedException.expectMessage("Tensor label is not a string (VALUE_NUMBER_INT)"); validateTensorJson( TensorType.fromSpec("tensor(x[])"), @@ -157,7 +157,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_non_integer_strings_in_address_points_are_disallowed_bounded() { expectedException.expect(InvalidConstantTensor.class); - expectedException.expectMessage("Coordinate \"a\" for dimension x is not an integer"); + expectedException.expectMessage("Index 'a' for dimension 'x' is not an integer"); validateTensorJson( TensorType.fromSpec("tensor(x[5])"), @@ -175,7 +175,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_missing_coordinates_fail() { expectedException.expect(InvalidConstantTensor.class); - expectedException.expectMessage("Tensor address missing dimension(s): y, z"); + expectedException.expectMessage("Tensor address missing dimension(s) y, z"); validateTensorJson( TensorType.fromSpec("tensor(x[], y[], z[])"), @@ -211,7 +211,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_extra_dimensions_are_disallowed() { expectedException.expect(InvalidConstantTensor.class); - expectedException.expectMessage("Tensor dimension \"z\" does not exist"); + expectedException.expectMessage("Tensor dimension 'z' does not exist"); validateTensorJson( TensorType.fromSpec("tensor(x[], y[])"), @@ -229,7 +229,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_duplicate_dimensions_are_disallowed() { expectedException.expect(InvalidConstantTensor.class); - expectedException.expectMessage("Duplicate tensor dimension \"y\""); + expectedException.expectMessage("Duplicate tensor dimension 'y'"); validateTensorJson( TensorType.fromSpec("tensor(x[], y[])"), @@ -265,7 +265,7 @@ public class ConstantTensorJsonValidatorTest { @Test public void ensure_that_invalid_json_not_in_tensor_format_fails() { expectedException.expect(InvalidConstantTensor.class); - expectedException.expectMessage("Expected field name \"cells\", got \"stats\""); + expectedException.expectMessage("Expected field name 'cells', got 'stats'"); validateTensorJson(TensorType.fromSpec("tensor(x[], y[])"), inputJsonToReader( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java index d99fd93d5eb..dbf6013b167 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java @@ -9,6 +9,7 @@ import org.junit.rules.ExpectedException; import static com.yahoo.vespa.model.application.validation.RankingConstantsValidator.TensorValidationFailed; public class RankingConstantsValidatorTest { + @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -20,10 +21,11 @@ public class RankingConstantsValidatorTest { @Test public void ensure_that_failing_ranking_constants_fails() { expectedException.expect(TensorValidationFailed.class); - expectedException.expectMessage("Ranking constant \"constant_tensor_2\" (tensors/constant_tensor_2.json): Tensor coordinate is not a string (VALUE_NUMBER_INT)"); - expectedException.expectMessage("Ranking constant \"constant_tensor_3\" (tensors/constant_tensor_3.json): Tensor dimension \"cd\" does not exist"); - expectedException.expectMessage("Ranking constant \"constant_tensor_4\" (tensors/constant_tensor_4.json): Tensor dimension \"z\" does not exist"); + expectedException.expectMessage("Ranking constant 'constant_tensor_2' (tensors/constant_tensor_2.json): Tensor label is not a string (VALUE_NUMBER_INT)"); + expectedException.expectMessage("Ranking constant 'constant_tensor_3' (tensors/constant_tensor_3.json): Tensor dimension 'cd' does not exist"); + expectedException.expectMessage("Ranking constant 'constant_tensor_4' (tensors/constant_tensor_4.json): Tensor dimension 'z' does not exist"); new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_fail/").create(); } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java index a704938610b..5da36d82a62 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java @@ -119,12 +119,12 @@ public class AttributeChangeValidatorTest { "Field 'f1' changed: tensor type: 'tensor(x[2])' -> 'tensor(x[3])'", Instant.now())); new Fixture( - "field f1 type tensor(x[]) { indexing: attribute }", + "field f1 type tensor(x[5]) { indexing: attribute }", "field f1 type tensor(x[3]) { indexing: attribute }") .assertValidation(newRefeedAction( "tensor-type-change", ValidationOverrides.empty, - "Field 'f1' changed: tensor type: 'tensor(x[])' -> 'tensor(x[3])'", Instant.now())); + "Field 'f1' changed: tensor type: 'tensor(x[5])' -> 'tensor(x[3])'", Instant.now())); } @Test @@ -168,10 +168,18 @@ public class AttributeChangeValidatorTest { } @Test + public void changing_distance_metric_without_hnsw_index_enabled_is_ok() throws Exception { + new Fixture("field f1 type tensor(x[2]) { indexing: attribute }", + "field f1 type tensor(x[2]) { indexing: attribute \n attribute { " + + "distance-metric: geodegrees \n } }"). + assertValidation(); + } + + @Test public void changing_distance_metric_with_hnsw_index_enabled_requires_restart() throws Exception { new Fixture("field f1 type tensor(x[2]) { indexing: attribute | index \n index { hnsw } }", - "field f1 type tensor(x[2]) { indexing: attribute | index \n index { " + - "distance-metric: geodegrees \n hnsw } }"). + "field f1 type tensor(x[2]) { indexing: attribute | index \n attribute { " + + "distance-metric: geodegrees \n } }"). assertValidation(newRestartAction("Field 'f1' changed: change property " + "'distance-metric' from 'EUCLIDEAN' to 'GEODEGREES'")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 3dfcef70aba..4bba66f0fb3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -201,7 +201,6 @@ public class DocumentTypeChangeValidatorTest { return new NewDocumentType( new NewDocumentType.Name("mydoc"), headerfields, - new StructDataType("bodyfields"), new FieldSets(), Collections.emptySet(), Collections.emptySet()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 0f94df80421..739ae055ec7 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -167,7 +167,6 @@ public class ContainerClusterTest { ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(10, threadpoolConfig.maxthreads()); assertEquals(0, threadpoolConfig.queueSize()); - assertEquals(0, threadpoolConfig.softStartSeconds(), 0); } @Test @@ -206,22 +205,6 @@ public class ContainerClusterTest { } @Test - public void requireThatSoftStartSecondsCanBeControlledByProperties() { - DeployState state = new DeployState.Builder().properties(new TestProperties().setSoftStartSeconds(300.0)) - .build(); - MockRoot root = new MockRoot("foo", state); - ApplicationContainerCluster cluster = createContainerCluster(root, false); - addContainer(root.deployLogger(), cluster, "c1", "host-c1"); - - ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); - cluster.getConfig(tpBuilder); - ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); - assertEquals(500, threadpoolConfig.maxthreads()); - assertEquals(0, threadpoolConfig.queueSize()); - assertEquals(300.0, threadpoolConfig.softStartSeconds(), 0.0); - } - - @Test public void requireThatPoolAndQueueCanNotBeControlledByPropertiesWhenNoFlavor() { DeployState state = new DeployState.Builder().properties(new TestProperties() .setThreadPoolSizeFactor(8.5) @@ -236,22 +219,20 @@ public class ContainerClusterTest { ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(500, threadpoolConfig.maxthreads()); assertEquals(0, threadpoolConfig.queueSize()); - assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); } @Test public void requireThatPoolAndQueueCanBeControlledByPropertiesAndFlavor() { FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder().name("my_flavor").minCpuCores(3); - NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder))) + NodeResourcesTuning nodeResourcesTuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources()) .setThreadPoolSizeFactor(13.3) .setQueueSizeFactor(17.5); ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); - nodeFlavorTuning.getConfig(tpBuilder); + nodeResourcesTuning.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(40, threadpoolConfig.maxthreads()); assertEquals(700, threadpoolConfig.queueSize()); - assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); } @Test @@ -265,7 +246,6 @@ public class ContainerClusterTest { ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(500, threadpoolConfig.maxthreads()); assertEquals(0, threadpoolConfig.queueSize()); - assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java index 98bc4210602..b144ea19980 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java @@ -26,24 +26,27 @@ public class MockSearchClusters { @Override protected AbstractSearchCluster.IndexingMode getIndexingMode() { return streaming ? AbstractSearchCluster.IndexingMode.STREAMING : AbstractSearchCluster.IndexingMode.REALTIME; } - @Override - protected void assureSdConsistent() {} @Override public void getConfig(DocumentdbInfoConfig.Builder builder) { } + @Override public void getConfig(IndexInfoConfig.Builder builder) { } + @Override public void getConfig(IlscriptsConfig.Builder builder) { } + @Override public void getConfig(AttributesConfig.Builder builder) { } + @Override public void getConfig(RankProfilesConfig.Builder builder) { } + } public static AbstractSearchCluster mockSearchCluster(AbstractConfigProducerRoot root, String clusterName, int clusterIndex, boolean isStreaming) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 15966694f8d..fdd7ae57f0f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -667,7 +667,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { String servicesXml = "<services>" + "<admin version='3.0'>" + - " <nodes count='1'/>" + + " <nodes count='2'/>" + "</admin>" + "<container id ='default' version='1.0'>" + " <nodes>" + @@ -766,7 +766,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertEquals("KMP_SETTING=1 KMP_AFFINITY=granularity=fine,verbose,compact,1,0 ", qrStartConfig.qrs().env()); } - private void verifyAvailableprocessors(boolean isHosted, Flavor flavor, int expectProcessors) throws IOException, SAXException { + private void verifyAvailableprocessors(boolean isHosted, Flavor flavor, int expectProcessors) { DeployState deployState = new DeployState.Builder() .modelHostProvisioner(flavor != null ? new SingleNodeProvisioner(flavor) : new SingleNodeProvisioner()) .properties(new TestProperties() @@ -790,7 +790,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void requireThatAvailableProcessorsFollowFlavor() throws IOException, SAXException { + public void requireThatAvailableProcessorsFollowFlavor() { verifyAvailableprocessors(false, null,0); verifyAvailableprocessors(true, null,0); verifyAvailableprocessors(true, new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()), 9); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index 484709a0c18..294df42bd77 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -1,7 +1,4 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ package com.yahoo.vespa.model.container.xml; @@ -11,10 +8,7 @@ import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; + import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ContainerCluster; @@ -50,7 +44,6 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) .deployLogger(logger) - .properties(new TestProperties().setHostedVespa(true)) .build()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); model.getConfig(qrStartBuilder, "container/container.0"); @@ -85,11 +78,11 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { } private static void verifyIgnoreJvmGCOptions(boolean isHosted) throws IOException, SAXException { - verifyIgnoreJvmGCOptionsIfJvmArgs(isHosted, "jvmargs", ContainerCluster.G1GC); - verifyIgnoreJvmGCOptionsIfJvmArgs(isHosted, "jvm-options", "-XX:+UseG1GC"); + verifyIgnoreJvmGCOptionsIfJvmArgs("jvmargs", ContainerCluster.G1GC); + verifyIgnoreJvmGCOptionsIfJvmArgs( "jvm-options", "-XX:+UseG1GC"); } - private static void verifyIgnoreJvmGCOptionsIfJvmArgs(boolean isHosted, String jvmOptionsName, String expectedGC) throws IOException, SAXException { + private static void verifyIgnoreJvmGCOptionsIfJvmArgs(String jvmOptionsName, String expectedGC) throws IOException, SAXException { String servicesXml = "<container version='1.0'>" + " <nodes jvm-gc-options='-XX:+UseG1GC' " + jvmOptionsName + "='-XX:+UseParNewGC'>" + @@ -102,8 +95,6 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) .deployLogger(logger) - .zone(new Zone(SystemName.cd, Environment.dev, RegionName.from("here"))) - .properties(new TestProperties().setHostedVespa(isHosted)) .build()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); model.getConfig(qrStartBuilder, "container/container.0"); @@ -117,7 +108,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { verifyIgnoreJvmGCOptions(true); } - private void verifyJvmGCOptions(boolean isHosted, String override, Zone zone, String expected) throws IOException, SAXException { + private void verifyJvmGCOptions(boolean isHosted, String featureFlagDefault, String override, String expected) throws IOException, SAXException { String servicesXml = "<container version='1.0'>" + " <nodes " + ((override == null) ? ">" : ("jvm-gc-options='" + override + "'>")) + @@ -130,8 +121,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) .deployLogger(logger) - .zone(zone) - .properties(new TestProperties().setHostedVespa(isHosted)) + .properties(new TestProperties().setJvmGCOptions(featureFlagDefault).setHostedVespa(isHosted)) .build()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); model.getConfig(qrStartBuilder, "container/container.0"); @@ -139,18 +129,16 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { assertEquals(expected, qrStartConfig.jvm().gcopts()); } - private void verifyJvmGCOptions(boolean isHosted, Zone zone, String expected) throws IOException, SAXException { - verifyJvmGCOptions(isHosted, null, zone, expected); - verifyJvmGCOptions(isHosted, "-XX:+UseG1GC", zone, "-XX:+UseG1GC"); - Zone DEV = new Zone(SystemName.dev, zone.environment(), zone.region()); - verifyJvmGCOptions(isHosted, null, DEV, ContainerCluster.G1GC); - verifyJvmGCOptions(isHosted, "-XX:+UseConcMarkSweepGC", DEV, "-XX:+UseConcMarkSweepGC"); - } - @Test public void requireThatJvmGCOptionsIsHonoured() throws IOException, SAXException { - verifyJvmGCOptions(false, Zone.defaultZone(),ContainerCluster.G1GC); - verifyJvmGCOptions(true, Zone.defaultZone(), ContainerCluster.G1GC); + verifyJvmGCOptions(false, null,null, ContainerCluster.G1GC); + verifyJvmGCOptions(true, null,null, ContainerCluster.CMS); + verifyJvmGCOptions(true, "",null, ContainerCluster.CMS); + verifyJvmGCOptions(false, "-XX:+UseConcMarkSweepGC",null, "-XX:+UseConcMarkSweepGC"); + verifyJvmGCOptions(true, "-XX:+UseConcMarkSweepGC",null, "-XX:+UseConcMarkSweepGC"); + verifyJvmGCOptions(false, null,"-XX:+UseG1GC", "-XX:+UseG1GC"); + verifyJvmGCOptions(false, "-XX:+UseConcMarkSweepGC","-XX:+UseG1GC", "-XX:+UseG1GC"); + verifyJvmGCOptions(false, null,"-XX:+UseConcMarkSweepGC", "-XX:+UseConcMarkSweepGC"); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index 47f37a75fd0..a2d13439f4e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.component.ComponentId; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; @@ -16,6 +17,7 @@ import org.w3c.dom.Element; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; import static com.yahoo.test.Matchers.hasItemWithMethod; +import static com.yahoo.vespa.model.container.search.ContainerSearch.QUERY_PROFILE_REGISTRY_CLASS; import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_BINDING; import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS; import static org.hamcrest.CoreMatchers.is; @@ -127,6 +129,14 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { assertThat(chainsConfig().chains(), hasItemWithMethod("vespa", "id")); } + @Test + public void query_profiles_registry_component_is_added() { + createClusterWithOnlyDefaultChains(); + ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); + var queryProfileRegistryId = ComponentId.fromString(QUERY_PROFILE_REGISTRY_CLASS); + assertTrue(cluster.getComponentsMap().containsKey(queryProfileRegistryId)); + } + private void createClusterWithOnlyDefaultChains() { Element containerElem = DomBuilderTest.parse( "<container id='default' version='1.0'>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 802082cc2ff..5b3c42df869 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -310,14 +310,14 @@ public class ContentClusterTest extends ContentBaseTest { StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); model.getConfig(builder, "bar/distributor/0"); StorDistributormanagerConfig config = new StorDistributormanagerConfig(builder); - assertEquals(false, config.inlinebucketsplitting()); + assertFalse(config.inlinebucketsplitting()); } { StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); model.getConfig(builder, "bar/storage/0"); StorFilestorConfig config = new StorFilestorConfig(builder); - assertEquals(false, config.enable_multibit_split_optimalization()); + assertFalse(config.enable_multibit_split_optimalization()); } } @@ -343,7 +343,7 @@ public class ContentClusterTest extends ContentBaseTest { List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2"); try{ new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); - assertTrue("Deploying without redundancy should fail", false); + fail("Deploying without redundancy should fail"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage(), e.getMessage().contains("missing required element \"redundancy\"")); } @@ -773,7 +773,7 @@ public class ContentClusterTest extends ContentBaseTest { " <documents/>" + " <engine>" + " <proton>" + - " <flush-on-shutdown>" + Boolean.toString(flushOnShutdown) + "</flush-on-shutdown>" + + " <flush-on-shutdown>" + flushOnShutdown + "</flush-on-shutdown>" + " </proton>" + " </engine>" + " <group>" + @@ -933,43 +933,20 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(distributionBits, storDistributormanagerConfig.minsplitcount()); } - private void verifyTopKProbabilityPropertiesControl(double topKProbability) { - VespaModel model = createEnd2EndOneNode(new TestProperties().setTopKProbability(topKProbability)); - - ContentCluster cc = model.getContentClusters().get("storage"); - DispatchConfig.Builder builder = new DispatchConfig.Builder(); - cc.getSearch().getConfig(builder); - - DispatchConfig cfg = new DispatchConfig(builder); - assertEquals(topKProbability, cfg.topKProbability(), 0.0); - } - - private void verifyRoundRobinPropertiesControl(boolean useAdaptiveDispatch) { - VespaModel model = createEnd2EndOneNode(new TestProperties().setUseAdaptiveDispatch(useAdaptiveDispatch)); + private void verifyTopKProbabilityPropertiesControl() { + VespaModel model = createEnd2EndOneNode(new TestProperties()); ContentCluster cc = model.getContentClusters().get("storage"); DispatchConfig.Builder builder = new DispatchConfig.Builder(); cc.getSearch().getConfig(builder); DispatchConfig cfg = new DispatchConfig(builder); - if (useAdaptiveDispatch) { - assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, cfg.distributionPolicy()); - } else { - assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, cfg.distributionPolicy()); - } - } - - @Test - public void default_dispatch_controlled_by_properties() { - verifyRoundRobinPropertiesControl(false); - verifyRoundRobinPropertiesControl(true); + assertEquals(0.9999, cfg.topKProbability(), 0.0); } @Test public void default_topKprobability_controlled_by_properties() { - verifyTopKProbabilityPropertiesControl(1.0); - verifyTopKProbabilityPropertiesControl(0.999); - verifyTopKProbabilityPropertiesControl(0.77); + verifyTopKProbabilityPropertiesControl(); } private boolean resolveDistributorBtreeDbConfigWithFeatureFlag(boolean flagEnabledBtreeDb) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index fdf911231d5..96fe7fc5775 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -143,7 +143,7 @@ public class StorageClusterTest { assertEquals(7, config.num_threads()); assertFalse(config.enable_multibit_split_optimalization()); - assertEquals(0, config.num_response_threads()); + assertEquals(1, config.num_response_threads()); } { assertEquals(1, stc.getChildren().size()); @@ -168,13 +168,13 @@ public class StorageClusterTest { " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + " </group>" + "</cluster>", - new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()), - new TestProperties().setDefaultNumResponseThreads(3) + new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build()) ); StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); stc.getConfig(builder); StorFilestorConfig config = new StorFilestorConfig(builder); - assertEquals(3, config.num_response_threads()); + assertEquals(1, config.num_response_threads()); + assertEquals(7, config.num_threads()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 852844fe451..7c93b4ef02b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -65,7 +65,7 @@ public class ClusterTest { assertEquals(0.23, config.minWaitAfterCoverageFactor(), DELTA); assertEquals(0.58, config.maxWaitAfterCoverageFactor(), DELTA); assertEquals(2, config.searchableCopies()); - assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, config.distributionPolicy()); + assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, config.distributionPolicy()); } @Test @@ -74,9 +74,10 @@ public class ClusterTest { "", joinLines( "<max-hits-per-partition>77</max-hits-per-partition>", - "<dispatch-policy>adaptive</dispatch-policy>", + "<dispatch-policy>round-robin</dispatch-policy>", "<min-group-coverage>13</min-group-coverage>", - "<min-active-docs-coverage>93</min-active-docs-coverage>"), + "<min-active-docs-coverage>93</min-active-docs-coverage>", + "<top-k-probability>0.777</top-k-probability>"), false); DispatchConfig.Builder builder = new DispatchConfig.Builder(); cluster.getSearch().getConfig(builder); @@ -84,8 +85,9 @@ public class ClusterTest { assertEquals(2, config.searchableCopies()); assertEquals(93.0, config.minActivedocsPercentage(), DELTA); assertEquals(13.0, config.minGroupCoverage(), DELTA); - assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, config.distributionPolicy()); + assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, config.distributionPolicy()); assertEquals(77, config.maxHitsPerNode()); + assertEquals(0.777, config.topKProbability(), DELTA); } @Test @@ -96,7 +98,7 @@ public class ClusterTest { cluster.getSearch().getConfig(builder); DispatchConfig config = new DispatchConfig(builder); assertEquals(2, config.searchableCopies()); - assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, config.distributionPolicy()); + assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, config.distributionPolicy()); assertEquals(0, config.maxNodesDownPerGroup()); assertEquals(1.0, config.maxWaitAfterCoverageFactor(), DELTA); assertEquals(0, config.minWaitAfterCoverageFactor(), DELTA); @@ -105,6 +107,7 @@ public class ClusterTest { assertEquals(100.0, config.minSearchCoverage(), DELTA); assertEquals(97.0, config.minActivedocsPercentage(), DELTA); assertEquals(100.0, config.minGroupCoverage(), DELTA); + assertEquals(0.9999, config.topKProbability(), DELTA); assertEquals(3, config.node().size()); assertEquals(0, config.node(0).key()); assertEquals(1, config.node(1).key()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java index 57dbf132883..5b38e09537d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java @@ -63,17 +63,17 @@ public class MlModelsTest { private final String testProfile = "rankingExpression(input).rankingScript: attribute(argument)\n" + - "rankingExpression(input).type: tensor<float>(d0[],d1[784])\n" + - "rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add).rankingScript: join(reduce(join(rename(rankingExpression(input), (d0, d1), (d0, d4)), constant(mnist_saved_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(mnist_saved_dnn_hidden1_bias_read), f(a,b)(a + b))\n" + + "rankingExpression(input).type: tensor<float>(d0[1],d1[784])\n" + + "rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add).rankingScript: join(reduce(join(reduce(rename(rankingExpression(input), (d0, d1), (d0, d4)), sum, d0), constant(mnist_saved_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(mnist_saved_dnn_hidden1_bias_read), f(a,b)(a + b))\n" + "rankingExpression(mnist_tensorflow).rankingScript: join(reduce(join(map(join(reduce(join(join(join(0.009999999776482582, rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add), f(a,b)(a * b)), rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add), f(a,b)(max(a,b))), constant(mnist_saved_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(mnist_saved_dnn_hidden2_bias_read), f(a,b)(a + b)), f(a)(1.0507009873554805 * if (a >= 0, a, 1.6732632423543772 * (exp(a) - 1)))), constant(mnist_saved_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(mnist_saved_dnn_outputs_bias_read), f(a,b)(a + b))\n" + "rankingExpression(Placeholder).rankingScript: attribute(argument)\n" + - "rankingExpression(Placeholder).type: tensor<float>(d0[],d1[784])\n" + - "rankingExpression(mnist_softmax_tensorflow).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))\n" + - "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" + + "rankingExpression(Placeholder).type: tensor<float>(d0[1],d1[784])\n" + + "rankingExpression(mnist_softmax_tensorflow).rankingScript: join(join(reduce(join(reduce(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), sum, d0), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))\n" + + "rankingExpression(mnist_softmax_onnx).rankingScript: join(join(reduce(join(reduce(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), sum, d0), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b)), tensor<float>(d0[1])(1.0), f(a,b)(a * b))\n" + "rankingExpression(my_xgboost).rankingScript: if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)\n" + "rankingExpression(my_lightgbm).rankingScript: if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)\n" + "vespa.rank.firstphase: rankingExpression(firstphase)\n" + "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost) + rankingExpression(my_lightgbm)\n" + - "vespa.type.attribute.argument: tensor<float>(d0[],d1[784])\n"; + "vespa.type.attribute.argument: tensor<float>(d0[1],d1[784])\n"; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index 023b7249939..5df522e1b9d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -11,13 +11,13 @@ import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; -import static com.yahoo.vespa.model.search.NodeFlavorTuning.MB; -import static com.yahoo.vespa.model.search.NodeFlavorTuning.GB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.MB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; /** * @author geirst */ -public class NodeFlavorTuningTest { +public class NodeResourcesTuningTest { private static double delta = 0.00001; @@ -29,8 +29,9 @@ public class NodeFlavorTuningTest { @Test public void require_that_hwinfo_memory_size_is_set() { - ProtonConfig cfg = configFromMemorySetting(24); - assertEquals(24 * GB, cfg.hwinfo().memory().size()); + double combinedFactor = 1 - 17.0/100; + assertEquals(24 * GB, configFromMemorySetting(24, false).hwinfo().memory().size()); + assertEquals(combinedFactor * 24 * GB, configFromMemorySetting(24, true).hwinfo().memory().size(), 1000); } private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, int gb, int redundancy, int searchableCopies) { @@ -43,6 +44,7 @@ public class NodeFlavorTuningTest { } return configFromMemorySetting(gb, builder, redundancy, searchableCopies); } + private void verify_that_initial_numdocs_is_dependent_of_mode(int redundancy, int searchablecopies) { int divisor = Math.max(redundancy, searchablecopies); ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24, redundancy, searchablecopies); @@ -54,6 +56,7 @@ public class NodeFlavorTuningTest { assertEquals(402653184/divisor, cfg.documentdb(2).allocation().initialnumdocs()); assertEquals("c", cfg.documentdb(2).inputdoctypename()); } + @Test public void require_that_initial_numdocs_is_dependent_of_mode_and_searchablecopies() { verify_that_initial_numdocs_is_dependent_of_mode(2,0); @@ -145,14 +148,20 @@ public class NodeFlavorTuningTest { @Test public void require_that_summary_cache_max_bytes_is_set_based_on_memory() { - assertEquals(1*GB/20, configFromMemorySetting(1).summary().cache().maxbytes()); - assertEquals(256*GB/20, configFromMemorySetting(256).summary().cache().maxbytes()); + assertEquals(1*GB / 20, configFromMemorySetting(1, false).summary().cache().maxbytes()); + assertEquals(256*GB / 20, configFromMemorySetting(256, false).summary().cache().maxbytes()); + } + + @Test + public void require_that_summary_cache_memory_is_reduced_with_combined_cluster() { + double combinedFactor = 1 - 17.0/100; + assertEquals(combinedFactor * 1*GB / 20, configFromMemorySetting(1, true).summary().cache().maxbytes(), 1000); + assertEquals(combinedFactor * 256*GB / 20, configFromMemorySetting(256, true).summary().cache().maxbytes(), 1000); } @Test public void require_that_docker_node_is_tagged_with_shared_disk() { assertSharedDisk(true, true); - assertSharedDisk(false, false); } @Test @@ -164,12 +173,12 @@ public class NodeFlavorTuningTest { } private static void assertDocumentStoreMaxFileSize(long expFileSizeBytes, int memoryGb) { - assertEquals(expFileSizeBytes, configFromMemorySetting(memoryGb).summary().log().maxfilesize()); + assertEquals(expFileSizeBytes, configFromMemorySetting(memoryGb, false).summary().log().maxfilesize()); } private static void assertFlushStrategyMemory(long expMemoryBytes, int memoryGb) { - assertEquals(expMemoryBytes, configFromMemorySetting(memoryGb).flush().memory().maxmemory()); - assertEquals(expMemoryBytes, configFromMemorySetting(memoryGb).flush().memory().each().maxmemory()); + assertEquals(expMemoryBytes, configFromMemorySetting(memoryGb, false).flush().memory().maxmemory()); + assertEquals(expMemoryBytes, configFromMemorySetting(memoryGb, false).flush().memory().each().maxmemory()); } private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) { @@ -189,56 +198,63 @@ public class NodeFlavorTuningTest { } private static void assertWriteFilter(double expMemoryLimit, int memoryGb) { - assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb).writefilter().memorylimit(), delta); + assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb, false).writefilter().memorylimit(), delta); } private static ProtonConfig configFromDiskSetting(boolean fastDisk) { - return getConfig(new FlavorsConfig.Flavor.Builder(). - fastDisk(fastDisk)); + return getConfig(new FlavorsConfig.Flavor.Builder().fastDisk(fastDisk), false); } private static ProtonConfig configFromDiskSetting(int diskGb) { - return getConfig(new FlavorsConfig.Flavor.Builder(). - minDiskAvailableGb(diskGb)); + return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), false); } - private static ProtonConfig configFromMemorySetting(int memoryGb) { - return getConfig(new FlavorsConfig.Flavor.Builder(). - minMainMemoryAvailableGb(memoryGb)); + private static ProtonConfig configFromMemorySetting(int memoryGb, boolean combined) { + return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), combined); } + private static ProtonConfig configFromMemorySetting(int memoryGb, ProtonConfig.Builder builder, int redundancy, int searchableCopies) { - return getConfig(new FlavorsConfig.Flavor.Builder(). - minMainMemoryAvailableGb(memoryGb), builder, redundancy, searchableCopies); + return getConfig(new FlavorsConfig.Flavor.Builder() + .minMainMemoryAvailableGb(memoryGb), builder, redundancy, searchableCopies, false); } private static ProtonConfig configFromNumCoresSetting(double numCores) { - return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores)); + return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), false); } private static ProtonConfig configFromNumCoresSetting(double numCores, int numThreadsPerSearch) { - return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), new ProtonConfig.Builder(), 1, 1, numThreadsPerSearch); + return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), + new ProtonConfig.Builder(), 1, 1, numThreadsPerSearch, false); } private static ProtonConfig configFromEnvironmentType(boolean docker) { String environment = (docker ? "DOCKER_CONTAINER" : "undefined"); - return getConfig(new FlavorsConfig.Flavor.Builder().environment(environment)); + return getConfig(new FlavorsConfig.Flavor.Builder().environment(environment), false); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder) { - return getConfig(flavorBuilder, new ProtonConfig.Builder()); + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, boolean combined) { + return getConfig(flavorBuilder, new ProtonConfig.Builder(), combined); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) { - return getConfig(flavorBuilder, protonBuilder, 1,1); + + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, + ProtonConfig.Builder protonBuilder, boolean combined) { + return getConfig(flavorBuilder, protonBuilder, 1, 1, combined); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int redundancy, int searchableCopies) { + + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, + int redundancy, int searchableCopies, boolean combined) { flavorBuilder.name("my_flavor"); - NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)), redundancy, searchableCopies); + NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), + redundancy, searchableCopies, 1, combined); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int redundancy, int searchableCopies, int numThreadsPerSearch) { + + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, + int redundancy, int searchableCopies, int numThreadsPerSearch, boolean combined) { flavorBuilder.name("my_flavor"); - NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)), redundancy, searchableCopies, numThreadsPerSearch); + NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), + redundancy, searchableCopies, numThreadsPerSearch, combined); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 97417f5a522..4005418f8dc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -116,13 +116,13 @@ public class DocumentDatabaseTestCase { @Test public void requireThatConcurrencyIsReflectedCorrectlyForDefault() { - verifyConcurrency("index", "", 0.40, 0.40); - verifyConcurrency("streaming", "", 0.8, 0.0); - verifyConcurrency("store-only", "", 0.8, 0.0); + verifyConcurrency("index", "", 0.50, 0.50); + verifyConcurrency("streaming", "", 1.0, 0.0); + verifyConcurrency("store-only", "", 1.0, 0.0); } @Test public void requireThatMixedModeConcurrencyIsReflectedCorrectlyForDefault() { - verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 0.8, Arrays.asList(0.40, 0.0)); + verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 1.0, Arrays.asList(0.50, 0.0)); } @Test public void requireThatMixedModeConcurrencyIsReflected() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index 9e26fa9e657..9c69ba8f212 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -48,14 +48,14 @@ public class SearchNodeTest { } private static SearchNode createSearchNode(AbstractConfigProducer parent, String name, int distributionKey, - NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted) { - return SearchNode.create(parent, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, Optional.empty(), Optional.empty(), isHosted); + NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted, boolean combined) { + return SearchNode.create(parent, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, Optional.empty(), Optional.empty(), isHosted, combined); } @Test public void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false); prepare(root, node); assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node); } @@ -63,7 +63,7 @@ public class SearchNodeTest { @Test public void requireThatPreShutdownCommandIsEmptyWhenNotActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false); node.setHostResource(new HostResource(new Host(node, "mynbode"))); node.initService(root.deployLogger()); assertFalse(node.getPreShutdownCommand().isPresent()); @@ -72,7 +72,7 @@ public class SearchNodeTest { @Test public void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted()); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted(), false); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.deployLogger()); assertTrue(node.getPreShutdownCommand().isPresent()); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index f723575c342..1a1cd243f89 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -90,7 +90,7 @@ public final class Capacity { /** Creates this from a node type */ public static Capacity fromRequiredNodeType(NodeType type) { - return from(new ClusterResources(0, 0, NodeResources.unspecified), true, false, type); + return from(new ClusterResources(0, 0, NodeResources.unspecified()), true, false, type); } private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Cloud.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Cloud.java index b1df63dae44..24ec4a7ab70 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Cloud.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Cloud.java @@ -1,6 +1,8 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import java.util.Objects; + /** * Represents a cloud service and its supported features. * @@ -15,9 +17,9 @@ public class Cloud { private final boolean reprovisionToUpgradeOs; private final boolean requireAccessControl; - protected Cloud(CloudName name, boolean dynamicProvisioning, boolean allowHostSharing, boolean reprovisionToUpgradeOs, - boolean requireAccessControl) { - this.name = name; + private Cloud(CloudName name, boolean dynamicProvisioning, boolean allowHostSharing, boolean reprovisionToUpgradeOs, + boolean requireAccessControl) { + this.name = Objects.requireNonNull(name); this.dynamicProvisioning = dynamicProvisioning; this.allowHostSharing = allowHostSharing; this.reprovisionToUpgradeOs = reprovisionToUpgradeOs; @@ -49,25 +51,54 @@ public class Cloud { return requireAccessControl; } - public Cloud withDynamicProvisioning(boolean dynamicProvisioning) { - return new Cloud(name, dynamicProvisioning, allowHostSharing, reprovisionToUpgradeOs, requireAccessControl); + /** For testing purposes only */ + public static Cloud defaultCloud() { + return new Builder().name(CloudName.defaultName()).build(); } - public Cloud withAllowHostSharing(boolean allowHostSharing) { - return new Cloud(name, dynamicProvisioning, allowHostSharing, reprovisionToUpgradeOs, requireAccessControl); + public static Builder builder() { + return new Builder(); } - public Cloud withReprovisionToUpgradeOs(boolean reprovisionToUpgradeOs) { - return new Cloud(name, dynamicProvisioning, allowHostSharing, reprovisionToUpgradeOs, requireAccessControl); - } + public static class Builder { - public Cloud withRequireAccessControl(boolean requireAccessControl) { - return new Cloud(name, dynamicProvisioning, allowHostSharing, reprovisionToUpgradeOs, requireAccessControl); - } + private CloudName name = CloudName.defaultName(); + private boolean dynamicProvisioning = false; + private boolean allowHostSharing = true; + private boolean reprovisionToUpgradeOs = false; + private boolean requireAccessControl = false; + + private Builder() {} + + public Builder name(CloudName name) { + this.name = name; + return this; + } + + public Builder dynamicProvisioning(boolean dynamicProvisioning) { + this.dynamicProvisioning = dynamicProvisioning; + return this; + } + + public Builder allowHostSharing(boolean allowHostSharing) { + this.allowHostSharing = allowHostSharing; + return this; + } + + public Builder reprovisionToUpgradeOs(boolean reprovisionToUpgradeOs) { + this.reprovisionToUpgradeOs = reprovisionToUpgradeOs; + return this; + } + + public Builder requireAccessControl(boolean requireAccessControl) { + this.requireAccessControl = requireAccessControl; + return this; + } + + public Cloud build() { + return new Cloud(name, dynamicProvisioning, allowHostSharing, reprovisionToUpgradeOs, requireAccessControl); + } - /** For testing purposes only */ - public static Cloud defaultCloud() { - return new Cloud(CloudName.defaultName(), false, true, false, false); } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index f7030535573..7c2c491ccf1 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -31,13 +31,14 @@ public final class ClusterSpec { this.groupId = groupId; this.vespaVersion = Objects.requireNonNull(vespaVersion); this.exclusive = exclusive; - // TODO(mpolden): Require combinedId to always be present for type combined after April 2020 - if (type != Type.combined && combinedId.isPresent()) { - throw new IllegalArgumentException("combinedId must be empty for cluster of type " + type); + if (type == Type.combined) { + if (combinedId.isEmpty()) throw new IllegalArgumentException("combinedId must be set for cluster of type " + type); + } else { + if (combinedId.isPresent()) throw new IllegalArgumentException("combinedId must be empty for cluster of type " + type); } this.combinedId = combinedId; if (dockerImageRepo.isPresent() && dockerImageRepo.get().tag().isPresent()) - throw new IllegalArgumentException("dockerimageRepo is not allowed to have a tag"); + throw new IllegalArgumentException("dockerImageRepo is not allowed to have a tag"); this.dockerImageRepo = dockerImageRepo; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java index 8d767e9f4ad..50156d1baa1 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java @@ -109,15 +109,25 @@ public class Flavor { public Optional<FlavorOverrides> flavorOverrides() { return flavorOverrides; } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public double getMinMainMemoryAvailableGb() { return resources.memoryGb(); } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public double getMinDiskAvailableGb() { return resources.diskGb(); } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public boolean hasFastDisk() { return resources.diskSpeed() == NodeResources.DiskSpeed.fast; } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public double getBandwidthGbps() { return resources.bandwidthGbps(); } /** Returns the number of cores available in this flavor, not scaled for speed. */ + // TODO: Remove when models older than 7.226 are gone + @Deprecated public double getMinCpuCores() { return minCpuCores; } public Type getType() { return type; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index 2a5d27a0fe7..fa684ff6046 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -22,73 +22,167 @@ public class HostSpec implements Comparable<HostSpec> { /** Aliases of this host */ private final List<String> aliases; + private final NodeResources realResources; + private final NodeResources advertisedResources; + private final NodeResources requestedResources; + /** The current membership role of this host in the cluster it belongs to */ private final Optional<ClusterMembership> membership; - private final Optional<Flavor> flavor; - private final Optional<Version> version; private final Optional<DockerImage> dockerImageRepo; private final Optional<NetworkPorts> networkPorts; - private final Optional<NodeResources> requestedResources; + /** Create a host in a non-cloud system, where hosts are specified in config */ + public HostSpec(String hostname, List<String> aliases, Optional<NetworkPorts> networkPorts) { + this(hostname, aliases, + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + Optional.empty(), Optional.empty(), networkPorts, Optional.empty()); + } + + /** Create a host in a hosted system */ + public HostSpec(String hostname, + NodeResources realResources, + NodeResources advertisedResurces, + NodeResources requestedResources, + ClusterMembership membership, + Optional<Version> version, + Optional<NetworkPorts> networkPorts, + Optional<DockerImage> dockerImageRepo) { + this(hostname, List.of(), + realResources, + advertisedResurces, + requestedResources, + Optional.of(membership), + version, + networkPorts, + dockerImageRepo); + } + + private HostSpec(String hostname, + List<String> aliases, + NodeResources realResources, + NodeResources advertisedResurces, + NodeResources requestedResources, + Optional<ClusterMembership> membership, + Optional<Version> version, + Optional<NetworkPorts> networkPorts, + Optional<DockerImage> dockerImageRepo) { + if (hostname == null || hostname.isEmpty()) throw new IllegalArgumentException("Hostname must be specified"); + this.hostname = hostname; + this.aliases = List.copyOf(aliases); + this.realResources = Objects.requireNonNull(realResources); + this.advertisedResources = Objects.requireNonNull(advertisedResurces); + this.requestedResources = Objects.requireNonNull(requestedResources, "RequestedResources cannot be null"); + this.membership = Objects.requireNonNull(membership); + this.version = Objects.requireNonNull(version, "Version cannot be null but can be empty"); + this.networkPorts = Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty"); + this.dockerImageRepo = Objects.requireNonNull(dockerImageRepo, "Docker image repo cannot be null but can be empty"); + } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public HostSpec(String hostname, Optional<ClusterMembership> membership) { - this(hostname, new ArrayList<>(), Optional.empty(), membership); + this(hostname, new ArrayList<>(), + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + membership, + Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public HostSpec(String hostname, ClusterMembership membership, Flavor flavor, Optional<Version> version) { - this(hostname, new ArrayList<>(), Optional.of(flavor), Optional.of(membership), version); + this(hostname, new ArrayList<>(), + flavor.resources(), flavor.resources(), NodeResources.unspecified(), + Optional.of(membership), version, Optional.empty(), Optional.empty()); } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public HostSpec(String hostname, List<String> aliases) { - this(hostname, aliases, Optional.empty(), Optional.empty()); + this(hostname, aliases, + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public HostSpec(String hostname, List<String> aliases, Flavor flavor) { - this(hostname, aliases, Optional.of(flavor), Optional.empty()); + this(hostname, aliases, + flavor.resources(), flavor.resources(), NodeResources.unspecified(), + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public HostSpec(String hostname, List<String> aliases, ClusterMembership membership) { - this(hostname, aliases, Optional.empty(), Optional.of(membership)); + this(hostname, aliases, + NodeResources.unspecified(), NodeResources.unspecified(), NodeResources.unspecified(), + Optional.of(membership), + Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership) { - this(hostname, aliases, flavor, membership, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + NodeResources.unspecified(), + membership, Optional.empty(), Optional.empty(), Optional.empty()); } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version) { - this(hostname, aliases, flavor, membership, version, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + NodeResources.unspecified(), + membership, version, + Optional.empty(), Optional.empty()); } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version, Optional<NetworkPorts> networkPorts) { - this(hostname, aliases, flavor, membership, version, networkPorts, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + NodeResources.unspecified(), + membership, version, networkPorts, + Optional.empty()); } - public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, + // TODO: Remove when models older than 7.226 are gone + @Deprecated + public HostSpec(String hostname, List<String> aliases, + Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version, Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources) { - this(hostname, aliases, flavor, membership, version, networkPorts, requestedResources, Optional.empty()); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + requestedResources.orElse(NodeResources.unspecified()), + membership, version, networkPorts, Optional.empty()); } + // TODO: Remove when models older than 7.226 are gone + @Deprecated public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor, Optional<ClusterMembership> membership, Optional<Version> version, Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources, Optional<DockerImage> dockerImageRepo) { - if (hostname == null || hostname.isEmpty()) throw new IllegalArgumentException("Hostname must be specified"); - this.hostname = hostname; - this.aliases = List.copyOf(aliases); - this.flavor = flavor; - this.membership = membership; - this.version = Objects.requireNonNull(version, "Version cannot be null but can be empty"); - this.networkPorts = Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty"); - this.requestedResources = Objects.requireNonNull(requestedResources, "RequestedResources cannot be null"); - this.dockerImageRepo = Objects.requireNonNull(dockerImageRepo, "Docker image repo cannot be null but can be empty"); + this(hostname, aliases, + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + flavor.map(f -> f.resources()).orElse(NodeResources.unspecified()), + requestedResources.orElse(NodeResources.unspecified()), + membership, version, networkPorts, dockerImageRepo); } /** Returns the name identifying this host */ @@ -97,7 +191,17 @@ public class HostSpec implements Comparable<HostSpec> { /** Returns the aliases of this host as an immutable list. This may be empty but never null. */ public List<String> aliases() { return aliases; } - public Optional<Flavor> flavor() { return flavor; } + /** The real resources available for Vespa processes on this node, after subtracting infrastructure overhead. */ + public NodeResources realResources() { return realResources; } + + /** The total advertised resources of this node, typically matching what's requested. */ + public NodeResources advertisedResources() { return advertisedResources; } + + /** A flavor contained the advertised resources of this host */ + // TODO: Remove when models older than 7.226 are gone + public Optional<Flavor> flavor() { + return advertisedResources.asOptional().map(resources -> new Flavor(resources)); + } /** Returns the current version of Vespa running on this node, or empty if not known */ public Optional<com.yahoo.component.Version> version() { return version; } @@ -108,13 +212,13 @@ public class HostSpec implements Comparable<HostSpec> { /** Returns the network port allocations on this host, or empty if not present */ public Optional<NetworkPorts> networkPorts() { return networkPorts; } - /** Returns the requested resources leading to this host being provisioned, or empty if not known */ - public Optional<NodeResources> requestedResources() { return requestedResources; } + /** Returns the requested resources leading to this host being provisioned, or empty if unspecified */ + public Optional<NodeResources> requestedResources() { return requestedResources.asOptional(); } public Optional<DockerImage> dockerImageRepo() { return dockerImageRepo; } public HostSpec withPorts(Optional<NetworkPorts> ports) { - return new HostSpec(hostname, aliases, flavor, membership, version, ports, requestedResources, dockerImageRepo); + return new HostSpec(hostname, aliases, realResources, advertisedResources, requestedResources, membership, version, ports, dockerImageRepo); } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 18dc0b217a3..9cae0a08360 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -1,7 +1,9 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import java.util.Locale; import java.util.Objects; +import java.util.Optional; /** * The node resources required by an application cluster @@ -11,10 +13,11 @@ import java.util.Objects; public class NodeResources { // Standard unit cost in dollars per hour - private static final double cpuUnitCost = 12.0; - private static final double memoryUnitCost = 1.2; - private static final double diskUnitCost = 0.045; + private static final double cpuUnitCost = 0.12; + private static final double memoryUnitCost = 0.012; + private static final double diskUnitCost = 0.0004; + // TODO: Remove when models older than 7.226 are gone public static final NodeResources unspecified = new NodeResources(0, 0, 0, 0); public enum DiskSpeed { @@ -198,10 +201,10 @@ public class NodeResources { if (o == this) return true; if ( ! (o instanceof NodeResources)) return false; NodeResources other = (NodeResources)o; - if (this.vcpu != other.vcpu) return false; - if (this.memoryGb != other.memoryGb) return false; - if (this.diskGb != other.diskGb) return false; - if (this.bandwidthGbps != other.bandwidthGbps) return false; + if ( ! equal(this.vcpu, other.vcpu)) return false; + if ( ! equal(this.memoryGb, other.memoryGb)) return false; + if ( ! equal(this.diskGb, other.diskGb)) return false; + if ( ! equal(this.bandwidthGbps, other.bandwidthGbps)) return false; if (this.diskSpeed != other.diskSpeed) return false; if (this.storageType != other.storageType) return false; return true; @@ -214,7 +217,7 @@ public class NodeResources { @Override public String toString() { - return String.format("[vcpu: %1$.1f, memory: %2$.1f Gb, disk %3$.1f Gb" + + return String.format(Locale.ENGLISH, "[vcpu: %1$.1f, memory: %2$.1f Gb, disk %3$.1f Gb" + (bandwidthGbps > 0 ? ", bandwidth: %4$.1f Gbps" : "") + ( ! diskSpeed.isDefault() ? ", disk speed: " + diskSpeed : "") + ( ! storageType.isDefault() ? ", storage type: " + storageType : "") + "]", @@ -241,17 +244,28 @@ public class NodeResources { /** Returns true if all the resources of this are the same as or compatible with the given resources */ public boolean compatibleWith(NodeResources other) { - if (this.vcpu != other.vcpu) return false; - if (this.memoryGb != other.memoryGb) return false; - if (this.diskGb != other.diskGb) return false; - if (this.bandwidthGbps != other.bandwidthGbps) return false; + if ( ! equal(this.vcpu, other.vcpu)) return false; + if ( ! equal(this.memoryGb, other.memoryGb)) return false; + if ( ! equal(this.diskGb, other.diskGb)) return false; + if ( ! equal(this.bandwidthGbps, other.bandwidthGbps)) return false; if ( ! this.diskSpeed.compatibleWith(other.diskSpeed)) return false; if ( ! this.storageType.compatibleWith(other.storageType)) return false; return true; } - public boolean isUnspecified() { return this == unspecified; } + public static NodeResources unspecified() { return unspecified; } + + public boolean isUnspecified() { return this.equals(unspecified); } + + /** Returns this.isUnspecified() ? Optional.empty() : Optional.of(this) */ + public Optional<NodeResources> asOptional() { + return this.isUnspecified() ? Optional.empty() : Optional.of(this); + } + + private boolean equal(double a, double b) { + return Math.abs(a - b) < 0.00000001; + } /** * Create this from serial form. diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java index 5b6485de2b4..f616174de1a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java @@ -25,9 +25,13 @@ public class Zone { @Inject public Zone(ConfigserverConfig configserverConfig, NodeFlavors nodeFlavors, CloudConfig cloudConfig) { - this(new Cloud(CloudName.from(configserverConfig.cloud()), cloudConfig.dynamicProvisioning(), - cloudConfig.allowHostSharing(), cloudConfig.reprovisionToUpgradeOs(), - cloudConfig.requireAccessControl()), + this(Cloud.builder() + .name(CloudName.from(configserverConfig.cloud())) + .dynamicProvisioning(cloudConfig.dynamicProvisioning()) + .allowHostSharing(cloudConfig.allowHostSharing()) + .reprovisionToUpgradeOs(cloudConfig.reprovisionToUpgradeOs()) + .requireAccessControl(cloudConfig.requireAccessControl()) + .build(), SystemName.from(configserverConfig.system()), Environment.from(configserverConfig.environment()), RegionName.from(configserverConfig.region()), diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java b/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java index c85c00d8ebe..1eebf6388c7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java @@ -2,5 +2,4 @@ @ExportPackage package com.yahoo.config.provision; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index 9ba26be072c..96b189cefe6 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -45,6 +45,9 @@ public class AllocatedHostsSerializer { private static final String aliasesKey = "aliases"; private static final String hostSpecMembershipKey = "membership"; + private static final String realResourcesKey = "realResources"; + private static final String advertisedResourcesKey = "advertisedResources"; + // Flavor can be removed when all allocated nodes are docker nodes private static final String flavorKey = "flavor"; @@ -90,7 +93,9 @@ public class AllocatedHostsSerializer { object.setString(hostSpecDockerImageRepoKey, repo.repository()); }); }); - host.flavor().ifPresent(flavor -> toSlime(flavor, object)); + host.flavor().ifPresent(flavor -> toSlime(flavor, object)); // TODO: Remove this line after June 2020 + toSlime(host.realResources(), object.setObject(realResourcesKey)); + toSlime(host.advertisedResources(), object.setObject(advertisedResourcesKey)); host.requestedResources().ifPresent(resources -> toSlime(resources, object.setObject(requestedResourcesKey))); host.version().ifPresent(version -> object.setString(hostSpecCurrentVespaVersionKey, version.toFullString())); host.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray(hostSpecNetworkPortsKey))); @@ -133,18 +138,25 @@ public class AllocatedHostsSerializer { } private static HostSpec hostFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { - return new HostSpec(object.field(hostSpecHostNameKey).asString(), - aliasesFromSlime(object), - flavorFromSlime(object, nodeFlavors), - object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(), - optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new), - NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)), - nodeResourcesFromSlime(object.field(requestedResourcesKey)), - optionalDockerImage(object.field(hostSpecDockerImageRepoKey))); + if (object.field(hostSpecMembershipKey).valid()) { // Hosted + return new HostSpec(object.field(hostSpecHostNameKey).asString(), + nodeResourcesFromSlime(object.field(realResourcesKey), object, nodeFlavors), + nodeResourcesFromSlime(object.field(advertisedResourcesKey), object, nodeFlavors), + optionalNodeResourcesFromSlime(object.field(requestedResourcesKey)), // TODO: Make non-optional after June 2020 + membershipFromSlime(object), + optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new), + NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)), + optionalDockerImage(object.field(hostSpecDockerImageRepoKey))); + } + else { + return new HostSpec(object.field(hostSpecHostNameKey).asString(), + aliasesFromSlime(object), + NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey))); + } } private static List<String> aliasesFromSlime(Inspector object) { - if ( ! object.field(aliasesKey).valid()) return Collections.emptyList(); + if ( ! object.field(aliasesKey).valid()) return List.of(); List<String> aliases = new ArrayList<>(); object.field(aliasesKey).traverse((ArrayTraverser)(index, alias) -> aliases.add(alias.asString())); return aliases; @@ -154,17 +166,28 @@ public class AllocatedHostsSerializer { if (object.field(flavorKey).valid() && nodeFlavors.isPresent() && nodeFlavors.get().exists(object.field(flavorKey).asString())) return nodeFlavors.get().getFlavor(object.field(flavorKey).asString()); else - return nodeResourcesFromSlime(object.field(resourcesKey)).map(resources -> new Flavor(resources)); + return Optional.empty(); + } + + private static NodeResources nodeResourcesFromSlime(Inspector resources) { + return new NodeResources(resources.field(vcpuKey).asDouble(), + resources.field(memoryKey).asDouble(), + resources.field(diskKey).asDouble(), + resources.field(bandwidthKey).asDouble(), + diskSpeedFromSlime(resources.field(diskSpeedKey)), + storageTypeFromSlime(resources.field(storageTypeKey))); + } + + private static NodeResources optionalNodeResourcesFromSlime(Inspector resources) { + if ( ! resources.valid()) return NodeResources.unspecified(); + return nodeResourcesFromSlime(resources); } - private static Optional<NodeResources> nodeResourcesFromSlime(Inspector resources) { - if ( ! resources.valid()) return Optional.empty(); - return Optional.of(new NodeResources(resources.field(vcpuKey).asDouble(), - resources.field(memoryKey).asDouble(), - resources.field(diskKey).asDouble(), - resources.field(bandwidthKey).asDouble(), - diskSpeedFromSlime(resources.field(diskSpeedKey)), - storageTypeFromSlime(resources.field(storageTypeKey)))); + private static NodeResources nodeResourcesFromSlime(Inspector resources, Inspector parent, + Optional<NodeFlavors> nodeFlavors) { + if ( ! resources.valid()) // TODO: Remove the fallback using nodeFlavors after June 2020 + return flavorFromSlime(parent, nodeFlavors).map(f -> f.resources()).orElse(NodeResources.unspecified); + return nodeResourcesFromSlime(resources); } private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java index 5ef23c80eac..faa617430e9 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java @@ -28,6 +28,9 @@ public interface ZoneFilter { /** Zones where config servers are up and running. */ ZoneList reachable(); + /** Zones where hosts must be reprovisioned to upgrade their OS */ + ZoneList reprovisionToUpgradeOs(); + /** All zones from the initial pool. */ ZoneList all(); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java index b0ac59718aa..5795770c294 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java @@ -1,10 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision.zone; -import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; import java.util.Objects; diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java index 40ed7500269..be4179e8b03 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java @@ -53,10 +53,13 @@ public class ClusterSpecTest { } private static ClusterSpec spec(ClusterSpec.Type type, String id) { - return ClusterSpec.specification(type, ClusterSpec.Id.from(id)) - .group(ClusterSpec.Group.from(1)) - .vespaVersion(Version.emptyVersion) - .build(); + ClusterSpec.Builder builder = ClusterSpec.specification(type, ClusterSpec.Id.from(id)) + .group(ClusterSpec.Group.from(1)) + .vespaVersion(Version.emptyVersion); + if (type == ClusterSpec.Type.combined) { + builder = builder.combinedId(Optional.of(ClusterSpec.Id.from("combined"))); + } + return builder.build(); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java index 31eec77de6b..e184e85c399 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeFlavorsTest.java @@ -34,7 +34,7 @@ public class NodeFlavorsTest { NodeFlavors nodeFlavors = new NodeFlavors(config); Flavor banana = nodeFlavors.getFlavor("banana").get(); assertEquals(3, banana.cost()); - assertEquals(10, banana.getMinCpuCores(), delta); + assertEquals(13, banana.resources().vcpu(), delta); assertEquals("10 * 1.3", 13, banana.resources().vcpu(), delta); } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java index 33157a01c07..30810b79104 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java @@ -14,7 +14,6 @@ import com.yahoo.config.provisioning.FlavorsConfig; import org.junit.Test; import java.io.IOException; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; @@ -33,34 +32,44 @@ public class AllocatedHostsSerializerTest { public void testAllocatedHostsSerialization() throws IOException { NodeFlavors configuredFlavors = configuredFlavorsFrom("C/12/45/100", 12, 45, 100, 50, Flavor.Type.BARE_METAL); Set<HostSpec> hosts = new LinkedHashSet<>(); - hosts.add(new HostSpec("empty", - Optional.empty())); - hosts.add(new HostSpec("with-aliases", - List.of("alias1", "alias2"))); + hosts.add(new HostSpec("empty", List.of(), Optional.empty())); + hosts.add(new HostSpec("with-aliases", List.of("alias1", "alias2"), Optional.empty())); hosts.add(new HostSpec("allocated", - List.of(), + NodeResources.unspecified(), + NodeResources.unspecified(), + NodeResources.unspecified(), + ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))), + Optional.empty(), Optional.empty(), - Optional.of(ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), - Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar")))), - Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar")))); - hosts.add(new HostSpec("flavor-from-resources-1", - Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4, 1)))); hosts.add(new HostSpec("flavor-from-resources-2", - Collections.emptyList(), - Optional.of(new Flavor(new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.slow))), + new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.slow), + new NodeResources(1.0, 6.2, 8, 2, NodeResources.DiskSpeed.slow), + new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.any), + ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.empty()), Optional.empty(), Optional.empty(), - Optional.empty(), - Optional.of(new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.any)))); - hosts.add(new HostSpec("configured-flavor", - Collections.emptyList(), configuredFlavors.getFlavorOrThrow("C/12/45/100"))); + Optional.empty())); hosts.add(new HostSpec("with-version", - Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.of(Version.fromString("3.4.5")))); + NodeResources.unspecified(), + NodeResources.unspecified(), + NodeResources.unspecified(), + ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.empty()), + Optional.of(Version.fromString("3.4.5")), + Optional.empty(), Optional.empty())); hosts.add(new HostSpec("with-ports", - Collections.emptyList(), Optional.empty(), Optional.empty(), Optional.empty(), + NodeResources.unspecified(), + NodeResources.unspecified(), + NodeResources.unspecified(), + ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.empty()), + Optional.empty(), Optional.of(new NetworkPorts(List.of(new NetworkPorts.Allocation(1234, "service1", "configId1", "suffix1"), - new NetworkPorts.Allocation(4567, "service2", "configId2", "suffix2")))))); + new NetworkPorts.Allocation(4567, "service2", "configId2", "suffix2")))), + Optional.empty())); assertAllocatedHosts(AllocatedHosts.withHosts(hosts), configuredFlavors); } @@ -73,11 +82,12 @@ public class AllocatedHostsSerializerTest { HostSpec deserializedHost = requireHost(expectedHost.hostname(), deserializedHosts); assertEquals(expectedHost.hostname(), deserializedHost.hostname()); assertEquals(expectedHost.membership(), deserializedHost.membership()); - assertEquals(expectedHost.flavor(), deserializedHost.flavor()); + assertEquals(expectedHost.realResources(), deserializedHost.realResources()); + assertEquals(expectedHost.advertisedResources(), deserializedHost.advertisedResources()); + assertEquals(expectedHost.requestedResources(), deserializedHost.requestedResources()); assertEquals(expectedHost.version(), deserializedHost.version()); assertEquals(expectedHost.networkPorts(), deserializedHost.networkPorts()); assertEquals(expectedHost.aliases(), deserializedHost.aliases()); - assertEquals(expectedHost.requestedResources(), deserializedHost.requestedResources()); assertEquals(expectedHost.dockerImageRepo(), deserializedHost.dockerImageRepo()); } } diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index 604ddd40930..3602c2b0fd7 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -31,8 +31,13 @@ attribute[].tensortype string default="" # Whether this is an imported attribute (from parent document db) or not. attribute[].imported bool default=false +# The distance metric to use for nearest neighbor search. +# Is only used when the attribute is a 1-dimensional indexed tensor. +attribute[].distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES } default=EUCLIDEAN + # Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. attribute[].index.hnsw.enabled bool default=false attribute[].index.hnsw.maxlinkspernode int default=16 +# Deprecated: Remove on Vespa 8 or before when possible. attribute[].index.hnsw.distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES } default=EUCLIDEAN attribute[].index.hnsw.neighborstoexploreatinsert int default=200 diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index e6fef0c454b..88a637e3ecb 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -64,6 +64,9 @@ sleepTimeWhenRedeployingFails long default=30 # Features (to be overridden in configserver-config.xml if needed) buildMinimalSetOfConfigModels bool default=true throwIfBootstrappingTenantRepoFails bool default=true + +# Unused, remove in Vespa 8 throwIfActiveSessionCannotBeLoaded bool default=true + canReturnEmptySentinelConfig bool default=false serverNodeType enum {config, controller} default=config diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index 0776e648ad7..aa40c317d75 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -14,7 +14,7 @@ minGroupCoverage double default=100 maxNodesDownPerGroup int default=0 # Distribution policy for group selection -distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ROUNDROBIN +distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ADAPTIVE ## Maximum number of hits that will be requested from a single node ## in this dataset. If not set, there is no limit. Using this option @@ -29,8 +29,8 @@ maxHitsPerNode int default=2147483647 ## will give you the globally K best hits according to this formula with the desired probability. ## q = k/n + qT (p',30) x √(k × (1/n) × (1 − 1/n)) ## With a probability of 0.999 and K=200 and N=10 will give a Q of 38, meaning that you only need to fetch 19% compared to -## default setting of 1.0. This is a significant optimisation with with very little loss in presicion. -topKProbability double default=1.0 +## a setting of 1.0. This is a significant optimisation with with very little loss in presicion. +topKProbability double default=0.9999 # Is multi-level dispatch configured for this cluster # Deprecated, will go away soon, NOOP diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def index 80ddc0931f4..5335f4f162f 100644 --- a/configdefinitions/src/vespa/stor-filestor.def +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -29,7 +29,7 @@ num_threads int default=8 restart ## Number of threads for response processing and delivery ## 0 will give legacy sync behavior. ## Negative number will choose a good number based on # cores. -num_response_threads int default=0 +num_response_threads int default=1 ## When merging, if we find more than this number of documents that exist on all ## of the same copies, send a separate apply bucket diff with these entries diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 40f223b47d3..c44a839d24d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.common.io.Files; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; @@ -24,7 +23,6 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; @@ -51,14 +49,14 @@ import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.metrics.ApplicationMetricsRetriever; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.RemoteSession; -import com.yahoo.vespa.config.server.session.RemoteSessionRepo; import com.yahoo.vespa.config.server.session.Session; -import com.yahoo.vespa.config.server.session.SessionFactory; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.config.server.session.SilentDeployLogger; +import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Lock; @@ -69,24 +67,29 @@ import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; import java.net.URI; +import java.nio.file.Files; import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; +import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk; import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT; +import static com.yahoo.yolean.Exceptions.uncheck; import static java.nio.file.Files.readAttributes; /** @@ -212,27 +215,18 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Slime deployLog = createDeployLog(); DeployLogger logger = new DeployHandlerLogger(deployLog.get().setArray("log"), prepareParams.isVerbose(), applicationId); try (ActionTimer timer = timerFor(applicationId, "deployment.prepareMillis")) { - ConfigChangeActions actions = session.prepare(logger, prepareParams, currentActiveApplicationSet, tenant.getPath(), now); + SessionRepository sessionRepository = tenant.getSessionRepository(); + ConfigChangeActions actions = sessionRepository.prepareLocalSession(session, logger, prepareParams, + currentActiveApplicationSet, tenant.getPath(), now); logConfigChangeActions(actions, logger); log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); return new PrepareResult(sessionId, actions, deployLog); } } - public PrepareResult prepareAndActivate(Tenant tenant, long sessionId, PrepareParams prepareParams, - boolean ignoreSessionStaleFailure, Instant now) { - PrepareResult result = prepare(tenant, sessionId, prepareParams, now); - activate(tenant, sessionId, prepareParams.getTimeoutBudget(), ignoreSessionStaleFailure); - return result; - } - - public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams) { - return deploy(in, prepareParams, false, clock.instant()); - } - public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams, boolean ignoreSessionStaleFailure, Instant now) { - File tempDir = Files.createTempDir(); + File tempDir = uncheck(() -> Files.createTempDirectory("deploy")).toFile(); PrepareResult prepareResult; try { prepareResult = deploy(decompressApplication(in, tempDir), prepareParams, ignoreSessionStaleFailure, now); @@ -251,7 +245,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye ApplicationId applicationId = prepareParams.getApplicationId(); long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); - return prepareAndActivate(tenant, sessionId, prepareParams, ignoreSessionStaleFailure, now); + PrepareResult result = prepare(tenant, sessionId, prepareParams, now); + activate(tenant, sessionId, prepareParams.getTimeoutBudget(), ignoreSessionStaleFailure); + return result; } /** @@ -306,11 +302,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Tenant tenant = tenantRepository.getTenant(application.tenant()); if (tenant == null) return Optional.empty(); - LocalSession activeSession = getActiveSession(tenant, application); + LocalSession activeSession = getActiveLocalSession(tenant, application); if (activeSession == null) return Optional.empty(); TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - LocalSession newSession = tenant.getSessionFactory().createSessionFromExisting(activeSession, logger, true, timeoutBudget); - tenant.getLocalSessionRepo().addSession(newSession); + LocalSession newSession = tenant.getSessionRepository().createSessionFromExisting(activeSession, logger, true, timeoutBudget); + tenant.getSessionRepository().addSession(newSession); return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock, false /* don't validate as this is already deployed */, bootstrap)); @@ -320,9 +316,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public Optional<Instant> lastDeployTime(ApplicationId application) { Tenant tenant = tenantRepository.getTenant(application.tenant()); if (tenant == null) return Optional.empty(); - LocalSession activeSession = getActiveSession(tenant, application); + RemoteSession activeSession = getActiveSession(tenant, application); if (activeSession == null) return Optional.empty(); - return Optional.of(Instant.ofEpochSecond(activeSession.getCreateTime())); + return Optional.of(activeSession.getCreateTime()); } public ApplicationId activate(Tenant tenant, @@ -394,6 +390,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye NestedTransaction transaction = new NestedTransaction(); transaction.add(new ContainerEndpointsCache(tenant.getPath(), tenant.getCurator()).delete(applicationId)); // TODO: Not unit tested + // Delete any application roles + transaction.add(new ApplicationRolesStore(tenant.getCurator(), tenant.getPath()).delete(applicationId)); + // Delete endpoint certificates + transaction.add(new EndpointCertificateMetadataStore(tenant.getCurator(), tenant.getPath()).delete(applicationId)); // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server // this is done asynchronously in application maintenance by the node repository) transaction.add(tenantApplications.createDeleteTransaction(applicationId)); @@ -438,24 +438,21 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Set<String> fileReferencesInUse = new HashSet<>(); // Intentionally skip applications that we for some reason do not find // or that we fail to get file references for (they will be retried on the next run) - for (var application : listApplications()) { + for (var applicationId : listApplications()) { try { - Optional<Application> app = getOptionalApplication(application); + Optional<Application> app = getOptionalApplication(applicationId); if (app.isEmpty()) continue; fileReferencesInUse.addAll(app.get().getModel().fileReferences().stream() .map(FileReference::value) .collect(Collectors.toSet())); } catch (Exception e) { - log.log(Level.WARNING, "Getting file references in use for '" + application + "' failed", e); + log.log(Level.WARNING, "Getting file references in use for '" + applicationId + "' failed", e); } } log.log(Level.FINE, "File references in use : " + fileReferencesInUse); // Find those on disk that are not in use - Set<String> fileReferencesOnDisk = new HashSet<>(); - File[] filesOnDisk = fileReferencesPath.listFiles(); - if (filesOnDisk != null) - fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet())); + Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath); log.log(Level.FINE, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); Instant instant = Instant.now().minus(keepFileReferences); @@ -493,7 +490,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); long sessionId = getSessionIdForApplication(tenant, applicationId); - RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId); + RemoteSession session = tenant.getSessionRepo().getRemoteSession(sessionId); if (session == null) throw new NotFoundException("Remote session " + sessionId + " not found"); return session.ensureApplicationLoaded().getForVersionOrLatest(version, clock.instant()); } catch (NotFoundException e) { @@ -505,6 +502,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } + // Will return Optional.empty() if getting application fails (instead of throwing an exception) private Optional<Application> getOptionalApplication(ApplicationId applicationId) { try { return Optional.of(getApplication(applicationId)); @@ -513,7 +511,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - Set<ApplicationId> listApplications() { + public Set<ApplicationId> listApplications() { return tenantRepository.getAllTenants().stream() .flatMap(tenant -> tenant.getApplicationRepo().activeApplications().stream()) .collect(Collectors.toSet()); @@ -530,10 +528,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private boolean localSessionHasBeenDeleted(ApplicationId applicationId, long sessionId, Duration waitTime) { - RemoteSessionRepo remoteSessionRepo = tenantRepository.getTenant(applicationId.tenant()).getRemoteSessionRepo(); + SessionRepository sessionRepository = tenantRepository.getTenant(applicationId.tenant()).getSessionRepo(); Instant end = Instant.now().plus(waitTime); do { - if (remoteSessionRepo.getSession(sessionId) == null) return true; + if (sessionRepository.getRemoteSession(sessionId) == null) return true; try { Thread.sleep(10); } catch (InterruptedException e) { /* ignored */} } while (Instant.now().isBefore(end)); @@ -559,7 +557,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return logRetriever.getLogs(logServerURI); } - // ---------------- Methods to do call against tester containers in hosted ------------------------------ public HttpResponse getTesterStatus(ApplicationId applicationId) { @@ -604,7 +601,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * * @return the active session, or null if there is no active session for the given application id. */ - public LocalSession getActiveSession(ApplicationId applicationId) { + public RemoteSession getActiveSession(ApplicationId applicationId) { return getActiveSession(tenantRepository.getTenant(applicationId.tenant()), applicationId); } @@ -640,16 +637,15 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye boolean internalRedeploy, TimeoutBudget timeoutBudget) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); - LocalSessionRepo localSessionRepo = tenant.getLocalSessionRepo(); - SessionFactory sessionFactory = tenant.getSessionFactory(); - LocalSession fromSession = getExistingSession(tenant, applicationId); - LocalSession session = sessionFactory.createSessionFromExisting(fromSession, logger, internalRedeploy, timeoutBudget); - localSessionRepo.addSession(session); + SessionRepository sessionRepository = tenant.getSessionRepository(); + RemoteSession fromSession = getExistingSession(tenant, applicationId); + LocalSession session = sessionRepository.createSessionFromExisting(fromSession, logger, internalRedeploy, timeoutBudget); + sessionRepository.addSession(session); return session.getSessionId(); } public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, InputStream in, String contentType) { - File tempDir = Files.createTempDir(); + File tempDir = uncheck(() -> Files.createTempDirectory("deploy")).toFile(); long sessionId; try { sessionId = createSession(applicationId, timeoutBudget, decompressApplication(in, contentType, tempDir)); @@ -662,15 +658,29 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File applicationDirectory) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); tenant.getApplicationRepo().createApplication(applicationId); - LocalSessionRepo localSessionRepo = tenant.getLocalSessionRepo(); - SessionFactory sessionFactory = tenant.getSessionFactory(); - LocalSession session = sessionFactory.createSession(applicationDirectory, applicationId, timeoutBudget); - localSessionRepo.addSession(session); + Optional<Long> activeSessionId = tenant.getApplicationRepo().activeSessionOf(applicationId); + LocalSession session = tenant.getSessionRepository().createSession(applicationDirectory, + applicationId, + timeoutBudget, + activeSessionId); + tenant.getSessionRepository().addSession(session); return session.getSessionId(); } public void deleteExpiredLocalSessions() { - tenantRepository.getAllTenants().forEach(tenant -> tenant.getLocalSessionRepo().purgeOldSessions()); + Map<Tenant, List<LocalSession>> sessionsPerTenant = new HashMap<>(); + tenantRepository.getAllTenants().forEach(tenant -> sessionsPerTenant.put(tenant, tenant.getSessionRepository().getLocalSessions())); + + Set<ApplicationId> applicationIds = new HashSet<>(); + sessionsPerTenant.values().forEach(sessionList -> sessionList.forEach(s -> applicationIds.add(s.getApplicationId()))); + + Map<ApplicationId, Long> activeSessions = new HashMap<>(); + applicationIds.forEach(applicationId -> { + RemoteSession activeSession = getActiveSession(applicationId); + if (activeSession != null) + activeSessions.put(applicationId, activeSession.getSessionId()); + }); + sessionsPerTenant.keySet().forEach(tenant -> tenant.getSessionRepository().deleteExpiredSessions(activeSessions)); } public int deleteExpiredRemoteSessions(Duration expiryTime) { @@ -680,7 +690,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { return tenantRepository.getAllTenants() .stream() - .map(tenant -> tenant.getRemoteSessionRepo().deleteExpiredSessions(clock, expiryTime)) + .map(tenant -> tenant.getSessionRepo().deleteExpiredRemoteSessions(clock, expiryTime)) .mapToInt(i -> i) .sum(); } @@ -724,7 +734,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Misc operations ---------------------------------------------------------------- - public ApplicationMetaData getMetadataFromSession(Tenant tenant, long sessionId) { + public ApplicationMetaData getMetadataFromLocalSession(Tenant tenant, long sessionId) { return getLocalSession(tenant, sessionId).getMetaData(); } @@ -732,6 +742,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return configserverConfig; } + public ApplicationId getApplicationIdForHostname(String hostname) { + Optional<ApplicationId> applicationId = tenantRepository.getAllTenantNames().stream() + .map(tenantName -> tenantRepository.getTenant(tenantName).getApplicationRepo().getApplicationIdForHostName(hostname)) + .filter(Objects::nonNull) + .findFirst(); + return applicationId.orElse(null); + } + private void validateThatLocalSessionIsNotActive(Tenant tenant, long sessionId) { LocalSession session = getLocalSession(tenant, sessionId); if (Session.Status.ACTIVATE.equals(session.getStatus())) { @@ -740,28 +758,26 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private LocalSession getLocalSession(Tenant tenant, long sessionId) { - LocalSession session = tenant.getLocalSessionRepo().getSession(sessionId); + LocalSession session = tenant.getSessionRepository().getLocalSession(sessionId); if (session == null) throw new NotFoundException("Session " + sessionId + " was not found"); return session; } private RemoteSession getRemoteSession(Tenant tenant, long sessionId) { - RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId); + RemoteSession session = tenant.getSessionRepo().getRemoteSession(sessionId); if (session == null) throw new NotFoundException("Session " + sessionId + " was not found"); return session; } - private Optional<ApplicationSet> getCurrentActiveApplicationSet(Tenant tenant, ApplicationId appId) { + public Optional<ApplicationSet> getCurrentActiveApplicationSet(Tenant tenant, ApplicationId appId) { Optional<ApplicationSet> currentActiveApplicationSet = Optional.empty(); TenantApplications applicationRepo = tenant.getApplicationRepo(); try { long currentActiveSessionId = applicationRepo.requireActiveSessionOf(appId); RemoteSession currentActiveSession = getRemoteSession(tenant, currentActiveSessionId); - if (currentActiveSession != null) { - currentActiveApplicationSet = Optional.ofNullable(currentActiveSession.ensureApplicationLoaded()); - } + currentActiveApplicationSet = Optional.ofNullable(currentActiveSession.ensureApplicationLoaded()); } catch (IllegalArgumentException e) { // Do nothing if we have no currently active session } @@ -792,15 +808,23 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - private LocalSession getExistingSession(Tenant tenant, ApplicationId applicationId) { + private RemoteSession getExistingSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); - return getLocalSession(tenant, applicationRepo.requireActiveSessionOf(applicationId)); + return getRemoteSession(tenant, applicationRepo.requireActiveSessionOf(applicationId)); + } + + private RemoteSession getActiveSession(Tenant tenant, ApplicationId applicationId) { + TenantApplications applicationRepo = tenant.getApplicationRepo(); + if (applicationRepo.activeApplications().contains(applicationId)) { + return tenant.getSessionRepo().getRemoteSession(applicationRepo.requireActiveSessionOf(applicationId)); + } + return null; } - private LocalSession getActiveSession(Tenant tenant, ApplicationId applicationId) { + public LocalSession getActiveLocalSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); if (applicationRepo.activeApplications().contains(applicationId)) { - return tenant.getLocalSessionRepo().getSession(applicationRepo.requireActiveSessionOf(applicationId)); + return tenant.getSessionRepository().getLocalSession(applicationRepo.requireActiveSessionOf(applicationId)); } return null; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java index 42c80acd80d..657e113475b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java @@ -16,7 +16,6 @@ import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.server.model.SuperModelConfigProvider; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; -import java.io.IOException; import java.io.StringReader; /** @@ -74,7 +73,7 @@ public class SuperModelController { long getGeneration() { return generation; } - public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) throws IOException { + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) { return model.getConfig(configClass, applicationId, configId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java index 6adbcc8dae9..6d5c4a81c92 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java @@ -2,11 +2,9 @@ package com.yahoo.vespa.config.server; import com.yahoo.path.Path; -import com.yahoo.transaction.AbstractTransaction; -import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.GenerationCounter; -import com.yahoo.vespa.curator.recipes.CuratorCounter; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.recipes.CuratorCounter; /** * Distributed global generation counter for the super model. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index 6fcfde80510..aa06c07f8af 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -3,21 +3,20 @@ package com.yahoo.vespa.config.server; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; import com.yahoo.config.ConfigInstance; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.component.Version; -import java.util.logging.Level; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; -import java.io.IOException; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; /** * Handles request for supermodel config. @@ -84,7 +83,7 @@ public class SuperModelRequestHandler implements RequestHandler { return null; } - public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) throws IOException { + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) { return handler.getConfig(configClass, applicationId, configId); } @@ -128,4 +127,5 @@ public class SuperModelRequestHandler implements RequestHandler { superModelManager.markAsComplete(); updateHandler(); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index eeb4d31fe12..dae5c6aecef 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -2,7 +2,11 @@ package com.yahoo.vespa.config.server.application; import com.google.common.io.ByteStreams; -import com.google.common.io.Files; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.logging.Level; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.InternalServerException; @@ -12,10 +16,11 @@ import org.apache.commons.compress.archivers.ArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; -import java.io.*; import java.util.logging.Logger; import java.util.zip.GZIPInputStream; +import static com.yahoo.yolean.Exceptions.uncheck; + /** * A compressed application points to an application package that can be decompressed. * @@ -74,7 +79,7 @@ public class CompressedApplicationInputStream implements AutoCloseable { } File decompress() throws IOException { - return decompress(Files.createTempDir()); + return decompress(uncheck(() -> java.nio.file.Files.createTempDirectory("decompress")).toFile()); } public File decompress(File dir) throws IOException { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 4f04724a0a8..795c6398354 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -1,16 +1,29 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; import com.yahoo.concurrent.StripedExecutor; +import com.yahoo.config.FileReference; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.NotFoundException; import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.vespa.config.server.ReloadListener; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.host.HostValidator; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; @@ -19,15 +32,23 @@ import com.yahoo.vespa.curator.transaction.CuratorTransaction; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Clock; import java.time.Duration; +import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static java.util.stream.Collectors.toSet; + /** * The applications of a tenant, backed by ZooKeeper. * @@ -38,7 +59,7 @@ import java.util.stream.Collectors; * @author Ulf Lilleengen * @author jonmv */ -public class TenantApplications { +public class TenantApplications implements RequestHandler, ReloadHandler, HostValidator<ApplicationId> { private static final Logger log = Logger.getLogger(TenantApplications.class.getName()); @@ -46,24 +67,48 @@ public class TenantApplications { private final Path applicationsPath; private final Path locksPath; private final Curator.DirectoryCache directoryCache; - private final ReloadHandler reloadHandler; private final Executor zkWatcherExecutor; + private final Metrics metrics; + private final TenantName tenant; + private final ReloadListener reloadListener; + private final ConfigResponseFactory responseFactory; + private final HostRegistry<ApplicationId> hostRegistry; + private final ApplicationMapper applicationMapper = new ApplicationMapper(); + private final MetricUpdater tenantMetricUpdater; + private final Clock clock = Clock.systemUTC(); + private final TenantFileSystemDirs tenantFileSystemDirs; - private TenantApplications(Curator curator, ReloadHandler reloadHandler, TenantName tenant, - ExecutorService zkCacheExecutor, StripedExecutor<TenantName> zkWatcherExecutor) { + public TenantApplications(TenantName tenant, Curator curator, StripedExecutor<TenantName> zkWatcherExecutor, + ExecutorService zkCacheExecutor, Metrics metrics, ReloadListener reloadListener, + ConfigserverConfig configserverConfig, HostRegistry<ApplicationId> hostRegistry, + TenantFileSystemDirs tenantFileSystemDirs) { this.curator = curator; this.applicationsPath = TenantRepository.getApplicationsPath(tenant); this.locksPath = TenantRepository.getLocksPath(tenant); - this.reloadHandler = reloadHandler; + this.tenant = tenant; this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenant, command); this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, zkCacheExecutor); this.directoryCache.start(); this.directoryCache.addListener(this::childEvent); + this.metrics = metrics; + this.reloadListener = reloadListener; + this.responseFactory = ConfigResponseFactory.create(configserverConfig); + this.tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); + this.hostRegistry = hostRegistry; + this.tenantFileSystemDirs = tenantFileSystemDirs; } - public static TenantApplications create(GlobalComponentRegistry registry, ReloadHandler reloadHandler, TenantName tenant) { - return new TenantApplications(registry.getCurator(), reloadHandler, tenant, - registry.getZkCacheExecutor(), registry.getZkWatcherExecutor()); + // For testing only + public static TenantApplications create(GlobalComponentRegistry componentRegistry, TenantName tenantName) { + return new TenantApplications(tenantName, + componentRegistry.getCurator(), + componentRegistry.getZkWatcherExecutor(), + componentRegistry.getZkCacheExecutor(), + componentRegistry.getMetrics(), + componentRegistry.getReloadListener(), + componentRegistry.getConfigserverConfig(), + componentRegistry.getHostRegistries().createApplicationHostRegistry(tenantName), + new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName)); } /** @@ -86,10 +131,14 @@ public class TenantApplications { /** Returns the id of the currently active session for the given application, if any. Throws on unknown applications. */ public Optional<Long> activeSessionOf(ApplicationId id) { String data = curator.getData(applicationPath(id)).map(Utf8::toString) - .orElseThrow(() -> new NotFoundException("No such application id: '" + id + "'")); + .orElseThrow(() -> new NotFoundException("No active session found for application id: '" + id + "'")); return data.isEmpty() ? Optional.empty() : Optional.of(Long.parseLong(data)); } + public boolean hasLocalSession(long sessionId) { + return Files.exists(Paths.get(tenantFileSystemDirs.sessionsPath().getAbsolutePath(), String.valueOf(sessionId))); + } + /** * Returns a transaction which writes the given session id as the currently active for the given application. * @@ -132,7 +181,7 @@ public class TenantApplications { * Removes all applications not known to this from the config server state. */ public void removeUnusedApplications() { - reloadHandler.removeApplicationsExcept(Set.copyOf(activeApplications())); + removeApplicationsExcept(Set.copyOf(activeApplications())); } /** @@ -164,13 +213,13 @@ public class TenantApplications { break; } // We may have lost events and may need to remove applications. - // New applications are added when session is added, not here. See RemoteSessionRepo. + // New applications are added when session is added, not here. See SessionRepository. removeUnusedApplications(); }); } private void applicationRemoved(ApplicationId applicationId) { - reloadHandler.removeApplication(applicationId); + removeApplication(applicationId); log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Application removed: " + applicationId); } @@ -186,4 +235,201 @@ public class TenantApplications { return locksPath.append(id.serializedForm()); } + + /** + * Gets a config for the given app, or null if not found + */ + @Override + public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); + } + return application.resolveConfig(req, responseFactory); + } + + private void notifyReloadListeners(ApplicationSet applicationSet) { + reloadListener.hostsUpdated(tenant, hostRegistry.getAllHosts()); + reloadListener.configActivated(applicationSet); + } + + /** + * Activates the config of the given app. Notifies listeners + * + * @param applicationSet the {@link ApplicationSet} to be reloaded + */ + @Override + public void reloadConfig(ApplicationSet applicationSet) { + ApplicationId id = applicationSet.getId(); + try (Lock lock = lock(id)) { + if ( ! exists(id)) + return; // Application was deleted before activation. + if (applicationSet.getApplicationGeneration() != requireActiveSessionOf(id)) + return; // Application activated a new session before we got here. + + setLiveApp(applicationSet); + notifyReloadListeners(applicationSet); + } + } + + @Override + public void removeApplication(ApplicationId applicationId) { + try (Lock lock = lock(applicationId)) { + if (exists(applicationId)) + return; // Application was deployed again. + + if (applicationMapper.hasApplication(applicationId, clock.instant())) { + applicationMapper.remove(applicationId); + hostRegistry.removeHostsForKey(applicationId); + reloadListenersOnRemove(applicationId); + tenantMetricUpdater.setApplications(applicationMapper.numApplications()); + metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); + } + } + } + + @Override + public void removeApplicationsExcept(Set<ApplicationId> applications) { + for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { + if ( ! applications.contains(activeApplication)) { + log.log(Level.INFO, "Will remove deleted application " + activeApplication.toShortString()); + removeApplication(activeApplication); + } + } + } + + private void reloadListenersOnRemove(ApplicationId applicationId) { + reloadListener.hostsUpdated(tenant, hostRegistry.getAllHosts()); + reloadListener.applicationRemoved(applicationId); + } + + private void setLiveApp(ApplicationSet applicationSet) { + ApplicationId id = applicationSet.getId(); + Collection<String> hostsForApp = applicationSet.getAllHosts(); + hostRegistry.update(id, hostsForApp); + applicationSet.updateHostMetrics(); + tenantMetricUpdater.setApplications(applicationMapper.numApplications()); + applicationMapper.register(id, applicationSet); + } + + @Override + public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> keyToMatch, boolean recursive) { + Application application = getApplication(appId, vespaVersion); + return listConfigs(application, keyToMatch, recursive); + } + + private Set<ConfigKey<?>> listConfigs(Application application, ConfigKey<?> keyToMatch, boolean recursive) { + Set<ConfigKey<?>> ret = new LinkedHashSet<>(); + for (ConfigKey<?> key : application.allConfigsProduced()) { + String configId = key.getConfigId(); + if (recursive) { + key = new ConfigKey<>(key.getName(), configId, key.getNamespace()); + } else { + // Include first part of id as id + key = new ConfigKey<>(key.getName(), configId.split("/")[0], key.getNamespace()); + } + if (keyToMatch != null) { + String n = key.getName(); // Never null + String ns = key.getNamespace(); // Never null + if (n.equals(keyToMatch.getName()) && + ns.equals(keyToMatch.getNamespace()) && + configId.startsWith(keyToMatch.getConfigId()) && + !(configId.equals(keyToMatch.getConfigId()))) { + + if (!recursive) { + // For non-recursive, include the id segment we were searching for, and first part of the rest + key = new ConfigKey<>(key.getName(), appendOneLevelOfId(keyToMatch.getConfigId(), configId), key.getNamespace()); + } + ret.add(key); + } + } else { + ret.add(key); + } + } + return ret; + } + + @Override + public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive) { + Application application = getApplication(appId, vespaVersion); + return listConfigs(application, null, recursive); + } + + /** + * Given baseIdSegment search/ and id search/qrservers/default.0, return search/qrservers + * @return id segment with one extra level from the id appended + */ + String appendOneLevelOfId(String baseIdSegment, String id) { + if ("".equals(baseIdSegment)) return id.split("/")[0]; + String theRest = id.substring(baseIdSegment.length()); + if ("".equals(theRest)) return id; + theRest = theRest.replaceFirst("/", ""); + String theRestFirstSeg = theRest.split("/")[0]; + return baseIdSegment+"/"+theRestFirstSeg; + } + + @Override + public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + return application.allConfigsProduced(); + } + + private Application getApplication(ApplicationId appId, Optional<Version> vespaVersion) { + try { + return applicationMapper.getForVersion(appId, vespaVersion, clock.instant()); + } catch (VersionDoesNotExistException ex) { + throw new NotFoundException(String.format("%sNo such application (id %s): %s", TenantRepository.logPre(tenant), appId, ex.getMessage())); + } + } + + @Override + public Set<String> allConfigIds(ApplicationId appId, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + return application.allConfigIds(); + } + + @Override + public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { + return hasHandler(appId, vespaVersion); + } + + private boolean hasHandler(ApplicationId appId, Optional<Version> vespaVersion) { + return applicationMapper.hasApplicationForVersion(appId, vespaVersion, clock.instant()); + } + + @Override + public ApplicationId resolveApplicationId(String hostName) { + ApplicationId applicationId = hostRegistry.getKeyForHost(hostName); + if (applicationId == null) { + applicationId = ApplicationId.defaultId(); + } + return applicationId; + } + + @Override + public Set<FileReference> listFileReferences(ApplicationId applicationId) { + return applicationMapper.listApplications(applicationId).stream() + .flatMap(app -> app.getModel().fileReferences().stream()) + .collect(toSet()); + } + + @Override + public void verifyHosts(ApplicationId key, Collection<String> newHosts) { + hostRegistry.verifyHosts(key, newHosts); + reloadListener.verifyHostsAreAvailable(tenant, newHosts); + } + + public HostValidator<ApplicationId> getHostValidator() { + return this; + } + + public HostRegistry<ApplicationId> getApplicationHostRegistry() { + return hostRegistry; + } + + public ApplicationId getApplicationIdForHostName(String hostname) { + return hostRegistry.getKeyForHost(hostname); + } + + public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index af4e85109e6..def629f738c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -15,9 +15,11 @@ import com.yahoo.vespa.config.server.ActivationConflictException; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.Tenant; @@ -30,7 +32,7 @@ import java.util.logging.Logger; /** * The process of deploying an application. - * Deployments are created by a {@link ApplicationRepository}. + * Deployments are created by an {@link ApplicationRepository}. * Instances of this are not multithread safe. * * @author Ulf Lilleengen @@ -117,7 +119,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { .isBootstrap(isBootstrap); dockerImageRepository.ifPresent(params::dockerImageRepository); athenzDomain.ifPresent(params::athenzDomain); - session.prepare(logger, params.build(), Optional.empty(), tenant.getPath(), clock.instant()); + Optional<ApplicationSet> activeApplicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, session.getApplicationId()); + tenant.getSessionRepository().prepareLocalSession(session, logger, params.build(), activeApplicationSet, + tenant.getPath(), clock.instant()); this.prepared = true; } } @@ -130,9 +134,11 @@ public class Deployment implements com.yahoo.config.provision.Deployment { try (ActionTimer timer = applicationRepository.timerFor(session.getApplicationId(), "deployment.activateMillis")) { TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - ApplicationId applicationId = session.getApplicationId(); - LocalSession previousActiveSession; + + if ( ! timeoutBudget.hasTimeLeft()) throw new RuntimeException("Timeout exceeded when trying to activate '" + applicationId + "'"); + + RemoteSession previousActiveSession; try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { validateSessionStatus(session); NestedTransaction transaction = new NestedTransaction(); @@ -149,13 +155,11 @@ public class Deployment implements com.yahoo.config.provision.Deployment { } session.waitUntilActivated(timeoutBudget); - - log.log(Level.INFO, session.logPre() + "Session " + session.getSessionId() + - " activated successfully using " + - (hostProvisioner.isPresent() ? hostProvisioner.get().getClass().getSimpleName() : "no host provisioner") + - ". Config generation " + session.getMetaData().getGeneration() + - (previousActiveSession != null ? ". Activated session based on previous active session " + previousActiveSession.getSessionId() : "") + - ". File references used: " + applicationRepository.getFileReferences(applicationId)); + log.log(Level.INFO, session.logPre() + "Session " + session.getSessionId() + " activated successfully using " + + hostProvisioner.map(provisioner -> provisioner.getClass().getSimpleName()).orElse("no host provisioner") + + ". Config generation " + session.getMetaData().getGeneration() + + (previousActiveSession != null ? ". Based on previous active session " + previousActiveSession.getSessionId() : "") + + ". File references: " + applicationRepository.getFileReferences(applicationId)); } } @@ -181,7 +185,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { } } - private Transaction deactivateCurrentActivateNew(LocalSession active, LocalSession prepared, boolean ignoreStaleSessionFailure) { + private static Transaction deactivateCurrentActivateNew(Session active, LocalSession prepared, boolean ignoreStaleSessionFailure) { Transaction transaction = prepared.createActivateTransaction(); if (isValidSession(active)) { checkIfActiveHasChanged(prepared, active, ignoreStaleSessionFailure); @@ -191,11 +195,11 @@ public class Deployment implements com.yahoo.config.provision.Deployment { return transaction; } - private boolean isValidSession(LocalSession session) { + private static boolean isValidSession(Session session) { return session != null; } - private void checkIfActiveHasChanged(LocalSession session, LocalSession currentActiveSession, boolean ignoreStaleSessionFailure) { + private static void checkIfActiveHasChanged(LocalSession session, Session currentActiveSession, boolean ignoreStaleSessionFailure) { long activeSessionAtCreate = session.getActiveSessionAtCreate(); log.log(Level.FINE, currentActiveSession.logPre() + "active session id at create time=" + activeSessionAtCreate); if (activeSessionAtCreate == 0) return; // No active session at create @@ -220,7 +224,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { // As of now, config generation is based on session id, and config generation must be a monotonically // increasing number - private void checkIfActiveIsNewerThanSessionToBeActivated(long sessionId, long currentActiveSessionId) { + private static void checkIfActiveIsNewerThanSessionToBeActivated(long sessionId, long currentActiveSessionId) { if (sessionId < currentActiveSessionId) { throw new ActivationConflictException("It is not possible to activate session " + sessionId + ", because it is older than current active session (" + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index fa7a107f953..74a9e72e255 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; @@ -24,6 +25,7 @@ import com.yahoo.vespa.flags.Flags; import java.io.File; import java.net.URI; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.Set; @@ -145,17 +147,16 @@ public class ModelContextImpl implements ModelContext { private final Set<ContainerEndpoint> endpoints; private final boolean isBootstrap; private final boolean isFirstTimeDeployment; - private final boolean useAdaptiveDispatch; - private final double defaultTopKprobability; private final boolean useDistributorBtreeDb; private final boolean useThreePhaseUpdates; private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final double defaultTermwiseLimit; - private final double defaultSoftStartSeconds; private final double threadPoolSizeFactor; private final double queueSizefactor; - private final int defaultNumResponseThreads; + private final String jvmGCOPtions; private final Optional<AthenzDomain> athenzDomain; + private final Optional<ApplicationRoles> applicationRoles; + private final int jdiscHealthCheckProxyClientTimeout; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -170,7 +171,8 @@ public class ModelContextImpl implements ModelContext { boolean isFirstTimeDeployment, FlagSource flagSource, Optional<EndpointCertificateSecrets> endpointCertificateSecrets, - Optional<AthenzDomain> athenzDomain) { + Optional<AthenzDomain> athenzDomain, + Optional<ApplicationRoles> applicationRoles) { this.applicationId = applicationId; this.multitenant = multitenantFromConfig || hostedVespa || Boolean.getBoolean("multitenant"); this.configServerSpecs = configServerSpecs; @@ -182,15 +184,9 @@ public class ModelContextImpl implements ModelContext { this.endpoints = endpoints; this.isBootstrap = isBootstrap; this.isFirstTimeDeployment = isFirstTimeDeployment; - this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.endpointCertificateSecrets = endpointCertificateSecrets; defaultTermwiseLimit = Flags.DEFAULT_TERM_WISE_LIMIT.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - defaultSoftStartSeconds = Flags.DEFAULT_SOFT_START_SECONDS.bindTo(flagSource) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - defaultTopKprobability = Flags.DEFAULT_TOP_K_PROBABILITY.bindTo(flagSource) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); useDistributorBtreeDb = Flags.USE_DISTRIBUTOR_BTREE_DB.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); useThreePhaseUpdates = Flags.USE_THREE_PHASE_UPDATES.bindTo(flagSource) @@ -199,9 +195,12 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - defaultNumResponseThreads = Flags.DEFAULT_NUM_RESPONSE_THREADS.bindTo(flagSource) + jvmGCOPtions = Flags.JVM_GC_OPTIONS.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.athenzDomain = athenzDomain; + this.applicationRoles = applicationRoles; + jdiscHealthCheckProxyClientTimeout = Flags.JDISC_HEALTH_CHECK_PROXY_CLIENT_TIMEOUT.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); } @Override @@ -242,9 +241,6 @@ public class ModelContextImpl implements ModelContext { public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } @Override - public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } - - @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } @Override @@ -260,15 +256,6 @@ public class ModelContextImpl implements ModelContext { return queueSizefactor; } - public double defaultSoftStartSeconds() { - return defaultSoftStartSeconds; - } - - @Override - public double defaultTopKProbability() { - return defaultTopKprobability; - } - @Override public boolean useDistributorBtreeDb() { return useDistributorBtreeDb; @@ -280,12 +267,15 @@ public class ModelContextImpl implements ModelContext { } @Override - public int defaultNumResponseThreads() { - return defaultNumResponseThreads; - } + public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } @Override - public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } + public Optional<ApplicationRoles> applicationRoles() { + return applicationRoles; + } + + @Override public Duration jdiscHealthCheckProxyClientTimeout() { return Duration.ofMillis(jdiscHealthCheckProxyClientTimeout); } + @Override public String jvmGCOptions() { return jvmGCOPtions; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index e2cf84d6715..7e83d7013e0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.Reader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 9f65c0e3c6e..07a686a63bd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -1,9 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.io.IOUtils; import java.util.logging.Level; import com.yahoo.text.Utf8; @@ -57,7 +55,7 @@ public class FileDirectory { return root.getAbsolutePath() + "/" + ref.value(); } - File getFile(FileReference reference) { + public File getFile(FileReference reference) { ensureRootExist(); File dir = new File(getPath(reference)); if (!dir.exists()) { @@ -179,4 +177,10 @@ public class FileDirectory { destChannel.transferFrom(sourceChannel, 0, sourceChannel.size()); } } + + @Override + public String toString() { + return "root dir: " + root.getAbsolutePath(); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java new file mode 100644 index 00000000000..c06e4da2b7b --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java @@ -0,0 +1,78 @@ +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.net.HostName; +import com.yahoo.vespa.config.Connection; +import com.yahoo.vespa.config.ConnectionPool; +import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.config.server.ConfigServerSpec; + +import java.io.File; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Utilities related to file distribution on config servers. + * + * @author musum + * @author gjoranv + */ +public class FileDistributionUtil { + + /** + * Returns all files in the given directory, non-recursive. + */ + public static Set<String> getFileReferencesOnDisk(File directory) { + Set<String> fileReferencesOnDisk = new HashSet<>(); + File[] filesOnDisk = directory.listFiles(); + if (filesOnDisk != null) + fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet())); + return fileReferencesOnDisk; + } + + /** + * Returns a connection pool with all config servers except this one, or an empty pool if there + * is only one config server. + */ + public static ConnectionPool createConnectionPool(ConfigserverConfig configserverConfig) { + List<String> configServers = ConfigServerSpec.fromConfig(configserverConfig) + .stream() + .filter(spec -> !spec.getHostName().equals(HostName.getLocalhost())) + .map(spec -> "tcp/" + spec.getHostName() + ":" + spec.getConfigServerPort()) + .collect(Collectors.toList()); + + return configServers.size() > 0 ? new JRTConnectionPool(new ConfigSourceSet(configServers)) : emptyConnectionPool(); + } + + static ConnectionPool emptyConnectionPool() { + return new EmptyConnectionPool(); + } + + private static class EmptyConnectionPool implements ConnectionPool { + + @Override + public void close() {} + + @Override + public void setError(Connection connection, int i) {} + + @Override + public Connection getCurrent() { return null; } + + @Override + public Connection setNewCurrentConnection() { return null; } + + @Override + public int getSize() { return 0; } + + @Override + public Supervisor getSupervisor() { return new Supervisor(new Transport()); } + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 805ee2bef95..99cdb0a74dc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -4,18 +4,10 @@ package com.yahoo.vespa.config.server.filedistribution; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.FileReference; -import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; -import com.yahoo.jrt.Supervisor; -import com.yahoo.jrt.Transport; -import java.util.logging.Level; -import com.yahoo.net.HostName; -import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; -import com.yahoo.vespa.config.JRTConnectionPool; -import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.filedistribution.CompressedFileReference; import com.yahoo.vespa.filedistribution.FileDownloader; @@ -27,12 +19,14 @@ import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; -import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; + +import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.createConnectionPool; +import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.emptyConnectionPool; public class FileServer { private static final Logger log = Logger.getLogger(FileServer.class.getName()); @@ -79,7 +73,7 @@ public class FileServer { // For testing only public FileServer(File rootDir) { - this(new EmptyConnectionPool(), rootDir); + this(emptyConnectionPool(), rootDir); } private FileServer(ConnectionPool connectionPool, File rootDir) { @@ -199,35 +193,4 @@ public class FileServer { downloader.close(); } - // Connection pool with all config servers except this one (might be an empty pool if there is only one config server) - private static ConnectionPool createConnectionPool(ConfigserverConfig configserverConfig) { - List<String> configServers = ConfigServerSpec.fromConfig(configserverConfig) - .stream() - .filter(spec -> !spec.getHostName().equals(HostName.getLocalhost())) - .map(spec -> "tcp/" + spec.getHostName() + ":" + spec.getConfigServerPort()) - .collect(Collectors.toList()); - - return configServers.size() > 0 ? new JRTConnectionPool(new ConfigSourceSet(configServers)) : new EmptyConnectionPool(); - } - - private static class EmptyConnectionPool implements ConnectionPool { - - @Override - public void close() {} - - @Override - public void setError(Connection connection, int i) {} - - @Override - public Connection getCurrent() { return null; } - - @Override - public Connection setNewCurrentConnection() { return null; } - - @Override - public int getSize() { return 0; } - - @Override - public Supervisor getSupervisor() { return new Supervisor(new Transport()); } - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostValidator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostValidator.java index 658440c3683..ea7d15b32e9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostValidator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostValidator.java @@ -8,7 +8,6 @@ import java.util.Collection; * strings. * * @author Ulf Lilleengen - * @since 5.9 */ public interface HostValidator<T> { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index ef510b7dc37..dedd96da6f3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.application.BindingMatch; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream; import com.yahoo.vespa.config.server.http.SessionHandler; @@ -18,6 +19,10 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.time.Duration; import java.time.Instant; +import static com.yahoo.vespa.config.server.application.CompressedApplicationInputStream.createFromCompressedStream; +import static com.yahoo.vespa.config.server.http.Utils.checkThatTenantExists; +import static com.yahoo.vespa.config.server.http.v2.SessionCreateHandler.validateDataAndHeader; + /** * * The implementation of the /application/v2 API. * @@ -46,31 +51,16 @@ public class ApplicationApiHandler extends SessionHandler { } @Override - protected HttpResponse handlePUT(HttpRequest request) { - Tenant tenant = getExistingTenant(request); - TenantName tenantName = tenant.getName(); - long sessionId = getSessionIdV2(request); - PrepareParams prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); - - PrepareResult result = applicationRepository.prepareAndActivate(tenant, sessionId, prepareParams, - shouldIgnoreSessionStaleFailure(request), - Instant.now()); - return new SessionPrepareAndActivateResponse(result, tenantName, request, prepareParams.getApplicationId(), zone); - } - - @Override protected HttpResponse handlePOST(HttpRequest request) { - Tenant tenant = getExistingTenant(request); - TenantName tenantName = tenant.getName(); - PrepareParams prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); - SessionCreateHandler.validateDataAndHeader(request); - - PrepareResult result = - applicationRepository.deploy(CompressedApplicationInputStream.createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader)), - prepareParams, - shouldIgnoreSessionStaleFailure(request), - Instant.now()); - return new SessionPrepareAndActivateResponse(result, tenantName, request, prepareParams.getApplicationId(), zone); + validateDataAndHeader(request); + Tenant tenant = validateTenant(request); + PrepareParams prepareParams = PrepareParams.fromHttpRequest(request, tenant.getName(), zookeeperBarrierTimeout); + CompressedApplicationInputStream compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader)); + PrepareResult result = applicationRepository.deploy(compressedStream, + prepareParams, + shouldIgnoreSessionStaleFailure(request), + Instant.now()); + return new SessionPrepareAndActivateResponse(result, request, prepareParams.getApplicationId(), zone); } @Override @@ -78,10 +68,15 @@ public class ApplicationApiHandler extends SessionHandler { return zookeeperBarrierTimeout.plus(Duration.ofSeconds(10)); } - private Tenant getExistingTenant(HttpRequest request) { - TenantName tenantName = Utils.getTenantNameFromSessionRequest(request); - Utils.checkThatTenantExists(tenantRepository, tenantName); + private Tenant validateTenant(HttpRequest request) { + TenantName tenantName = getTenantNameFromRequest(request); + checkThatTenantExists(tenantRepository, tenantName); return tenantRepository.getTenant(tenantName); } + public static TenantName getTenantNameFromRequest(HttpRequest request) { + BindingMatch<?> bm = Utils.getBindingMatch(request, "http://*/application/v2/tenant/*/prepareandactivate*"); + return TenantName.from(bm.group(2)); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java index ffe3d39b524..1ea41b85983 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -3,60 +3,37 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingMatch; -import java.util.logging.Level; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.host.HostRegistries; -import com.yahoo.vespa.config.server.host.HostRegistry; -import com.yahoo.vespa.config.server.http.*; - -import java.util.concurrent.Executor; - +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; +import com.yahoo.vespa.config.server.http.HttpHandler; +import com.yahoo.vespa.config.server.http.JSONResponse; /** * Handler for getting tenant and application for a given hostname. * * @author hmusum - * @since 5.19 */ public class HostHandler extends HttpHandler { - final HostRegistries hostRegistries; - private final Zone zone; + private final ApplicationRepository applicationRepository; @Inject - public HostHandler(HttpHandler.Context ctx, - GlobalComponentRegistry globalComponentRegistry) { + public HostHandler(HttpHandler.Context ctx, ApplicationRepository applicationRepository) { super(ctx); - this.hostRegistries = globalComponentRegistry.getHostRegistries(); - this.zone = globalComponentRegistry.getZone(); + this.applicationRepository = applicationRepository; } @Override public HttpResponse handleGET(HttpRequest request) { String hostname = getBindingMatch(request).group(2); - log.log(Level.FINE, "hostname=" + hostname); - - HostRegistry<TenantName> tenantHostRegistry = hostRegistries.getTenantHostRegistry(); - log.log(Level.FINE, "hosts in tenant host registry '" + tenantHostRegistry + "' " + tenantHostRegistry.getAllHosts()); - TenantName tenant = tenantHostRegistry.getKeyForHost(hostname); - if (tenant == null) return createError(hostname); - log.log(Level.FINE, "tenant=" + tenant); - HostRegistry<ApplicationId> applicationIdHostRegistry = hostRegistries.getApplicationHostRegistry(tenant); - ApplicationId applicationId; - if (applicationIdHostRegistry == null) return createError(hostname); - applicationId = applicationIdHostRegistry.getKeyForHost(hostname); - log.log(Level.FINE, "applicationId=" + applicationId); - if (applicationId == null) { - return createError(hostname); - } else { - log.log(Level.FINE, "hosts in application host registry '" + applicationIdHostRegistry + "' " + applicationIdHostRegistry.getAllHosts()); - return new HostResponse(Response.Status.OK, applicationId, zone); - } + ApplicationId applicationId = applicationRepository.getApplicationIdForHostname(hostname); + return (applicationId == null) + ? createError(hostname) + : new HostResponse(Response.Status.OK, applicationId, applicationRepository.zone()); } private HttpErrorResponse createError(String hostname) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index 4cf59135026..124902c988a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -48,7 +48,7 @@ public class SessionActiveHandler extends SessionHandler { final Long sessionId = getSessionIdV2(request); ApplicationId applicationId = applicationRepository.activate(tenant, sessionId, timeoutBudget, shouldIgnoreSessionStaleFailure(request)); - ApplicationMetaData metaData = applicationRepository.getMetadataFromSession(tenant, sessionId); + ApplicationMetaData metaData = applicationRepository.getMetadataFromLocalSession(tenant, sessionId); return new SessionActiveResponse(metaData.getSlime(), request, applicationId, sessionId, zone); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index 286adbd256d..5aee711b379 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -63,7 +63,7 @@ public class SessionCreateHandler extends SessionHandler { return createResponse(request, tenantName, deployLog, sessionId); } - private static ApplicationId getFromApplicationId(HttpRequest request) { + static ApplicationId getFromApplicationId(HttpRequest request) { String from = request.getProperty("from"); if (from == null || "".equals(from)) { throw new BadRequestException("Parameter 'from' has illegal value '" + from + "'"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java index 4665a33c647..7d9a0b11c28 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java @@ -5,8 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; import com.yahoo.vespa.config.server.http.SessionResponse; @@ -17,17 +15,10 @@ import com.yahoo.vespa.config.server.http.SessionResponse; */ class SessionPrepareAndActivateResponse extends SessionResponse { - SessionPrepareAndActivateResponse(PrepareResult result, TenantName tenantName, HttpRequest request, - ApplicationId applicationId, Zone zone) { - this(result.deployLog(), tenantName, request, result.sessionId(), result.configChangeActions(), - zone, applicationId); - } - - private SessionPrepareAndActivateResponse(Slime deployLog, TenantName tenantName, HttpRequest request, - long sessionId, ConfigChangeActions actions, Zone zone, - ApplicationId applicationId) { - super(deployLog, deployLog.get()); - String message = "Session " + sessionId + " for tenant '" + tenantName.value() + "' prepared and activated."; + SessionPrepareAndActivateResponse(PrepareResult result, HttpRequest request, ApplicationId applicationId, Zone zone) { + super(result.deployLog(), result.deployLog().get()); + TenantName tenantName = applicationId.tenant(); + String message = "Session " + result.sessionId() + " for tenant '" + tenantName.value() + "' prepared and activated."; this.root.setString("tenant", tenantName.value()); root.setString("url", "http://" + request.getHost() + ":" + request.getPort() + "/application/v2/tenant/" + tenantName + @@ -36,7 +27,7 @@ class SessionPrepareAndActivateResponse extends SessionResponse { "/region/" + zone.region().value() + "/instance/" + applicationId.instance().value()); root.setString("message", message); - new ConfigChangeActionsSlimeConverter(actions).toSlime(root); + new ConfigChangeActionsSlimeConverter(result.configChangeActions()).toSlime(root); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java new file mode 100644 index 00000000000..b7b5c6380ef --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -0,0 +1,75 @@ +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.FileReference; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.session.RemoteSession; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.filedistribution.FileDownloader; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; + +import java.io.File; +import java.time.Duration; +import java.util.Set; +import java.util.logging.Logger; + +import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk; +import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.createConnectionPool; + +/** + * Verifies that all active sessions has an application package on local disk. + * If not, the package is downloaded with file distribution. This can happen e.g. + * if a configserver is down when the application is deployed. + * + * @author gjoranv + */ +public class ApplicationPackageMaintainer extends ConfigServerMaintainer { + private static final Logger log = Logger.getLogger(ApplicationPackageMaintainer.class.getName()); + + private final ApplicationRepository applicationRepository; + private final ConfigserverConfig configserverConfig; + private final File downloadDirectory; + private final BooleanFlag distributeApplicationPackage; + + ApplicationPackageMaintainer(ApplicationRepository applicationRepository, + Curator curator, + Duration interval, + ConfigserverConfig configserverConfig, + FlagSource flagSource) { + super(applicationRepository, curator, interval, interval); + this.applicationRepository = applicationRepository; + this.configserverConfig = configserverConfig; + + distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); + downloadDirectory = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())); + } + + @Override + protected void maintain() { + if (! distributeApplicationPackage.value()) return; + + try (var fileDownloader = new FileDownloader(createConnectionPool(configserverConfig), downloadDirectory)){ + for (var applicationId : applicationRepository.listApplications()) { + RemoteSession session = applicationRepository.getActiveSession(applicationId); + FileReference applicationPackage = session.getApplicationPackageReference(); + + if (applicationPackage != null && missingOnDisk(applicationPackage)) { + log.fine(() -> "Downloading missing application package for application " + applicationId + " - session " + session.getSessionId()); + + if (fileDownloader.getFile(applicationPackage).isEmpty()) { + log.warning("Failed to download application package for application " + applicationId + " - session " + session.getSessionId()); + } + } + } + } + } + + private boolean missingOnDisk(FileReference applicationPackageReference) { + Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(downloadDirectory); + return ! fileReferencesOnDisk.contains(applicationPackageReference.value()); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 502cf280e60..4e6a541793d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.maintenance; +import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.SystemName; @@ -23,8 +24,9 @@ public class ConfigServerMaintenance extends AbstractComponent { //private final TenantsMaintainer tenantsMaintainer; private final FileDistributionMaintainer fileDistributionMaintainer; private final SessionsMaintainer sessionsMaintainer; + private final ApplicationPackageMaintainer applicationPackageMaintainer; - @SuppressWarnings("unused") // instantiated by Dependency Injection + @Inject public ConfigServerMaintenance(ConfigserverConfig configserverConfig, ApplicationRepository applicationRepository, Curator curator, @@ -34,7 +36,8 @@ public class ConfigServerMaintenance extends AbstractComponent { // TODO: Disabled until we have application metadata //tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval); fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig); - sessionsMaintainer = new SessionsMaintainer(applicationRepository, curator, defaults.defaultInterval, flagSource); + sessionsMaintainer = new SessionsMaintainer(applicationRepository, curator, defaults.defaultInterval); + applicationPackageMaintainer = new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofMinutes(1), configserverConfig, flagSource); } @Override @@ -42,6 +45,7 @@ public class ConfigServerMaintenance extends AbstractComponent { //tenantsMaintainer.close(); fileDistributionMaintainer.close(); sessionsMaintainer.close(); + applicationPackageMaintainer.close(); } /* diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java index fb4445b0f02..4975b82a801 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -4,9 +4,6 @@ package com.yahoo.vespa.config.server.maintenance; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.LongFlag; import java.time.Duration; @@ -19,14 +16,12 @@ import java.time.Duration; */ public class SessionsMaintainer extends ConfigServerMaintainer { private final boolean hostedVespa; - private final LongFlag expiryTimeFlag; - SessionsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, FlagSource flagSource) { + SessionsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { // Start this maintainer immediately. It frees disk space, so if disk goes full and config server // restarts this makes sure that cleanup will happen as early as possible super(applicationRepository, curator, Duration.ZERO, interval); this.hostedVespa = applicationRepository.configserverConfig().hostedVespa(); - this.expiryTimeFlag = Flags.CONFIGSERVER_SESSIONS_EXPIRY_INTERVAL_IN_DAYS.bindTo(flagSource); } @Override @@ -36,9 +31,9 @@ public class SessionsMaintainer extends ConfigServerMaintainer { // Expired remote sessions are sessions that belong to an application that have external deployments that // are no longer active if (hostedVespa) { - Duration expiryTime = Duration.ofDays(expiryTimeFlag.value()); + Duration expiryTime = Duration.ofDays(1); int deleted = applicationRepository.deleteExpiredRemoteSessions(expiryTime); - log.log(LogLevel.INFO, "Deleted " + deleted + " expired remote sessions, expiry time " + expiryTime); + log.log(LogLevel.FINE, "Deleted " + deleted + " expired remote sessions, expiry time " + expiryTime); } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 0b83e927c39..6b42ca7fa95 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -16,7 +16,6 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.secretstore.SecretStore; -import java.util.logging.Level; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ServerCache; @@ -28,17 +27,18 @@ import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.session.SilentDeployLogger; +import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; import java.net.URI; -import java.time.Instant; import java.util.Map; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -87,8 +87,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { ApplicationId applicationId, Optional<DockerImage> wantedDockerImageRepository, Version wantedNodeVespaVersion, - Optional<AllocatedHosts> ignored, // Ignored since we have this in the app package for activated models - Instant now) { + Optional<AllocatedHosts> ignored // Ignored since we have this in the app package for activated models + ) { log.log(Level.FINE, String.format("Loading model version %s for session %s application %s", modelFactory.version(), appGeneration, applicationId)); ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId); @@ -147,7 +147,9 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { new EndpointCertificateMetadataStore(curator, TenantRepository.getTenantPath(tenant)) .readEndpointCertificateMetadata(applicationId) .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets), - zkClient.readAthenzDomain()); + zkClient.readAthenzDomain(), + new ApplicationRolesStore(curator, TenantRepository.getTenantPath(tenant)) + .readApplicationRoles(applicationId)); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 455731d9cb6..245b9db020b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -166,8 +166,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { applicationId, wantedDockerImageRepository, wantedNodeVespaVersion, - allocatedHosts.asOptional(), - now); + allocatedHosts.asOptional()); allocatedHosts.set(latestModelVersion.getModel().allocatedHosts()); // Update with additional clusters allocated allApplicationVersions.add(latestModelVersion); } @@ -189,8 +188,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { applicationId, wantedDockerImageRepository, wantedNodeVespaVersion, - allocatedHosts.asOptional(), - now); + allocatedHosts.asOptional()); allocatedHosts.set(modelVersion.getModel().allocatedHosts()); // Update with additional clusters allocated allApplicationVersions.add(modelVersion); } catch (RuntimeException e) { @@ -245,8 +243,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { protected abstract MODELRESULT buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, ApplicationId applicationId, Optional<DockerImage> dockerImageRepository, - Version wantedNodeVespaVersion, Optional<AllocatedHosts> allocatedHosts, - Instant now); + Version wantedNodeVespaVersion, Optional<AllocatedHosts> allocatedHosts); /** * Returns a host provisioner returning the previously allocated hosts if available and when on hosted Vespa, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index b8e7d68ea5e..2397dba6b5e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -2,10 +2,12 @@ package com.yahoo.vespa.config.server.modelfactory; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; @@ -15,28 +17,24 @@ import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.api.ValidationParameters.IgnoreValidationErrors; import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; -import java.util.logging.Level; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; +import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.provision.StaticProvisioner; -import com.yahoo.vespa.config.server.session.FileDistributionFactory; import com.yahoo.vespa.config.server.session.PrepareParams; -import com.yahoo.vespa.config.server.session.SessionContext; import java.io.File; import java.io.IOException; -import java.time.Instant; import java.util.List; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -49,19 +47,19 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P private final PermanentApplicationPackage permanentApplicationPackage; private final ConfigDefinitionRepo configDefinitionRepo; - private final SessionContext context; + private final HostValidator<ApplicationId> hostValidator; private final DeployLogger logger; private final PrepareParams params; - private final FileDistributionFactory fileDistributionFactory; + private final FileDistributionProvider fileDistributionProvider; private final Optional<ApplicationSet> currentActiveApplicationSet; private final ModelContext.Properties properties; public PreparedModelsBuilder(ModelFactoryRegistry modelFactoryRegistry, PermanentApplicationPackage permanentApplicationPackage, ConfigDefinitionRepo configDefinitionRepo, - FileDistributionFactory fileDistributionFactory, + FileDistributionProvider fileDistributionProvider, HostProvisionerProvider hostProvisionerProvider, - SessionContext context, + HostValidator<ApplicationId> hostValidator, DeployLogger logger, PrepareParams params, Optional<ApplicationSet> currentActiveApplicationSet, @@ -70,28 +68,23 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P super(modelFactoryRegistry, configserverConfig, properties.zone(), hostProvisionerProvider); this.permanentApplicationPackage = permanentApplicationPackage; this.configDefinitionRepo = configDefinitionRepo; - - this.fileDistributionFactory = fileDistributionFactory; - - this.context = context; + this.fileDistributionProvider = fileDistributionProvider; + this.hostValidator = hostValidator; this.logger = logger; this.params = params; this.currentActiveApplicationSet = currentActiveApplicationSet; - this.properties = properties; } @Override - protected PreparedModelResult buildModelVersion(ModelFactory modelFactory, + protected PreparedModelResult buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, ApplicationId applicationId, Optional<DockerImage> wantedDockerImageRepository, Version wantedNodeVespaVersion, - Optional<AllocatedHosts> allocatedHosts, - Instant now) { + Optional<AllocatedHosts> allocatedHosts) { Version modelVersion = modelFactory.version(); log.log(Level.FINE, "Building model " + modelVersion + " for " + applicationId); - FileDistributionProvider fileDistributionProvider = fileDistributionFactory.createProvider(context.getServerDBSessionDir()); // Use empty on non-hosted systems, use already allocated hosts if available, create connection to a host provisioner otherwise Provisioned provisioned = new Provisioned(); @@ -114,13 +107,13 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P ValidationParameters validationParameters = new ValidationParameters(params.ignoreValidationErrors() ? IgnoreValidationErrors.TRUE : IgnoreValidationErrors.FALSE); ModelCreateResult result = modelFactory.createAndValidateModel(modelContext, validationParameters); - validateModelHosts(context.getHostValidator(), applicationId, result.getModel()); + validateModelHosts(hostValidator, applicationId, result.getModel()); log.log(Level.FINE, "Done building model " + modelVersion + " for " + applicationId); return new PreparedModelsBuilder.PreparedModelResult(modelVersion, result.getModel(), fileDistributionProvider, result.getConfigChangeActions()); } private Optional<Model> modelOf(Version version) { - if ( ! currentActiveApplicationSet.isPresent()) return Optional.empty(); + if (currentActiveApplicationSet.isEmpty()) return Optional.empty(); return currentActiveApplicationSet.get().get(version).map(Application::getModel); } @@ -130,12 +123,12 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P Provisioned provisioned) { Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(properties.applicationId(), provisioned); - if ( ! allocatedHosts.isPresent()) return nodeRepositoryProvisioner; + if (allocatedHosts.isEmpty()) return nodeRepositoryProvisioner; Optional<HostProvisioner> staticProvisioner = createStaticProvisioner(allocatedHosts, properties.applicationId(), provisioned); - if ( ! staticProvisioner.isPresent()) return Optional.empty(); // Since we have hosts allocated this means we are on non-hosted + if (staticProvisioner.isEmpty()) return Optional.empty(); // Since we have hosts allocated this means we are on non-hosted // Nodes are already allocated by a model and we should use them unless this model requests hosts from a // previously unallocated cluster. This allows future models to stop allocate certain clusters. @@ -153,7 +146,8 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P } private void validateModelHosts(HostValidator<ApplicationId> hostValidator, ApplicationId applicationId, Model model) { - hostValidator.verifyHosts(applicationId, model.getHosts().stream().map(hostInfo -> hostInfo.getHostname()) + hostValidator.verifyHosts(applicationId, model.getHosts().stream() + .map(HostInfo::getHostname) .collect(Collectors.toList())); } @@ -162,11 +156,11 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P public final Version version; public final Model model; - public final FileDistributionProvider fileDistributionProvider; + public final com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider fileDistributionProvider; public final List<ConfigChangeAction> actions; public PreparedModelResult(Version version, Model model, - FileDistributionProvider fileDistributionProvider, List<ConfigChangeAction> actions) { + com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider fileDistributionProvider, List<ConfigChangeAction> actions) { this.version = version; this.model = model; this.fileDistributionProvider = fileDistributionProvider; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java index 62da6fcffbe..52ca73c68b9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java @@ -14,8 +14,6 @@ import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Timer; -import java.util.TimerTask; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java index eb14947f73f..13c21a065ff 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java @@ -5,10 +5,8 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.provision.Provisioner; -import java.util.logging.Level; import java.util.Optional; -import java.util.logging.Logger; /** * This class is necessary to support both having and not having a host provisioner. We inject diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java index 307ec5c0c3c..6a681ae143d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; -import com.yahoo.config.provision.NetworkPorts; import java.util.*; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 7809000695a..62f7d3ce5d0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -3,10 +3,9 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.collections.Pair; -import com.yahoo.config.provision.TenantName; import com.yahoo.component.Version; +import com.yahoo.config.provision.TenantName; import com.yahoo.jrt.Request; -import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ErrorCode; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DefaultRpcAuthorizerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DefaultRpcAuthorizerProvider.java index 5c760f0a25a..8d1d4f58e37 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DefaultRpcAuthorizerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DefaultRpcAuthorizerProvider.java @@ -8,8 +8,6 @@ import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.security.tls.TransportSecurityUtils; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.rpc.RequestHandlerProvider; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; /** * A provider for {@link RpcAuthorizer}. The instance provided is dependent on the configuration of the configserver. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index 4946c24efbc..73e7b36c381 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -1,31 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ApplicationMetaData; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.transaction.AbstractTransaction; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.transaction.Transaction; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.*; -import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.path.Path; +import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.curator.Curator; - -import java.io.File; -import java.time.Instant; -import java.util.Optional; /** * A LocalSession is a session that has been created locally on this configserver. A local session can be edited and @@ -36,40 +19,22 @@ import java.util.Optional; */ // This is really the store of an application, whether it is active or in an edit session // TODO: Separate the "application store" and "session" aspects - the latter belongs in the HTTP layer -bratseth -public class LocalSession extends Session implements Comparable<LocalSession> { +public class LocalSession extends Session { protected final ApplicationPackage applicationPackage; private final TenantApplications applicationRepo; - private final SessionPreparer sessionPreparer; - private final SessionContext sessionContext; - private final File serverDB; /** - * Create a session. This involves loading the application, validating it and distributing it. + * Creates a session. This involves loading the application, validating it and distributing it. * * @param sessionId The session id for this session. */ - public LocalSession(TenantName tenant, long sessionId, SessionPreparer sessionPreparer, SessionContext sessionContext) { - super(tenant, sessionId, sessionContext.getSessionZooKeeperClient()); - this.serverDB = sessionContext.getServerDBSessionDir(); - this.applicationPackage = sessionContext.getApplicationPackage(); - this.applicationRepo = sessionContext.getApplicationRepo(); - this.sessionPreparer = sessionPreparer; - this.sessionContext = sessionContext; - } - - public ConfigChangeActions prepare(DeployLogger logger, - PrepareParams params, - Optional<ApplicationSet> currentActiveApplicationSet, - Path tenantPath, - Instant now) { - applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( - Curator.CompletionWaiter waiter = zooKeeperClient.createPrepareWaiter(); - ConfigChangeActions actions = sessionPreparer.prepare(sessionContext, logger, params, - currentActiveApplicationSet, tenantPath, now); - setPrepared(); - waiter.awaitCompletion(params.getTimeoutBudget().timeLeft()); - return actions; + public LocalSession(TenantName tenant, long sessionId, ApplicationPackage applicationPackage, + SessionZooKeeperClient sessionZooKeeperClient, + TenantApplications applicationRepo) { + super(tenant, sessionId, sessionZooKeeperClient); + this.applicationPackage = applicationPackage; + this.applicationRepo = applicationRepo; } public ApplicationFile getApplicationFile(Path relativePath, Mode mode) { @@ -79,29 +44,25 @@ public class LocalSession extends Session implements Comparable<LocalSession> { return applicationPackage.getFile(relativePath); } - private void setPrepared() { + void setPrepared() { setStatus(Session.Status.PREPARE); } private Transaction createSetStatusTransaction(Status status) { - return zooKeeperClient.createWriteStatusTransaction(status); + return sessionZooKeeperClient.createWriteStatusTransaction(status); } private void setStatus(Session.Status newStatus) { - zooKeeperClient.writeStatus(newStatus); + sessionZooKeeperClient.writeStatus(newStatus); } public Transaction createActivateTransaction() { - zooKeeperClient.createActiveWaiter(); + sessionZooKeeperClient.createActiveWaiter(); Transaction transaction = createSetStatusTransaction(Status.ACTIVATE); - transaction.add(applicationRepo.createPutTransaction(zooKeeperClient.readApplicationId(), getSessionId()).operations()); + transaction.add(applicationRepo.createPutTransaction(sessionZooKeeperClient.readApplicationId(), getSessionId()).operations()); return transaction; } - public Transaction createDeactivateTransaction() { - return createSetStatusTransaction(Status.DEACTIVATE); - } - private void markSessionEdited() { setStatus(Session.Status.NEW); } @@ -110,130 +71,16 @@ public class LocalSession extends Session implements Comparable<LocalSession> { return applicationPackage.getMetaData().getPreviousActiveGeneration(); } - // Note: Assumes monotonically increasing session ids - public boolean isNewerThan(long sessionId) { - return getSessionId() > sessionId; - } - - /** Add transactions to delete this session to the given nested transaction */ - public void delete(NestedTransaction transaction) { - transaction.add(zooKeeperClient.deleteTransaction(), FileTransaction.class); - transaction.add(FileTransaction.from(FileOperations.delete(serverDB.getAbsolutePath()))); - } - - @Override - public int compareTo(LocalSession rhs) { - Long lhsId = getSessionId(); - Long rhsId = rhs.getSessionId(); - return lhsId.compareTo(rhsId); - } - public void waitUntilActivated(TimeoutBudget timeoutBudget) { - zooKeeperClient.getActiveWaiter().awaitCompletion(timeoutBudget.timeLeft()); - } - - public void setApplicationId(ApplicationId applicationId) { - zooKeeperClient.writeApplicationId(applicationId); - } - - public void setVespaVersion(Version version) { - zooKeeperClient.writeVespaVersion(version); - } - - public void setDockerImageRepository(Optional<DockerImage> dockerImageRepository) { - zooKeeperClient.writeDockerImageRepository(dockerImageRepository); - } - - public void setAthenzDomain(Optional<AthenzDomain> athenzDomain) { - zooKeeperClient.writeAthenzDomain(athenzDomain); + sessionZooKeeperClient.getActiveWaiter().awaitCompletion(timeoutBudget.timeLeft()); } public enum Mode { READ, WRITE } - public ApplicationMetaData getMetaData() { - return applicationPackage.getMetaData(); - } - - public ApplicationId getApplicationId() { return zooKeeperClient.readApplicationId(); } + public ApplicationMetaData getMetaData() { return applicationPackage.getMetaData(); } - public Optional<DockerImage> getDockerImageRepository() { return zooKeeperClient.readDockerImageRepository(); } - - public Version getVespaVersion() { return zooKeeperClient.readVespaVersion(); } - - public Optional<AthenzDomain> getAthenzDomain() { return zooKeeperClient.readAthenzDomain(); } - - public AllocatedHosts getAllocatedHosts() { - return zooKeeperClient.getAllocatedHosts(); - } - - public TenantName getTenantName() { return tenant; } - - @Override - public String logPre() { - if (getApplicationId().equals(ApplicationId.defaultId())) { - return TenantRepository.logPre(getTenant()); - } else { - return TenantRepository.logPre(getApplicationId()); - } - } - - // The rest of this class should be moved elsewhere ... - - private static class FileTransaction extends AbstractTransaction { - - public static FileTransaction from(FileOperation operation) { - FileTransaction transaction = new FileTransaction(); - transaction.add(operation); - return transaction; - } - - @Override - public void prepare() { } - - @Override - public void commit() { - for (Operation operation : operations()) - ((FileOperation)operation).commit(); - } - - } - - /** Factory for file operations */ - private static class FileOperations { - - /** Creates an operation which recursively deletes the given path */ - public static DeleteOperation delete(String pathToDelete) { - return new DeleteOperation(pathToDelete); - } - - } - - private interface FileOperation extends Transaction.Operation { - - void commit(); - - } - - /** - * Recursively deletes this path and everything below. - * Succeeds with no action if the path does not exist. - */ - private static class DeleteOperation implements FileOperation { - - private final String pathToDelete; - - DeleteOperation(String pathToDelete) { - this.pathToDelete = pathToDelete; - } - - @Override - public void commit() { - // TODO: Check delete access in prepare() - IOUtils.recursiveDeleteDir(new File(pathToDelete)); - } - - } + public ApplicationPackage getApplicationPackage() { return applicationPackage; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionLoader.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionLoader.java deleted file mode 100644 index ce8ba291c28..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionLoader.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -/** - * Interface of a component that is able to load a session given a session id. - * - * @author Ulf Lilleengen - * @since 5.1 - */ -public interface LocalSessionLoader { - - LocalSession loadSession(long sessionId); - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java deleted file mode 100644 index 97c4530e447..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.provision.TenantName; -import java.util.logging.Level; -import com.yahoo.path.Path; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; - -import java.io.File; -import java.io.FilenameFilter; -import java.time.Clock; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -/** - * File-based session repository for LocalSessions. Contains state for the local instance of the configserver. - * - * @author Ulf Lilleengen - */ -public class LocalSessionRepo extends SessionRepo<LocalSession> { - - private static final Logger log = Logger.getLogger(LocalSessionRepo.class.getName()); - private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); - - private final Map<Long, LocalSessionStateWatcher> sessionStateWatchers = new HashMap<>(); - private final long sessionLifetime; // in seconds - private final Clock clock; - private final Curator curator; - private final Executor zkWatcherExecutor; - private final TenantFileSystemDirs tenantFileSystemDirs; - - public LocalSessionRepo(TenantName tenantName, GlobalComponentRegistry componentRegistry, LocalSessionLoader loader) { - this(tenantName, componentRegistry); - loadSessions(loader); - } - - // Constructor public only for testing - public LocalSessionRepo(TenantName tenantName, GlobalComponentRegistry componentRegistry) { - this.clock = componentRegistry.getClock(); - this.curator = componentRegistry.getCurator(); - this.sessionLifetime = componentRegistry.getConfigserverConfig().sessionLifetime(); - this.zkWatcherExecutor = command -> componentRegistry.getZkWatcherExecutor().execute(tenantName, command); - this.tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName); - } - - @Override - public synchronized void addSession(LocalSession session) { - super.addSession(session); - Path sessionsPath = TenantRepository.getSessionsPath(session.getTenantName()); - long sessionId = session.getSessionId(); - Curator.FileCache fileCache = curator.createFileCache(sessionsPath.append(String.valueOf(sessionId)).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); - sessionStateWatchers.put(sessionId, new LocalSessionStateWatcher(fileCache, session, this, zkWatcherExecutor)); - } - - private void loadSessions(LocalSessionLoader loader) { - File[] sessions = tenantFileSystemDirs.sessionsPath().listFiles(sessionApplicationsFilter); - if (sessions == null) { - return; - } - for (File session : sessions) { - try { - addSession(loader.loadSession(Long.parseLong(session.getName()))); - } catch (IllegalArgumentException e) { - log.log(Level.WARNING, "Could not load session '" + - session.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); - } - } - } - - public void purgeOldSessions() { - log.log(Level.FINE, "Purging old sessions"); - try { - List<LocalSession> sessions = new ArrayList<>(listSessions()); - for (LocalSession candidate : sessions) { - if (hasExpired(candidate) && !isActiveSession(candidate)) { - deleteSession(candidate); - } - } - // Make sure to catch here, to avoid executor just dying in case of issues ... - } catch (Throwable e) { - log.log(Level.WARNING, "Error when purging old sessions ", e); - } - log.log(Level.FINE, "Done purging old sessions"); - } - - private boolean hasExpired(LocalSession candidate) { - return (candidate.getCreateTime() + sessionLifetime) <= TimeUnit.MILLISECONDS.toSeconds(clock.millis()); - } - - private boolean isActiveSession(LocalSession candidate) { - return candidate.getStatus() == Session.Status.ACTIVATE; - } - - void deleteSession(LocalSession session) { - long sessionId = session.getSessionId(); - log.log(Level.FINE, "Deleting local session " + sessionId); - LocalSessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); - if (watcher != null) watcher.close(); - removeSession(sessionId); - NestedTransaction transaction = new NestedTransaction(); - session.delete(transaction); - transaction.commit(); - } - - public void close() { - deleteAllSessions(); - tenantFileSystemDirs.delete(); - } - - private void deleteAllSessions() { - List<LocalSession> sessions = new ArrayList<>(listSessions()); - for (LocalSession session : sessions) { - deleteSession(session); - } - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java index 662094fc0ca..acbb1dc81ce 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java @@ -21,14 +21,14 @@ public class LocalSessionStateWatcher { private final Curator.FileCache fileCache; private final LocalSession session; - private final LocalSessionRepo localSessionRepo; + private final SessionRepository sessionRepository; private final Executor zkWatcherExecutor; LocalSessionStateWatcher(Curator.FileCache fileCache, LocalSession session, - LocalSessionRepo localSessionRepo, Executor zkWatcherExecutor) { + SessionRepository sessionRepository, Executor zkWatcherExecutor) { this.fileCache = fileCache; this.session = session; - this.localSessionRepo = localSessionRepo; + this.sessionRepository = sessionRepository; this.zkWatcherExecutor = zkWatcherExecutor; this.fileCache.start(); this.fileCache.addListener(this::nodeChanged); @@ -40,9 +40,9 @@ public class LocalSessionStateWatcher { log.log(status == Session.Status.DELETE ? Level.INFO : Level.FINE, session.logPre() + "Session change: Local session " + sessionId + " changed status to " + status); - if (status.equals(Session.Status.DELETE) && localSessionRepo.getSession(sessionId) != null) { + if (status.equals(Session.Status.DELETE) && sessionRepository.getLocalSession(sessionId) != null) { log.log(Level.FINE, session.logPre() + "Deleting session " + sessionId); - localSessionRepo.deleteSession(session); + sessionRepository.deleteLocalSession(session); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index e5d4cf9b1ff..2e101762fc4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; +import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.provision.ApplicationId; @@ -38,6 +39,8 @@ public final class PrepareParams { static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata"; static final String DOCKER_IMAGE_REPOSITORY = "dockerImageRepository"; static final String ATHENZ_DOMAIN = "athenzDomain"; + static final String APPLICATION_HOST_ROLE = "applicationHostRole"; + static final String APPLICATION_CONTAINER_ROLE = "applicationContainerRole"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -51,12 +54,14 @@ public final class PrepareParams { private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private final Optional<DockerImage> dockerImageRepository; private final Optional<AthenzDomain> athenzDomain; + private final Optional<ApplicationRoles> applicationRoles; private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, - Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain) { + Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain, + Optional<ApplicationRoles> applicationRoles) { this.timeoutBudget = timeoutBudget; this.applicationId = applicationId; this.ignoreValidationErrors = ignoreValidationErrors; @@ -69,6 +74,7 @@ public final class PrepareParams { this.endpointCertificateMetadata = endpointCertificateMetadata; this.dockerImageRepository = dockerImageRepository; this.athenzDomain = athenzDomain; + this.applicationRoles = applicationRoles; } public static class Builder { @@ -78,13 +84,14 @@ public final class PrepareParams { private boolean verbose = false; private boolean isBootstrap = false; private ApplicationId applicationId = ApplicationId.defaultId(); - private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(30)); + private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60)); private Optional<Version> vespaVersion = Optional.empty(); - private List<ContainerEndpoint> containerEndpoints = List.of(); + private List<ContainerEndpoint> containerEndpoints = null; private Optional<String> tlsSecretsKeyName = Optional.empty(); private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty(); private Optional<DockerImage> dockerImageRepository = Optional.empty(); private Optional<AthenzDomain> athenzDomain = Optional.empty(); + private Optional<ApplicationRoles> applicationRoles = Optional.empty(); public Builder() { } @@ -174,12 +181,17 @@ public final class PrepareParams { return this; } + public Builder applicationRoles(ApplicationRoles applicationRoles) { + this.applicationRoles = Optional.ofNullable(applicationRoles); + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, - endpointCertificateMetadata, dockerImageRepository, athenzDomain); + endpointCertificateMetadata, dockerImageRepository, athenzDomain, + applicationRoles); } - } public static PrepareParams fromHttpRequest(HttpRequest request, TenantName tenant, Duration barrierTimeout) { @@ -194,6 +206,7 @@ public final class PrepareParams { .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME)) .dockerImageRepository(request.getProperty(DOCKER_IMAGE_REPOSITORY)) .athenzDomain(request.getProperty(ATHENZ_DOMAIN)) + .applicationRoles(ApplicationRoles.fromString(request.getProperty(APPLICATION_HOST_ROLE), request.getProperty(APPLICATION_CONTAINER_ROLE))) .build(); } @@ -261,4 +274,7 @@ public final class PrepareParams { public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } + public Optional<ApplicationRoles> applicationRoles() { + return applicationRoles; + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 9da3ad2fd7a..763c77f2088 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -1,23 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.lang.SettableOptional; -import java.util.logging.Level; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.modelfactory.ActivatedModelsBuilder; -import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import org.apache.zookeeper.KeeperException; import java.time.Clock; -import java.time.Instant; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -51,20 +50,20 @@ public class RemoteSession extends Session { } void loadPrepared() { - Curator.CompletionWaiter waiter = zooKeeperClient.getPrepareWaiter(); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.getPrepareWaiter(); ensureApplicationLoaded(); notifyCompletion(waiter); } private ApplicationSet loadApplication() { - ApplicationPackage applicationPackage = zooKeeperClient.loadApplicationPackage(); + ApplicationPackage applicationPackage = sessionZooKeeperClient.loadApplicationPackage(); // Read hosts allocated on the config server instance which created this Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts(); - return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(), - zooKeeperClient.readDockerImageRepository(), - zooKeeperClient.readVespaVersion(), + return ApplicationSet.fromList(applicationLoader.buildModels(sessionZooKeeperClient.readApplicationId(), + sessionZooKeeperClient.readDockerImageRepository(), + sessionZooKeeperClient.readVespaVersion(), applicationPackage, new SettableOptional<>(allocatedHosts), clock.instant())); @@ -74,20 +73,16 @@ public class RemoteSession extends Session { return applicationSet == null ? applicationSet = loadApplication() : applicationSet; } - public Session.Status getStatus() { - return zooKeeperClient.readStatus(); - } - public synchronized void deactivate() { applicationSet = null; } public Transaction createDeleteTransaction() { - return zooKeeperClient.createWriteStatusTransaction(Status.DELETE); + return sessionZooKeeperClient.createWriteStatusTransaction(Status.DELETE); } void makeActive(ReloadHandler reloadHandler) { - Curator.CompletionWaiter waiter = zooKeeperClient.getActiveWaiter(); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.getActiveWaiter(); log.log(Level.FINE, () -> logPre() + "Getting session from repo: " + getSessionId()); ApplicationSet app = ensureApplicationLoaded(); log.log(Level.FINE, () -> logPre() + "Reloading config for " + getSessionId()); @@ -97,17 +92,8 @@ public class RemoteSession extends Session { log.log(Level.INFO, logPre() + "Session activated: " + getSessionId()); } - @Override - public String logPre() { - if (applicationSet != null) { - return TenantRepository.logPre(applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()).getId()); - } - - return TenantRepository.logPre(getTenant()); - } - void confirmUpload() { - Curator.CompletionWaiter waiter = zooKeeperClient.getUploadWaiter(); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.getUploadWaiter(); log.log(Level.FINE, "Notifying upload waiter for session " + getSessionId()); notifyCompletion(waiter); log.log(Level.FINE, "Done notifying upload for session " + getSessionId()); @@ -130,9 +116,13 @@ public class RemoteSession extends Session { } public void delete() { - Transaction transaction = zooKeeperClient.deleteTransaction(); + Transaction transaction = sessionZooKeeperClient.deleteTransaction(); transaction.commit(); transaction.close(); } + public ApplicationMetaData getMetaData() { + return sessionZooKeeperClient.loadApplicationPackage().getMetaData(); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java deleted file mode 100644 index 0707260dffd..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; - -/** - * @author Ulf Lilleengen - */ -public class RemoteSessionFactory { - - private final GlobalComponentRegistry componentRegistry; - private final Curator curator; - private final ConfigCurator configCurator; - private final Path sessionsPath; - private final TenantName tenant; - private final ConfigserverConfig configserverConfig; - - public RemoteSessionFactory(GlobalComponentRegistry componentRegistry, TenantName tenant) { - this.componentRegistry = componentRegistry; - this.curator = componentRegistry.getCurator(); - this.configCurator = componentRegistry.getConfigCurator(); - this.sessionsPath = TenantRepository.getSessionsPath(tenant); - this.tenant = tenant; - this.configserverConfig = componentRegistry.getConfigserverConfig(); - } - - public RemoteSession createSession(long sessionId) { - Path sessionPath = this.sessionsPath.append(String.valueOf(sessionId)); - SessionZooKeeperClient sessionZKClient = new SessionZooKeeperClient(curator, - configCurator, - sessionPath, - configserverConfig.serverId(), - componentRegistry.getZone().nodeFlavors()); - return new RemoteSession(tenant, sessionId, componentRegistry, sessionZKClient); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java deleted file mode 100644 index c27b7c6802b..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.google.common.collect.HashMultiset; -import com.google.common.collect.Multiset; -import com.yahoo.concurrent.StripedExecutor; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; - -import java.time.Clock; -import java.util.logging.Level; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.ReloadHandler; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; - -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Will watch/prepare sessions (applications) based on watched nodes in ZooKeeper. The zookeeper state watched in - * this class is shared between all config servers, so it should not modify any global state, because the operation - * will be performed on all servers. The repo can be regarded as read only from the POV of the configserver. - * - * @author Vegard Havdal - * @author Ulf Lilleengen - */ -public class RemoteSessionRepo extends SessionRepo<RemoteSession> { - - private static final Logger log = Logger.getLogger(RemoteSessionRepo.class.getName()); - - private final GlobalComponentRegistry componentRegistry; - private final Curator curator; - private final Path sessionsPath; - private final RemoteSessionFactory remoteSessionFactory; - private final Map<Long, RemoteSessionStateWatcher> sessionStateWatchers = new HashMap<>(); - private final ReloadHandler reloadHandler; - private final TenantName tenantName; - private final MetricUpdater metrics; - private final Curator.DirectoryCache directoryCache; - private final TenantApplications applicationRepo; - private final Executor zkWatcherExecutor; - - public RemoteSessionRepo(GlobalComponentRegistry componentRegistry, - RemoteSessionFactory remoteSessionFactory, - ReloadHandler reloadHandler, - TenantName tenantName, - TenantApplications applicationRepo) { - this.componentRegistry = componentRegistry; - this.curator = componentRegistry.getCurator(); - this.sessionsPath = TenantRepository.getSessionsPath(tenantName); - this.applicationRepo = applicationRepo; - this.remoteSessionFactory = remoteSessionFactory; - this.reloadHandler = reloadHandler; - this.tenantName = tenantName; - this.metrics = componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); - StripedExecutor<TenantName> zkWatcherExecutor = componentRegistry.getZkWatcherExecutor(); - this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenantName, command); - initializeSessions(); - this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, componentRegistry.getZkCacheExecutor()); - this.directoryCache.addListener(this::childEvent); - this.directoryCache.start(); - } - - public List<Long> getSessions() { - return getSessionList(curator.getChildren(sessionsPath)); - } - - public int deleteExpiredSessions(Clock clock, Duration expiryTime) { - int deleted = 0; - for (long sessionId : getSessions()) { - RemoteSession session = getSession(sessionId); - if (session == null) continue; // Internal sessions not in synch with zk, continue - if (session.getStatus() == Session.Status.ACTIVATE) continue; - Instant created = Instant.ofEpochSecond(session.getCreateTime()); - if (sessionHasExpired(created, expiryTime, clock)) { - log.log(Level.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); - session.delete(); - deleted++; - } - } - return deleted; - } - - private boolean sessionHasExpired(Instant created, Duration expiryTime, Clock clock) { - return (created.plus(expiryTime).isBefore(clock.instant())); - } - - private List<Long> getSessionListFromDirectoryCache(List<ChildData> children) { - return getSessionList(children.stream() - .map(child -> Path.fromString(child.getPath()).getName()) - .collect(Collectors.toList())); - } - - private List<Long> getSessionList(List<String> children) { - return children.stream().map(Long::parseLong).collect(Collectors.toList()); - } - - private void initializeSessions() throws NumberFormatException { - getSessions().forEach(this::sessionAdded); - } - - private synchronized void sessionsChanged() throws NumberFormatException { - List<Long> sessions = getSessionListFromDirectoryCache(directoryCache.getCurrentData()); - checkForRemovedSessions(sessions); - checkForAddedSessions(sessions); - } - - private void checkForRemovedSessions(List<Long> sessions) { - for (RemoteSession session : listSessions()) - if ( ! sessions.contains(session.getSessionId())) - sessionRemoved(session.getSessionId()); - } - - private void checkForAddedSessions(List<Long> sessions) { - for (Long sessionId : sessions) - if (getSession(sessionId) == null) - sessionAdded(sessionId); - } - - /** - * A session for which we don't have a watcher, i.e. hitherto unknown to us. - * - * @param sessionId session id for the new session - */ - private void sessionAdded(long sessionId) { - log.log(Level.FINE, () -> "Adding session to RemoteSessionRepo: " + sessionId); - try { - RemoteSession session = remoteSessionFactory.createSession(sessionId); - Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); - Curator.FileCache fileCache = curator.createFileCache(sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); - fileCache.addListener(this::nodeChanged); - loadSessionIfActive(session); - addSession(session); - metrics.incAddedSessions(); - sessionStateWatchers.put(sessionId, new RemoteSessionStateWatcher(fileCache, reloadHandler, session, metrics, zkWatcherExecutor)); - } catch (Exception e) { - if (componentRegistry.getConfigserverConfig().throwIfActiveSessionCannotBeLoaded()) throw e; - log.log(Level.WARNING, "Failed loading session " + sessionId + ": No config for this session can be served", e); - } - } - - private void sessionRemoved(long sessionId) { - RemoteSessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); - if (watcher != null) watcher.close(); - removeSession(sessionId); - metrics.incRemovedSessions(); - } - - private void loadSessionIfActive(RemoteSession session) { - for (ApplicationId applicationId : applicationRepo.activeApplications()) { - if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { - log.log(Level.FINE, () -> "Found active application for session " + session.getSessionId() + " , loading it"); - reloadHandler.reloadConfig(session.ensureApplicationLoaded()); - log.log(Level.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); - return; - } - } - } - - public synchronized void close() { - try { - if (directoryCache != null) { - directoryCache.close(); - } - } catch (Exception e) { - log.log(Level.WARNING, "Exception when closing path cache", e); - } finally { - checkForRemovedSessions(new ArrayList<>()); - } - } - - private void nodeChanged() { - zkWatcherExecutor.execute(() -> { - Multiset<Session.Status> sessionMetrics = HashMultiset.create(); - for (RemoteSession session : listSessions()) { - sessionMetrics.add(session.getStatus()); - } - metrics.setNewSessions(sessionMetrics.count(Session.Status.NEW)); - metrics.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE)); - metrics.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE)); - metrics.setDeactivatedSessions(sessionMetrics.count(Session.Status.DEACTIVATE)); - }); - } - - @SuppressWarnings("unused") - private void childEvent(CuratorFramework ignored, PathChildrenCacheEvent event) { - zkWatcherExecutor.execute(() -> { - log.log(Level.FINE, () -> "Got child event: " + event); - switch (event.getType()) { - case CHILD_ADDED: - sessionsChanged(); - synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); - break; - case CHILD_REMOVED: - sessionsChanged(); - break; - case CONNECTION_RECONNECTED: - sessionsChanged(); - break; - } - }); - } - - private void synchronizeOnNew(List<Long> sessionList) { - for (long sessionId : sessionList) { - RemoteSession session = getSession(sessionId); - if (session == null) continue; // session might have been deleted after getting session list - log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); - session.confirmUpload(); - } - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java index 6a56be89c6a..93e6c83fe5b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java @@ -20,7 +20,6 @@ import java.util.logging.Logger; public class RemoteSessionStateWatcher { private static final Logger log = Logger.getLogger(RemoteSessionStateWatcher.class.getName()); - // One thread pool for all instances of this class private final Curator.FileCache fileCache; private final ReloadHandler reloadHandler; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index 64ecc510fe9..1e832548342 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -1,9 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.component.Version; +import com.yahoo.config.FileReference; +import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.TenantName; +import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import java.time.Instant; +import java.util.Optional; + /** * A session represents an instance of an application that can be edited, prepared and activated. This * class represents the common stuff between sessions working on the local file @@ -11,16 +21,16 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; * * @author Ulf Lilleengen */ -public abstract class Session { +public abstract class Session implements Comparable<Session> { private final long sessionId; protected final TenantName tenant; - protected final SessionZooKeeperClient zooKeeperClient; + protected final SessionZooKeeperClient sessionZooKeeperClient; - protected Session(TenantName tenant, long sessionId, SessionZooKeeperClient zooKeeperClient) { + protected Session(TenantName tenant, long sessionId, SessionZooKeeperClient sessionZooKeeperClient) { this.tenant = tenant; this.sessionId = sessionId; - this.zooKeeperClient = zooKeeperClient; + this.sessionZooKeeperClient = sessionZooKeeperClient; } /** * Retrieve the session id for this session. @@ -31,7 +41,7 @@ public abstract class Session { } public Session.Status getStatus() { - return zooKeeperClient.readStatus(); + return sessionZooKeeperClient.readStatus(); } @Override @@ -54,22 +64,77 @@ public abstract class Session { return Status.NEW; } } - - public TenantName getTenant() { - return tenant; - } + + public TenantName getTenantName() { return tenant; } /** * Helper to provide a log message preamble for code dealing with sessions * @return log preamble */ public String logPre() { - return TenantRepository.logPre(getTenant()); + if (getApplicationId().equals(ApplicationId.defaultId())) { + return TenantRepository.logPre(getTenantName()); + } else { + return TenantRepository.logPre(getApplicationId()); + } + } + + public Instant getCreateTime() { + return sessionZooKeeperClient.readCreateTime(); + } + + public void setApplicationId(ApplicationId applicationId) { + sessionZooKeeperClient.writeApplicationId(applicationId); } - // in seconds - public long getCreateTime() { - return zooKeeperClient.readCreateTime(); + void setApplicationPackageReference(FileReference applicationPackageReference) { + if (applicationPackageReference == null) throw new IllegalArgumentException(String.format( + "Null application package FileReference for tenant: %s, session: %d", tenant, sessionId)); + sessionZooKeeperClient.writeApplicationPackageReference(applicationPackageReference); + } + + public void setVespaVersion(Version version) { + sessionZooKeeperClient.writeVespaVersion(version); + } + + public void setDockerImageRepository(Optional<DockerImage> dockerImageRepository) { + sessionZooKeeperClient.writeDockerImageRepository(dockerImageRepository); + } + + public void setAthenzDomain(Optional<AthenzDomain> athenzDomain) { + sessionZooKeeperClient.writeAthenzDomain(athenzDomain); + } + + public ApplicationId getApplicationId() { return sessionZooKeeperClient.readApplicationId(); } + + public FileReference getApplicationPackageReference() {return sessionZooKeeperClient.readApplicationPackageReference(); } + + public Optional<DockerImage> getDockerImageRepository() { return sessionZooKeeperClient.readDockerImageRepository(); } + + public Version getVespaVersion() { return sessionZooKeeperClient.readVespaVersion(); } + + public Optional<AthenzDomain> getAthenzDomain() { return sessionZooKeeperClient.readAthenzDomain(); } + + public AllocatedHosts getAllocatedHosts() { + return sessionZooKeeperClient.getAllocatedHosts(); + } + + public Transaction createDeactivateTransaction() { + return createSetStatusTransaction(Status.DEACTIVATE); + } + + private Transaction createSetStatusTransaction(Status status) { + return sessionZooKeeperClient.createWriteStatusTransaction(status); + } + + // Note: Assumes monotonically increasing session ids + public boolean isNewerThan(long sessionId) { return getSessionId() > sessionId; } + + @Override + public int compareTo(Session rhs) { + Long lhsId = getSessionId(); + Long rhsId = rhs.getSessionId(); + return lhsId.compareTo(rhsId); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java index 3400504fb58..501d4918996 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java @@ -1,24 +1,22 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; +import java.util.List; /** - * A generic session repository that can store any type of session that extends the abstract interface. + * A session cache that can store any type of {@link Session}. * * @author Ulf Lilleengen + * @author hmusum */ -// TODO: This is a ZK cache. We should probably remove it, or make that explicit -public class SessionRepo<SESSIONTYPE extends Session> { +public class SessionCache<SESSIONTYPE extends Session> { private final HashMap<Long, SESSIONTYPE> sessions = new HashMap<>(); public synchronized void addSession(SESSIONTYPE session) { - if (sessions.containsKey(session.getSessionId())) - throw new IllegalArgumentException("There already exists a session with id '" + session.getSessionId() + "'"); - sessions.put(session.getSessionId(), session); + sessions.putIfAbsent(session.getSessionId(), session); } synchronized void removeSession(long id) { @@ -37,7 +35,7 @@ public class SessionRepo<SESSIONTYPE extends Session> { return sessions.get(id); } - public synchronized Collection<SESSIONTYPE> listSessions() { + public synchronized List<SESSIONTYPE> getSessions() { return new ArrayList<>(sessions.values()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java deleted file mode 100644 index 0495a51514c..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.server.SuperModelGenerationCounter; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.host.HostValidator; -import com.yahoo.vespa.flags.FlagSource; - -import java.io.File; - -/** - * The dependencies needed for a local session to be edited and prepared. - * - * @author Ulf Lilleengen - */ -public class SessionContext { - - private final ApplicationPackage applicationPackage; - private final SessionZooKeeperClient sessionZooKeeperClient; - private final File serverDBSessionDir; - private final TenantApplications applicationRepo; - private final HostValidator<ApplicationId> hostRegistry; - private final FlagSource flagSource; - - public SessionContext(ApplicationPackage applicationPackage, SessionZooKeeperClient sessionZooKeeperClient, - File serverDBSessionDir, TenantApplications applicationRepo, - HostValidator<ApplicationId> hostRegistry, - FlagSource flagSource) { - this.applicationPackage = applicationPackage; - this.sessionZooKeeperClient = sessionZooKeeperClient; - this.serverDBSessionDir = serverDBSessionDir; - this.applicationRepo = applicationRepo; - this.hostRegistry = hostRegistry; - this.flagSource = flagSource; - } - - public ApplicationPackage getApplicationPackage() { - return applicationPackage; - } - - public SessionZooKeeperClient getSessionZooKeeperClient() { - return sessionZooKeeperClient; - } - - public File getServerDBSessionDir() { - return serverDBSessionDir; - } - - public TenantApplications getApplicationRepo() { - return applicationRepo; - } - - public HostValidator<ApplicationId> getHostValidator() { return hostRegistry; } - - public FlagSource getFlagSource() { - return flagSource; - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java deleted file mode 100644 index 5527d3060f7..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.server.TimeoutBudget; - -import java.io.File; - -/** - * A session factory responsible for creating deploy sessions. - * - * @author Ulf Lilleengen - */ -public interface SessionFactory { - - /** - * Creates a new deployment session from an application package. - * - * @param applicationDirectory a File pointing to an application. - * @param applicationId application id for this new session. - * @param timeoutBudget Timeout for creating session and waiting for other servers. - * @return a new session - */ - LocalSession createSession(File applicationDirectory, ApplicationId applicationId, TimeoutBudget timeoutBudget); - - /** - * Creates a new deployment session from an already existing session. - * - * @param existingSession the session to use as base - * @param logger a deploy logger where the deploy log will be written. - * @param internalRedeploy whether this session is for a system internal redeploy — not an application package change - * @param timeoutBudget timeout for creating session and waiting for other servers. - * @return a new session - */ - LocalSession createSessionFromExisting(LocalSession existingSession, DeployLogger logger, - boolean internalRedeploy, TimeoutBudget timeoutBudget); - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java deleted file mode 100644 index 58b643ec82c..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.*; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.io.IOUtils; -import java.util.logging.Level; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.*; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; -import com.yahoo.vespa.config.server.host.HostValidator; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.SessionCounter; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.FlagSource; - -import java.io.File; -import java.time.Clock; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -/** - * Serves as the factory of sessions. Takes care of copying files to the correct folder and initializing the - * session state. - * - * @author Ulf Lilleengen - */ -public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { - - private static final Logger log = Logger.getLogger(SessionFactoryImpl.class.getName()); - private static final long nonExistingActiveSession = 0; - - private final SessionPreparer sessionPreparer; - private final Curator curator; - private final ConfigCurator configCurator; - private final SessionCounter sessionCounter; - private final TenantApplications applicationRepo; - private final Path sessionsPath; - private final TenantFileSystemDirs tenantFileSystemDirs; - private final HostValidator<ApplicationId> hostRegistry; - private final TenantName tenant; - private final String serverId; - private final Optional<NodeFlavors> nodeFlavors; - private final Clock clock; - private final FlagSource flagSource; - - public SessionFactoryImpl(GlobalComponentRegistry globalComponentRegistry, - TenantApplications applicationRepo, - HostValidator<ApplicationId> hostRegistry, - TenantName tenant) { - this.hostRegistry = hostRegistry; - this.tenant = tenant; - this.sessionPreparer = globalComponentRegistry.getSessionPreparer(); - this.curator = globalComponentRegistry.getCurator(); - this.configCurator = globalComponentRegistry.getConfigCurator(); - this.sessionCounter = new SessionCounter(globalComponentRegistry.getConfigCurator(), tenant); - this.sessionsPath = TenantRepository.getSessionsPath(tenant); - this.applicationRepo = applicationRepo; - this.tenantFileSystemDirs = new TenantFileSystemDirs(globalComponentRegistry.getConfigServerDB(), tenant); - this.serverId = globalComponentRegistry.getConfigserverConfig().serverId(); - this.nodeFlavors = globalComponentRegistry.getZone().nodeFlavors(); - this.clock = globalComponentRegistry.getClock(); - this.flagSource = globalComponentRegistry.getFlagSource(); - } - - /** Create a session for a true application package change */ - @Override - public LocalSession createSession(File applicationFile, - ApplicationId applicationId, - TimeoutBudget timeoutBudget) { - return create(applicationFile, applicationId, nonExistingActiveSession, false, timeoutBudget); - } - - private void ensureZKPathDoesNotExist(Path sessionPath) { - if (configCurator.exists(sessionPath.getAbsolute())) { - throw new IllegalArgumentException("Path " + sessionPath.getAbsolute() + " already exists in ZooKeeper"); - } - } - - private ApplicationPackage createApplication(File userDir, - File configApplicationDir, - ApplicationId applicationId, - long sessionId, - long currentlyActiveSessionId, - boolean internalRedeploy) { - long deployTimestamp = System.currentTimeMillis(); - String user = System.getenv("USER"); - if (user == null) { - user = "unknown"; - } - DeployData deployData = new DeployData(user, userDir.getAbsolutePath(), applicationId, deployTimestamp, internalRedeploy, sessionId, currentlyActiveSessionId); - return FilesApplicationPackage.fromFileWithDeployData(configApplicationDir, deployData); - } - - private LocalSession createSessionFromApplication(ApplicationPackage applicationPackage, - long sessionId, - SessionZooKeeperClient sessionZKClient, - TimeoutBudget timeoutBudget, - Clock clock) { - log.log(Level.FINE, TenantRepository.logPre(tenant) + "Creating session " + sessionId + " in ZooKeeper"); - sessionZKClient.createNewSession(clock.instant().toEpochMilli(), TimeUnit.MILLISECONDS); - log.log(Level.FINE, TenantRepository.logPre(tenant) + "Creating upload waiter for session " + sessionId); - Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); - log.log(Level.FINE, TenantRepository.logPre(tenant) + "Done creating upload waiter for session " + sessionId); - SessionContext context = new SessionContext(applicationPackage, sessionZKClient, getSessionAppDir(sessionId), applicationRepo, hostRegistry, flagSource); - LocalSession session = new LocalSession(tenant, sessionId, sessionPreparer, context); - log.log(Level.FINE, TenantRepository.logPre(tenant) + "Waiting on upload waiter for session " + sessionId); - waiter.awaitCompletion(timeoutBudget.timeLeft()); - log.log(Level.FINE, TenantRepository.logPre(tenant) + "Done waiting on upload waiter for session " + sessionId); - return session; - } - - @Override - public LocalSession createSessionFromExisting(LocalSession existingSession, - DeployLogger logger, - boolean internalRedeploy, - TimeoutBudget timeoutBudget) { - File existingApp = getSessionAppDir(existingSession.getSessionId()); - ApplicationId existingApplicationId = existingSession.getApplicationId(); - - long activeSessionId = getActiveSessionId(existingApplicationId); - logger.log(Level.FINE, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); - LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); - // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() - session.setApplicationId(existingApplicationId); - session.setVespaVersion(existingSession.getVespaVersion()); - session.setDockerImageRepository(existingSession.getDockerImageRepository()); - session.setAthenzDomain(existingSession.getAthenzDomain()); - return session; - } - - private LocalSession create(File applicationFile, ApplicationId applicationId, long currentlyActiveSessionId, - boolean internalRedeploy, TimeoutBudget timeoutBudget) { - long sessionId = sessionCounter.nextSessionId(); - Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId)); - try { - ensureZKPathDoesNotExist(sessionIdPath); - SessionZooKeeperClient sessionZooKeeperClient = new SessionZooKeeperClient(curator, - configCurator, - sessionIdPath, - serverId, - nodeFlavors); - File userApplicationDir = tenantFileSystemDirs.getUserApplicationDir(sessionId); - IOUtils.copyDirectory(applicationFile, userApplicationDir); - ApplicationPackage applicationPackage = createApplication(applicationFile, - userApplicationDir, - applicationId, - sessionId, - currentlyActiveSessionId, - internalRedeploy); - applicationPackage.writeMetaData(); - return createSessionFromApplication(applicationPackage, sessionId, sessionZooKeeperClient, timeoutBudget, clock); - } catch (Exception e) { - throw new RuntimeException("Error creating session " + sessionIdPath, e); - } - } - - private File getSessionAppDir(long sessionId) { - File appDir = tenantFileSystemDirs.getUserApplicationDir(sessionId); - if (!appDir.exists() || !appDir.isDirectory()) { - throw new IllegalArgumentException("Unable to find correct application directory for session " + sessionId); - } - return appDir; - } - - @Override - public LocalSession loadSession(long sessionId) { - File sessionDir = getSessionAppDir(sessionId); - ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); - Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId)); - SessionZooKeeperClient sessionZKClient = new SessionZooKeeperClient(curator, - configCurator, - sessionIdPath, - serverId, - nodeFlavors); - SessionContext context = new SessionContext(applicationPackage, sessionZKClient, sessionDir, applicationRepo, - hostRegistry, flagSource); - return new LocalSession(tenant, sessionId, sessionPreparer, context); - } - - private long getActiveSessionId(ApplicationId applicationId) { - List<ApplicationId> applicationIds = applicationRepo.activeApplications(); - if (applicationIds.contains(applicationId)) { - return applicationRepo.requireActiveSessionOf(applicationId); - } - return nonExistingActiveSession; - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 00cafb77f6d..4542b8267e8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -2,19 +2,21 @@ package com.yahoo.vespa.config.server.session; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.Vtag; +import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; @@ -23,7 +25,6 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.lang.SettableOptional; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -31,27 +32,35 @@ import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; +import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; +import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; +import java.io.File; import java.io.IOException; import java.net.URI; import java.time.Instant; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -72,8 +81,9 @@ public class SessionPreparer { private final ConfigDefinitionRepo configDefinitionRepo; private final Curator curator; private final Zone zone; - private final FlagSource flagSource; private final SecretStore secretStore; + private final BooleanFlag distributeApplicationPackage; + private final FlagSource flagSource; @Inject public SessionPreparer(ModelFactoryRegistry modelFactoryRegistry, @@ -94,32 +104,40 @@ public class SessionPreparer { this.configDefinitionRepo = configDefinitionRepo; this.curator = curator; this.zone = zone; - this.flagSource = flagSource; this.secretStore = secretStore; + this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); + this.flagSource = flagSource; } /** * Prepares a session (validates, builds model, writes to zookeeper and distributes files) * - * @param context Contains classes needed to read/write session data. + * @param hostValidator host validator * @param logger For storing logs returned in response to client. * @param params parameters controlling behaviour of prepare. * @param currentActiveApplicationSet Set of currently active applications. - * @param tenantPath Zookeeper path for the tenant for this session + * @param tenantPath Zookeeper path for the tenant for this session * @return the config change actions that must be done to handle the activation of the models prepared. */ - public ConfigChangeActions prepare(SessionContext context, DeployLogger logger, PrepareParams params, - Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, - Instant now) { - Preparation preparation = new Preparation(context, logger, params, currentActiveApplicationSet, tenantPath); + public ConfigChangeActions prepare(HostValidator<ApplicationId> hostValidator, DeployLogger logger, PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, + Instant now, File serverDbSessionDir, ApplicationPackage applicationPackage, + SessionZooKeeperClient sessionZooKeeperClient) { + Preparation preparation = new Preparation(hostValidator, logger, params, currentActiveApplicationSet, + tenantPath, serverDbSessionDir, applicationPackage, sessionZooKeeperClient); + preparation.preprocess(); + + var distributedApplicationPackage = preparation.distributeApplicationPackage(); + try { AllocatedHosts allocatedHosts = preparation.buildModels(now); preparation.makeResult(allocatedHosts); if ( ! params.isDryRun()) { - preparation.writeStateZK(); + preparation.writeStateZK(distributedApplicationPackage); preparation.writeEndpointCertificateMetadataZK(); preparation.writeContainerEndpointsZK(); + preparation.writeApplicationRoles(); preparation.distribute(); } log.log(Level.FINE, () -> "time used " + params.getTimeoutBudget().timesUsed() + @@ -133,12 +151,9 @@ public class SessionPreparer { private class Preparation { - final SessionContext context; final DeployLogger logger; final PrepareParams params; - final Optional<ApplicationSet> currentActiveApplicationSet; - final Path tenantPath; final ApplicationId applicationId; /** The repository part of docker image to be used for this deployment */ @@ -147,46 +162,48 @@ public class SessionPreparer { /** The version of Vespa the application to be prepared specifies for its nodes */ final Version vespaVersion; - final ContainerEndpointsCache containerEndpoints; - final Set<ContainerEndpoint> endpointsSet; + final ContainerEndpointsCache containerEndpointsCache; + final List<ContainerEndpoint> containerEndpoints; final ModelContext.Properties properties; private final EndpointCertificateMetadataStore endpointCertificateMetadataStore; - private final EndpointCertificateRetriever endpointCertificateRetriever; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; - private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final Optional<AthenzDomain> athenzDomain; + private final ApplicationRolesStore applicationRolesStore; + private final Optional<ApplicationRoles> applicationRoles; + private final ApplicationPackage applicationPackage; + private final SessionZooKeeperClient sessionZooKeeperClient; - private ApplicationPackage applicationPackage; + private ApplicationPackage preprocessedApplicationPackage; private List<PreparedModelsBuilder.PreparedModelResult> modelResultList; private PrepareResult prepareResult; private final PreparedModelsBuilder preparedModelsBuilder; + private final FileDistributionProvider fileDistributionProvider; - Preparation(SessionContext context, DeployLogger logger, PrepareParams params, - Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath) { - this.context = context; + Preparation(HostValidator<ApplicationId> hostValidator, DeployLogger logger, PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, + File serverDbSessionDir, ApplicationPackage preprocessedApplicationPackage, + SessionZooKeeperClient sessionZooKeeperClient) { this.logger = logger; this.params = params; - this.currentActiveApplicationSet = currentActiveApplicationSet; - this.tenantPath = tenantPath; - + this.applicationPackage = preprocessedApplicationPackage; + this.sessionZooKeeperClient = sessionZooKeeperClient; this.applicationId = params.getApplicationId(); this.dockerImageRepository = params.dockerImageRepository(); this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion); - this.containerEndpoints = new ContainerEndpointsCache(tenantPath, curator); + this.containerEndpointsCache = new ContainerEndpointsCache(tenantPath, curator); this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); - this.endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); - + EndpointCertificateRetriever endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); this.endpointCertificateMetadata = params.endpointCertificateMetadata() .or(() -> params.tlsSecretsKeyName().map(EndpointCertificateMetadataSerializer::fromString)); - - endpointCertificateSecrets = endpointCertificateMetadata + Optional<EndpointCertificateSecrets> endpointCertificateSecrets = endpointCertificateMetadata .or(() -> endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId)) .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); - - this.endpointsSet = getEndpoints(params.containerEndpoints()); + this.containerEndpoints = readEndpointsIfNull(params.containerEndpoints()); this.athenzDomain = params.athenzDomain(); - + this.applicationRolesStore = new ApplicationRolesStore(curator, tenantPath); + this.applicationRoles = params.applicationRoles() + .or(() -> applicationRolesStore.readApplicationRoles(applicationId)); this.properties = new ModelContextImpl.Properties(params.getApplicationId(), configserverConfig.multitenant(), ConfigServerSpec.fromConfig(configserverConfig), @@ -195,18 +212,19 @@ public class SessionPreparer { configserverConfig.athenzDnsSuffix(), configserverConfig.hostedVespa(), zone, - endpointsSet, + Set.copyOf(containerEndpoints), params.isBootstrap(), - ! currentActiveApplicationSet.isPresent(), - context.getFlagSource(), + currentActiveApplicationSet.isEmpty(), + flagSource, endpointCertificateSecrets, - athenzDomain); + athenzDomain, applicationRoles); + this.fileDistributionProvider = fileDistributionFactory.createProvider(serverDbSessionDir); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, - fileDistributionFactory, + fileDistributionProvider, hostProvisionerProvider, - context, + hostValidator, logger, params, currentActiveApplicationSet, @@ -221,9 +239,23 @@ public class SessionPreparer { } } + FileReference distributeApplicationPackage() { + if ( ! distributeApplicationPackage.value()) return null; + + FileRegistry fileRegistry = fileDistributionProvider.getFileRegistry(); + FileReference fileReference = fileRegistry.addApplicationPackage(); + FileDistribution fileDistribution = fileDistributionProvider.getFileDistribution(); + log.log(Level.INFO, "Distribute application package for " + applicationId + " (" + fileReference + ") to other config servers"); + properties.configServerSpecs().stream() + .filter(spec -> ! spec.getHostName().equals(fileRegistry.fileSourceHost())) + .forEach(spec -> fileDistribution.startDownload(spec.getHostName(), spec.getConfigServerPort(), Set.of(fileReference))); + + return fileReference; + } + void preprocess() { try { - this.applicationPackage = context.getApplicationPackage().preprocess(properties.zone(), logger); + this.preprocessedApplicationPackage = applicationPackage.preprocess(properties.zone(), logger); } catch (IOException | TransformerException | ParserConfigurationException | SAXException e) { throw new IllegalArgumentException("Error preprocessing application package for " + applicationId, e); } @@ -233,7 +265,7 @@ public class SessionPreparer { AllocatedHosts buildModels(Instant now) { SettableOptional<AllocatedHosts> allocatedHosts = new SettableOptional<>(); this.modelResultList = preparedModelsBuilder.buildModels(applicationId, dockerImageRepository, vespaVersion, - applicationPackage, allocatedHosts, now); + preprocessedApplicationPackage, allocatedHosts, now); checkTimeout("build models"); return allocatedHosts.get(); } @@ -243,11 +275,12 @@ public class SessionPreparer { checkTimeout("making result from models"); } - void writeStateZK() { + void writeStateZK(FileReference distributedApplicationPackage) { log.log(Level.FINE, "Writing application package state to zookeeper"); - writeStateToZooKeeper(context.getSessionZooKeeperClient(), - applicationPackage, + writeStateToZooKeeper(sessionZooKeeperClient, + preprocessedApplicationPackage, applicationId, + distributedApplicationPackage, dockerImageRepository, vespaVersion, logger, @@ -264,12 +297,16 @@ public class SessionPreparer { } void writeContainerEndpointsZK() { - if (!params.containerEndpoints().isEmpty()) { // Use endpoints from parameter when explicitly given - containerEndpoints.write(applicationId, params.containerEndpoints()); - } + containerEndpointsCache.write(applicationId, containerEndpoints); checkTimeout("write container endpoints to zookeeper"); } + void writeApplicationRoles() { + applicationRoles.ifPresent(roles -> + applicationRolesStore.writeApplicationRoles(applicationId, roles)); + checkTimeout("write application roles to zookeeper"); + } + void distribute() { prepareResult.asList().forEach(modelResult -> modelResult.model .distributeFiles(modelResult.fileDistributionProvider.getFileDistribution())); @@ -280,11 +317,11 @@ public class SessionPreparer { return prepareResult.getConfigChangeActions(); } - private Set<ContainerEndpoint> getEndpoints(List<ContainerEndpoint> endpoints) { - if (endpoints == null || endpoints.isEmpty()) { - endpoints = this.containerEndpoints.read(applicationId); + private List<ContainerEndpoint> readEndpointsIfNull(List<ContainerEndpoint> endpoints) { + if (endpoints == null) { // endpoints are only set when prepared via HTTP + endpoints = this.containerEndpointsCache.read(applicationId); } - return ImmutableSet.copyOf(endpoints); + return List.copyOf(endpoints); } } @@ -292,6 +329,7 @@ public class SessionPreparer { private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient, ApplicationPackage applicationPackage, ApplicationId applicationId, + FileReference distributedApplicationPackage, Optional<DockerImage> dockerImageRepository, Version vespaVersion, DeployLogger deployLogger, @@ -303,6 +341,7 @@ public class SessionPreparer { zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); // Note: When changing the below you need to also change similar calls in SessionFactoryImpl.createSessionFromExisting() zooKeeperClient.writeApplicationId(applicationId); + if (distributeApplicationPackage.value()) zooKeeperClient.writeApplicationPackageReference(distributedApplicationPackage); zooKeeperClient.writeVespaVersion(vespaVersion); zooKeeperClient.writeDockerImageRepository(dockerImageRepository); zooKeeperClient.writeAthenzDomain(athenzDomain); @@ -345,7 +384,7 @@ public class SessionPreparer { */ public ConfigChangeActions getConfigChangeActions() { return new ConfigChangeActions(results.stream().map(result -> result.actions) - .flatMap(actions -> actions.stream()) + .flatMap(Collection::stream) .collect(Collectors.toList())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java new file mode 100644 index 00000000000..c7dc295c42d --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -0,0 +1,658 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.DeployData; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.TenantName; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.transaction.AbstractTransaction; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.TenantApplications; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.vespa.config.server.filedistribution.FileDirectory; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.server.zookeeper.SessionCounter; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * + * Session repository for a tenant. Stores session state in zookeeper and file system. There are two + * different session types (RemoteSession and LocalSession). + * + * @author Ulf Lilleengen + * @author hmusum + * + */ +public class SessionRepository { + + private static final Logger log = Logger.getLogger(SessionRepository.class.getName()); + private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); + private static final long nonExistingActiveSession = 0; + + private final SessionCache<LocalSession> localSessionCache = new SessionCache<>(); + private final SessionCache<RemoteSession> remoteSessionCache = new SessionCache<>(); + private final Map<Long, LocalSessionStateWatcher> localSessionStateWatchers = new HashMap<>(); + private final Map<Long, RemoteSessionStateWatcher> remoteSessionStateWatchers = new HashMap<>(); + private final Duration sessionLifetime; + private final Clock clock; + private final Curator curator; + private final Executor zkWatcherExecutor; + private final TenantFileSystemDirs tenantFileSystemDirs; + private final BooleanFlag distributeApplicationPackage; + private final ReloadHandler reloadHandler; + private final MetricUpdater metrics; + private final Curator.DirectoryCache directoryCache; + private final TenantApplications applicationRepo; + private final SessionPreparer sessionPreparer; + private final Path sessionsPath; + private final TenantName tenantName; + private final GlobalComponentRegistry componentRegistry; + + public SessionRepository(TenantName tenantName, + GlobalComponentRegistry componentRegistry, + TenantApplications applicationRepo, + ReloadHandler reloadHandler, + FlagSource flagSource, + SessionPreparer sessionPreparer) { + this.tenantName = tenantName; + this.componentRegistry = componentRegistry; + this.sessionsPath = TenantRepository.getSessionsPath(tenantName); + this.clock = componentRegistry.getClock(); + this.curator = componentRegistry.getCurator(); + this.sessionLifetime = Duration.ofSeconds(componentRegistry.getConfigserverConfig().sessionLifetime()); + this.zkWatcherExecutor = command -> componentRegistry.getZkWatcherExecutor().execute(tenantName, command); + this.tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName); + this.applicationRepo = applicationRepo; + this.sessionPreparer = sessionPreparer; + this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); + this.reloadHandler = reloadHandler; + this.metrics = componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); + + loadLocalSessions(); + initializeRemoteSessions(); + this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, componentRegistry.getZkCacheExecutor()); + this.directoryCache.addListener(this::childEvent); + this.directoryCache.start(); + } + + // ---------------- Local sessions ---------------------------------------------------------------- + + public synchronized void addSession(LocalSession session) { + localSessionCache.addSession(session); + Path sessionsPath = TenantRepository.getSessionsPath(session.getTenantName()); + long sessionId = session.getSessionId(); + Curator.FileCache fileCache = curator.createFileCache(sessionsPath.append(String.valueOf(sessionId)).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); + localSessionStateWatchers.put(sessionId, new LocalSessionStateWatcher(fileCache, session, this, zkWatcherExecutor)); + } + + public LocalSession getLocalSession(long sessionId) { + return localSessionCache.getSession(sessionId); + } + + public List<LocalSession> getLocalSessions() { + return localSessionCache.getSessions(); + } + + private void loadLocalSessions() { + File[] sessions = tenantFileSystemDirs.sessionsPath().listFiles(sessionApplicationsFilter); + if (sessions == null) { + return; + } + for (File session : sessions) { + try { + addSession(createSessionFromId(Long.parseLong(session.getName()))); + } catch (IllegalArgumentException e) { + log.log(Level.WARNING, "Could not load session '" + + session.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); + } + } + } + + public ConfigChangeActions prepareLocalSession(LocalSession session, + DeployLogger logger, + PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, + Path tenantPath, + Instant now) { + applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( + logger.log(Level.FINE, "Created application " + params.getApplicationId()); + long sessionId = session.getSessionId(); + SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.createPrepareWaiter(); + ConfigChangeActions actions = sessionPreparer.prepare(applicationRepo.getHostValidator(), logger, params, + currentActiveApplicationSet, tenantPath, now, + getSessionAppDir(sessionId), + session.getApplicationPackage(), sessionZooKeeperClient); + session.setPrepared(); + waiter.awaitCompletion(params.getTimeoutBudget().timeLeft()); + return actions; + } + + public void deleteExpiredSessions(Map<ApplicationId, Long> activeSessions) { + log.log(Level.FINE, "Purging old sessions"); + try { + for (LocalSession candidate : localSessionCache.getSessions()) { + Instant createTime = candidate.getCreateTime(); + log.log(Level.FINE, "Candidate session for deletion: " + candidate.getSessionId() + ", created: " + createTime); + + // Sessions with state other than ACTIVATED + if (hasExpired(candidate) && !isActiveSession(candidate)) { + deleteLocalSession(candidate); + } else if (createTime.plus(Duration.ofDays(1)).isBefore(clock.instant())) { + // Sessions with state ACTIVATE, but which are not actually active + ApplicationId applicationId = candidate.getApplicationId(); + Long activeSession = activeSessions.get(applicationId); + if (activeSession == null || activeSession != candidate.getSessionId()) { + deleteLocalSession(candidate); + log.log(Level.INFO, "Deleted inactive session " + candidate.getSessionId() + " created " + + createTime + " for '" + applicationId + "'"); + } + } + } + // Make sure to catch here, to avoid executor just dying in case of issues ... + } catch (Throwable e) { + log.log(Level.WARNING, "Error when purging old sessions ", e); + } + log.log(Level.FINE, "Done purging old sessions"); + } + + private boolean hasExpired(LocalSession candidate) { + return (candidate.getCreateTime().plus(sessionLifetime).isBefore(clock.instant())); + } + + private boolean isActiveSession(LocalSession candidate) { + return candidate.getStatus() == Session.Status.ACTIVATE; + } + + public void deleteLocalSession(LocalSession session) { + long sessionId = session.getSessionId(); + log.log(Level.FINE, "Deleting local session " + sessionId); + LocalSessionStateWatcher watcher = localSessionStateWatchers.remove(sessionId); + if (watcher != null) watcher.close(); + localSessionCache.removeSession(sessionId); + NestedTransaction transaction = new NestedTransaction(); + deleteLocalSession(session, transaction); + transaction.commit(); + } + + /** Add transactions to delete this session to the given nested transaction */ + public void deleteLocalSession(LocalSession session, NestedTransaction transaction) { + long sessionId = session.getSessionId(); + SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); + transaction.add(sessionZooKeeperClient.deleteTransaction(), FileTransaction.class); + transaction.add(FileTransaction.from(FileOperations.delete(getSessionAppDir(sessionId).getAbsolutePath()))); + } + + public void close() { + deleteAllSessions(); + tenantFileSystemDirs.delete(); + try { + if (directoryCache != null) { + directoryCache.close(); + } + } catch (Exception e) { + log.log(Level.WARNING, "Exception when closing path cache", e); + } finally { + checkForRemovedSessions(new ArrayList<>()); + } + } + + private void deleteAllSessions() { + List<LocalSession> sessions = new ArrayList<>(localSessionCache.getSessions()); + for (LocalSession session : sessions) { + deleteLocalSession(session); + } + } + + // ---------------- Remote sessions ---------------------------------------------------------------- + + public RemoteSession getRemoteSession(long sessionId) { + return remoteSessionCache.getSession(sessionId); + } + + public List<Long> getRemoteSessions() { + return getSessionList(curator.getChildren(sessionsPath)); + } + + public void addRemoteSession(RemoteSession session) { + remoteSessionCache.addSession(session); + metrics.incAddedSessions(); + } + + public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { + int deleted = 0; + for (long sessionId : getRemoteSessions()) { + RemoteSession session = remoteSessionCache.getSession(sessionId); + if (session == null) continue; // Internal sessions not in synch with zk, continue + if (session.getStatus() == Session.Status.ACTIVATE) continue; + if (sessionHasExpired(session.getCreateTime(), expiryTime, clock)) { + log.log(Level.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); + session.delete(); + deleted++; + } + } + return deleted; + } + + private boolean sessionHasExpired(Instant created, Duration expiryTime, Clock clock) { + return (created.plus(expiryTime).isBefore(clock.instant())); + } + + private List<Long> getSessionListFromDirectoryCache(List<ChildData> children) { + return getSessionList(children.stream() + .map(child -> Path.fromString(child.getPath()).getName()) + .collect(Collectors.toList())); + } + + private List<Long> getSessionList(List<String> children) { + return children.stream().map(Long::parseLong).collect(Collectors.toList()); + } + + private void initializeRemoteSessions() throws NumberFormatException { + getRemoteSessions().forEach(this::sessionAdded); + } + + private synchronized void sessionsChanged() throws NumberFormatException { + List<Long> sessions = getSessionListFromDirectoryCache(directoryCache.getCurrentData()); + checkForRemovedSessions(sessions); + checkForAddedSessions(sessions); + } + + private void checkForRemovedSessions(List<Long> sessions) { + for (RemoteSession session : remoteSessionCache.getSessions()) + if ( ! sessions.contains(session.getSessionId())) + sessionRemoved(session.getSessionId()); + } + + private void checkForAddedSessions(List<Long> sessions) { + for (Long sessionId : sessions) + if (remoteSessionCache.getSession(sessionId) == null) + sessionAdded(sessionId); + } + + /** + * A session for which we don't have a watcher, i.e. hitherto unknown to us. + * + * @param sessionId session id for the new session + */ + public void sessionAdded(long sessionId) { + log.log(Level.FINE, () -> "Adding session to SessionRepository: " + sessionId); + RemoteSession session = createRemoteSession(sessionId); + Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); + Curator.FileCache fileCache = curator.createFileCache(sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); + fileCache.addListener(this::nodeChanged); + loadSessionIfActive(session); + addRemoteSession(session); + remoteSessionStateWatchers.put(sessionId, new RemoteSessionStateWatcher(fileCache, reloadHandler, session, metrics, zkWatcherExecutor)); + if (distributeApplicationPackage.value()) { + Optional<LocalSession> localSession = createLocalSessionUsingDistributedApplicationPackage(sessionId); + localSession.ifPresent(this::addSession); + } + } + + private void sessionRemoved(long sessionId) { + RemoteSessionStateWatcher watcher = remoteSessionStateWatchers.remove(sessionId); + if (watcher != null) watcher.close(); + remoteSessionCache.removeSession(sessionId); + metrics.incRemovedSessions(); + } + + private void loadSessionIfActive(RemoteSession session) { + for (ApplicationId applicationId : applicationRepo.activeApplications()) { + if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { + log.log(Level.FINE, () -> "Found active application for session " + session.getSessionId() + " , loading it"); + reloadHandler.reloadConfig(session.ensureApplicationLoaded()); + log.log(Level.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); + return; + } + } + } + + private void nodeChanged() { + zkWatcherExecutor.execute(() -> { + Multiset<Session.Status> sessionMetrics = HashMultiset.create(); + for (RemoteSession session : remoteSessionCache.getSessions()) { + sessionMetrics.add(session.getStatus()); + } + metrics.setNewSessions(sessionMetrics.count(Session.Status.NEW)); + metrics.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE)); + metrics.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE)); + metrics.setDeactivatedSessions(sessionMetrics.count(Session.Status.DEACTIVATE)); + }); + } + + @SuppressWarnings("unused") + private void childEvent(CuratorFramework ignored, PathChildrenCacheEvent event) { + zkWatcherExecutor.execute(() -> { + log.log(Level.FINE, () -> "Got child event: " + event); + switch (event.getType()) { + case CHILD_ADDED: + sessionsChanged(); + synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); + break; + case CHILD_REMOVED: + case CONNECTION_RECONNECTED: + sessionsChanged(); + break; + } + }); + } + + private void synchronizeOnNew(List<Long> sessionList) { + for (long sessionId : sessionList) { + RemoteSession session = remoteSessionCache.getSession(sessionId); + if (session == null) continue; // session might have been deleted after getting session list + log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); + session.confirmUpload(); + } + } + + /** + * Creates a new deployment session from an application package. + * + * @param applicationDirectory a File pointing to an application. + * @param applicationId application id for this new session. + * @param timeoutBudget Timeout for creating session and waiting for other servers. + * @return a new session + */ + public LocalSession createSession(File applicationDirectory, ApplicationId applicationId, + TimeoutBudget timeoutBudget, Optional<Long> activeSessionId) { + return create(applicationDirectory, applicationId, activeSessionId, false, timeoutBudget); + } + + public RemoteSession createRemoteSession(long sessionId) { + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + return new RemoteSession(tenantName, sessionId, componentRegistry, sessionZKClient); + } + + private void ensureSessionPathDoesNotExist(long sessionId) { + Path sessionPath = getSessionPath(sessionId); + if (componentRegistry.getConfigCurator().exists(sessionPath.getAbsolute())) { + throw new IllegalArgumentException("Path " + sessionPath.getAbsolute() + " already exists in ZooKeeper"); + } + } + + private ApplicationPackage createApplication(File userDir, + File configApplicationDir, + ApplicationId applicationId, + long sessionId, + Optional<Long> currentlyActiveSessionId, + boolean internalRedeploy) { + long deployTimestamp = System.currentTimeMillis(); + String user = System.getenv("USER"); + if (user == null) { + user = "unknown"; + } + DeployData deployData = new DeployData(user, userDir.getAbsolutePath(), applicationId, deployTimestamp, + internalRedeploy, sessionId, currentlyActiveSessionId.orElse(nonExistingActiveSession)); + return FilesApplicationPackage.fromFileWithDeployData(configApplicationDir, deployData); + } + + private LocalSession createSessionFromApplication(ApplicationPackage applicationPackage, + long sessionId, + TimeoutBudget timeoutBudget, + Clock clock) { + log.log(Level.FINE, TenantRepository.logPre(tenantName) + "Creating session " + sessionId + " in ZooKeeper"); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + sessionZKClient.createNewSession(clock.instant()); + Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); + LocalSession session = new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient, applicationRepo); + waiter.awaitCompletion(timeoutBudget.timeLeft()); + return session; + } + + /** + * Creates a new deployment session from an already existing session. + * + * @param existingSession the session to use as base + * @param logger a deploy logger where the deploy log will be written. + * @param internalRedeploy whether this session is for a system internal redeploy — not an application package change + * @param timeoutBudget timeout for creating session and waiting for other servers. + * @return a new session + */ + public LocalSession createSessionFromExisting(Session existingSession, + DeployLogger logger, + boolean internalRedeploy, + TimeoutBudget timeoutBudget) { + File existingApp = getSessionAppDir(existingSession.getSessionId()); + ApplicationId existingApplicationId = existingSession.getApplicationId(); + + Optional<Long> activeSessionId = getActiveSessionId(existingApplicationId); + logger.log(Level.FINE, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); + LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); + // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() + session.setApplicationId(existingApplicationId); + if (distributeApplicationPackage.value() && existingSession.getApplicationPackageReference() != null) { + session.setApplicationPackageReference(existingSession.getApplicationPackageReference()); + } + session.setVespaVersion(existingSession.getVespaVersion()); + session.setDockerImageRepository(existingSession.getDockerImageRepository()); + session.setAthenzDomain(existingSession.getAthenzDomain()); + return session; + } + + private LocalSession create(File applicationFile, ApplicationId applicationId, Optional<Long> currentlyActiveSessionId, + boolean internalRedeploy, TimeoutBudget timeoutBudget) { + long sessionId = getNextSessionId(); + try { + ensureSessionPathDoesNotExist(sessionId); + ApplicationPackage app = createApplicationPackage(applicationFile, applicationId, + sessionId, currentlyActiveSessionId, internalRedeploy); + return createSessionFromApplication(app, sessionId, timeoutBudget, clock); + } catch (Exception e) { + throw new RuntimeException("Error creating session " + sessionId, e); + } + } + + /** + * This method is used when creating a session based on a remote session and the distributed application package + * It does not wait for session being created on other servers + */ + private LocalSession createLocalSession(File applicationFile, ApplicationId applicationId, + long sessionId, Optional<Long> currentlyActiveSessionId) { + try { + ApplicationPackage applicationPackage = createApplicationPackage(applicationFile, applicationId, + sessionId, currentlyActiveSessionId, false); + SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); + return new LocalSession(tenantName, sessionId, applicationPackage, sessionZooKeeperClient, applicationRepo); + } catch (Exception e) { + throw new RuntimeException("Error creating session " + sessionId, e); + } + } + + private ApplicationPackage createApplicationPackage(File applicationFile, ApplicationId applicationId, + long sessionId, Optional<Long> currentlyActiveSessionId, + boolean internalRedeploy) throws IOException { + File userApplicationDir = getSessionAppDir(sessionId); + IOUtils.copyDirectory(applicationFile, userApplicationDir); + ApplicationPackage applicationPackage = createApplication(applicationFile, + userApplicationDir, + applicationId, + sessionId, + currentlyActiveSessionId, + internalRedeploy); + applicationPackage.writeMetaData(); + return applicationPackage; + } + + /** + * Returns a new session instance for the given session id. + */ + LocalSession createSessionFromId(long sessionId) { + File sessionDir = getAndValidateExistingSessionAppDir(sessionId); + ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + return new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient, applicationRepo); + } + + /** + * Returns a new session instance for the given session id. + */ + Optional<LocalSession> createLocalSessionUsingDistributedApplicationPackage(long sessionId) { + if (applicationRepo.hasLocalSession(sessionId)) { + log.log(Level.FINE, "Local session for session id " + sessionId + " already exists"); + return Optional.of(createSessionFromId(sessionId)); + } + + log.log(Level.INFO, "Creating local session for session id " + sessionId); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + FileReference fileReference = sessionZKClient.readApplicationPackageReference(); + log.log(Level.FINE, "File reference for session id " + sessionId + ": " + fileReference); + if (fileReference != null) { + File rootDir = new File(Defaults.getDefaults().underVespaHome(componentRegistry.getConfigserverConfig().fileReferencesDir())); + File sessionDir; + FileDirectory fileDirectory = new FileDirectory(rootDir); + try { + sessionDir = fileDirectory.getFile(fileReference); + } catch (IllegalArgumentException e) { + // We cannot be guaranteed that the file reference exists (it could be that it has not + // been downloaded yet), and e.g when bootstrapping we cannot throw an exception in that case + log.log(Level.INFO, "File reference for session id " + sessionId + ": " + fileReference + " not found in " + fileDirectory); + return Optional.empty(); + } + ApplicationId applicationId = sessionZKClient.readApplicationId(); + return Optional.of(createLocalSession(sessionDir, + applicationId, + sessionId, + getActiveSessionId(applicationId))); + } + return Optional.empty(); + } + + private Optional<Long> getActiveSessionId(ApplicationId applicationId) { + List<ApplicationId> applicationIds = applicationRepo.activeApplications(); + return applicationIds.contains(applicationId) + ? Optional.of(applicationRepo.requireActiveSessionOf(applicationId)) + : Optional.empty(); + } + + private long getNextSessionId() { + return new SessionCounter(componentRegistry.getConfigCurator(), tenantName).nextSessionId(); + } + + private Path getSessionPath(long sessionId) { + return sessionsPath.append(String.valueOf(sessionId)); + } + + + private SessionZooKeeperClient createSessionZooKeeperClient(long sessionId) { + String serverId = componentRegistry.getConfigserverConfig().serverId(); + Optional<NodeFlavors> nodeFlavors = componentRegistry.getZone().nodeFlavors(); + Path sessionPath = getSessionPath(sessionId); + return new SessionZooKeeperClient(curator, componentRegistry.getConfigCurator(), sessionPath, serverId, nodeFlavors); + } + + private File getAndValidateExistingSessionAppDir(long sessionId) { + File appDir = getSessionAppDir(sessionId); + if (!appDir.exists() || !appDir.isDirectory()) { + throw new IllegalArgumentException("Unable to find correct application directory for session " + sessionId); + } + return appDir; + } + + private File getSessionAppDir(long sessionId) { + return new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName).getUserApplicationDir(sessionId); + } + + @Override + public String toString() { + return getLocalSessions().toString(); + } + + private static class FileTransaction extends AbstractTransaction { + + public static FileTransaction from(FileOperation operation) { + FileTransaction transaction = new FileTransaction(); + transaction.add(operation); + return transaction; + } + + @Override + public void prepare() { } + + @Override + public void commit() { + for (Operation operation : operations()) + ((FileOperation)operation).commit(); + } + + } + + /** Factory for file operations */ + private static class FileOperations { + + /** Creates an operation which recursively deletes the given path */ + public static DeleteOperation delete(String pathToDelete) { + return new DeleteOperation(pathToDelete); + } + + } + + private interface FileOperation extends Transaction.Operation { + + void commit(); + + } + + /** + * Recursively deletes this path and everything below. + * Succeeds with no action if the path does not exist. + */ + private static class DeleteOperation implements FileOperation { + + private final String pathToDelete; + + DeleteOperation(String pathToDelete) { + this.pathToDelete = pathToDelete; + } + + @Override + public void commit() { + // TODO: Check delete access in prepare() + IOUtils.recursiveDeleteDir(new File(pathToDelete)); + } + + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index c200825e810..807629a2148 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; import com.yahoo.component.Vtag; +import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigDefinitionRepo; @@ -11,10 +12,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeFlavors; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.Utf8; -import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.UserConfigDefinitionRepo; import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; @@ -25,8 +24,9 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import java.time.Instant; import java.util.Optional; -import java.util.concurrent.TimeUnit; +import java.util.logging.Level; /** * Zookeeper client for a specific session. Path for a session is /config/v2/tenants/<tenant>/sessions/<sessionid> @@ -41,6 +41,7 @@ public class SessionZooKeeperClient { // NOTE: Any state added here MUST also be propagated in com.yahoo.vespa.config.server.deploy.Deployment.prepare() static final String APPLICATION_ID_PATH = "applicationId"; + static final String APPLICATION_PACKAGE_REFERENCE_PATH = "applicationPackageReference"; private static final String VERSION_PATH = "version"; private static final String CREATE_TIME_PATH = "createTime"; private static final String DOCKER_IMAGE_REPOSITORY_PATH = "dockerImageRepository"; @@ -54,7 +55,7 @@ public class SessionZooKeeperClient { // Only for testing when cache loader does not need cache entries. public SessionZooKeeperClient(Curator curator, Path sessionPath) { - this(curator, ConfigCurator.create(curator), sessionPath, "", Optional.empty()); + this(curator, ConfigCurator.create(curator), sessionPath, "1", Optional.empty()); } public SessionZooKeeperClient(Curator curator, @@ -127,18 +128,6 @@ public class SessionZooKeeperClient { return curator.getCompletionWaiter(path, getNumberOfMembers(), serverId); } - public void delete(NestedTransaction transaction ) { - try { - log.log(Level.FINE, "Deleting " + sessionPath.getAbsolute()); - CuratorTransaction curatorTransaction = new CuratorTransaction(curator); - CuratorOperations.deleteAll(sessionPath.getAbsolute(), curator).forEach(curatorTransaction::add); - transaction.add(curatorTransaction); - transaction.commit(); - } catch (RuntimeException e) { - log.log(Level.INFO, "Error deleting session (" + sessionPath.getAbsolute() + ") from zookeeper", e); - } - } - /** Returns a transaction deleting this session on commit */ public CuratorTransaction deleteTransaction() { return CuratorTransaction.from(CuratorOperations.deleteAll(sessionPath.getAbsolute(), curator), curator); @@ -165,6 +154,19 @@ public class SessionZooKeeperClient { return ApplicationId.fromSerializedForm(configCurator.getData(applicationIdPath())); } + void writeApplicationPackageReference(FileReference applicationPackageReference) { + configCurator.putData(applicationPackageReferencePath(), applicationPackageReference.value()); + } + + FileReference readApplicationPackageReference() { + if ( ! configCurator.exists(applicationPackageReferencePath())) return null; // This should not happen. + return new FileReference(configCurator.getData(applicationPackageReferencePath())); + } + + private String applicationPackageReferencePath() { + return sessionPath.append(APPLICATION_PACKAGE_REFERENCE_PATH).getAbsolute(); + } + private String versionPath() { return sessionPath.append(VERSION_PATH).getAbsolute(); } @@ -196,11 +198,10 @@ public class SessionZooKeeperClient { dockerImageRepository.ifPresent(repo -> configCurator.putData(dockerImageRepositoryPath(), repo.repository())); } - // in seconds - public long readCreateTime() { + public Instant readCreateTime() { String path = getCreateTimePath(); - if ( ! configCurator.exists(path)) return 0L; - return Long.parseLong(configCurator.getData(path)); + if ( ! configCurator.exists(path)) return Instant.EPOCH; + return Instant.ofEpochSecond(Long.parseLong(configCurator.getData(path))); } private String getCreateTimePath() { @@ -243,14 +244,13 @@ public class SessionZooKeeperClient { * Create necessary paths atomically for a new session. * * @param createTime Time of session creation. - * @param timeUnit Time unit of createTime. */ - public void createNewSession(long createTime, TimeUnit timeUnit) { + public void createNewSession(Instant createTime) { CuratorTransaction transaction = new CuratorTransaction(curator); transaction.add(CuratorOperations.create(sessionPath.getAbsolute())); transaction.add(CuratorOperations.create(sessionPath.append(UPLOAD_BARRIER).getAbsolute())); transaction.add(createWriteStatusTransaction(Session.Status.NEW).operations()); - transaction.add(CuratorOperations.create(getCreateTimePath(), Utf8.toBytes(String.valueOf(timeUnit.toSeconds(createTime))))); + transaction.add(CuratorOperations.create(getCreateTimePath(), Utf8.toBytes(String.valueOf(createTime.getEpochSecond())))); transaction.commit(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ApplicationRolesSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ApplicationRolesSerializer.java new file mode 100644 index 00000000000..132828de7b4 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ApplicationRolesSerializer.java @@ -0,0 +1,27 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.ApplicationRoles; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; + +/** + * @author mortent + */ +public class ApplicationRolesSerializer { + + private final static String hostRoleField = "applicationHostRole"; + private final static String containerRoleField = "applicationContainerRole"; + + + public static void toSlime(ApplicationRoles applicationRoles, Cursor object) { + object.setString(hostRoleField, applicationRoles.applicationHostRole()); + object.setString(containerRoleField, applicationRoles.applicationContainerRole()); + } + + public static ApplicationRoles fromSlime(Inspector inspector) { + return new ApplicationRoles(inspector.field(hostRoleField).asString(), + inspector.field(containerRoleField).asString()); + + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ApplicationRolesStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ApplicationRolesStore.java new file mode 100644 index 00000000000..a41e5465509 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ApplicationRolesStore.java @@ -0,0 +1,64 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.ApplicationRoles; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.path.Path; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.transaction.CuratorOperations; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; + +import java.util.Optional; + +/** + * Stores application roles for an application. + * + * @author mortent + */ +public class ApplicationRolesStore { + + private final Path path; + private final Curator curator; + + public ApplicationRolesStore(Curator curator, Path tenantPath) { + this.curator = curator; + this.path = tenantPath.append("applicationRoles/"); + } + + /** Reads the application roles from ZooKeeper, if it exists */ + public Optional<ApplicationRoles> readApplicationRoles(ApplicationId application) { + try { + Optional<byte[]> data = curator.getData(applicationRolesPath(application)); + if (data.isEmpty() || data.get().length == 0) return Optional.empty(); + Slime slime = SlimeUtils.jsonToSlime(data.get()); + ApplicationRoles applicationRoles = ApplicationRolesSerializer.fromSlime(slime.get()); + return Optional.of(applicationRoles); + } catch (Exception e) { + throw new RuntimeException("Error reading application roles of " + application, e); + } + } + + /** Writes the application roles to ZooKeeper */ + public void writeApplicationRoles(ApplicationId application, ApplicationRoles applicationRoles) { + try { + Slime slime = new Slime(); + ApplicationRolesSerializer.toSlime(applicationRoles, slime.setObject()); + curator.set(applicationRolesPath(application), SlimeUtils.toJsonBytes(slime)); + } catch (Exception e) { + throw new RuntimeException("Could not write application roles of " + application, e); + } + } + + /** Returns a transaction which deletes application roles if they exist */ + public CuratorTransaction delete(ApplicationId application) { + if (!curator.exists(applicationRolesPath(application))) return CuratorTransaction.empty(curator); + return CuratorTransaction.from(CuratorOperations.delete(applicationRolesPath(application).getAbsolute()), curator); + } + + /** Returns the path storing the application roles for an application */ + private Path applicationRolesPath(ApplicationId application) { + return path.append(application.serializedForm()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java index 97179e5234b..0d277eeb1d1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java @@ -41,10 +41,7 @@ public class ContainerEndpointsCache { } public void write(ApplicationId applicationId, List<ContainerEndpoint> endpoints) { - if (endpoints.isEmpty()) return; - var slime = ContainerEndpointSerializer.endpointListToSlime(endpoints); - try { var bytes = SlimeUtils.toJsonBytes(slime); curator.set(containerEndpointsPath(applicationId), bytes); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java index bbd3ae55f10..f7c8ae9d5c3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java @@ -6,9 +6,7 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; -import com.yahoo.vespa.config.server.session.RemoteSessionRepo; -import com.yahoo.vespa.config.server.session.SessionFactory; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.curator.Curator; import org.apache.zookeeper.data.Stat; @@ -28,31 +26,24 @@ public class Tenant implements TenantHandlerProvider { static final String APPLICATIONS = "applications"; private final TenantName name; - private final RemoteSessionRepo remoteSessionRepo; private final Path path; - private final SessionFactory sessionFactory; - private final LocalSessionRepo localSessionRepo; + private final SessionRepository sessionRepository; private final TenantApplications applicationRepo; private final RequestHandler requestHandler; private final ReloadHandler reloadHandler; private final Curator curator; Tenant(TenantName name, - Path path, - SessionFactory sessionFactory, - LocalSessionRepo localSessionRepo, - RemoteSessionRepo remoteSessionRepo, + SessionRepository sessionRepository, RequestHandler requestHandler, ReloadHandler reloadHandler, TenantApplications applicationRepo, Curator curator) { this.name = name; - this.path = path; + this.path = TenantRepository.getTenantPath(name); this.requestHandler = requestHandler; this.reloadHandler = reloadHandler; - this.remoteSessionRepo = remoteSessionRepo; - this.sessionFactory = sessionFactory; - this.localSessionRepo = localSessionRepo; + this.sessionRepository = sessionRepository; this.applicationRepo = applicationRepo; this.curator = curator; } @@ -75,13 +66,8 @@ public class Tenant implements TenantHandlerProvider { return requestHandler; } - /** - * The RemoteSessionRepo for this - * - * @return repo - */ - public RemoteSessionRepo getRemoteSessionRepo() { - return remoteSessionRepo; + public SessionRepository getSessionRepo() { + return sessionRepository; } public TenantName getName() { @@ -92,13 +78,7 @@ public class Tenant implements TenantHandlerProvider { return path; } - public SessionFactory getSessionFactory() { - return sessionFactory; - } - - public LocalSessionRepo getLocalSessionRepo() { - return localSessionRepo; - } + public SessionRepository getSessionRepository() { return sessionRepository; } @Override public String toString() { @@ -141,9 +121,8 @@ public class Tenant implements TenantHandlerProvider { * Called by watchers as a reaction to {@link #delete()}. */ void close() { - remoteSessionRepo.close(); // Closes watchers and clears memory. applicationRepo.close(); // Closes watchers. - localSessionRepo.close(); // Closes watchers, clears memory, and deletes local files and ZK session state. + sessionRepository.close(); // Closes watchers, clears memory, and deletes local files and ZK session state. } /** Deletes the tenant tree from ZooKeeper (application and session status for the tenant) and triggers {@link #close()}. */ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java deleted file mode 100644 index 108892803c1..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.host.HostValidator; -import com.yahoo.vespa.config.server.RequestHandler; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.session.*; - -import java.util.Collections; - -/** - * Builder for helping out with tenant creation. Each of a tenants dependencies may be overridden for testing. - * - * @author Ulf Lilleengen - */ -public class TenantBuilder { - - private final GlobalComponentRegistry componentRegistry; - private final TenantName tenant; - private RemoteSessionRepo remoteSessionRepo; - private LocalSessionRepo localSessionRepo; - private SessionFactory sessionFactory; - private LocalSessionLoader localSessionLoader; - private TenantApplications applicationRepo; - private TenantRequestHandler reloadHandler; - private RequestHandler requestHandler; - private RemoteSessionFactory remoteSessionFactory; - private HostValidator<ApplicationId> hostValidator; - - private TenantBuilder(GlobalComponentRegistry componentRegistry, TenantName tenant) { - this.componentRegistry = componentRegistry; - this.tenant = tenant; - } - - public static TenantBuilder create(GlobalComponentRegistry componentRegistry, TenantName tenant) { - return new TenantBuilder(componentRegistry, tenant); - } - - public TenantBuilder withSessionFactory(SessionFactory sessionFactory) { - this.sessionFactory = sessionFactory; - return this; - } - - public TenantBuilder withLocalSessionRepo(LocalSessionRepo localSessionRepo) { - this.localSessionRepo = localSessionRepo; - return this; - } - - public TenantBuilder withApplicationRepo(TenantApplications applicationRepo) { - this.applicationRepo = applicationRepo; - return this; - } - - public TenantBuilder withRequestHandler(RequestHandler requestHandler) { - this.requestHandler = requestHandler; - return this; - } - - /** - * Create a real tenant from the properties given by this builder. - * - * @return a new {@link Tenant} instance. - */ - public Tenant build() { - createTenantRequestHandler(); - createApplicationRepo(); - createRemoteSessionFactory(); - createRemoteSessionRepo(); - createSessionFactory(); - createLocalSessionRepo(); - return new Tenant(tenant, - TenantRepository.getTenantPath(tenant), - sessionFactory, - localSessionRepo, - remoteSessionRepo, - requestHandler, - reloadHandler, - applicationRepo, - componentRegistry.getCurator()); - } - - private void createLocalSessionRepo() { - if (localSessionRepo == null) { - localSessionRepo = new LocalSessionRepo(tenant, componentRegistry, localSessionLoader); - } - } - - private void createSessionFactory() { - if (sessionFactory == null || localSessionLoader == null) { - SessionFactoryImpl impl = new SessionFactoryImpl(componentRegistry, applicationRepo, hostValidator, tenant); - if (sessionFactory == null) { - sessionFactory = impl; - } - if (localSessionLoader == null) { - localSessionLoader = impl; - } - } - } - - private void createApplicationRepo() { - if (applicationRepo == null) { - applicationRepo = reloadHandler.applications(); - } - } - - private void createTenantRequestHandler() { - if (requestHandler == null || reloadHandler == null) { - TenantRequestHandler impl = new TenantRequestHandler(componentRegistry.getMetrics(), - tenant, - Collections.singletonList(componentRegistry.getReloadListener()), - ConfigResponseFactory.create(componentRegistry.getConfigserverConfig()), - componentRegistry); - this.hostValidator = impl; - if (requestHandler == null) { - requestHandler = impl; - } - reloadHandler = impl; - } - } - - private void createRemoteSessionFactory() { - if (remoteSessionFactory == null) { - remoteSessionFactory = new RemoteSessionFactory(componentRegistry, tenant); - } - } - - private void createRemoteSessionRepo() { - remoteSessionRepo = new RemoteSessionRepo(componentRegistry, - remoteSessionFactory, - reloadHandler, - tenant, - applicationRepo); - - } - - public TenantName getTenantName() { return tenant; } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 04deed40c54..304fbb6786a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -7,10 +7,14 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.application.TenantApplications; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -43,7 +47,7 @@ import java.util.stream.Collectors; * This component will monitor the set of tenants in the config server by watching in ZooKeeper. * It will set up Tenant objects accordingly, which will manage the config sessions per tenant. * This class will read the preexisting set of tenants from ZooKeeper at startup. (For now it will also - * create a default tenant since that will be used for API that do no know about tenants or have not yet + * create a default tenant since that will be used for APIs that do no know about tenants or have not yet * implemented support for it). * * This instance is called from two different threads, the http handler threads and the zookeeper watcher threads. @@ -68,7 +72,7 @@ public class TenantRepository { private static final Logger log = Logger.getLogger(TenantRepository.class.getName()); private final Map<TenantName, Tenant> tenants = Collections.synchronizedMap(new LinkedHashMap<>()); - private final GlobalComponentRegistry globalComponentRegistry; + private final GlobalComponentRegistry componentRegistry; private final List<TenantListener> tenantListeners = Collections.synchronizedList(new ArrayList<>()); private final Curator curator; @@ -83,30 +87,30 @@ public class TenantRepository { /** * Creates a new tenant repository * - * @param globalComponentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} + * @param componentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} */ @Inject - public TenantRepository(GlobalComponentRegistry globalComponentRegistry) { - this(globalComponentRegistry, true); + public TenantRepository(GlobalComponentRegistry componentRegistry) { + this(componentRegistry, true); } /** * Creates a new tenant repository * - * @param globalComponentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} + * @param componentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} * @param useZooKeeperWatchForTenantChanges set to false for tests where you want to control adding and deleting * tenants yourself */ - public TenantRepository(GlobalComponentRegistry globalComponentRegistry, boolean useZooKeeperWatchForTenantChanges) { - this.globalComponentRegistry = globalComponentRegistry; - ConfigserverConfig configserverConfig = globalComponentRegistry.getConfigserverConfig(); + public TenantRepository(GlobalComponentRegistry componentRegistry, boolean useZooKeeperWatchForTenantChanges) { + this.componentRegistry = componentRegistry; + ConfigserverConfig configserverConfig = componentRegistry.getConfigserverConfig(); this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders()); this.throwExceptionIfBootstrappingFails = configserverConfig.throwIfBootstrappingTenantRepoFails(); - this.curator = globalComponentRegistry.getCurator(); - metricUpdater = globalComponentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); - this.tenantListeners.add(globalComponentRegistry.getTenantListener()); - this.zkCacheExecutor = globalComponentRegistry.getZkCacheExecutor(); - this.zkWatcherExecutor = globalComponentRegistry.getZkWatcherExecutor(); + this.curator = componentRegistry.getCurator(); + metricUpdater = componentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); + this.tenantListeners.add(componentRegistry.getTenantListener()); + this.zkCacheExecutor = componentRegistry.getZkCacheExecutor(); + this.zkWatcherExecutor = componentRegistry.getZkWatcherExecutor(); curator.framework().getConnectionStateListenable().addListener(this::stateChanged); curator.create(tenantsPath); @@ -138,12 +142,13 @@ public class TenantRepository { } public synchronized void addTenant(TenantName tenantName) { - addTenant(TenantBuilder.create(globalComponentRegistry, tenantName)); + addTenant(tenantName, null, null); } - public synchronized void addTenant(TenantBuilder builder) { - writeTenantPath(builder.getTenantName()); - createTenant(builder); + public synchronized void addTenant(TenantName tenantName, RequestHandler requestHandler, + ReloadHandler reloadHandler) { + writeTenantPath(tenantName); + createTenant(tenantName, requestHandler, reloadHandler); } private static Set<TenantName> readTenantsFromZooKeeper(Curator curator) { @@ -194,17 +199,35 @@ public class TenantRepository { } } - private void createTenant(TenantName tenantName) { - createTenant(TenantBuilder.create(globalComponentRegistry, tenantName)); + protected void createTenant(TenantName tenantName) { + createTenant(tenantName, null, null); } // Creates tenant and all its dependencies. This also includes loading active applications - protected void createTenant(TenantBuilder builder) { - TenantName tenantName = builder.getTenantName(); + private void createTenant(TenantName tenantName, RequestHandler requestHandler, ReloadHandler reloadHandler) { if (tenants.containsKey(tenantName)) return; + TenantApplications applicationRepo = + new TenantApplications(tenantName, + curator, + componentRegistry.getZkWatcherExecutor(), + componentRegistry.getZkCacheExecutor(), + componentRegistry.getMetrics(), + componentRegistry.getReloadListener(), + componentRegistry.getConfigserverConfig(), + componentRegistry.getHostRegistries().createApplicationHostRegistry(tenantName), + new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName)); + if (requestHandler == null) + requestHandler = applicationRepo; + if (reloadHandler == null) + reloadHandler = applicationRepo; + SessionRepository sessionRepository = new SessionRepository(tenantName, componentRegistry, + applicationRepo, reloadHandler, + componentRegistry.getFlagSource(), + componentRegistry.getSessionPreparer()); log.log(Level.INFO, "Creating tenant '" + tenantName + "'"); - Tenant tenant = builder.build(); + Tenant tenant = new Tenant(tenantName, sessionRepository, requestHandler, + reloadHandler, applicationRepo, componentRegistry.getCurator()); notifyNewTenant(tenant); tenants.putIfAbsent(tenantName, tenant); } @@ -218,7 +241,6 @@ public class TenantRepository { return tenants.get(DEFAULT_TENANT); } - private void removeUnusedApplications() { getAllTenants().forEach(tenant -> tenant.getApplicationRepo().removeUnusedApplications()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java deleted file mode 100644 index 1fb09b993fb..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.component.Version; -import com.yahoo.concurrent.StripedExecutor; -import com.yahoo.config.FileReference; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import java.util.logging.Level; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.GetConfigRequest; -import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.NotFoundException; -import com.yahoo.vespa.config.server.ReloadHandler; -import com.yahoo.vespa.config.server.ReloadListener; -import com.yahoo.vespa.config.server.RequestHandler; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.application.ApplicationMapper; -import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.application.VersionDoesNotExistException; -import com.yahoo.vespa.config.server.host.HostRegistries; -import com.yahoo.vespa.config.server.host.HostRegistry; -import com.yahoo.vespa.config.server.host.HostValidator; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.Lock; - -import java.time.Clock; -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ExecutorService; - -import static java.util.stream.Collectors.toSet; - -/** - * A per tenant request handler, for handling reload (activate application) and getConfig requests for - * a set of applications belonging to a tenant. - * - * @author Harald Musum - */ -public class TenantRequestHandler implements RequestHandler, ReloadHandler, HostValidator<ApplicationId> { - - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(TenantRequestHandler.class.getName()); - - private final Metrics metrics; - private final TenantName tenant; - private final List<ReloadListener> reloadListeners; - private final ConfigResponseFactory responseFactory; - private final HostRegistry<ApplicationId> hostRegistry; - private final ApplicationMapper applicationMapper = new ApplicationMapper(); - private final MetricUpdater tenantMetricUpdater; - private final Clock clock = Clock.systemUTC(); - private final TenantApplications applications; - - public TenantRequestHandler(Metrics metrics, - TenantName tenant, - List<ReloadListener> reloadListeners, - ConfigResponseFactory responseFactory, - GlobalComponentRegistry registry) { // TODO jvenstad: Merge this class with TenantApplications, and straighten this out. - this.metrics = metrics; - this.tenant = tenant; - this.reloadListeners = List.copyOf(reloadListeners); - this.responseFactory = responseFactory; - this.tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); - this.hostRegistry = registry.getHostRegistries().createApplicationHostRegistry(tenant); - this.applications = TenantApplications.create(registry, this, tenant); - - } - - /** - * Gets a config for the given app, or null if not found - */ - @Override - public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); - } - return application.resolveConfig(req, responseFactory); - } - - // For testing only - long getApplicationGeneration(ApplicationId appId, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - return application.getApplicationGeneration(); - } - - private void notifyReloadListeners(ApplicationSet applicationSet) { - for (ReloadListener reloadListener : reloadListeners) { - reloadListener.hostsUpdated(tenant, hostRegistry.getAllHosts()); - reloadListener.configActivated(applicationSet); - } - } - - /** - * Activates the config of the given app. Notifies listeners - * - * @param applicationSet the {@link ApplicationSet} to be reloaded - */ - @Override - public void reloadConfig(ApplicationSet applicationSet) { - ApplicationId id = applicationSet.getId(); - try (Lock lock = applications.lock(id)) { - if ( ! applications.exists(id)) - return; // Application was deleted before activation. - if (applicationSet.getApplicationGeneration() != applications.requireActiveSessionOf(id)) - return; // Application activated a new session before we got here. - - setLiveApp(applicationSet); - notifyReloadListeners(applicationSet); - } - } - - @Override - public void removeApplication(ApplicationId applicationId) { - try (Lock lock = applications.lock(applicationId)) { - if (applications.exists(applicationId)) - return; // Application was deployed again. - - if (applicationMapper.hasApplication(applicationId, clock.instant())) { - applicationMapper.remove(applicationId); - hostRegistry.removeHostsForKey(applicationId); - reloadListenersOnRemove(applicationId); - tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); - } - } - } - - @Override - public void removeApplicationsExcept(Set<ApplicationId> applications) { - for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { - if ( ! applications.contains(activeApplication)) { - log.log(Level.INFO, "Will remove deleted application " + activeApplication.toShortString()); - removeApplication(activeApplication); - } - } - } - - private void reloadListenersOnRemove(ApplicationId applicationId) { - for (ReloadListener listener : reloadListeners) { - listener.hostsUpdated(tenant, hostRegistry.getAllHosts()); - listener.applicationRemoved(applicationId); - } - } - - private void setLiveApp(ApplicationSet applicationSet) { - ApplicationId id = applicationSet.getId(); - Collection<String> hostsForApp = applicationSet.getAllHosts(); - hostRegistry.update(id, hostsForApp); - applicationSet.updateHostMetrics(); - tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - applicationMapper.register(id, applicationSet); - } - - @Override - public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> keyToMatch, boolean recursive) { - Application application = getApplication(appId, vespaVersion); - return listConfigs(application, keyToMatch, recursive); - } - - private Set<ConfigKey<?>> listConfigs(Application application, ConfigKey<?> keyToMatch, boolean recursive) { - Set<ConfigKey<?>> ret = new LinkedHashSet<>(); - for (ConfigKey<?> key : application.allConfigsProduced()) { - String configId = key.getConfigId(); - if (recursive) { - key = new ConfigKey<>(key.getName(), configId, key.getNamespace()); - } else { - // Include first part of id as id - key = new ConfigKey<>(key.getName(), configId.split("/")[0], key.getNamespace()); - } - if (keyToMatch != null) { - String n = key.getName(); // Never null - String ns = key.getNamespace(); // Never null - if (n.equals(keyToMatch.getName()) && - ns.equals(keyToMatch.getNamespace()) && - configId.startsWith(keyToMatch.getConfigId()) && - !(configId.equals(keyToMatch.getConfigId()))) { - - if (!recursive) { - // For non-recursive, include the id segment we were searching for, and first part of the rest - key = new ConfigKey<>(key.getName(), appendOneLevelOfId(keyToMatch.getConfigId(), configId), key.getNamespace()); - } - ret.add(key); - } - } else { - ret.add(key); - } - } - return ret; - } - - @Override - public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive) { - Application application = getApplication(appId, vespaVersion); - return listConfigs(application, null, recursive); - } - - /** - * Given baseIdSegment search/ and id search/qrservers/default.0, return search/qrservers - * @return id segment with one extra level from the id appended - */ - String appendOneLevelOfId(String baseIdSegment, String id) { - if ("".equals(baseIdSegment)) return id.split("/")[0]; - String theRest = id.substring(baseIdSegment.length()); - if ("".equals(theRest)) return id; - theRest = theRest.replaceFirst("/", ""); - String theRestFirstSeg = theRest.split("/")[0]; - return baseIdSegment+"/"+theRestFirstSeg; - } - - @Override - public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - return application.allConfigsProduced(); - } - - private Application getApplication(ApplicationId appId, Optional<Version> vespaVersion) { - try { - return applicationMapper.getForVersion(appId, vespaVersion, clock.instant()); - } catch (VersionDoesNotExistException ex) { - throw new NotFoundException(String.format("%sNo such application (id %s): %s", TenantRepository.logPre(tenant), appId, ex.getMessage())); - } - } - - @Override - public Set<String> allConfigIds(ApplicationId appId, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - return application.allConfigIds(); - } - - @Override - public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { - return hasHandler(appId, vespaVersion); - } - - private boolean hasHandler(ApplicationId appId, Optional<Version> vespaVersion) { - return applicationMapper.hasApplicationForVersion(appId, vespaVersion, clock.instant()); - } - - @Override - public ApplicationId resolveApplicationId(String hostName) { - ApplicationId applicationId = hostRegistry.getKeyForHost(hostName); - if (applicationId == null) { - applicationId = ApplicationId.defaultId(); - } - return applicationId; - } - - @Override - public Set<FileReference> listFileReferences(ApplicationId applicationId) { - return applicationMapper.listApplications(applicationId).stream() - .flatMap(app -> app.getModel().fileReferences().stream()) - .collect(toSet()); - } - - @Override - public void verifyHosts(ApplicationId key, Collection<String> newHosts) { - hostRegistry.verifyHosts(key, newHosts); - for (ReloadListener reloadListener : reloadListeners) { - reloadListener.verifyHostsAreAvailable(tenant, newHosts); - } - } - - TenantApplications applications() { return applications; } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java index f3f9c914be8..20ac4b65c64 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -3,16 +3,11 @@ package com.yahoo.vespa.config.server.zookeeper; import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.io.IOUtils; -import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.Curator; -import java.io.File; -import java.io.FilenameFilter; -import java.io.IOException; -import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; /** * A (stateful) curator wrapper for the config server. This simplifies Curator method calls used by the config server @@ -49,8 +44,6 @@ public class ConfigCurator { /** Path for session state */ public static final String SESSIONSTATE_ZK_SUBPATH = "/sessionState"; - private static final FilenameFilter acceptsAllFileNameFilter = (dir, name) -> true; - private final Curator curator; public static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigCurator.class.getName()); @@ -111,11 +104,6 @@ public class ConfigCurator { return (data == null) ? null : Utf8.toString(data); } - /** Returns the data at a path and node. Replaces / by # in node names. Returns null if the path doesn't exist. */ - public byte[] getBytes(String path, String node) { - return getBytes(createFullPath(path, node)); - } - /** * Returns the data at a path, or null if the path does not exist. * @@ -234,65 +222,6 @@ public class ConfigCurator { putData(path, name, data); } - /** - * Takes for instance the dir /app and puts the contents into the given ZK path. Ignores files starting with dot, - * and dirs called CVS. - * - * @param dir directory which holds the summary class part files - * @param path zookeeper path - * @param filenameFilter A FilenameFilter which decides which files in dir are fed to zookeeper - * @param recurse recurse subdirectories - */ - void feedZooKeeper(File dir, String path, FilenameFilter filenameFilter, boolean recurse) { - try { - if (filenameFilter == null) { - filenameFilter = acceptsAllFileNameFilter; - } - if (!dir.isDirectory()) { - log.fine(dir.getCanonicalPath() + " is not a directory. Not feeding the files into ZooKeeper."); - return; - } - for (File file : listFiles(dir, filenameFilter)) { - if (file.getName().startsWith(".")) continue; //.svn , .git ... - if ("CVS".equals(file.getName())) continue; - if (file.isFile()) { - byte[] contents = IOUtils.readFileBytes(file); - putData(path, file.getName(), contents); - } else if (recurse && file.isDirectory()) { - createNode(path, file.getName()); - feedZooKeeper(file, path + '/' + file.getName(), filenameFilter, recurse); - } - } - } - catch (IOException e) { - throw new RuntimeException("Exception feeding ZooKeeper at path " + path, e); - } - } - - /** - * Same as normal listFiles, but use the filter only for normal files - * - * @param dir directory to list files in - * @param filter A FilenameFilter which decides which files in dir are listed - * @return an array of Files - */ - protected File[] listFiles(File dir, FilenameFilter filter) { - File[] rawList = dir.listFiles(); - List<File> ret = new ArrayList<>(); - if (rawList != null) { - for (File f : rawList) { - if (f.isDirectory()) { - ret.add(f); - } else { - if (filter.accept(dir, f.getName())) { - ret.add(f); - } - } - } - } - return ret.toArray(new File[0]); - } - /** Deletes the node at the given path, and any children it may have. If the node does not exist this does nothing */ public void deleteRecurse(String path) { try { diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 0cd283ca62b..d3b39e572ee 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -106,7 +106,6 @@ </handler> <handler id='com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/prepareandactivate</binding> - <binding>http://*/application/v2/tenant/*/session/*/prepareandactivate</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.SessionContentHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/session/*/content/*</binding> diff --git a/configserver/src/test/apps/hosted/services.xml b/configserver/src/test/apps/hosted/services.xml index a5d2aae8de9..a5c8fa1d26f 100644 --- a/configserver/src/test/apps/hosted/services.xml +++ b/configserver/src/test/apps/hosted/services.xml @@ -9,7 +9,7 @@ <container version="1.0"> <http> <filtering> - <access-control domain="foo" write="true" /> + <access-control domain="myDomain" write="true" /> </filtering> <server id="foo"/> </http> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 9ae079c87e0..f0aa38a228e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -1,14 +1,22 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.FileReference; +import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.application.api.ApplicationMetaData; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationRoles; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Deployment; +import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; @@ -17,21 +25,41 @@ import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.application.OrchestratorMock; +import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployTester; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.config.server.session.RemoteSession; +import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.session.SilentDeployLogger; +import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.model.VespaModelFactory; +import org.hamcrest.core.Is; +import org.jetbrains.annotations.NotNull; import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import java.io.File; @@ -43,15 +71,19 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -64,6 +96,8 @@ public class ApplicationRepositoryTest { private final static File testAppJdiscOnly = new File("src/test/apps/app-jdisc-only"); private final static File testAppJdiscOnlyRestart = new File("src/test/apps/app-jdisc-only-restart"); private final static File testAppLogServerWithContainer = new File("src/test/apps/app-logserver-with-container"); + private final static File app1 = new File("src/test/apps/cs1"); + private final static File app2 = new File("src/test/apps/cs2"); private final static TenantName tenant1 = TenantName.from("test1"); private final static TenantName tenant2 = TenantName.from("test2"); @@ -75,16 +109,33 @@ public class ApplicationRepositoryTest { private SessionHandlerTest.MockProvisioner provisioner; private OrchestratorMock orchestrator; private TimeoutBudget timeoutBudget; + private ConfigCurator configCurator; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + @Before - public void setup() { + public void setup() throws IOException { + setup(new InMemoryFlagSource()); + } + + public void setup(FlagSource flagSource) throws IOException { Curator curator = new MockCurator(); - tenantRepository = new TenantRepository(new TestComponentRegistry.Builder() - .curator(curator) - .build()); + configCurator = ConfigCurator.create(curator); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build()) + .flagSource(flagSource) + .build(); + tenantRepository = new TenantRepository(componentRegistry, false); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); tenantRepository.addTenant(tenant1); tenantRepository.addTenant(tenant2); @@ -99,16 +150,22 @@ public class ApplicationRepositoryTest { } @Test - public void prepareAndActivate() throws IOException { - PrepareResult result = prepareAndActivateApp(testApp); + public void prepareAndActivate() { + PrepareResult result = prepareAndActivate(testApp); assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); assertTrue(result.configChangeActions().getRestartActions().isEmpty()); + + TenantName tenantName = applicationId().tenant(); + Tenant tenant = tenantRepository.getTenant(tenantName); + LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo() + .requireActiveSessionOf(applicationId())); + session.getAllocatedHosts(); } @Test - public void prepareAndActivateWithRestart() throws IOException { - prepareAndActivateApp(testAppJdiscOnly); - PrepareResult result = prepareAndActivateApp(testAppJdiscOnlyRestart); + public void prepareAndActivateWithRestart() { + prepareAndActivate(testAppJdiscOnly); + PrepareResult result = prepareAndActivate(testAppJdiscOnlyRestart); assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); assertFalse(result.configChangeActions().getRestartActions().isEmpty()); } @@ -121,6 +178,23 @@ public class ApplicationRepositoryTest { } @Test + public void redeploy() { + PrepareResult result = deployApp(testApp); + + long firstSessionId = result.sessionId(); + + PrepareResult result2 = deployApp(testApp); + long secondSessionId = result2.sessionId(); + assertNotEquals(firstSessionId, secondSessionId); + + TenantName tenantName = applicationId().tenant(); + Tenant tenant = tenantRepository.getTenant(tenantName); + LocalSession session = tenant.getSessionRepository().getLocalSession( + tenant.getApplicationRepo().requireActiveSessionOf(applicationId())); + assertEquals(firstSessionId, session.getMetaData().getPreviousActiveGeneration()); + } + + @Test public void createFromActiveSession() { PrepareResult result = deployApp(testApp); long sessionId = applicationRepository.createSessionFromExisting(applicationId(), @@ -228,24 +302,32 @@ public class ApplicationRepositoryTest { @Test public void delete() { + TenantName tenantName = applicationId().tenant(); + Tenant tenant = tenantRepository.getTenant(tenantName); { PrepareResult result = deployApp(testApp); long sessionId = result.sessionId(); - Tenant tenant = tenantRepository.getTenant(applicationId().tenant()); - LocalSession applicationData = tenant.getLocalSessionRepo().getSession(sessionId); + LocalSession applicationData = tenant.getSessionRepository().getLocalSession(sessionId); assertNotNull(applicationData); assertNotNull(applicationData.getApplicationId()); - assertNotNull(tenant.getRemoteSessionRepo().getSession(sessionId)); + assertNotNull(tenant.getSessionRepo().getLocalSession(sessionId)); assertNotNull(applicationRepository.getActiveSession(applicationId())); + String sessionNode = TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId)).getAbsolute(); + assertTrue(configCurator.exists(sessionNode)); + TenantFileSystemDirs tenantFileSystemDirs = tenant.getApplicationRepo().getTenantFileSystemDirs(); + File sessionFile = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); + assertTrue(sessionFile.exists()); // Delete app and verify that it has been deleted from repos and provisioner assertTrue(applicationRepository.delete(applicationId())); assertNull(applicationRepository.getActiveSession(applicationId())); - assertNull(tenant.getLocalSessionRepo().getSession(sessionId)); - assertNull(tenant.getRemoteSessionRepo().getSession(sessionId)); + assertNull(tenant.getSessionRepository().getLocalSession(sessionId)); + assertNull(tenant.getSessionRepo().getLocalSession(sessionId)); assertTrue(provisioner.removed); assertEquals(tenant.getName(), provisioner.lastApplicationId.tenant()); assertEquals(applicationId(), provisioner.lastApplicationId); + assertFalse(configCurator.exists(sessionNode)); + assertFalse(sessionFile.exists()); assertFalse(applicationRepository.delete(applicationId())); } @@ -280,22 +362,21 @@ public class ApplicationRepositoryTest { // No active session or remote session (deleted in step above), but an exception was thrown above // A new delete should cleanup and be successful - LocalSession activeSession = applicationRepository.getActiveSession(applicationId()); + RemoteSession activeSession = applicationRepository.getActiveSession(applicationId()); assertNull(activeSession); - Tenant tenant = tenantRepository.getTenant(applicationId().tenant()); - assertNull(tenant.getRemoteSessionRepo().getSession(prepareResult.sessionId())); + assertNull(tenant.getSessionRepo().getLocalSession(prepareResult.sessionId())); assertTrue(applicationRepository.delete(applicationId())); } } @Test - public void testDeletingInactiveSessions() { + public void testDeletingInactiveSessions() throws IOException { ManualClock clock = new ManualClock(Instant.now()); ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(Files.createTempDir().getAbsolutePath()) - .configDefinitionsDir(Files.createTempDir().getAbsolutePath()) + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) .sessionLifetime(60)); DeployTester tester = new DeployTester(configserverConfig, clock); tester.deployApp("src/test/apps/app", clock.instant()); // session 2 (numbering starts at 2) @@ -316,18 +397,33 @@ public class ApplicationRepositoryTest { assertNotEquals(activeSessionId, deployment3session); // No change to active session id assertEquals(activeSessionId, tester.tenant().getApplicationRepo().requireActiveSessionOf(tester.applicationId())); - assertEquals(3, tester.tenant().getLocalSessionRepo().listSessions().size()); + SessionRepository sessionRepository = tester.tenant().getSessionRepository(); + assertEquals(3, sessionRepository.getLocalSessions().size()); clock.advance(Duration.ofHours(1)); // longer than session lifetime // All sessions except 3 should be removed after the call to deleteExpiredLocalSessions tester.applicationRepository().deleteExpiredLocalSessions(); - Collection<LocalSession> sessions = tester.tenant().getLocalSessionRepo().listSessions(); + Collection<LocalSession> sessions = sessionRepository.getLocalSessions(); assertEquals(1, sessions.size()); - assertEquals(3, new ArrayList<>(sessions).get(0).getSessionId()); + ArrayList<LocalSession> localSessions = new ArrayList<>(sessions); + LocalSession localSession = localSessions.get(0); + assertEquals(3, localSession.getSessionId()); // There should be no expired remote sessions in the common case assertEquals(0, tester.applicationRepository().deleteExpiredRemoteSessions(clock, Duration.ofSeconds(0))); + + // Deploy, but do not activate + Optional<com.yahoo.config.provision.Deployment> deployment4 = tester.redeployFromLocalActive(); + assertTrue(deployment4.isPresent()); + deployment4.get().prepare(); // session 5 (not activated) + + assertEquals(2, sessionRepository.getLocalSessions().size()); + sessionRepository.deleteLocalSession(localSession); + assertEquals(1, sessionRepository.getLocalSessions().size()); + + // Check that trying to expire when there are no active sessions works + tester.applicationRepository().deleteExpiredLocalSessions(); } @Test @@ -352,6 +448,242 @@ public class ApplicationRepositoryTest { assertEquals(expected.values, actual.values); } + @Test + public void deletesApplicationRoles() { + var tenant = tenantRepository.getTenant(tenant1); + var applicationId = applicationId(tenant1); + var prepareParams = new PrepareParams.Builder().applicationId(applicationId) + .applicationRoles(ApplicationRoles.fromString("hostRole","containerRole")).build(); + deployApp(testApp, prepareParams); + var approlesStore = new ApplicationRolesStore(tenant.getCurator(), tenant.getPath()); + var appRoles = approlesStore.readApplicationRoles(applicationId); + + // App roles present after deploy + assertTrue(appRoles.isPresent()); + assertEquals("hostRole", appRoles.get().applicationHostRole()); + assertEquals("containerRole", appRoles.get().applicationContainerRole()); + + assertTrue(applicationRepository.delete(applicationId)); + + // App roles deleted on application delete + assertTrue(approlesStore.readApplicationRoles(applicationId).isEmpty()); + } + + @Test + public void require_that_provision_info_can_be_read() { + prepareAndActivate(testAppJdiscOnly); + + TenantName tenantName = applicationId().tenant(); + Tenant tenant = tenantRepository.getTenant(tenantName); + LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo().requireActiveSessionOf(applicationId())); + + List<NetworkPorts.Allocation> list = new ArrayList<>(); + list.add(new NetworkPorts.Allocation(8080, "container", "container/container.0", "http")); + list.add(new NetworkPorts.Allocation(19070, "configserver", "admin/configservers/configserver.0", "rpc")); + list.add(new NetworkPorts.Allocation(19071, "configserver", "admin/configservers/configserver.0", "http")); + list.add(new NetworkPorts.Allocation(19080, "logserver", "admin/logserver", "rpc")); + list.add(new NetworkPorts.Allocation(19081, "logserver", "admin/logserver", "unused/1")); + list.add(new NetworkPorts.Allocation(19082, "logserver", "admin/logserver", "unused/2")); + list.add(new NetworkPorts.Allocation(19083, "logserver", "admin/logserver", "unused/3")); + list.add(new NetworkPorts.Allocation(19089, "logd", "hosts/mytesthost/logd", "http")); + list.add(new NetworkPorts.Allocation(19090, "configproxy", "hosts/mytesthost/configproxy", "rpc")); + list.add(new NetworkPorts.Allocation(19092, "metricsproxy-container", "admin/metrics/mytesthost", "http")); + list.add(new NetworkPorts.Allocation(19093, "metricsproxy-container", "admin/metrics/mytesthost", "http/1")); + list.add(new NetworkPorts.Allocation(19094, "metricsproxy-container", "admin/metrics/mytesthost", "rpc/admin")); + list.add(new NetworkPorts.Allocation(19095, "metricsproxy-container", "admin/metrics/mytesthost", "rpc/metrics")); + list.add(new NetworkPorts.Allocation(19097, "config-sentinel", "hosts/mytesthost/sentinel", "rpc")); + list.add(new NetworkPorts.Allocation(19098, "config-sentinel", "hosts/mytesthost/sentinel", "http")); + list.add(new NetworkPorts.Allocation(19099, "slobrok", "admin/slobrok.0", "rpc")); + list.add(new NetworkPorts.Allocation(19100, "container", "container/container.0", "http/1")); + list.add(new NetworkPorts.Allocation(19101, "container", "container/container.0", "messaging")); + list.add(new NetworkPorts.Allocation(19102, "container", "container/container.0", "rpc/admin")); + list.add(new NetworkPorts.Allocation(19103, "slobrok", "admin/slobrok.0", "http")); + + AllocatedHosts info = session.getAllocatedHosts(); + assertNotNull(info); + assertThat(info.getHosts().size(), is(1)); + assertTrue(info.getHosts().contains(new HostSpec("mytesthost", + Collections.emptyList(), + Optional.empty()))); + Optional<NetworkPorts> portsCopy = info.getHosts().iterator().next().networkPorts(); + assertTrue(portsCopy.isPresent()); + assertThat(portsCopy.get().allocations(), is(list)); + } + + @Test + public void testActivationOfUnpreparedSession() { + // Needed so we can test that the original active session is still active after a failed activation + PrepareResult result = deployApp(testApp); + long firstSession = result.sessionId(); + + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, testAppJdiscOnly); + exceptionRule.expect(IllegalStateException.class); + exceptionRule.expectMessage(containsString("tenant:test1 Session 3 is not prepared")); + applicationRepository.activate(tenantRepository.getTenant(tenant1), sessionId, timeoutBudget, false); + + RemoteSession activeSession = applicationRepository.getActiveSession(applicationId()); + assertEquals(firstSession, activeSession.getSessionId()); + assertEquals(Session.Status.ACTIVATE, activeSession.getStatus()); + } + + @Test + public void testActivationTimesOut() { + // Needed so we can test that the original active session is still active after a failed activation + PrepareResult result = deployApp(testAppJdiscOnly); + long firstSession = result.sessionId(); + + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, testAppJdiscOnly); + applicationRepository.prepare(tenantRepository.getTenant(tenant1), sessionId, prepareParams(), clock.instant()); + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage(containsString("Timeout exceeded when trying to activate 'test1.testapp'")); + applicationRepository.activate(tenantRepository.getTenant(tenant1), sessionId, new TimeoutBudget(clock, Duration.ofSeconds(0)), false); + + RemoteSession activeSession = applicationRepository.getActiveSession(applicationId()); + assertEquals(firstSession, activeSession.getSessionId()); + assertEquals(Session.Status.ACTIVATE, activeSession.getStatus()); + } + + @Test + public void testActivationOfSessionCreatedFromNoLongerActiveSessionFails() { + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); + + PrepareResult result1 = deployApp(testAppJdiscOnly); + result1.sessionId(); + + long sessionId2 = applicationRepository.createSessionFromExisting(applicationId(), + new BaseDeployLogger(), + false, + timeoutBudget); + // Deploy and activate another session + PrepareResult result2 = deployApp(testAppJdiscOnly); + result2.sessionId(); + + applicationRepository.prepare(tenantRepository.getTenant(tenant1), sessionId2, prepareParams(), clock.instant()); + exceptionRule.expect(ActivationConflictException.class); + exceptionRule.expectMessage(containsString("tenant:test1 app:testapp:default Cannot activate session 3 because the currently active session (4) has changed since session 3 was created (was 2 at creation time)")); + applicationRepository.activate(tenantRepository.getTenant(tenant1), sessionId2, timeoutBudget, false); + } + + @Test + public void testPrepareAndActivateAlreadyActivatedSession() { + PrepareResult result = deployApp(testAppJdiscOnly); + long sessionId = result.sessionId(); + + exceptionRule.expect(IllegalStateException.class); + exceptionRule.expectMessage(containsString("Session is active: 2")); + applicationRepository.prepare(tenantRepository.getTenant(tenant1), sessionId, prepareParams(), clock.instant()); + + exceptionRule.expect(IllegalStateException.class); + exceptionRule.expectMessage(containsString("tenant:test1 app:testapp:default Session 2 is already active")); + applicationRepository.activate(tenantRepository.getTenant(tenant1), sessionId, timeoutBudget, false); + } + + @Test + public void testThatPreviousSessionIsDeactivated() { + deployApp(testAppJdiscOnly); + Session firstSession = applicationRepository.getActiveSession(applicationId()); + + deployApp(testAppJdiscOnly); + + assertEquals(Session.Status.DEACTIVATE, firstSession.getStatus()); + } + + @Test + public void testResolveForAppId() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337 , config.intval()); + } + + @Test + public void testResolveConfigForMultipleApps() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + ApplicationId appId2 = new ApplicationId.Builder() + .tenant(tenant1) + .applicationName("myapp2") + .instanceName("default") + .build(); + applicationRepository.deploy(app2, new PrepareParams.Builder() + .applicationId(appId2) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337, config.intval()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler2 = reloadConfig(appId2); + SimpletypesConfig config2 = resolve(SimpletypesConfig.class, requestHandler2, appId2, vespaVersion); + assertEquals(1330, config2.intval()); + + assertTrue(requestHandler.hasApplication(applicationId(), Optional.of(vespaVersion))); + assertThat(requestHandler.resolveApplicationId("doesnotexist"), Is.is(ApplicationId.defaultId())); + assertThat(requestHandler.resolveApplicationId("mytesthost"), Is.is(new ApplicationId.Builder() + .tenant(tenant1) + .applicationName("testapp").build())); // Host set in application package. + } + + @Test + public void testResolveMultipleVersions() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337, config.intval()); + + // TODO: Revisit this test, I cannot see that we create a model for version 3.2.1 + config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), new Version(3, 2, 1)); + assertThat(config.intval(), Is.is(1337)); + } + + @Test + public void testResolveForDeletedApp() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337 , config.intval()); + + applicationRepository.delete(applicationId()); + + exceptionRule.expect(com.yahoo.vespa.config.server.NotFoundException.class); + exceptionRule.expectMessage(containsString("No such application id: test1.testapp")); + resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + } + + @Test + public void testDistributionOfApplicationPackage() throws IOException { + FlagSource flagSource = new InMemoryFlagSource() + .withBooleanFlag(Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.id(), true); + setup(flagSource); + applicationRepository.deploy(app1, prepareParams()); + } + private ApplicationRepository createApplicationRepository() { return new ApplicationRepository(tenantRepository, provisioner, @@ -363,12 +695,8 @@ public class ApplicationRepositoryTest { new NullMetric()); } - private PrepareResult prepareAndActivateApp(File application) throws IOException { - FilesApplicationPackage appDir = FilesApplicationPackage.fromFile(application); - ApplicationId applicationId = applicationId(); - long sessionId = applicationRepository.createSession(applicationId, timeoutBudget, appDir.getAppDir()); - return applicationRepository.prepareAndActivate(tenantRepository.getTenant(applicationId.tenant()), - sessionId, prepareParams(), false, Instant.now()); + private PrepareResult prepareAndActivate(File application) { + return applicationRepository.deploy(application, prepareParams(), false, Instant.now()); } private PrepareResult deployApp(File applicationPackage) { @@ -393,10 +721,9 @@ public class ApplicationRepositoryTest { private ApplicationMetaData getApplicationMetaData(ApplicationId applicationId, long sessionId) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); - return applicationRepository.getMetadataFromSession(tenant, sessionId); + return applicationRepository.getMetadataFromLocalSession(tenant, sessionId); } - /** Stores all added or set values for each metric and context. */ static class MockMetric implements Metric { @@ -418,7 +745,6 @@ public class ApplicationRepositoryTest { return new Context(properties); } - private static class Context implements Metric.Context { private final Map<String, ?> point; @@ -431,4 +757,48 @@ public class ApplicationRepositoryTest { } + private <T extends ConfigInstance> T resolve(Class<T> clazz, + RequestHandler applications, + ApplicationId appId, + Version vespaVersion) { + String configId = ""; + ConfigResponse response = getConfigResponse(clazz, applications, appId, vespaVersion, configId); + return ConfigPayload.fromUtf8Array(response.getPayload()).toInstance(clazz, configId); + } + + private <T extends ConfigInstance> ConfigResponse getConfigResponse(Class<T> clazz, + RequestHandler applications, + ApplicationId appId, + Version vespaVersion, + String configId) { + return applications.resolveConfig(appId, new GetConfigRequest() { + @Override + public ConfigKey<T> getConfigKey() { + return new ConfigKey<>(clazz, configId); + } + + @Override + public DefContent getDefContent() { + return DefContent.fromClass(clazz); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return Optional.of(VespaVersion.fromString(vespaVersion.toFullString())); + } + + @Override + public boolean noCache() { + return false; + } + }, Optional.empty()); + } + + @NotNull + private RequestHandler reloadConfig(ApplicationId applicationId) { + RequestHandler requestHandler = tenantRepository.getTenant(applicationId.tenant()).getRequestHandler(); + ((TenantApplications) requestHandler).reloadConfig(applicationRepository.getActiveSession(applicationId).ensureApplicationLoaded()); + return requestHandler; + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MiscTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MiscTestCase.java index f8777f4c477..4df39f72ce9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MiscTestCase.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MiscTestCase.java @@ -1,15 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import static org.junit.Assert.*; -import java.io.*; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.ArrayList; -import org.junit.Test; -import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.config.AppConfig; import com.yahoo.config.Md5testConfig; +import com.yahoo.vespa.config.util.ConfigUtils; +import org.junit.Test; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; /** * Tests that does not yet have a specific home due to removed classes, obsolete features etc. @@ -25,8 +31,7 @@ public class MiscTestCase { */ @Test public void testGetDefMd5() throws IOException { - System.out.println("\nStarting testGetDefMd5"); - final String defDir = "src/test/resources/configdefinitions/"; + String defDir = "src/test/resources/configdefinitions/"; assertEquals(AppConfig.CONFIG_DEF_MD5, ConfigUtils.getDefMd5(file2lines(new File(defDir + "app.def")))); assertEquals(Md5testConfig.CONFIG_DEF_MD5, ConfigUtils.getDefMd5(file2lines(new File(defDir + "md5test.def")))); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index 20e8524b9b8..158b8ea55d2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -62,6 +62,7 @@ public class ModelContextImplTest { false, flagSource, null, + Optional.empty(), Optional.empty()), Optional.empty(), Optional.empty(), @@ -85,8 +86,6 @@ public class ModelContextImplTest { assertEquals(new Version(7), context.modelVespaVersion()); assertEquals(new Version(8), context.wantedNodeVespaVersion()); assertEquals(1.0, context.properties().defaultTermwiseLimit(), 0.0); - assertEquals(1.0, context.properties().defaultTopKProbability(), 0.0); - assertFalse(context.properties().useAdaptiveDispatch()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index 561422c1cf8..965374f2aa4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -147,10 +147,8 @@ public class SuperModelControllerTest { Applications.Hosts hosts = app.hosts(host); assertThat(hosts.hostname(), is(host)); for (Map.Entry<String, Applications.Hosts.Services> e : app.hosts(host).services().entrySet()) { - System.out.println(e.getKey()); if (QRSERVER.serviceName.equals(e.getKey())) { Applications.Hosts.Services s = e.getValue(); - System.out.println(s); assertThat(s.type(), is("qrserver")); assertThat(s.ports().size(), is(4)); assertThat(s.ports().get(0).number(), is(8000)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index 4346d83e85e..f91205af44d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -2,19 +2,16 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.component.Version; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; -import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; - import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -28,7 +25,11 @@ import java.util.Arrays; import java.util.Optional; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -129,10 +130,6 @@ public class SuperModelRequestHandlerTest { } - public static NodeFlavors emptyNodeFlavors() { - return new NodeFlavors(new FlavorsConfig(new FlavorsConfig.Builder())); - } - private ApplicationId applicationId(String tenantName, String applicationName) { return ApplicationId.from(tenantName, applicationName, "default"); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index a304f74858b..bc16f44b405 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.concurrent.StripedExecutor; @@ -12,6 +11,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.application.TenantApplicationsTest; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.Metrics; @@ -21,7 +21,6 @@ import com.yahoo.vespa.config.server.session.MockFileDistributionFactory; import com.yahoo.vespa.config.server.session.SessionPreparer; import com.yahoo.vespa.config.server.tenant.MockTenantListener; import com.yahoo.vespa.config.server.tenant.TenantListener; -import com.yahoo.vespa.config.server.tenant.TenantRequestHandlerTest; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -29,11 +28,13 @@ import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModelFactory; +import java.nio.file.Files; import java.time.Clock; import java.util.Collections; import java.util.Optional; import java.util.concurrent.ExecutorService; +import static com.yahoo.yolean.Exceptions.uncheck; /** * @author Ulf Lilleengen @@ -50,7 +51,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private final TenantListener tenantListener; private final PermanentApplicationPackage permanentApplicationPackage; private final HostRegistries hostRegistries; - private final FileDistributionFactory fileDistributionFactory; + private final FileDistributionFactory fileDistributionProvider; private final ModelFactoryRegistry modelFactoryRegistry; private final Optional<Provisioner> hostProvisioner; private final Zone zone; @@ -59,11 +60,12 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private final StripedExecutor<TenantName> zkWatcherExecutor; private final ExecutorService zkCacheExecutor; private final SecretStore secretStore; + private final FlagSource flagSource; private TestComponentRegistry(Curator curator, ConfigCurator configCurator, Metrics metrics, ModelFactoryRegistry modelFactoryRegistry, PermanentApplicationPackage permanentApplicationPackage, - FileDistributionFactory fileDistributionFactory, + FileDistributionFactory fileDistributionProvider, HostRegistries hostRegistries, ConfigserverConfig configserverConfig, SessionPreparer sessionPreparer, @@ -73,7 +75,8 @@ public class TestComponentRegistry implements GlobalComponentRegistry { TenantListener tenantListener, Zone zone, Clock clock, - SecretStore secretStore) { + SecretStore secretStore, + FlagSource flagSource) { this.curator = curator; this.configCurator = configCurator; this.metrics = metrics; @@ -83,7 +86,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { this.defRepo = defRepo; this.permanentApplicationPackage = permanentApplicationPackage; this.hostRegistries = hostRegistries; - this.fileDistributionFactory = fileDistributionFactory; + this.fileDistributionProvider = fileDistributionProvider; this.modelFactoryRegistry = modelFactoryRegistry; this.hostProvisioner = hostProvisioner; this.sessionPreparer = sessionPreparer; @@ -93,6 +96,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { this.zkWatcherExecutor = new StripedExecutor<>(new InThreadExecutorService()); this.zkCacheExecutor = new InThreadExecutorService(); this.secretStore = secretStore; + this.flagSource = flagSource; } public static class Builder { @@ -101,11 +105,11 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private Metrics metrics = Metrics.createTestMetrics(); private ConfigserverConfig configserverConfig = new ConfigserverConfig( new ConfigserverConfig.Builder() - .configServerDBDir(Files.createTempDir().getAbsolutePath()) - .sessionLifetime(5) - .configDefinitionsDir(Files.createTempDir().getAbsolutePath())); + .configServerDBDir(uncheck(() -> Files.createTempDirectory("serverdb")).toString()) + .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString()) + .sessionLifetime(5)); private ConfigDefinitionRepo defRepo = new StaticConfigDefinitionRepo(); - private TenantRequestHandlerTest.MockReloadListener reloadListener = new TenantRequestHandlerTest.MockReloadListener(); + private ReloadListener reloadListener = new TenantApplicationsTest.MockReloadListener(); private MockTenantListener tenantListener = new MockTenantListener(); private Optional<PermanentApplicationPackage> permanentApplicationPackage = Optional.empty(); private HostRegistries hostRegistries = new HostRegistries(); @@ -114,6 +118,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private Optional<Provisioner> hostProvisioner = Optional.empty(); private Zone zone = Zone.defaultZone(); private Clock clock = Clock.systemUTC(); + private FlagSource flagSource = new InMemoryFlagSource(); public Builder configServerConfig(ConfigserverConfig configserverConfig) { this.configserverConfig = configserverConfig; @@ -155,22 +160,32 @@ public class TestComponentRegistry implements GlobalComponentRegistry { return this; } + public Builder reloadListener(ReloadListener reloadListener) { + this.reloadListener = reloadListener; + return this; + } + + public Builder flagSource(FlagSource flagSource) { + this.flagSource = flagSource; + return this; + } + public TestComponentRegistry build() { final PermanentApplicationPackage permApp = this.permanentApplicationPackage .orElse(new PermanentApplicationPackage(configserverConfig)); - FileDistributionFactory fileDistributionFactory = this.fileDistributionFactory + FileDistributionFactory fileDistributionProvider = this.fileDistributionFactory .orElse(new MockFileDistributionFactory(configserverConfig)); HostProvisionerProvider hostProvisionerProvider = hostProvisioner. map(HostProvisionerProvider::withProvisioner).orElseGet(HostProvisionerProvider::empty); SecretStore secretStore = new MockSecretStore(); - SessionPreparer sessionPreparer = new SessionPreparer(modelFactoryRegistry, fileDistributionFactory, + SessionPreparer sessionPreparer = new SessionPreparer(modelFactoryRegistry, fileDistributionProvider, hostProvisionerProvider, permApp, configserverConfig, defRepo, curator, - zone, new InMemoryFlagSource(), secretStore); + zone, flagSource, secretStore); return new TestComponentRegistry(curator, ConfigCurator.create(curator), metrics, modelFactoryRegistry, - permApp, fileDistributionFactory, hostRegistries, configserverConfig, + permApp, fileDistributionProvider, hostRegistries, configserverConfig, sessionPreparer, hostProvisioner, defRepo, reloadListener, tenantListener, - zone, clock, secretStore); + zone, clock, secretStore, flagSource); } } @@ -215,7 +230,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { } @Override - public FlagSource getFlagSource() { return new InMemoryFlagSource(); } + public FlagSource getFlagSource() { return flagSource; } @Override public ExecutorService getZkCacheExecutor() { @@ -227,6 +242,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { return secretStore; } - public FileDistributionFactory getFileDistributionFactory() { return fileDistributionFactory; } + public FileDistributionFactory getFileDistributionProvider() { return fileDistributionProvider; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg deleted file mode 100644 index 8b43ff9c793..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg +++ /dev/null @@ -1,78 +0,0 @@ -accesslog "/home/vespa/logs/vespa/foo.log" -partialsd "sd" -partialsd2 "global2" -asyncfetchocc 10 -a 0 -b 1 -c 2 -d 3 -e 4 -onlyindef 45 -listenport 13700 -rangecheck2 10 -rangecheck3 10 -kanon -78.56 -rangecheck1 10.0 -testref search/cluster.music/c0/r0/indexer.4 -testref2 some/babbel -mode BATCH -functionmodules[0] -storage[2] -storage[0].feeder[1] -storage[0].feeder[0] "test" -storage[1].id search/cluster.music/c0/r0/indexer.4 -storage[1].id2 pjatt -storage[1].feeder[2] -storage[1].feeder[0] "me" -storage[1].feeder[1] "now" -search[3] -search[0].feeder[1] -search[0].feeder[0] "foofeeder" -search[1].feeder[4] -search[1].feeder[0] "barfeeder1_1" -search[1].feeder[1] "barfeeder2" -search[1].feeder[2] "" -search[1].feeder[3] "barfeeder2_1" -search[2].feeder[2] -search[2].feeder[0] "" -search[2].feeder[1] "bazfeeder" -f[1] -f[0].a "A" -f[0].b "B" -f[0].c "C" -f[0].h "H" -f[0].f "F" -config[1] -config[0].role "rtx" -config[0].usewrapper false -config[0].id search/cluster.music/rtx/0 -routingtable[1] -routingtable[0].hop[3] -routingtable[0].hop[0].name "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].hop[0].selector "docproc/cluster.music.indexing/*/chain.music.indexing" -routingtable[0].hop[0].recipient[0] -routingtable[0].hop[1].name "search/cluster.music" -routingtable[0].hop[1].selector "search/cluster.music/[SearchColumn]/[SearchRow]/feed-destination" -routingtable[0].hop[1].recipient[1] -routingtable[0].hop[1].recipient[0] "search/cluster.music/c0/r0/feed-destination" -routingtable[0].hop[2].selector "[DocumentRouteSelector]" -routingtable[0].hop[2].name "indexing" -routingtable[0].hop[2].recipient[1] -routingtable[0].hop[2].recipient[0] "search/cluster.music" -speciallog[1] -speciallog[0].filehandler.name "QueryAccessLog" -speciallog[0].filehandler.pattern "logs/vespa/qrs/QueryAccessLog.%Y%m%d%H%M%S" -speciallog[0].filehandler.rotation "0 1 ..." -speciallog[0].cachehandler.name "QueryAccessLog" -speciallog[0].name "QueryAccessLog" -speciallog[0].type "file" -speciallog[0].cachehandler.size 1000 -rulebase[4] -rulebase[0].name "cjk" -rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# ä½³:\u4f73\n# 能:\u80fd\n# ç´¢:\u7d22\n# å°¼:\u5c3c\n# æƒ :\u60e0\n# æ™®:\u666e\n\n@default\n\naç´¢ -> ç´¢a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,æƒ æ™®,佳能;\n" -rulebase[1].name "common" -rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" -rulebase[2].name "egyik" -rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" -rulebase[3].name "masik" -rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg deleted file mode 100644 index 927ff8a26c9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg +++ /dev/null @@ -1,48 +0,0 @@ -accesslog "/home/vespa/logs/vespa/foo.log" -partialsd "global" -partialsd2 "global2" -asyncfetchocc 10 -a 0 -b 1 -c 67 -d 89 -e 4 -onlyindef 45 -listenport 13700 -rangecheck2 10 -rangecheck3 10 -rangecheck1 10.0 -mode BATCH -functionmodules[0] -storage[0] -search[3] -search[0].feeder[1] -search[0].feeder[0] "foofeeder" -search[1].feeder[4] -search[1].feeder[0] "barfeeder1_1" -search[1].feeder[1] "sportsfeeder1" -search[1].feeder[2] "" -search[1].feeder[3] "barfeeder2_1" -search[2].feeder[2] -search[2].feeder[0] "" -search[2].feeder[1] "bazfeeder" -f[0] -config[0] -routingtable[0] -speciallog[1] -speciallog[0].filehandler.name "QueryAccessLog" -speciallog[0].filehandler.pattern "logs/vespa/qrs/QueryAccessLog.%Y%m%d%H%M%S" -speciallog[0].filehandler.rotation "0 1 ..." -speciallog[0].cachehandler.name "QueryAccessLog" -speciallog[0].name "QueryAccessLog" -speciallog[0].type "file" -speciallog[0].cachehandler.size 1000 -rulebase[4] -rulebase[0].name "cjk" -rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# ä½³:\u4f73\n# 能:\u80fd\n# ç´¢:\u7d22\n# å°¼:\u5c3c\n# æƒ :\u60e0\n# æ™®:\u666e\n\n@default\n\naç´¢ -> ç´¢a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,æƒ æ™®,佳能;\n" -rulebase[1].name "common" -rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" -rulebase[2].name "egyik" -rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" -rulebase[3].name "masik" -rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar b/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar Binary files differdeleted file mode 100644 index 69f6e335092..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar +++ /dev/null diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml b/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml deleted file mode 100644 index b7924e73e7a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml +++ /dev/null @@ -1,18 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services version="1.0"> - - <admin version="2.0"> - <adminserver hostalias="node1"/> - </admin> - - <content version="1.0"> - <redundancy>1</redundancy> - <documents> - <document type="music" mode="index"/> - </documents> - <nodes>> - <node hostalias="node1" distribution-key="0"/> - </nodes> - </content> -</services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java index 4cccafef266..cf7740f133f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java @@ -1,25 +1,32 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.http.HttpFetcher; -import com.yahoo.vespa.config.server.http.StaticResponse; import com.yahoo.vespa.config.server.http.RequestTimeoutException; +import com.yahoo.vespa.config.server.http.StaticResponse; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import java.net.URL; +import java.util.Arrays; +import java.util.Collections; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; +import static com.yahoo.vespa.config.server.application.MockModel.createServiceInfo; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertSame; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class HttpProxyTest { + private final HttpFetcher fetcher = mock(HttpFetcher.class); private final HttpProxy proxy = new HttpProxy(fetcher); @@ -29,7 +36,7 @@ public class HttpProxyTest { @Before public void setup() { - Model modelMock = MockModel.createClusterController(hostname, port); + Model modelMock = createClusterController(); when(applicationMock.getModel()).thenReturn(modelMock); } @@ -52,14 +59,30 @@ public class HttpProxyTest { // The HttpResponse returned by the fetcher IS the same object as the one returned by the proxy, // when everything goes well. - assertTrue(actualResponse == response); + assertSame(actualResponse, response); } @Test(expected = RequestTimeoutException.class) public void testFetchException() { when(fetcher.get(any(), any())).thenThrow(new RequestTimeoutException("timed out")); - HttpResponse actualResponse = proxy.get(applicationMock, hostname, CLUSTERCONTROLLER_CONTAINER.serviceName, - "clustercontroller-status/v1/clusterName"); + proxy.get(applicationMock, hostname, CLUSTERCONTROLLER_CONTAINER.serviceName, + "clustercontroller-status/v1/clusterName"); } + + private static MockModel createClusterController() { + ServiceInfo container = createServiceInfo( + hostname, + "foo", // name + CLUSTERCONTROLLER_CONTAINER.serviceName, + ClusterSpec.Type.container, + port, + "state http external query"); + ServiceInfo serviceNoStatePort = createServiceInfo(hostname, "storagenode", "storagenode", + ClusterSpec.Type.content, 1234, "rpc"); + HostInfo hostInfo = new HostInfo(hostname, Arrays.asList(container, serviceNoStatePort)); + + return new MockModel(Collections.singleton(hostInfo)); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java index 0da96f9f01d..c9f25451ea4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java @@ -23,8 +23,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; - /** * Model with two services, one that does not have a state port * @@ -45,22 +43,6 @@ public class MockModel implements Model { return new HostInfo(hostname, Arrays.asList(container, serviceNoStatePort)); } - // TODO: Move to caller - static MockModel createClusterController(String hostname, int statePort) { - ServiceInfo container = createServiceInfo( - hostname, - "foo", // name - CLUSTERCONTROLLER_CONTAINER.serviceName, - ClusterSpec.Type.container, - statePort, - "state http external query"); - ServiceInfo serviceNoStatePort = createServiceInfo(hostname, "storagenode", "storagenode", - ClusterSpec.Type.content, 1234, "rpc"); - HostInfo hostInfo = new HostInfo(hostname, Arrays.asList(container, serviceNoStatePort)); - - return new MockModel(Collections.singleton(hostInfo)); - } - static MockModel createConfigProxies(List<String> hostnames, int rpcPort) { Set<HostInfo> hostInfos = new HashSet<>(); hostnames.forEach(hostname -> { @@ -71,7 +53,7 @@ public class MockModel implements Model { return new MockModel(hostInfos); } - static private ServiceInfo createServiceInfo( + static ServiceInfo createServiceInfo( String hostname, String name, String type, @@ -121,4 +103,5 @@ public class MockModel implements Model { public AllocatedHosts allocatedHosts() { throw new UnsupportedOperationException(); } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 33932a678b7..969040174dd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -1,23 +1,49 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Utf8; -import com.yahoo.vespa.config.server.MockReloadHandler; - +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.server.ReloadListener; +import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.model.TestModelFactory; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.VespaModelFactory; import org.apache.curator.framework.CuratorFramework; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; - +import org.junit.rules.TemporaryFolder; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -25,14 +51,34 @@ import static org.junit.Assert.*; public class TenantApplicationsTest { private static final TenantName tenantName = TenantName.from("tenant"); + private static final Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); - private Curator curator; + private MockReloadListener listener = new MockReloadListener(); private CuratorFramework curatorFramework; + private TestComponentRegistry componentRegistry; + private TenantApplications applications; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); @Before - public void setup() { - curator = new MockCurator(); + public void setup() throws IOException { + Curator curator = new MockCurator(); curatorFramework = curator.framework(); + componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) + .configServerDBDir(tempFolder.newFolder("configserverdb").getAbsolutePath()) + .configDefinitionsDir(tempFolder.newFolder("configdefinitions").getAbsolutePath()) + .build()) + .modelFactoryRegistry(createRegistry()) + .reloadListener(listener) + .build(); + TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); + tenantRepository.addTenant(tenantName); + applications = TenantApplications.create(componentRegistry, tenantName); } @Test @@ -94,29 +140,71 @@ public class TenantApplicationsTest { assertThat(repo.activeApplications().size(), is(0)); } - @Test - public void require_that_reload_handler_is_called_when_apps_are_removed() throws Exception { - ApplicationId foo = createApplicationId("foo"); - writeApplicationData(foo, 3L); - writeApplicationData(createApplicationId("bar"), 4L); - MockReloadHandler reloadHandler = new MockReloadHandler(); - TenantApplications repo = createZKAppRepo(reloadHandler); - assertNull(reloadHandler.lastRemoved); - repo.createDeleteTransaction(foo).commit(); - long endTime = System.currentTimeMillis() + 60_000; - while (System.currentTimeMillis() < endTime && reloadHandler.lastRemoved == null) { - Thread.sleep(100); + public static class MockReloadListener implements ReloadListener { + public AtomicInteger reloaded = new AtomicInteger(0); + AtomicInteger removed = new AtomicInteger(0); + Map<String, Collection<String>> tenantHosts = new LinkedHashMap<>(); + + @Override + public void configActivated(ApplicationSet application) { + reloaded.incrementAndGet(); + } + + @Override + public void hostsUpdated(TenantName tenant, Collection<String> newHosts) { + tenantHosts.put(tenant.value(), newHosts); + } + + @Override + public void verifyHostsAreAvailable(TenantName tenant, Collection<String> newHosts) { + } + + @Override + public void applicationRemoved(ApplicationId applicationId) { + removed.incrementAndGet(); } - assertNotNull(reloadHandler.lastRemoved); - assertThat(reloadHandler.lastRemoved.serializedForm(), is(foo.serializedForm())); } - private TenantApplications createZKAppRepo() { - return createZKAppRepo(new MockReloadHandler()); + private void assertdefaultAppNotFound() { + assertFalse(applications.hasApplication(ApplicationId.defaultId(), Optional.of(vespaVersion))); + } + + @Test + public void testListConfigs() throws IOException, SAXException { + assertdefaultAppNotFound(); + + VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); + applications.createApplication(ApplicationId.defaultId()); + applications.createPutTransaction(ApplicationId.defaultId(), 1).commit(); + applications.reloadConfig(ApplicationSet.fromSingle(new Application(model, + new ServerCache(), + 1, + false, + vespaVersion, + MetricUpdater.createTestUpdater(), + ApplicationId.defaultId()))); + Set<ConfigKey<?>> configNames = applications.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), false); + assertTrue(configNames.contains(new ConfigKey<>("sentinel", "hosts", "cloud.config"))); + + configNames = applications.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), true); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("documenttypes", "", "document"))); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("health-monitor", "container", "container.jdisc.config"))); + assertTrue(configNames.contains(new ConfigKey<>("specific", "container", "project"))); } - private TenantApplications createZKAppRepo(MockReloadHandler reloadHandler) { - return TenantApplications.create(new TestComponentRegistry.Builder().curator(curator).build(), reloadHandler, tenantName); + @Test + public void testAppendIdsInNonRecursiveListing() { + assertEquals(applications.appendOneLevelOfId("search/music", "search/music/qrservers/default/qr.0"), "search/music/qrservers"); + assertEquals(applications.appendOneLevelOfId("search", "search/music/qrservers/default/qr.0"), "search/music"); + assertEquals(applications.appendOneLevelOfId("search/music/qrservers/default/qr.0", "search/music/qrservers/default/qr.0"), "search/music/qrservers/default/qr.0"); + assertEquals(applications.appendOneLevelOfId("", "search/music/qrservers/default/qr.0"), "search"); + } + + private TenantApplications createZKAppRepo() { + return TenantApplications.create(componentRegistry, tenantName); } private static ApplicationId createApplicationId(String name) { @@ -134,4 +222,10 @@ public class TenantApplicationsTest { .forPath(TenantRepository.getApplicationsPath(tenantName).append(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId)); } + + private ModelFactoryRegistry createRegistry() { + return new ModelFactoryRegistry(Arrays.asList(new TestModelFactory(vespaVersion), + new TestModelFactory(new Version(3, 2, 1)))); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java index 2566b1029a8..e2c3369d49e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.configchange; import com.google.common.collect.ImmutableMap; @@ -6,7 +6,6 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 06db7018727..7a8bad3d199 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; -import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelRegistry; @@ -48,6 +47,7 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; import java.io.File; +import java.nio.file.Files; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -59,6 +59,8 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import static com.yahoo.yolean.Exceptions.uncheck; + /** * @author bratseth */ @@ -78,8 +80,8 @@ public class DeployTester { public DeployTester(List<ModelFactory> modelFactories) { this(modelFactories, new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(Files.createTempDir().getAbsolutePath()) - .configDefinitionsDir(Files.createTempDir().getAbsolutePath())), + .configServerDBDir(uncheck(() -> Files.createTempDirectory("serverdb")).toString()) + .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString())), Clock.systemUTC()); } @@ -247,7 +249,7 @@ public class DeployTester { public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { Tenant tenant = tenant(); - LocalSession session = tenant.getLocalSessionRepo().getSession(tenant.getApplicationRepo() + LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo() .requireActiveSessionOf(applicationId)); return session.getAllocatedHosts(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index d8534774e33..254fe62cba8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -96,14 +96,14 @@ public class HostedDeployTest { tester.deployApp("src/test/apps/hosted/", Instant.now(), new PrepareParams.Builder() .vespaVersion("4.5.6") .dockerImageRepository(dockerImageRepository) - .athenzDomain("foo")); + .athenzDomain("myDomain")); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(tester.applicationId()); assertTrue(deployment.isPresent()); deployment.get().activate(); assertEquals("4.5.6", ((Deployment) deployment.get()).session().getVespaVersion().toString()); assertEquals(DockerImage.fromString(dockerImageRepository), ((Deployment) deployment.get()).session().getDockerImageRepository().get()); - assertEquals("foo", ((Deployment) deployment.get()).session().getAthenzDomain().get().value()); + assertEquals("myDomain", ((Deployment) deployment.get()).session().getAthenzDomain().get().value()); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/MockDeployer.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/MockDeployer.java deleted file mode 100644 index 967e2321b95..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/MockDeployer.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.deploy; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Deployment; - -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; - -/** - * @author Ulf Lilleengen - */ -public class MockDeployer implements com.yahoo.config.provision.Deployer { - - @Override - public Optional<Deployment> deployFromLocalActive(ApplicationId application) { - return deployFromLocalActive(application, Duration.ofSeconds(60)); - } - - @Override - public Optional<Deployment> deployFromLocalActive(ApplicationId application, boolean bootstrap) { - return Optional.empty(); - } - - @Override - public Optional<Deployment> deployFromLocalActive(ApplicationId application, Duration timeout) { - return Optional.empty(); - } - - @Override - public Optional<Deployment> deployFromLocalActive(ApplicationId application, Duration timeout, boolean bootstrap) { - return Optional.empty(); - } - - @Override - public Optional<Instant> lastDeployTime(ApplicationId application) { - return Optional.empty(); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index 36467a2ca64..c07c7316930 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -33,11 +33,11 @@ public class RedeployTest { assertTrue(deployment.isPresent()); long activeSessionIdBefore = tester.applicationRepository().getActiveSession(tester.applicationId()).getSessionId(); - assertEquals(tester.applicationId(), tester.tenant().getLocalSessionRepo().getSession(activeSessionIdBefore).getApplicationId()); + assertEquals(tester.applicationId(), tester.tenant().getSessionRepository().getLocalSession(activeSessionIdBefore).getApplicationId()); deployment.get().activate(); long activeSessionIdAfter = tester.applicationRepository().getActiveSession(tester.applicationId()).getSessionId(); assertEquals(activeSessionIdAfter, activeSessionIdBefore + 1); - assertEquals(tester.applicationId(), tester.tenant().getLocalSessionRepo().getSession(activeSessionIdAfter).getApplicationId()); + assertEquals(tester.applicationId(), tester.tenant().getSessionRepository().getLocalSession(activeSessionIdAfter).getApplicationId()); } /** No deployment is done because there is no local active session. */ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index 14fa0cb2dbe..8394611737e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -193,8 +193,8 @@ public class ZooKeeperClientTest { Path app = Path.fromString("/1"); ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, true, app); zooKeeperClient.setupZooKeeper(); - HostSpec host1 = new HostSpec("host1.yahoo.com", Collections.emptyList()); - HostSpec host2 = new HostSpec("host2.yahoo.com", Collections.emptyList()); + HostSpec host1 = new HostSpec("host1.yahoo.com", Collections.emptyList(), Optional.empty()); + HostSpec host2 = new HostSpec("host2.yahoo.com", Collections.emptyList(), Optional.empty()); ImmutableSet<HostSpec> hosts = ImmutableSet.of(host1, host2); zooKeeperClient.write(AllocatedHosts.withHosts(hosts)); Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java deleted file mode 100644 index b6d9ab5d618..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import org.junit.Test; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; - -import static com.yahoo.jdisc.http.HttpResponse.Status.*; -import static com.yahoo.jdisc.http.HttpRequest.Method.*; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author hmusum - * @since 5.1.14 - */ -public class SessionExampleHandlerTest { - private static final String URI = "http://localhost:19071/session/example"; - - @Test - public void basicPut() throws IOException { - final SessionExampleHandler handler = new SessionExampleHandler(Executors.newCachedThreadPool()); - final HttpRequest request = HttpRequest.createTestRequest(URI, PUT); - HttpResponse response = handler.handle(request); - assertThat(response.getStatus(), is(OK)); - assertThat(SessionHandlerTest.getRenderedString(response), is("{\"test\":\"PUT received\"}")); - } - - @Test - public void invalidMethod() { - final SessionExampleHandler handler = new SessionExampleHandler(Executors.newCachedThreadPool()); - final HttpRequest request = HttpRequest.createTestRequest(URI, GET); - HttpResponse response = handler.handle(request); - assertThat(response.getStatus(), is(METHOD_NOT_ALLOWED)); - } - - - /** - * A handler that prepares a session given by an id in the request. - * - * @author hmusum - * @since 5.1.14 - */ - public static class SessionExampleHandler extends ThreadedHttpRequestHandler { - - public SessionExampleHandler(Executor executor) { - super(executor, null); - } - - @Override - public HttpResponse handle(HttpRequest request) { - final com.yahoo.jdisc.http.HttpRequest.Method method = request.getMethod(); - switch (method) { - case PUT: - return handlePUT(request); - case GET: - return new SessionExampleResponse(METHOD_NOT_ALLOWED, "Method '" + method + "' is not supported"); - default: - return new SessionExampleResponse(INTERNAL_SERVER_ERROR); - } - } - - @SuppressWarnings({"UnusedDeclaration"}) - HttpResponse handlePUT(HttpRequest request) { - return new SessionExampleResponse(OK, "PUT received"); - } - - private static class SessionExampleResponse extends HttpResponse { - private final Slime slime = new Slime(); - private final Cursor root = slime.setObject(); - private final String message; - - - private SessionExampleResponse(int status) { - this(status, ""); - headers().put("Cache-Control","max-age=120"); - } - - private SessionExampleResponse(int status, String message) { - super(status); - this.message = message; - } - - @Override - public void render(OutputStream outputStream) throws IOException { - root.setString("test", message); - new JsonFormat(true).encode(outputStream, slime); - } - } - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index a1e252c8130..0b9a780d9e1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -1,15 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import com.google.common.io.Files; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; @@ -17,26 +13,15 @@ import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.io.IOUtils; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; -import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.session.DummyTransaction; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.MockSessionZKClient; -import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; -import com.yahoo.vespa.config.server.session.SessionContext; -import com.yahoo.vespa.config.server.session.SessionFactory; -import com.yahoo.vespa.flags.InMemoryFlagSource; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -44,7 +29,6 @@ import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Optional; /** * Base class for session handler tests @@ -95,48 +79,21 @@ public class SessionHandlerTest { return baos.toString(StandardCharsets.UTF_8); } - public static class MockSession extends LocalSession { + public static class MockLocalSession extends LocalSession { - public boolean doVerboseLogging = false; public Session.Status status; - private ConfigChangeActions actions = new ConfigChangeActions(); - private long createTime = System.currentTimeMillis() / 1000; + private Instant createTime = Instant.now(); private ApplicationId applicationId; - private Optional<DockerImage> dockerImageRepository; - public MockSession(long id, ApplicationPackage app) { - this(id, app, new InMemoryFlagSource()); + public MockLocalSession(long sessionId, ApplicationPackage app) { + super(TenantName.defaultName(), sessionId, app, new MockSessionZKClient(app), null); } - private MockSession(long id, ApplicationPackage app, InMemoryFlagSource flagSource) { - super(TenantName.defaultName(), id, null, new SessionContext(app, new MockSessionZKClient(app), null, null, new HostRegistry<>(), flagSource)); - } - - public MockSession(long sessionId, ApplicationPackage applicationPackage, long createTime) { - this(sessionId, applicationPackage); - this.createTime = createTime; - } - - public MockSession(long sessionId, ApplicationPackage applicationPackage, ConfigChangeActions actions) { - this(sessionId, applicationPackage); - this.actions = actions; - } - - public MockSession(long sessionId, ApplicationPackage app, ApplicationId applicationId) { + public MockLocalSession(long sessionId, ApplicationPackage app, ApplicationId applicationId) { this(sessionId, app); this.applicationId = applicationId; } - @Override - public ConfigChangeActions prepare(DeployLogger logger, PrepareParams params, Optional<ApplicationSet> application, Path tenantPath, Instant now) { - status = Session.Status.PREPARE; - this.dockerImageRepository = params.dockerImageRepository(); - if (doVerboseLogging) { - logger.log(Level.FINE, "debuglog"); - } - return actions; - } - public void setStatus(Session.Status status) { this.status = status; } @@ -147,11 +104,6 @@ public class SessionHandlerTest { } @Override - public Transaction createDeactivateTransaction() { - return new DummyTransaction().add((DummyTransaction.RunnableOperation) () -> status = Status.DEACTIVATE); - } - - @Override public Transaction createActivateTransaction() { return new DummyTransaction().add((DummyTransaction.RunnableOperation) () -> status = Status.ACTIVATE); } @@ -167,17 +119,10 @@ public class SessionHandlerTest { } @Override - public long getCreateTime() { + public Instant getCreateTime() { return createTime; } - @Override - public void delete(NestedTransaction transaction) { } - - @Override - public Optional<DockerImage> getDockerImageRepository() { - return dockerImageRepository; - } } public enum Cmd { @@ -196,39 +141,6 @@ public class SessionHandlerTest { } } - public static class MockSessionFactory implements SessionFactory { - public boolean createCalled = false; - public boolean createFromCalled = false; - public boolean doThrow = false; - public File applicationPackage; - - @Override - public LocalSession createSession(File applicationDirectory, ApplicationId applicationId, TimeoutBudget timeoutBudget) { - createCalled = true; - if (doThrow) { - throw new RuntimeException("foo"); - } - final File tempDir = Files.createTempDir(); - try { - IOUtils.copyDirectory(applicationDirectory, tempDir); - } catch (IOException e) { - e.printStackTrace(); - } - this.applicationPackage = tempDir; - return new SessionHandlerTest.MockSession(0, FilesApplicationPackage.fromFile(applicationPackage)); - } - - @Override - public LocalSession createSessionFromExisting(LocalSession existingSession, DeployLogger logger, - boolean internalRedeploy, TimeoutBudget timeoutBudget) { - if (doThrow) { - throw new RuntimeException("foo"); - } - createFromCalled = true; - return new SessionHandlerTest.MockSession(existingSession.getSessionId() + 1, FilesApplicationPackage.fromFile(applicationPackage)); - } - } - public static class MockProvisioner implements Provisioner { public boolean activated = false; @@ -263,17 +175,4 @@ public class SessionHandlerTest { } - public static class FailingMockProvisioner extends MockProvisioner { - - @Override - public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { - throw new IllegalArgumentException("Cannot activate application"); - } - - @Override - public void remove(NestedTransaction transaction, ApplicationId application) { - throw new IllegalArgumentException("Cannot remove application"); - } - - } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index b7f55aa0670..4cf81d22e3c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -2,19 +2,18 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.Tenant; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; import org.junit.Test; @@ -45,23 +44,23 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { private ApplicationId idTenant2 = new ApplicationId.Builder() .tenant(tenantName2) .applicationName("foo").instanceName("quux").build(); - private MockSession session2; + private MockLocalSession session2; @Before public void setupHandler() { TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, tenantName1)); - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, tenantName2)); + tenantRepository.addTenant(tenantName1); + tenantRepository.addTenant(tenantName2); - session2 = new MockSession(2, FilesApplicationPackage.fromFile(new File("src/test/apps/content"))); + session2 = new MockLocalSession(2, FilesApplicationPackage.fromFile(new File("src/test/apps/content"))); Tenant tenant1 = tenantRepository.getTenant(tenantName1); - tenant1.getLocalSessionRepo().addSession(session2); + tenant1.getSessionRepository().addSession(session2); tenant1.getApplicationRepo().createApplication(idTenant1); tenant1.getApplicationRepo().createPutTransaction(idTenant1, 2).commit(); - MockSession session3 = new MockSession(3, FilesApplicationPackage.fromFile(new File("src/test/apps/content2"))); + MockLocalSession session3 = new MockLocalSession(3, FilesApplicationPackage.fromFile(new File("src/test/apps/content2"))); Tenant tenant2 = tenantRepository.getTenant(tenantName2); - tenant2.getLocalSessionRepo().addSession(session3); + tenant2.getSessionRepository().addSession(session3); tenant2.getApplicationRepo().createApplication(idTenant2); tenant2.getApplicationRepo().createPutTransaction(idTenant2, 3).commit(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 70f66cf8fde..06e404bee32 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; @@ -26,7 +27,6 @@ import com.yahoo.vespa.config.server.http.StaticResponse; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.After; import org.junit.Before; @@ -57,28 +57,27 @@ import static org.mockito.Mockito.when; */ public class ApplicationHandlerTest { - private static File testApp = new File("src/test/apps/app"); + private static final File testApp = new File("src/test/apps/app"); private final static TenantName mytenantName = TenantName.from("mytenant"); - private final static TenantName foobar = TenantName.from("foobar"); - private final static ApplicationId myTenantApplicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); - private final static ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(TenantName.defaultName()).build(); + private final static ApplicationId myTenantApplicationId = ApplicationId.from(mytenantName, ApplicationName.defaultName(), InstanceName.defaultName()); + private final static ApplicationId applicationId = ApplicationId.from(TenantName.defaultName(), ApplicationName.defaultName(), InstanceName.defaultName()); private final static MockTesterClient testerClient = new MockTesterClient(); private final static NullMetric metric = new NullMetric(); private final static ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); private static final MockLogRetriever logRetriever = new MockLogRetriever(); - private TestComponentRegistry componentRegistry; private TenantRepository tenantRepository; private ApplicationRepository applicationRepository; private SessionHandlerTest.MockProvisioner provisioner; - private MockStateApiFactory stateApiFactory = new MockStateApiFactory(); + private final MockStateApiFactory stateApiFactory = new MockStateApiFactory(); private OrchestratorMock orchestrator; @Before public void setup() { - componentRegistry = new TestComponentRegistry.Builder().build(); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().provisioner(provisioner).build(); tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(mytenantName); provisioner = new SessionHandlerTest.MockProvisioner(); orchestrator = new OrchestratorMock(); applicationRepository = new ApplicationRepository(tenantRepository, @@ -98,8 +97,8 @@ public class ApplicationHandlerTest { @Test public void testDelete() throws Exception { - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar)); - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName)); + TenantName foobar = TenantName.from("foobar"); + tenantRepository.addTenant(foobar); { applicationRepository.deploy(testApp, prepareParams(applicationId)); @@ -220,7 +219,7 @@ public class ApplicationHandlerTest { ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant("default").build(); HttpResponse responseForUnknown = fileDistributionStatus(unknown, zone); assertEquals(404, responseForUnknown.getStatus()); - assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: 'default.unknown'\"}", + assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No active session found for application id: 'default.unknown'\"}", getRenderedString(responseForUnknown)); } @@ -288,13 +287,13 @@ public class ApplicationHandlerTest { private void deleteAndAssertOKResponseMocked(ApplicationId applicationId, boolean fullAppIdInUrl) throws IOException { long sessionId = tenantRepository.getTenant(applicationId.tenant()).getApplicationRepo().requireActiveSessionOf(applicationId); deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, fullAppIdInUrl); - assertNull(tenantRepository.getTenant(applicationId.tenant()).getLocalSessionRepo().getSession(sessionId)); + assertNull(tenantRepository.getTenant(applicationId.tenant()).getSessionRepository().getLocalSession(sessionId)); } private void deleteAndAssertOKResponse(Tenant tenant, ApplicationId applicationId) throws IOException { long sessionId = tenant.getApplicationRepo().requireActiveSessionOf(applicationId); deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, true); - assertNull(tenant.getLocalSessionRepo().getSession(sessionId)); + assertNull(tenant.getSessionRepository().getLocalSession(sessionId)); } private void deleteAndAssertResponse(ApplicationId applicationId, Zone zone, int expectedStatus, HttpErrorResponse.errorCodes errorCode, boolean fullAppIdInUrl) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 1db70956407..37181abfcf4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -4,12 +4,17 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.*; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; -import com.yahoo.vespa.config.server.*; -import com.yahoo.vespa.config.server.host.HostRegistries; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -18,7 +23,6 @@ import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.session.MockSessionZKClient; import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.tenant.Tenant; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.model.VespaModelFactory; import org.junit.Before; @@ -26,14 +30,13 @@ import org.junit.Test; import java.io.File; import java.io.IOException; +import java.time.Clock; import java.util.Collections; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - /** * @author hmusum */ +// TODO: Try to move testing to ApplicationRepositoryTest and avoid all the low-level setup code here public class HostHandlerTest { private static final String urlPrefix = "http://myhost:14000/application/v2/host/"; private static File testApp = new File("src/test/apps/app"); @@ -41,50 +44,45 @@ public class HostHandlerTest { private HostHandler handler; private final static TenantName mytenant = TenantName.from("mytenant"); private final static String hostname = "testhost"; + private final static Zone zone = Zone.defaultZone(); private TenantRepository tenantRepository; - private HostRegistries hostRegistries; - private HostHandler hostHandler; static void addMockApplication(Tenant tenant, ApplicationId applicationId, long sessionId) { tenant.getApplicationRepo().createApplication(applicationId); tenant.getApplicationRepo().createPutTransaction(applicationId, sessionId).commit(); ApplicationPackage app = FilesApplicationPackage.fromFile(testApp); - tenant.getLocalSessionRepo().addSession(new SessionHandlerTest.MockSession(sessionId, app, applicationId)); + tenant.getSessionRepository().addSession(new SessionHandlerTest.MockLocalSession(sessionId, app, applicationId)); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .modelFactoryRegistry(new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())))) .build(); - tenant.getRemoteSessionRepo().addSession(new RemoteSession(tenant.getName(), sessionId, componentRegistry, new MockSessionZKClient(app))); + tenant.getSessionRepo().addRemoteSession(new RemoteSession(tenant.getName(), sessionId, componentRegistry, new MockSessionZKClient(app))); } @Before public void setup() { - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); - tenantRepository = new TenantRepository(componentRegistry, false); - TenantBuilder tb = TenantBuilder.create(componentRegistry, mytenant); - tenantRepository.addTenant(tb); - handler = createHostHandler(); - } - - private HostHandler createHostHandler() { final HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); hostRegistry.update(mytenant, Collections.singletonList(hostname)); - TestComponentRegistry testComponentRegistry = new TestComponentRegistry.Builder().build(); - hostRegistries = testComponentRegistry.getHostRegistries(); - hostRegistries.createApplicationHostRegistry(mytenant).update(ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()), Collections.singletonList(hostname)); - hostRegistries.getTenantHostRegistry().update(mytenant, Collections.singletonList(hostname)); - hostHandler = new HostHandler( - HostHandler.testOnlyContext(), - testComponentRegistry); - return hostHandler; + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .zone(zone) + .build(); + tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(mytenant); + Tenant tenant = tenantRepository.getTenant(mytenant); + HostRegistry<ApplicationId> applicationHostRegistry = tenant.getApplicationRepo().getApplicationHostRegistry(); + applicationHostRegistry.update(ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()), Collections.singletonList(hostname)); + ApplicationRepository applicationRepository = new ApplicationRepository(tenantRepository, + new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), + Clock.systemUTC()); + handler = new HostHandler(HostHandler.testOnlyContext(), applicationRepository); } @Test public void require_correct_tenant_and_application_for_hostname() throws Exception { - assertThat(hostRegistries, is(hostHandler.hostRegistries)); long sessionId = 1; ApplicationId id = ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()); addMockApplication(tenantRepository.getTenant(mytenant), id, sessionId); - assertApplicationForHost(hostname, mytenant, id, Zone.defaultZone()); + assertApplicationForHost(hostname, mytenant, id, zone); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index b72785876bc..46a17795acf 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.config.server.http.HttpConfigRequest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.rpc.MockRequestHandler; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; import org.junit.Test; @@ -48,7 +47,7 @@ public class HttpGetConfigHandlerTest { }} ); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, tenant).withRequestHandler(mockRequestHandler)); + tenantRepository.addTenant(tenant, mockRequestHandler, mockRequestHandler); handler = new HttpGetConfigHandler(HttpGetConfigHandler.testOnlyContext(), tenantRepository); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java index 7789c5d88db..e8484ad10fe 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java @@ -8,7 +8,6 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.rpc.MockRequestHandler; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -46,9 +45,7 @@ public class HttpListConfigsHandlerTest { }} ); TenantName tenantName = TenantName.from("mytenant"); TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); - TenantBuilder tenantBuilder = TenantBuilder.create(componentRegistry, tenantName) - .withRequestHandler(mockRequestHandler); - tenantRepository.addTenant(tenantBuilder); + tenantRepository.addTenant(tenantName, mockRequestHandler, mockRequestHandler); handler = new HttpListConfigsHandler(HttpListConfigsHandler.testOnlyContext(), tenantRepository, Zone.defaultZone()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java index f97bc443a38..bef6369beb7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java @@ -1,27 +1,32 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.yahoo.config.provision.*; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import org.junit.Test; import org.junit.Before; +import org.junit.Test; import java.io.IOException; +import static com.yahoo.jdisc.http.HttpRequest.Method.DELETE; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static com.yahoo.jdisc.http.HttpRequest.Method.POST; +import static com.yahoo.jdisc.http.HttpRequest.Method.PUT; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; -import static com.yahoo.jdisc.http.HttpRequest.Method.*; - /** * @author Ulf Lilleengen */ @@ -37,8 +42,8 @@ public class ListApplicationsHandlerTest { @Before public void setup() { TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenant)); - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar)); + tenantRepository.addTenant(mytenant); + tenantRepository.addTenant(foobar); applicationRepo = tenantRepository.getTenant(mytenant).getApplicationRepo(); applicationRepo2 = tenantRepository.getTenant(foobar).getApplicationRepo(); handler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 6ba3c33d37b..edfe74ef3f9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -1,54 +1,32 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationMetaData; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.DeployData; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.slime.JsonFormat; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; -import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.session.LocalSession; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; -import com.yahoo.vespa.config.server.session.MockSessionZKClient; -import com.yahoo.vespa.config.server.session.RemoteSession; -import com.yahoo.vespa.config.server.session.Session; -import com.yahoo.vespa.config.server.session.SessionContext; -import com.yahoo.vespa.config.server.session.SessionTest; -import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; +import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModelFactory; import org.hamcrest.core.Is; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -56,35 +34,33 @@ import org.junit.rules.TemporaryFolder; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.util.Collections; -import java.util.Optional; +import java.time.Duration; +import java.util.List; -import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; -import static com.yahoo.jdisc.Response.Status.CONFLICT; -import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; import static com.yahoo.jdisc.Response.Status.NOT_FOUND; import static com.yahoo.jdisc.Response.Status.OK; +import static com.yahoo.vespa.config.server.http.SessionHandlerTest.Cmd; +import static com.yahoo.vespa.config.server.http.SessionHandlerTest.createTestRequest; +import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; -public class SessionActiveHandlerTest extends SessionHandlerTest { +public class SessionActiveHandlerTest { private static final File testApp = new File("src/test/apps/app"); private static final String appName = "default"; private static final TenantName tenantName = TenantName.from("activatetest"); private static final String activatedMessage = " for tenant '" + tenantName + "' activated."; + private static final String pathPrefix = "/application/v2/tenant/" + tenantName + "/session/"; - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - private Curator curator; - private LocalSessionRepo localRepo; - private TenantApplications applicationRepo; - private MockProvisioner hostProvisioner; - private VespaModelFactory modelFactory; + private SessionHandlerTest.MockProvisioner hostProvisioner; private TestComponentRegistry componentRegistry; private TenantRepository tenantRepository; + private ApplicationRepository applicationRepository; private SessionActiveHandler handler; @Rule @@ -92,163 +68,35 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { @Before public void setup() { - curator = new MockCurator(); - modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); + VespaModelFactory modelFactory = new TestModelFactory(Version.fromString("7.222.2")); + hostProvisioner = new SessionHandlerTest.MockProvisioner(); componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) - .modelFactoryRegistry(new ModelFactoryRegistry(Collections.singletonList(modelFactory))) + .curator(new MockCurator()) + .modelFactoryRegistry(new ModelFactoryRegistry(List.of((modelFactory)))) .build(); tenantRepository = new TenantRepository(componentRegistry, false); - applicationRepo = TenantApplications.create(componentRegistry, new MockReloadHandler(), tenantName); - localRepo = new LocalSessionRepo(tenantName, componentRegistry); - pathPrefix = "/application/v2/tenant/" + tenantName + "/session/"; - hostProvisioner = new MockProvisioner(); - TenantBuilder tenantBuilder = TenantBuilder.create(componentRegistry, tenantName) - .withSessionFactory(new MockSessionFactory()) - .withLocalSessionRepo(localRepo) - .withApplicationRepo(applicationRepo); - tenantRepository.addTenant(tenantBuilder); + applicationRepository = new ApplicationRepository(tenantRepository, hostProvisioner, + new OrchestratorMock(), componentRegistry.getClock()); + tenantRepository.addTenant(tenantName); handler = createHandler(); } @Test - public void testThatPreviousSessionIsDeactivated() throws Exception { - RemoteSession firstSession = activateAndAssertOK(90, 0); - activateAndAssertOK(91, 90); - assertThat(firstSession.getStatus(), Is.is(Session.Status.DEACTIVATE)); - } - - @Test - public void testForceActivationWithActivationInBetween() throws Exception { - activateAndAssertOK(90, 0); - activateAndAssertOK(92, 89, "?force=true"); + public void testActivation() throws Exception { + activateAndAssertOK(); } @Test public void testUnknownSession() { - HttpResponse response = handler.handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, 9999L, "?timeout=1.0")); + HttpResponse response = handler.handle(createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, 9999L, "?timeout=1.0")); assertEquals(response.getStatus(), NOT_FOUND); } @Test - public void testActivationWithBarrierTimeout() throws Exception { - // Needed so we can test that previous active session is still active after a failed activation - activateAndAssertOK(90, 0); - ((MockCurator) curator).timeoutBarrierOnEnter(true); - ActivateRequest activateRequest = new ActivateRequest(91, 90, "").invoke(); - HttpResponse actResponse = activateRequest.getActResponse(); - assertThat(actResponse.getStatus(), Is.is(INTERNAL_SERVER_ERROR)); - } - - @Test - public void require_that_session_created_from_active_that_is_no_longer_active_cannot_be_activated() throws Exception { - long sessionId = 1; - activateAndAssertOK(1, 0); - sessionId++; - activateAndAssertOK(sessionId, 1); - - sessionId++; - ActivateRequest activateRequest = new ActivateRequest(sessionId, 1, "").invoke(); - HttpResponse actResponse = activateRequest.getActResponse(); - String message = getRenderedString(actResponse); - assertThat(message, actResponse.getStatus(), Is.is(CONFLICT)); - assertThat(message, - containsString("Cannot activate session 3 because the currently active session (2) has changed since session 3 was created (was 1 at creation time)")); - } - - @Test - public void testAlreadyActivatedSession() throws Exception { - activateAndAssertOK(1, 0); - HttpResponse response = handler.handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, 1L)); - String message = getRenderedString(response); - assertThat(message, response.getStatus(), Is.is(BAD_REQUEST)); - assertThat(message, containsString("Session 1 is already active")); - } - - @Test - public void testActivation() throws Exception { - activateAndAssertOK(1, 0); - } - - @Test - public void testActivationWithActivationInBetween() throws Exception { - activateAndAssertOK(90, 0); - activateAndAssertError(92, 89, - Response.Status.CONFLICT, HttpErrorResponse.errorCodes.ACTIVATION_CONFLICT, - "tenant:" + tenantName + " app:default:default Cannot activate session 92 because the currently active session (90) has changed since session 92 was created (was 89 at creation time)"); - } - - @Test - public void testActivationOfUnpreparedSession() throws Exception { - // Needed so we can test that previous active session is still active after a failed activation - RemoteSession firstSession = activateAndAssertOK(90, 0); - long sessionId = 91L; - ActivateRequest activateRequest = new ActivateRequest(sessionId, 0, Session.Status.NEW, "").invoke(); - HttpResponse actResponse = activateRequest.getActResponse(); - RemoteSession session = activateRequest.getSession(); - assertThat(actResponse.getStatus(), is(Response.Status.BAD_REQUEST)); - assertThat(getRenderedString(actResponse), is("{\"error-code\":\"BAD_REQUEST\",\"message\":\"tenant:"+ tenantName +" app:default:default Session " + sessionId + " is not prepared\"}")); - assertThat(session.getStatus(), is(not(Session.Status.ACTIVATE))); - assertThat(firstSession.getStatus(), is(Session.Status.ACTIVATE)); - } - - @Test public void require_that_handler_gives_error_for_unsupported_methods() throws Exception { - testUnsupportedMethod(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.POST, Cmd.PREPARED, 1L)); - testUnsupportedMethod(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.DELETE, Cmd.PREPARED, 1L)); - testUnsupportedMethod(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L)); - } - - @Test - @Ignore - public void require_that_handler_gives_error_when_provisioner_activated_fails() throws Exception { - hostProvisioner = new FailingMockProvisioner(); - hostProvisioner.activated = false; - activateAndAssertError(1, 0, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Cannot activate application"); - assertFalse(hostProvisioner.activated); - } - - private RemoteSession createRemoteSession(long sessionId, Session.Status status, SessionZooKeeperClient zkClient) throws IOException { - zkClient.writeStatus(status); - ZooKeeperClient zkC = new ZooKeeperClient(componentRegistry.getConfigCurator(), new BaseDeployLogger(), false, - TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId))); - zkC.write(Collections.singletonMap(modelFactory.version(), new MockFileRegistry())); - zkC.write(AllocatedHosts.withHosts(Collections.emptySet())); - return new RemoteSession(tenantName, sessionId, componentRegistry, zkClient); - } - - private void addLocalSession(long sessionId, DeployData deployData, SessionZooKeeperClient zkc) throws IOException { - writeApplicationId(zkc, deployData.getApplicationId()); - TenantFileSystemDirs tenantFileSystemDirs = new TenantFileSystemDirs(temporaryFolder.newFolder(), tenantName); - ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(testApp, deployData); - localRepo.addSession(new LocalSession(tenantName, sessionId, new SessionTest.MockSessionPreparer(), - new SessionContext(app, zkc, new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)), - applicationRepo, new HostRegistry<>(), - flagSource))); - } - - private ActivateRequest activateAndAssertOKPut(long sessionId, long previousSessionId, String subPath) throws Exception { - ActivateRequest activateRequest = new ActivateRequest(sessionId, previousSessionId, subPath); - activateRequest.invoke(); - HttpResponse actResponse = activateRequest.getActResponse(); - String message = getRenderedString(actResponse); - assertThat(message, actResponse.getStatus(), Is.is(OK)); - assertActivationMessageOK(activateRequest, message); - RemoteSession session = activateRequest.getSession(); - assertThat(session.getStatus(), Is.is(Session.Status.ACTIVATE)); - return activateRequest; - } - - private void activateAndAssertErrorPut(long sessionId, long previousSessionId, - int statusCode, HttpErrorResponse.errorCodes errorCode, String expectedError) throws Exception { - ActivateRequest activateRequest = new ActivateRequest(sessionId, previousSessionId, ""); - activateRequest.invoke(); - HttpResponse actResponse = activateRequest.getActResponse(); - RemoteSession session = activateRequest.getSession(); - assertThat(actResponse.getStatus(), Is.is(statusCode)); - String message = getRenderedString(actResponse); - assertThat(message, Is.is("{\"error-code\":\"" + errorCode.name() + "\",\"message\":\"" + expectedError + "\"}")); - assertThat(session.getStatus(), Is.is(Session.Status.PREPARE)); + testUnsupportedMethod(createTestRequest(pathPrefix, HttpRequest.Method.POST, Cmd.PREPARED, 1L)); + testUnsupportedMethod(createTestRequest(pathPrefix, HttpRequest.Method.DELETE, Cmd.PREPARED, 1L)); + testUnsupportedMethod(createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L)); } private void testUnsupportedMethod(com.yahoo.container.jdisc.HttpRequest request) throws Exception { @@ -262,72 +110,47 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { protected class ActivateRequest { private long sessionId; - private RemoteSession session; private HttpResponse actResponse; - private Session.Status initialStatus; - private DeployData deployData; private ApplicationMetaData metaData; private String subPath; - ActivateRequest(long sessionId, long previousSessionId, String subPath) { - this(sessionId, previousSessionId, Session.Status.PREPARE, subPath); - } - - ActivateRequest(long sessionId, long previousSessionId, Session.Status initialStatus, String subPath) { - this.sessionId = sessionId; - this.initialStatus = initialStatus; - this.deployData = new DeployData("foo", - "bar", - ApplicationId.from(tenantName.value(), appName, "default"), - 0L, - false, - sessionId, - previousSessionId); + ActivateRequest(String subPath) { this.subPath = subPath; } - public RemoteSession getSession() { - return session; - } - - public SessionHandler getHandler() { - return handler; - } + public SessionHandler getHandler() { return handler; } - HttpResponse getActResponse() { - return actResponse; - } + HttpResponse getActResponse() { return actResponse; } - public long getSessionId() { - return sessionId; - } + public long getSessionId() { return sessionId; } - ApplicationMetaData getMetaData() { - return metaData; - } + ApplicationMetaData getMetaData() { return metaData; } - ActivateRequest invoke() throws Exception { - SessionZooKeeperClient zkClient = - new MockSessionZKClient(curator, tenantName, sessionId, - Optional.of(AllocatedHosts.withHosts(Collections.singleton(new HostSpec("bar", Collections.emptyList()))))); - session = createRemoteSession(sessionId, initialStatus, zkClient); - addLocalSession(sessionId, deployData, zkClient); - tenantRepository.getTenant(tenantName).getApplicationRepo().createApplication(deployData.getApplicationId()); - metaData = localRepo.getSession(sessionId).getMetaData(); - actResponse = handler.handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, sessionId, subPath)); - return this; + void invoke() { + Tenant tenant = tenantRepository.getTenant(tenantName); + long sessionId = applicationRepository.createSession(applicationId(), + new TimeoutBudget(componentRegistry.getClock(), Duration.ofSeconds(10)), + testApp); + applicationRepository.prepare(tenant, + sessionId, + new PrepareParams.Builder().applicationId(applicationId()).build(), + componentRegistry.getClock().instant()); + actResponse = handler.handle(createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, sessionId, subPath)); + LocalSession session = applicationRepository.getActiveLocalSession(tenant, applicationId()); + metaData = session.getMetaData(); + this.sessionId = sessionId; } } - private RemoteSession activateAndAssertOK(long sessionId, long previousSessionId) throws Exception { - ActivateRequest activateRequest = activateAndAssertOKPut(sessionId, previousSessionId, ""); - return activateRequest.getSession(); + private void activateAndAssertOK() throws Exception { + ActivateRequest activateRequest = new ActivateRequest(""); + activateRequest.invoke(); + HttpResponse actResponse = activateRequest.getActResponse(); + String message = getRenderedString(actResponse); + assertThat(message, actResponse.getStatus(), Is.is(OK)); + assertActivationMessageOK(activateRequest, message); } - private void activateAndAssertOK(long sessionId, long previousSessionId, String subPath) throws Exception { - activateAndAssertOKPut(sessionId, previousSessionId, subPath); - } - private void assertActivationMessageOK(ActivateRequest activateRequest, String message) throws IOException { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); new JsonFormat(true).encode(byteArrayOutputStream, activateRequest.getMetaData().getSlime()); @@ -341,23 +164,14 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { assertThat(hostProvisioner.lastHosts.size(), is(1)); } - private void activateAndAssertError(long sessionId, long previousSessionId, int statusCode, HttpErrorResponse.errorCodes errorCode, String expectedError) throws Exception { - hostProvisioner.activated = false; - activateAndAssertErrorPut(sessionId, previousSessionId, statusCode, errorCode, expectedError); - assertFalse(hostProvisioner.activated); - } - - private void writeApplicationId(SessionZooKeeperClient zkc, ApplicationId id) { - zkc.writeApplicationId(id); - } - private SessionActiveHandler createHandler() { return new SessionActiveHandler(SessionActiveHandler.testOnlyContext(), - new ApplicationRepository(tenantRepository, - hostProvisioner, - new OrchestratorMock(), - componentRegistry.getClock()), + applicationRepository, Zone.defaultZone()); } + private ApplicationId applicationId() { + return ApplicationId.from(tenantName.value(), appName, "default"); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index a106f52e4d1..f639843ac08 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; import org.junit.Ignore; @@ -43,8 +42,8 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { @Before public void setupHandler() throws Exception { tenantRepository = new TenantRepository(componentRegistry, false); - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, tenant)); - tenantRepository.getTenant(tenant).getLocalSessionRepo().addSession(new MockSession(1L, FilesApplicationPackage.fromFile(createTestApp()))); + tenantRepository.addTenant(tenant); + tenantRepository.getTenant(tenant).getSessionRepository().addSession(new MockLocalSession(1L, FilesApplicationPackage.fromFile(createTestApp()))); handler = createHandler(); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; baseUrl = "http://foo:1337/application/v2/tenant/" + tenant + "/session/1/content/"; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 5ea2ce5266b..5c0b1764e70 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -1,22 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStreamTest; -import com.yahoo.vespa.config.server.http.HandlerTest; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; +import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; import org.junit.Ignore; @@ -25,6 +21,7 @@ import org.junit.Test; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileWriter; import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -36,7 +33,9 @@ import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; import static com.yahoo.jdisc.Response.Status.OK; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; import static com.yahoo.jdisc.http.HttpRequest.Method.POST; +import static com.yahoo.vespa.config.server.http.HandlerTest.assertHttpStatusCodeErrorCodeAndMessage; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -50,16 +49,13 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { private static final HashMap<String, String> postHeaders = new HashMap<>(); private final TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); + ApplicationRepository applicationRepository; private String pathPrefix = "/application/v2/session/"; private String createdMessage = " created.\""; private String tenantMessage = ""; public File testApp = new File("src/test/apps/app"); - private LocalSessionRepo localSessionRepo; - private TenantApplications applicationRepo; - private TenantRepository tenantRepository; - private MockSessionFactory sessionFactory; static { postHeaders.put(ApplicationApiHandler.contentTypeHeader, ApplicationApiHandler.APPLICATION_X_GZIP); @@ -67,15 +63,12 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { @Before public void setupRepo() { - applicationRepo = TenantApplications.create(componentRegistry, new MockReloadHandler(), tenant); - localSessionRepo = new LocalSessionRepo(tenant, componentRegistry); - tenantRepository = new TenantRepository(componentRegistry, false); - sessionFactory = new MockSessionFactory(); - TenantBuilder tenantBuilder = TenantBuilder.create(componentRegistry, tenant) - .withSessionFactory(sessionFactory) - .withLocalSessionRepo(localSessionRepo) - .withApplicationRepo(applicationRepo); - tenantRepository.addTenant(tenantBuilder); + TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); + applicationRepository = new ApplicationRepository(tenantRepository, + new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), + componentRegistry.getClock()); + tenantRepository.addTenant(tenant); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; createdMessage = " for tenant '" + tenant + "' created.\""; tenantMessage = ",\"tenant\":\"test\""; @@ -86,13 +79,13 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { public void require_that_from_parameter_cannot_be_set_if_data_in_request() throws IOException { HttpRequest request = post(Collections.singletonMap("from", "active")); HttpResponse response = createHandler().handle(request); - HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Parameter 'from' is illegal for POST"); + assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Parameter 'from' is illegal for POST"); } @Test public void require_that_post_request_must_contain_data() throws IOException { HttpResponse response = createHandler().handle(post()); - HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Request contains no data"); + assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Request contains no data"); } @Test @@ -100,26 +93,13 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { HashMap<String, String> headers = new HashMap<>(); // no Content-Type header File outFile = CompressedApplicationInputStreamTest.createTarFile(); HttpResponse response = createHandler().handle(post(outFile, headers, null)); - HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Request contains no Content-Type header"); - } - - private void assertFromParameter(String expected, String from) throws IOException { - HttpRequest request = post(Collections.singletonMap("from", from)); - sessionFactory.applicationPackage = testApp; - HttpResponse response = createHandler().handle(request); - assertNotNull(response); - assertThat(response.getStatus(), is(OK)); - assertTrue(sessionFactory.createFromCalled); - assertThat(SessionHandlerTest.getRenderedString(response), - is("{\"log\":[]" + tenantMessage + ",\"session-id\":\"" + expected + "\",\"prepared\":\"http://" + hostname + ":" + port + pathPrefix + - expected + "/prepared\",\"content\":\"http://" + hostname + ":" + port + pathPrefix + - expected + "/content/\",\"message\":\"Session " + expected + createdMessage + "}")); + assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Request contains no Content-Type header"); } private void assertIllegalFromParameter(String fromValue) throws IOException { File outFile = CompressedApplicationInputStreamTest.createTarFile(); HttpRequest request = post(outFile, postHeaders, Collections.singletonMap("from", fromValue)); - HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(createHandler().handle(request), BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Parameter 'from' has illegal value '" + fromValue + "'"); + assertHttpStatusCodeErrorCodeAndMessage(createHandler().handle(request), BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Parameter 'from' has illegal value '" + fromValue + "'"); } @Test @@ -130,76 +110,44 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { assertNotNull(response); assertThat(response.getStatus(), is(OK)); assertThat(SessionHandlerTest.getRenderedString(response), - is("{\"log\":[]" + tenantMessage + ",\"session-id\":\"0\",\"prepared\":\"http://" + - hostname + ":" + port + pathPrefix + "0/prepared\",\"content\":\"http://" + - hostname + ":" + port + pathPrefix + "0/content/\",\"message\":\"Session 0" + createdMessage + "}")); - } - - @Test - public void require_that_session_factory_is_called() throws IOException { - File outFile = CompressedApplicationInputStreamTest.createTarFile(); - createHandler().handle(post(outFile)); - assertTrue(sessionFactory.createCalled); + is("{\"log\":[]" + tenantMessage + ",\"session-id\":\"2\",\"prepared\":\"http://" + + hostname + ":" + port + pathPrefix + "2/prepared\",\"content\":\"http://" + + hostname + ":" + port + pathPrefix + "2/content/\",\"message\":\"Session 2" + createdMessage + "}")); } @Test public void require_that_handler_does_not_support_get() throws IOException { HttpResponse response = createHandler().handle(HttpRequest.createTestRequest(pathPrefix, GET)); - HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, METHOD_NOT_ALLOWED, + assertHttpStatusCodeErrorCodeAndMessage(response, METHOD_NOT_ALLOWED, HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, "Method 'GET' is not supported"); } @Test public void require_internal_error_when_exception() throws IOException { - sessionFactory.doThrow = true; File outFile = CompressedApplicationInputStreamTest.createTarFile(); + new FileWriter(outFile).write("rubbish"); HttpResponse response = createHandler().handle(post(outFile)); - HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, INTERNAL_SERVER_ERROR, - HttpErrorResponse.errorCodes.INTERNAL_SERVER_ERROR, - "foo"); + assertHttpStatusCodeErrorCodeAndMessage(response, INTERNAL_SERVER_ERROR, + HttpErrorResponse.errorCodes.INTERNAL_SERVER_ERROR, + "Unable to create compressed application stream"); } @Test public void require_that_handler_unpacks_application() throws IOException { File outFile = CompressedApplicationInputStreamTest.createTarFile(); createHandler().handle(post(outFile)); - assertTrue(sessionFactory.createCalled); - final File applicationPackage = sessionFactory.applicationPackage; - assertNotNull(applicationPackage); - assertTrue(applicationPackage.exists()); - final File[] files = applicationPackage.listFiles(); - assertNotNull(files); - assertThat(files.length, is(3)); - } - - @Test - public void require_that_session_is_stored_in_repo() throws IOException { - File outFile = CompressedApplicationInputStreamTest.createTarFile(); - createHandler().handle(post(outFile)); - assertNotNull(localSessionRepo.getSession(0)); + ApplicationFile applicationFile = applicationRepository.getApplicationFileFromSession(tenant, 2, "services.xml", LocalSession.Mode.READ); + assertTrue(applicationFile.exists()); } @Test public void require_that_application_urls_can_be_given_as_from_parameter() throws Exception { - localSessionRepo.addSession(new SessionHandlerTest.MockSession(2, FilesApplicationPackage.fromFile(testApp))); - ApplicationId fooId = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("foo") - .instanceName("quux") - .build(); - applicationRepo.createApplication(fooId); - applicationRepo.createPutTransaction(fooId, 2).commit(); - assertFromParameter("3", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foo/environment/test/region/baz/instance/quux"); - localSessionRepo.addSession(new SessionHandlerTest.MockSession(5, FilesApplicationPackage.fromFile(testApp))); - ApplicationId bioId = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("foobio") - .instanceName("quux") - .build(); - applicationRepo.createApplication(bioId); - applicationRepo.createPutTransaction(bioId, 5).commit(); - assertFromParameter("6", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foobio/environment/staging/region/baz/instance/quux"); + ApplicationId applicationId = ApplicationId.from(tenant.value(), "foo", "quux"); + HttpRequest request = post(Collections.singletonMap( + "from", + "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foo/environment/test/region/baz/instance/quux")); + assertEquals(applicationId, SessionCreateHandler.getFromApplicationId(request)); } @Test @@ -213,14 +161,9 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { } private SessionCreateHandler createHandler() { - return new SessionCreateHandler( - SessionCreateHandler.testOnlyContext(), - new ApplicationRepository(tenantRepository, - new SessionHandlerTest.MockProvisioner(), - new OrchestratorMock(), - componentRegistry.getClock()), - componentRegistry.getConfigserverConfig()); - + return new SessionCreateHandler(SessionCreateHandler.testOnlyContext(), + applicationRepository, + componentRegistry.getConfigserverConfig()); } private HttpRequest post() throws FileNotFoundException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 94276b30f42..535118ae5aa 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -1,37 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.UncheckedTimeoutException; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; +import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.http.HttpRequest; -import com.yahoo.path.Path; import com.yahoo.slime.JsonDecoder; import com.yahoo.slime.Slime; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.host.HostRegistry; -import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; -import com.yahoo.vespa.config.server.configchange.MockRefeedAction; -import com.yahoo.vespa.config.server.configchange.MockRestartAction; -import com.yahoo.vespa.config.server.http.*; -import com.yahoo.vespa.config.server.session.*; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; +import com.yahoo.vespa.config.server.http.SessionHandler; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -39,14 +28,11 @@ import org.junit.Before; import org.junit.Test; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.time.Clock; -import java.time.Instant; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.time.Duration; import java.util.Map; -import java.util.Optional; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; @@ -65,11 +51,13 @@ import static org.junit.Assert.assertThat; */ public class SessionPrepareHandlerTest extends SessionHandlerTest { private static final TenantName tenant = TenantName.from("test"); + private static final File app = new File("src/test/resources/deploy/validapp"); private Curator curator = new MockCurator(); private final TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); private final Clock clock = componentRegistry.getClock(); - private LocalSessionRepo localRepo; + private final TimeoutBudget timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); + private ApplicationRepository applicationRepository; private String preparedMessage = " prepared.\"}"; private String tenantMessage = ""; @@ -77,16 +65,15 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { @Before public void setupRepo() { - localRepo = new LocalSessionRepo(tenant, componentRegistry); + tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(tenant); + applicationRepository = new ApplicationRepository(tenantRepository, + new MockProvisioner(), + new OrchestratorMock(), + clock); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; preparedMessage = " for tenant '" + tenant + "' prepared.\""; tenantMessage = ",\"tenant\":\"" + tenant + "\""; - tenantRepository = new TenantRepository(componentRegistry, false); - TenantBuilder tenantBuilder = TenantBuilder.create(componentRegistry, tenant) - .withSessionFactory(new MockSessionFactory()) - .withLocalSessionRepo(localRepo) - .withApplicationRepo(TenantApplications.create(componentRegistry, new MockReloadHandler(), tenant)); - tenantRepository.addTenant(tenantBuilder); } @Test @@ -107,8 +94,8 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { @Test public void require_that_handler_gives_error_for_unsupported_methods() throws Exception { - testUnsupportedMethod(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.POST, Cmd.PREPARED, 1L)); - testUnsupportedMethod(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.DELETE, Cmd.PREPARED, 1L)); + testUnsupportedMethod(createTestRequest(pathPrefix, HttpRequest.Method.POST, Cmd.PREPARED, 1L)); + testUnsupportedMethod(createTestRequest(pathPrefix, HttpRequest.Method.DELETE, Cmd.PREPARED, 1L)); } private void testUnsupportedMethod(com.yahoo.container.jdisc.HttpRequest request) throws Exception { @@ -120,85 +107,61 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { @Test public void require_that_activate_url_is_returned_on_success() throws Exception { - MockSession session = new MockSession(1, null); - localRepo.addSession(session); - HttpResponse response = request(HttpRequest.Method.PUT, 1L); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); + HttpResponse response = request(HttpRequest.Method.PUT, sessionId); assertNotNull(response); assertThat(response.getStatus(), is(OK)); - assertResponseContains(response, "\"activate\":\"http://foo:1337" + pathPrefix + "1/active\",\"message\":\"Session 1" + preparedMessage); + assertResponseContains(response, "\"activate\":\"http://foo:1337" + pathPrefix + sessionId + + "/active\",\"message\":\"Session " + sessionId + preparedMessage); } @Test public void require_debug() throws Exception { HttpResponse response = createHandler().handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 9999L, "?debug=true")); + createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 9999L, "?debug=true")); assertThat(response.getStatus(), is(NOT_FOUND)); assertThat(SessionHandlerTest.getRenderedString(response), containsString("NotFoundException")); } @Test public void require_verbose() throws Exception { - MockSession session = new MockSession(1, null); - session.doVerboseLogging = true; - localRepo.addSession(session); + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); HttpResponse response = createHandler().handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L, "?verbose=true")); + createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, sessionId, "?verbose=true")); + System.out.println(getRenderedString(response)); assertThat(response.getStatus(), is(OK)); - assertThat(SessionHandlerTest.getRenderedString(response), containsString("debuglog")); - } - - private SessionZooKeeperClient createSessionZooKeeperClient(LocalSession session) { - return new MockSessionZKClient(curator, tenant, session.getSessionId()); + assertThat(getRenderedString(response), containsString("Created application ")); } @Test public void require_get_response_activate_url_on_ok() throws Exception { - MockSession session = new MockSession(1, null); - localRepo.addSession(session); - request(HttpRequest.Method.PUT, 1L); - session.setStatus(Session.Status.PREPARE); - SessionZooKeeperClient zooKeeperClient = createSessionZooKeeperClient(session); - zooKeeperClient.writeStatus(Session.Status.PREPARE); - HttpResponse getResponse = request(HttpRequest.Method.GET, 1L); + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); + request(HttpRequest.Method.PUT, sessionId); + HttpResponse getResponse = request(HttpRequest.Method.GET, sessionId); assertResponseContains(getResponse, "\"activate\":\"http://foo:1337" + pathPrefix + - "1/active\",\"message\":\"Session 1" + preparedMessage); + sessionId + "/active\",\"message\":\"Session " + sessionId + preparedMessage); } @Test public void require_get_response_error_on_not_prepared() throws Exception { - MockSession session = new MockSession(1, null); - localRepo.addSession(session); - session.setStatus(Session.Status.NEW); - SessionZooKeeperClient zooKeeperClient = createSessionZooKeeperClient(session); - zooKeeperClient.writeStatus(Session.Status.NEW); - HttpResponse getResponse = request(HttpRequest.Method.GET, 1L); - assertHttpStatusCodeErrorCodeAndMessage(getResponse, BAD_REQUEST, - HttpErrorResponse.errorCodes.BAD_REQUEST, - "Session not prepared: 1"); - session.setStatus(Session.Status.ACTIVATE); - zooKeeperClient.writeStatus(Session.Status.ACTIVATE); - getResponse = request(HttpRequest.Method.GET, 1L); + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); + + HttpResponse getResponse = request(HttpRequest.Method.GET, sessionId); assertHttpStatusCodeErrorCodeAndMessage(getResponse, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, - "Session is active: 1"); - } + "Session not prepared: " + sessionId); - @Test - public void require_cannot_prepare_active_session() throws Exception { - MockSession session = new MockSession(1, null); - localRepo.addSession(session); - session.setStatus(Session.Status.ACTIVATE); - HttpResponse putResponse = request(HttpRequest.Method.PUT, 1L); - assertHttpStatusCodeErrorCodeAndMessage(putResponse, BAD_REQUEST, + request(HttpRequest.Method.PUT, sessionId); + applicationRepository.activate(tenantRepository.getTenant(tenant), sessionId, timeoutBudget, false); + + getResponse = request(HttpRequest.Method.GET, sessionId); + assertHttpStatusCodeErrorCodeAndMessage(getResponse, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, - "Session is active: 1"); + "Session is active: " + sessionId); } @Test public void require_get_response_error_when_session_id_does_not_exist() throws Exception { - MockSession session = new MockSession(1, null); - localRepo.addSession(session); HttpResponse getResponse = request(HttpRequest.Method.GET, 9999L); assertHttpStatusCodeErrorCodeAndMessage(getResponse, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, @@ -207,137 +170,116 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { @Test public void require_that_tenant_is_in_response() throws Exception { - MockSession session = new MockSession(1, null); - localRepo.addSession(session); - HttpResponse response = request(HttpRequest.Method.PUT, 1L); + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); + HttpResponse response = request(HttpRequest.Method.PUT, sessionId); assertNotNull(response); assertThat(response.getStatus(), is(OK)); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); assertResponseContains(response, tenantMessage); } @Test public void require_that_preparing_with_multiple_tenants_work() throws Exception { - final TenantName defaultTenant = TenantName.from("test2"); - // Need different repo for 'test2' tenant - LocalSessionRepo localRepoDefault = new LocalSessionRepo(defaultTenant, componentRegistry); - TenantBuilder defaultTenantBuilder = TenantBuilder.create(componentRegistry, defaultTenant) - .withLocalSessionRepo(localRepoDefault) - .withSessionFactory(new MockSessionFactory()); - tenantRepository.addTenant(defaultTenantBuilder); - final SessionHandler handler = createHandler(); - - long sessionId = 1; - // Deploy with default tenant - MockSession session = new MockSession(sessionId, null); - localRepoDefault.addSession(session); - pathPrefix = "/application/v2/tenant/" + defaultTenant + "/session/"; + SessionHandler handler = createHandler(); + TenantName defaultTenant = TenantName.from("test2"); + tenantRepository.addTenant(defaultTenant); + ApplicationId applicationId1 = ApplicationId.from(defaultTenant, ApplicationName.from("app"), InstanceName.defaultName()); + long sessionId = applicationRepository.createSession(applicationId1, timeoutBudget, app); + + pathPrefix = "/application/v2/tenant/" + defaultTenant + "/session/"; HttpResponse response = request(HttpRequest.Method.PUT, sessionId); assertNotNull(response); assertThat(SessionHandlerTest.getRenderedString(response), response.getStatus(), is(OK)); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); - // Same session id, as this is for another tenant - session = new MockSession(sessionId, null); - localRepo.addSession(session); String applicationName = "myapp"; - pathPrefix = "/application/v2/tenant/" + tenant + "/session/" + sessionId + + ApplicationId applicationId2 = ApplicationId.from(tenant.value(), applicationName, "default"); + long sessionId2 = applicationRepository.createSession(applicationId2, timeoutBudget, app); + assertEquals(sessionId, sessionId2); // Want to test when they are equal (but for different tenants) + + pathPrefix = "/application/v2/tenant/" + tenant + "/session/" + sessionId2 + "/prepared?applicationName=" + applicationName; response = handler.handle(SessionHandlerTest.createTestRequest(pathPrefix)); assertNotNull(response); assertThat(SessionHandlerTest.getRenderedString(response), response.getStatus(), is(OK)); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); - sessionId++; - session = new MockSession(sessionId, null); - localRepo.addSession(session); - pathPrefix = "/application/v2/tenant/" + tenant + "/session/" + sessionId + + ApplicationId applicationId3 = ApplicationId.from(tenant.value(), applicationName, "quux"); + long sessionId3 = applicationRepository.createSession(applicationId3, timeoutBudget, app); + pathPrefix = "/application/v2/tenant/" + tenant + "/session/" + sessionId3 + "/prepared?applicationName=" + applicationName + "&instance=quux"; response = handler.handle(SessionHandlerTest.createTestRequest(pathPrefix)); assertNotNull(response); assertThat(SessionHandlerTest.getRenderedString(response), response.getStatus(), is(OK)); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); } @Test public void require_that_config_change_actions_are_in_response() throws Exception { - MockSession session = new MockSession(1, null); - localRepo.addSession(session); - HttpResponse response = request(HttpRequest.Method.PUT, 1L); + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); + HttpResponse response = request(HttpRequest.Method.PUT, sessionId); assertResponseContains(response, "\"configChangeActions\":{\"restart\":[],\"refeed\":[]}"); } @Test - public void require_that_config_change_actions_are_logged_if_existing() throws Exception { - List<ServiceInfo> services = Collections.singletonList( - new ServiceInfo("serviceName", "serviceType", null, - ImmutableMap.of("clustername", "foo", "clustertype", "bar"), "configId", "hostName")); - ConfigChangeActions actions = new ConfigChangeActions(Arrays.asList( - new MockRestartAction("change", services), - new MockRefeedAction("change-id", false, "other change", services, "test"))); - MockSession session = new MockSession(1, null, actions); - localRepo.addSession(session); - HttpResponse response = request(HttpRequest.Method.PUT, 1L); - assertResponseContains(response, - "Change(s) between active and new application that require restart:\\nIn cluster 'foo' of type 'bar"); - assertResponseContains(response, - "Change(s) between active and new application that may require re-feed:\\nchange-id: Consider removing data and re-feed document type 'test'"); - } - - @Test public void require_that_config_change_actions_are_not_logged_if_not_existing() throws Exception { - MockSession session = new MockSession(1, null); - localRepo.addSession(session); - HttpResponse response = request(HttpRequest.Method.PUT, 1L); + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); + HttpResponse response = request(HttpRequest.Method.PUT, sessionId); assertResponseNotContains(response, "Change(s) between active and new application that require restart"); assertResponseNotContains(response, "Change(s) between active and new application that require re-feed"); } @Test public void test_out_of_capacity_response() throws IOException { - String message = "Internal error"; - SessionThrowingException session = new SessionThrowingException(new OutOfCapacityException(message)); - localRepo.addSession(session); - HttpResponse response = request(HttpRequest.Method.PUT, 1L); + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); + String exceptionMessage = "Out of capacity"; + FailingSessionPrepareHandler handler = new FailingSessionPrepareHandler(SessionPrepareHandler.testOnlyContext(), + applicationRepository, + componentRegistry.getConfigserverConfig(), + new OutOfCapacityException(exceptionMessage)); + HttpResponse response = handler.handle(createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, sessionId)); assertEquals(400, response.getStatus()); Slime data = getData(response); assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.OUT_OF_CAPACITY.name())); - assertThat(data.get().field("message").asString(), is(message)); + assertThat(data.get().field("message").asString(), is(exceptionMessage)); } @Test public void test_that_nullpointerexception_gives_internal_server_error() throws IOException { - String message = "No nodes available"; - SessionThrowingException session = new SessionThrowingException(new NullPointerException(message)); - localRepo.addSession(session); - HttpResponse response = request(HttpRequest.Method.PUT, 1L); + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); + String exceptionMessage = "nullpointer thrown in test handler"; + FailingSessionPrepareHandler handler = new FailingSessionPrepareHandler(SessionPrepareHandler.testOnlyContext(), + applicationRepository, + componentRegistry.getConfigserverConfig(), + new NullPointerException(exceptionMessage)); + HttpResponse response = handler.handle(createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, sessionId)); assertEquals(500, response.getStatus()); Slime data = getData(response); assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.INTERNAL_SERVER_ERROR.name())); - assertThat(data.get().field("message").asString(), is(message)); + assertThat(data.get().field("message").asString(), is(exceptionMessage)); } @Test public void test_application_lock_failure() throws IOException { - String message = "Timed out after waiting PT1M to acquire lock '/provision/v1/locks/foo/bar/default'"; - SessionThrowingException session = - new SessionThrowingException(new ApplicationLockException(new UncheckedTimeoutException(message))); - localRepo.addSession(session); - HttpResponse response = request(HttpRequest.Method.PUT, 1L); + String exceptionMessage = "Timed out after waiting PT1M to acquire lock '/provision/v1/locks/foo/bar/default'"; + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); + FailingSessionPrepareHandler handler = new FailingSessionPrepareHandler(SessionPrepareHandler.testOnlyContext(), + applicationRepository, + componentRegistry.getConfigserverConfig(), + new ApplicationLockException(new UncheckedTimeoutException(exceptionMessage))); + HttpResponse response = handler.handle(createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, sessionId)); assertEquals(500, response.getStatus()); Slime data = getData(response); assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.APPLICATION_LOCK_FAILURE.name())); - assertThat(data.get().field("message").asString(), is(message)); + assertThat(data.get().field("message").asString(), is(exceptionMessage)); } @Test public void test_docker_image_repository() { - MockSession session = new MockSession(1, null); - localRepo.addSession(session); + long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); String dockerImageRepository = "https://foo.bar.com:4443/baz"; - request(HttpRequest.Method.PUT, 1L, Map.of("dockerImageRepository", dockerImageRepository)); - assertEquals(DockerImage.fromString(dockerImageRepository), localRepo.getSession(1).getDockerImageRepository().get()); + request(HttpRequest.Method.PUT, sessionId, Map.of("dockerImageRepository", dockerImageRepository, + "applicationName", applicationId().application().value())); + applicationRepository.activate(tenantRepository.getTenant(tenant), sessionId, timeoutBudget, false); + assertEquals(DockerImage.fromString(dockerImageRepository), + applicationRepository.getActiveSession(applicationId()).getDockerImageRepository().get()); } private Slime getData(HttpResponse response) throws IOException { @@ -359,12 +301,8 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { private SessionHandler createHandler() { return new SessionPrepareHandler( SessionPrepareHandler.testOnlyContext(), - new ApplicationRepository(tenantRepository, - new MockProvisioner(), - new OrchestratorMock(), - clock), + applicationRepository, componentRegistry.getConfigserverConfig()); - } private HttpResponse request(HttpRequest.Method put, long l) { @@ -375,60 +313,24 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { return createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, put, Cmd.PREPARED, l, "", null, requestParameters)); } - public static class SessionThrowingException extends LocalSession { - private final RuntimeException exception; - - SessionThrowingException(RuntimeException exception) { - super(TenantName.defaultName(), 1, null, - new SessionContext(null, - new MockSessionZKClient(MockApplicationPackage.createEmpty()), - null, - null, - new HostRegistry<>(), - null)); - this.exception = exception; - } - - @Override - public ConfigChangeActions prepare(DeployLogger logger, - PrepareParams params, - Optional<ApplicationSet> application, - Path tenantPath, - Instant now) { - throw exception; - } - - @Override - public Session.Status getStatus() { - return null; - } - - @Override - public Transaction createDeactivateTransaction() { - return null; - } + private ApplicationId applicationId() { + return ApplicationId.from(tenant.value(), "app", "default"); + } - @Override - public Transaction createActivateTransaction() { - return null; - } + private static class FailingSessionPrepareHandler extends SessionPrepareHandler { - @Override - public ApplicationFile getApplicationFile(Path relativePath, Mode mode) { - return null; - } + private final RuntimeException exceptionToBeThrown; - @Override - public ApplicationId getApplicationId() { - return null; + public FailingSessionPrepareHandler(Context ctx, ApplicationRepository applicationRepository, + ConfigserverConfig configserverConfig, RuntimeException exceptionToBeThrown) { + super(ctx, applicationRepository, configserverConfig); + this.exceptionToBeThrown = exceptionToBeThrown; } @Override - public long getCreateTime() { - return 0; + protected HttpResponse handlePUT(com.yahoo.container.jdisc.HttpRequest request) { + throw exceptionToBeThrown; } - - @Override - public void delete(NestedTransaction transaction) { } } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java index 492767728e5..49de9b41ca1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java @@ -1,7 +1,6 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.metrics; -import com.yahoo.component.Version; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.api.HostInfo; @@ -23,7 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** * @author olaa diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg deleted file mode 100644 index d3970ee48eb..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg +++ /dev/null @@ -1,44 +0,0 @@ -include: search/cluster.music -include: search/cluster.music -c 2 -storage[2] -storage[0].feeder[1] -storage[0].feeder[0] "test" -storage[1].feeder[2] -storage[1].feeder[0] "me" -storage[1].feeder[1] now -storage[1].id :parent: -storage[1].id2 pjatt -testref :parent: -testref2 some/babbel -config[1] -config[0].role "rtx" -#config[0].usewrapper false -config[0].id search/cluster.music/rtx/0 -f[1] -f[0].a "A" -f[0].b "B" -f[0].c "C" -f[0].h "H" -f[0].f "F" -f[0].notindef "notindef" -routingtable[1] -routingtable[0].hop[3] -routingtable[0].hop[0].name "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].hop[0].selector "docproc/cluster.music.indexing/*/chain.music.indexing" -routingtable[0].hop[1].name "search/cluster.music" -routingtable[0].hop[1].selector "search/cluster.music/[SearchColumn]/[SearchRow]/feed-destination" -routingtable[0].hop[1].recipient[1] -routingtable[0].hop[1].recipient[0] "search/cluster.music/c0/r0/feed-destination" -routingtable[0].hop[2].selector "[DocumentRouteSelector]" -routingtable[0].hop[2].name "indexing" -routingtable[0].hop[2].notindef "not in def" -routingtable[0].hop[2].recipient[1] -routingtable[0].hop[2].recipient[0] "search/cluster.music" -notindef "dfsd" -nopenotindef[0] "boo" -nadaindef[0].naw 98 -mode NOTINDEF -rangecheck1 100 -rangecheck2 10000 -rangecheck3 20 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg deleted file mode 100644 index 727a5052ed6..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg +++ /dev/null @@ -1,2 +0,0 @@ -include: search/cluster.sports -c 67 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg deleted file mode 100644 index f4996027f60..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg +++ /dev/null @@ -1 +0,0 @@ -model vespa diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg deleted file mode 100644 index d75d76810f9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg +++ /dev/null @@ -1,2 +0,0 @@ -foo "bar" -gaz -78 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg deleted file mode 100644 index 7ccdb73eb9a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg +++ /dev/null @@ -1,2 +0,0 @@ -include: search/cluster.logical/* -include: search/cluster.video/* diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg deleted file mode 100644 index 5b07d3a2890..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg +++ /dev/null @@ -1 +0,0 @@ -include: search/cluster.music diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java index 607a2dca6c6..76a97927e78 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java @@ -15,11 +15,15 @@ import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.concurrent.Executors; import java.util.function.Supplier; +import java.util.logging.Logger; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; public class ZKMetricUpdaterTest { + + private static final Logger LOG = Logger.getLogger(ZKMetricUpdaterTest.class.getName()); + private Thread serverThread; private int serverPort; @@ -81,13 +85,13 @@ public class ZKMetricUpdaterTest { output.close(); } } catch (IOException e) { - System.out.println("Error in fake ZK server: " + e.toString()); + LOG.severe("Error in fake ZK server: " + e.toString()); } } try { serverSocket.close(); } catch (IOException e) { - System.out.println("Error closing server socket in fake ZK server: " + e.toString()); + LOG.severe("Error closing server socket in fake ZK server: " + e.toString()); } }); serverThread.start(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java index 07f6e9cf222..d923f4c1856 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java @@ -55,14 +55,6 @@ public class MockRpc extends RpcServer { this(port, true, tempDir); } - /** Reset fields used to assert on the calls made to this */ - public void resetChecks() { - forced = false; - tryResolveConfig = false; - tryRespond = false; - latestRequest = null; - } - private static ConfigserverConfig createConfig(int port) { ConfigserverConfig.Builder b = new ConfigserverConfig.Builder(); b.rpcport(port); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg deleted file mode 100644 index f3acd4cf8b9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg +++ /dev/null @@ -1,5 +0,0 @@ -asyncfetchocc 9 -d 3 -kanon -78.56 - -partialsd "sd" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg deleted file mode 100644 index 5d8a01a18ea..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg +++ /dev/null @@ -1,2 +0,0 @@ -d 89 -search[1].feeder[1] "sportsfeeder1" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg deleted file mode 100644 index f6c35df398d..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg +++ /dev/null @@ -1 +0,0 @@ -gaff -89 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg deleted file mode 100644 index c3d9b1e45a1..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg +++ /dev/null @@ -1,8 +0,0 @@ -classes[1] -classes[logical].id 1906788747 -classes[logical].name logical -classes[logical].fields[2] -classes[logical].fields[0].name sddocnameNAM -classes[logical].fields[0].type longstring -classes[logical].fields[1].name title -classes[logical].fields[1].type longstringSTRIN diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg deleted file mode 100644 index 12a21671b4a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg +++ /dev/null @@ -1,8 +0,0 @@ -classes[1] -classes[music].id 1906788746 -classes[music].name music -classes[music].fields[2] -classes[music].fields[0].name sddocnameNAME -classes[music].fields[0].type longstring -classes[music].fields[1].name title -classes[music].fields[1].type longstringSTRING diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg deleted file mode 100644 index 4001c59adbc..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg +++ /dev/null @@ -1,14 +0,0 @@ -classes[0] -classes[smallsum614540714].id 614540714 -classes[smallsum614540714].name smallsum -classes[smallsum614540714].fields[5] -classes[smallsum614540714].fields[0].name s_13 -classes[smallsum614540714].fields[0].type longstring -classes[smallsum614540714].fields[1].name ranklog -classes[smallsum614540714].fields[1].type longstring -classes[smallsum614540714].fields[2].name rankfeatures -classes[smallsum614540714].fields[2].type longstring -classes[smallsum614540714].fields[3].name summaryfeatures -classes[smallsum614540714].fields[3].type longstring -classes[smallsum614540714].fields[4].name sddocname -classes[smallsum614540714].fields[4].type longstring diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg deleted file mode 100644 index 33d07b99ab6..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg +++ /dev/null @@ -1,14 +0,0 @@ -classes[0] -classes[smallsum507688128].id 507688128 -classes[smallsum507688128].name smallsum -classes[smallsum507688128].fields[5] -classes[smallsum507688128].fields[0].name title -classes[smallsum507688128].fields[0].type longstring -classes[smallsum507688128].fields[1].name ranklog -classes[smallsum507688128].fields[1].type longstring -classes[smallsum507688128].fields[2].name rankfeatures -classes[smallsum507688128].fields[2].type longstring -classes[smallsum507688128].fields[3].name summaryfeatures -classes[smallsum507688128].fields[3].type longstring -classes[smallsum507688128].fields[4].name sddocname -classes[smallsum507688128].fields[4].type longstring diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg deleted file mode 100644 index de9fbdd39f4..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg +++ /dev/null @@ -1,7 +0,0 @@ -rec 56 -national 77 -ilscript[1] -ilscript[music].name music -ilscript[music].doctype music -ilscript[music].content[1] -ilscript[music].content[0] "input year | summary s_3 | tokenize \"stemming,normalizing\" { index f_3 | index f_4; };" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg deleted file mode 100644 index e95f976a43a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg +++ /dev/null @@ -1,7 +0,0 @@ -ursive -50 -teatern 78 -ilscript[1] -ilscript[father].name father -ilscript[father].doctype father -ilscript[father].content[6] -ilscript[father].content[0] "input year | summary s_3 | tokenize \"stemming,normalizing\" { index f_3 | index f_5; };" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg deleted file mode 100644 index cea943d5bc9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg +++ /dev/null @@ -1,2 +0,0 @@ -include: search/cluster.music/conf1.sd.derived -include: search/cluster.music/conf2.sd.derived diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java deleted file mode 100644 index 9232109a89b..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.test.ManualClock; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.MockReloadHandler; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.io.IOUtils; -import com.yahoo.vespa.config.server.host.HostRegistry; -import com.yahoo.vespa.config.server.http.SessionHandlerTest; - -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.time.Instant; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; - -/** - * @author Ulf Lilleengen - */ -public class LocalSessionRepoTest { - - private File testApp = new File("src/test/apps/app"); - private LocalSessionRepo repo; - private ManualClock clock; - private static final TenantName tenantName = TenantName.defaultName(); - - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Before - public void setupSessions() throws Exception { - setupSessions(tenantName, true); - } - - private void setupSessions(TenantName tenantName, boolean createInitialSessions) throws Exception { - File configserverDbDir = temporaryFolder.newFolder().getAbsoluteFile(); - if (createInitialSessions) { - Path sessionsPath = Paths.get(configserverDbDir.getAbsolutePath(), "tenants", tenantName.value(), "sessions"); - IOUtils.copyDirectory(testApp, sessionsPath.resolve("1").toFile()); - IOUtils.copyDirectory(testApp, sessionsPath.resolve("2").toFile()); - IOUtils.copyDirectory(testApp, sessionsPath.resolve("3").toFile()); - } - clock = new ManualClock(Instant.ofEpochSecond(1)); - GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder() - .curator(new MockCurator()) - .clock(clock) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(configserverDbDir.getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .sessionLifetime(5) - .build()) - .build(); - LocalSessionLoader loader = new SessionFactoryImpl(globalComponentRegistry, - TenantApplications.create(globalComponentRegistry, new MockReloadHandler(), tenantName), - new HostRegistry<>(), - tenantName); - repo = new LocalSessionRepo(tenantName, globalComponentRegistry, loader); - } - - @Test - public void require_that_sessions_can_be_loaded_from_disk() { - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNotNull(repo.getSession(3L)); - assertNull(repo.getSession(4L)); - } - - @Test - public void require_that_old_sessions_are_purged() { - clock.advance(Duration.ofSeconds(1)); - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNotNull(repo.getSession(3L)); - clock.advance(Duration.ofSeconds(1)); - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNotNull(repo.getSession(3L)); - clock.advance(Duration.ofSeconds(1)); - addSession(4L, 6); - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNotNull(repo.getSession(3L)); - assertNotNull(repo.getSession(4L)); - clock.advance(Duration.ofSeconds(1)); - addSession(5L, 10); - repo.purgeOldSessions(); - assertNull(repo.getSession(1L)); - assertNull(repo.getSession(2L)); - assertNull(repo.getSession(3L)); - } - - @Test - public void require_that_all_sessions_are_deleted() { - repo.close(); - assertNull(repo.getSession(1L)); - assertNull(repo.getSession(2L)); - assertNull(repo.getSession(3L)); - } - - private void addSession(long sessionId, long createTime) { - repo.addSession(new SessionHandlerTest.MockSession(sessionId, FilesApplicationPackage.fromFile(testApp), createTime)); - } - - @Test - public void require_that_sessions_belong_to_a_tenant() { - // tenant is "default" - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNotNull(repo.getSession(3L)); - assertNull(repo.getSession(4L)); - - // tenant is "newTenant" - try { - setupSessions(TenantName.from("newTenant"), false); - } catch (Exception e) { - fail(); - } - assertNull(repo.getSession(1L)); - - repo.addSession(new SessionHandlerTest.MockSession(1L, FilesApplicationPackage.fromFile(testApp))); - repo.addSession(new SessionHandlerTest.MockSession(2L, FilesApplicationPackage.fromFile(testApp))); - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNull(repo.getSession(3L)); - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index b357f712004..c1377ae439b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -9,38 +9,30 @@ import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.slime.Slime; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; -import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; - -import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.IOException; import java.time.Instant; -import java.util.ArrayList; import java.util.Collections; -import java.util.List; import java.util.Optional; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -49,17 +41,30 @@ import static org.junit.Assert.assertTrue; */ public class LocalSessionTest { - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - private Path tenantPath = Path.createRoot(); + private static final File testApp = new File("src/test/apps/app"); + private static final TenantName tenantName = TenantName.from("test_tenant"); + private static final Path tenantPath = Path.createRoot(); + + private TenantRepository tenantRepository; private Curator curator; private ConfigCurator configCurator; - private TenantFileSystemDirs tenantFileSystemDirs; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void setupTest() { + public void setupTest() throws IOException { curator = new MockCurator(); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .build()) + .build(); + tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(tenantName); configCurator = ConfigCurator.create(curator); - tenantFileSystemDirs = new TenantFileSystemDirs(Files.createTempDir(), TenantName.from("test_tenant")); } @Test @@ -93,26 +98,6 @@ public class LocalSessionTest { } @Test - public void require_that_preparer_is_run() throws Exception { - SessionTest.MockSessionPreparer preparer = new SessionTest.MockSessionPreparer(); - LocalSession session = createSession(TenantName.defaultName(), 3, preparer); - assertFalse(preparer.isPrepared); - doPrepare(session); - assertTrue(preparer.isPrepared); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); - } - - @Test - public void require_that_session_status_can_be_deactivated() throws Exception { - SessionTest.MockSessionPreparer preparer = new SessionTest.MockSessionPreparer(); - LocalSession session = createSession(TenantName.defaultName(), 3, preparer); - session.createDeactivateTransaction().commit(); - assertThat(session.getStatus(), is(Session.Status.DEACTIVATE)); - } - - private File testApp = new File("src/test/apps/app"); - - @Test public void require_that_application_file_can_be_fetched() throws Exception { LocalSession session = createSession(TenantName.defaultName(), 3); ApplicationFile f1 = session.getApplicationFile(Path.fromString("services.xml"), LocalSession.Mode.READ); @@ -121,87 +106,28 @@ public class LocalSessionTest { assertFalse(f2.exists()); } - @Test - public void require_that_session_can_be_deleted() throws Exception { - TenantName tenantName = TenantName.defaultName(); - LocalSession session = createSession(tenantName, 3); - String sessionNode = TenantRepository.getSessionsPath(tenantName).append(String.valueOf(3)).getAbsolute(); - assertTrue(configCurator.exists(sessionNode)); - assertTrue(new File(tenantFileSystemDirs.sessionsPath(), "3").exists()); - NestedTransaction transaction = new NestedTransaction(); - session.delete(transaction); - transaction.commit(); - assertFalse(configCurator.exists(sessionNode)); - assertFalse(new File(tenantFileSystemDirs.sessionsPath(), "3").exists()); - } - @Test(expected = IllegalStateException.class) public void require_that_no_provision_info_throws_exception() throws Exception { createSession(TenantName.defaultName(), 3).getAllocatedHosts(); } - @Test - public void require_that_provision_info_can_be_read() throws Exception { - List<NetworkPorts.Allocation> list = new ArrayList<>(); - list.add(new NetworkPorts.Allocation(8080, "container", "default/0", "http")); - list.add(new NetworkPorts.Allocation(19101, "searchnode", "other/1", "rpc")); - NetworkPorts ports = new NetworkPorts(list); - - AllocatedHosts input = AllocatedHosts.withHosts(Collections.singleton( - new HostSpec("myhost", Collections.emptyList(), - Optional.empty(), Optional.empty(), Optional.empty(), - Optional.of(ports)))); - - LocalSession session = createSession(TenantName.defaultName(), 3, new SessionTest.MockSessionPreparer(), Optional.of(input)); - ApplicationId origId = new ApplicationId.Builder() - .tenant("tenant") - .applicationName("foo").instanceName("quux").build(); - doPrepare(session, new PrepareParams.Builder().applicationId(origId).build()); - AllocatedHosts info = session.getAllocatedHosts(); - assertNotNull(info); - assertThat(info.getHosts().size(), is(1)); - assertTrue(info.getHosts().contains(new HostSpec("myhost", Collections.emptyList()))); - Optional<NetworkPorts> portsCopy = info.getHosts().iterator().next().networkPorts(); - assertTrue(portsCopy.isPresent()); - assertThat(portsCopy.get().allocations(), is(list)); - } - - @Test - public void require_that_application_metadata_is_correct() throws Exception { - LocalSession session = createSession(TenantName.defaultName(), 3); - doPrepare(session, new PrepareParams.Builder().build()); - assertThat(session.getMetaData().toString(), is("n/a, n/a, 0, 0, , 0")); - } - private LocalSession createSession(TenantName tenant, long sessionId) throws Exception { - SessionTest.MockSessionPreparer preparer = new SessionTest.MockSessionPreparer(); - return createSession(tenant, sessionId, preparer); - } - - private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer) throws Exception { - return createSession(tenant, sessionId, preparer, Optional.empty()); + return createSession(tenant, sessionId, Optional.empty()); } - private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer, Optional<AllocatedHosts> allocatedHosts) throws Exception { + private LocalSession createSession(TenantName tenant, long sessionId, + Optional<AllocatedHosts> allocatedHosts) throws Exception { SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenant, sessionId, allocatedHosts); zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); - ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); + ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, + TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); if (allocatedHosts.isPresent()) { zkClient.write(allocatedHosts.get()); } zkClient.write(Collections.singletonMap(new Version(0, 0, 0), new MockFileRegistry())); - File sessionDir = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); - sessionDir.createNewFile(); - TenantApplications applications = TenantApplications.create(new TestComponentRegistry.Builder().curator(curator).build(), new MockReloadHandler(), tenant); + TenantApplications applications = tenantRepository.getTenant(tenantName).getApplicationRepo(); applications.createApplication(zkc.readApplicationId()); - return new LocalSession(tenant, sessionId, preparer, - new SessionContext( - FilesApplicationPackage.fromFile(testApp), - zkc, - sessionDir, - applications, - new HostRegistry<>(), - flagSource)); + return new LocalSession(tenant, sessionId, FilesApplicationPackage.fromFile(testApp), zkc, applications); } private void doPrepare(LocalSession session) { @@ -209,12 +135,13 @@ public class LocalSessionTest { } private void doPrepare(LocalSession session, PrepareParams params) { - session.prepare(getLogger(), params, Optional.empty(), tenantPath, Instant.now()); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + sessionRepository.prepareLocalSession(session, getLogger(), params, Optional.empty(), tenantPath, Instant.now()); } private DeployHandlerLogger getLogger() { return new DeployHandlerLogger(new Slime().get(), false, - new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build()); + new ApplicationId.Builder().tenant(tenantName).applicationName("testapp").build()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java index e1874c622c2..ae626499d30 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionProvider; import java.io.File; @@ -20,7 +19,7 @@ public class MockFileDistributionFactory extends FileDistributionFactory { } @Override - public FileDistributionProvider createProvider(File applicationFile) { + public com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider createProvider(File applicationFile) { return mockFileDistributionProvider; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java index 3f5d06b7071..bf21f800815 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -70,6 +71,19 @@ public class PrepareParamsTest { assertEquals(endpoints, prepareParams.containerEndpoints()); } + @Test + public void testCorrectParsingWithApplicationRoles() { + String req = request + "&" + + PrepareParams.APPLICATION_HOST_ROLE + "=hostRole&" + + PrepareParams.APPLICATION_CONTAINER_ROLE + "=containerRole"; + var prepareParams = createParams(req, TenantName.from("foo")); + + Optional<ApplicationRoles> applicationRoles = prepareParams.applicationRoles(); + assertTrue(applicationRoles.isPresent()); + assertEquals("hostRole", applicationRoles.get().applicationHostRole()); + assertEquals("containerRole", applicationRoles.get().applicationContainerRole()); + } + // Create PrepareParams from a request (based on uri and tenant name) private static PrepareParams createParams(String uri, TenantName tenantName) { return PrepareParams.fromHttpRequest( diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java deleted file mode 100644 index 552ae841c3f..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.text.Utf8; - -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.MockReloadHandler; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.tenant.Tenant; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Test; - -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; - -import java.time.Duration; -import java.util.concurrent.TimeUnit; -import java.util.function.LongPredicate; - -/** - * @author Ulf Lilleengen - */ -public class RemoteSessionRepoTest { - - private static final TenantName tenantName = TenantName.defaultName(); - - private RemoteSessionRepo remoteSessionRepo; - private Curator curator; - - @Before - public void setupFacade() { - curator = new MockCurator(); - Tenant tenant = TenantBuilder.create(new TestComponentRegistry.Builder() - .curator(curator) - .build(), - tenantName) - .build(); - this.remoteSessionRepo = tenant.getRemoteSessionRepo(); - curator.create(TenantRepository.getTenantPath(tenantName).append("/applications")); - curator.create(TenantRepository.getSessionsPath(tenantName)); - createSession(1L, false); - createSession(2L, false); - } - - private void createSession(long sessionId, boolean wait) { - createSession(sessionId, wait, tenantName); - } - - private void createSession(long sessionId, boolean wait, TenantName tenantName) { - Path sessionsPath = TenantRepository.getSessionsPath(tenantName); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, sessionsPath.append(String.valueOf(sessionId))); - zkc.createNewSession(System.currentTimeMillis(), TimeUnit.MILLISECONDS); - if (wait) { - Curator.CompletionWaiter waiter = zkc.getUploadWaiter(); - waiter.awaitCompletion(Duration.ofSeconds(120)); - } - } - - @Test - public void testInitialize() { - assertSessionExists(1L); - assertSessionExists(2L); - } - - @Test - public void testCreateSession() { - createSession(3L, true); - assertSessionExists(3L); - } - - @Test - public void testSessionStateChange() throws Exception { - long sessionId = 3L; - createSession(sessionId, true); - assertSessionStatus(sessionId, Session.Status.NEW); - assertStatusChange(sessionId, Session.Status.PREPARE); - assertStatusChange(sessionId, Session.Status.ACTIVATE); - - Path session = TenantRepository.getSessionsPath(tenantName).append("" + sessionId); - curator.delete(session); - assertSessionRemoved(sessionId); - assertNull(remoteSessionRepo.getSession(sessionId)); - } - - // If reading a session throws an exception it should be handled and not prevent other applications - // from loading. In this test we just show that we end up with one session in remote session - // repo even if it had bad data (by making getSessionIdForApplication() in FailingTenantApplications - // throw an exception). - @Test - public void testBadApplicationRepoOnActivate() { - long sessionId = 3L; - TenantName mytenant = TenantName.from("mytenant"); - GlobalComponentRegistry registry = new TestComponentRegistry.Builder().curator(curator).build(); - curator.set(TenantRepository.getApplicationsPath(mytenant).append("mytenant:appX:default"), new byte[0]); // Invalid data - Tenant tenant = TenantBuilder.create(registry, mytenant) - .build(); - curator.create(TenantRepository.getSessionsPath(mytenant)); - remoteSessionRepo = tenant.getRemoteSessionRepo(); - assertThat(remoteSessionRepo.listSessions().size(), is(0)); - createSession(sessionId, true, mytenant); - assertThat(remoteSessionRepo.listSessions().size(), is(1)); - } - - private void assertStatusChange(long sessionId, Session.Status status) throws Exception { - Path statePath = TenantRepository.getSessionsPath(tenantName).append("" + sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); - curator.create(statePath); - curator.framework().setData().forPath(statePath.getAbsolute(), Utf8.toBytes(status.toString())); - System.out.println("Setting status " + status + " for " + sessionId); - assertSessionStatus(sessionId, status); - } - - private void assertSessionRemoved(long sessionId) { - waitFor(p -> remoteSessionRepo.getSession(sessionId) == null, sessionId); - assertNull(remoteSessionRepo.getSession(sessionId)); - } - - private void assertSessionExists(long sessionId) { - assertSessionStatus(sessionId, Session.Status.NEW); - } - - private void assertSessionStatus(long sessionId, Session.Status status) { - waitFor(p -> remoteSessionRepo.getSession(sessionId) != null && - remoteSessionRepo.getSession(sessionId).getStatus() == status, sessionId); - assertNotNull(remoteSessionRepo.getSession(sessionId)); - assertThat(remoteSessionRepo.getSession(sessionId).getStatus(), is(status)); - } - - private void waitFor(LongPredicate predicate, long sessionId) { - long endTime = System.currentTimeMillis() + 60_000; - boolean ok; - do { - ok = predicate.test(sessionId); - try { - Thread.sleep(10); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } while (System.currentTimeMillis() < endTime && !ok); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java index 99ef1831744..e90f01e98c4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; @@ -24,8 +23,11 @@ import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.IOException; import java.time.Clock; import java.time.Instant; import java.time.LocalDate; @@ -53,6 +55,9 @@ public class RemoteSessionTest { private Curator curator; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before public void setupTest() { curator = new MockCurator(); @@ -172,9 +177,10 @@ public class RemoteSessionTest { } @Test - public void require_that_permanent_app_is_used() { + public void require_that_permanent_app_is_used() throws IOException { Optional<PermanentApplicationPackage> permanentApp = Optional.of(new PermanentApplicationPackage( - new ConfigserverConfig(new ConfigserverConfig.Builder().applicationDirectory(Files.createTempDir().getAbsolutePath())))); + new ConfigserverConfig(new ConfigserverConfig.Builder() + .applicationDirectory(temporaryFolder.newFolder("appdir").getAbsolutePath())))); MockModelFactory mockModelFactory = new MockModelFactory(); try { int sessionId = 3; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index cdf57b68814..e2ef115aad4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -20,7 +20,6 @@ import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.io.IOUtils; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; @@ -29,12 +28,10 @@ import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.MockSecretStore; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudgetTest; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; -import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.InvalidApplicationException; @@ -46,6 +43,7 @@ import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Before; import org.junit.Rule; @@ -66,7 +64,9 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; +import static com.yahoo.vespa.config.server.session.SessionZooKeeperClient.APPLICATION_PACKAGE_REFERENCE_PATH; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -104,7 +104,7 @@ public class SessionPreparerTest { curator = new MockCurator(); configCurator = ConfigCurator.create(curator); componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - fileDistributionFactory = (MockFileDistributionFactory)componentRegistry.getFileDistributionFactory(); + fileDistributionFactory = (MockFileDistributionFactory)componentRegistry.getFileDistributionProvider(); preparer = createPreparer(); } @@ -122,7 +122,7 @@ public class SessionPreparerTest { HostProvisionerProvider hostProvisionerProvider) { return new SessionPreparer( modelFactoryRegistry, - componentRegistry.getFileDistributionFactory(), + componentRegistry.getFileDistributionProvider(), hostProvisionerProvider, new PermanentApplicationPackage(componentRegistry.getConfigserverConfig()), componentRegistry.getConfigserverConfig(), @@ -164,21 +164,24 @@ public class SessionPreparerTest { @Test(expected = InvalidApplicationException.class) public void require_exception_for_overlapping_host() throws IOException { - SessionContext ctx = getContext(getApplicationPackage(testApp)); - ((HostRegistry<ApplicationId>)ctx.getHostValidator()).update(applicationId("foo"), Collections.singletonList("mytesthost")); - preparer.prepare(ctx, new BaseDeployLogger(), new PrepareParams.Builder().build(), Optional.empty(), tenantPath, Instant.now()); + FilesApplicationPackage app = getApplicationPackage(testApp); + HostRegistry<ApplicationId> hostValidator = new HostRegistry<>(); + hostValidator.update(applicationId("foo"), Collections.singletonList("mytesthost")); + preparer.prepare(hostValidator, new BaseDeployLogger(), new PrepareParams.Builder().build(), + Optional.empty(), tenantPath, Instant.now(), app.getAppDir(), app, new SessionZooKeeperClient(curator, sessionsPath)); } @Test public void require_no_warning_for_overlapping_host_for_same_appid() throws IOException { - SessionContext ctx = getContext(getApplicationPackage(testApp)); - ((HostRegistry<ApplicationId>)ctx.getHostValidator()).update(applicationId("default"), Collections.singletonList("mytesthost")); final StringBuilder logged = new StringBuilder(); DeployLogger logger = (level, message) -> { - System.out.println(level + ": "+message); if (level.equals(Level.WARNING) && message.contains("The host mytesthost is already in use")) logged.append("ok"); }; - preparer.prepare(ctx, logger, new PrepareParams.Builder().build(), Optional.empty(), tenantPath, Instant.now()); + FilesApplicationPackage app = getApplicationPackage(testApp); + HostRegistry<ApplicationId> hostValidator = new HostRegistry<>(); + hostValidator.update(applicationId("default"), Collections.singletonList("mytesthost")); + preparer.prepare(hostValidator, logger, new PrepareParams.Builder().build(), + Optional.empty(), tenantPath, Instant.now(), app.getAppDir(), app, new SessionZooKeeperClient(curator, sessionsPath)); assertEquals(logged.toString(), ""); } @@ -196,6 +199,13 @@ public class SessionPreparerTest { } @Test + public void require_that_file_reference_of_application_package_is_written_to_zk() throws Exception { + flagSource.withBooleanFlag(Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.id(), true); + prepare(testApp); + assertTrue(configCurator.exists(sessionsPath.append(APPLICATION_PACKAGE_REFERENCE_PATH).getAbsolute())); + } + + @Test public void require_that_container_endpoints_are_written_and_used() throws Exception { var modelFactory = new TestModelFactory(version123); preparer = createPreparer(new ModelFactoryRegistry(List.of(modelFactory)), HostProvisionerProvider.empty()); @@ -234,10 +244,16 @@ public class SessionPreparerTest { var containerEndpointsFromModel = modelContext.properties().endpoints(); assertEquals(Set.copyOf(expected), containerEndpointsFromModel); - // Writing empty container endpoints keeps old value + // Preparing with null container endpoints keeps old value. This is what happens when deployments happen from + // an existing session (e.g. internal redeployment). params = new PrepareParams.Builder().applicationId(applicationId).build(); prepare(new File("src/test/resources/deploy/hosted-app"), params); assertEquals(expected, readContainerEndpoints(applicationId)); + + // Preparing with empty container endpoints clears endpoints + params = new PrepareParams.Builder().applicationId(applicationId).containerEndpoints("[]").build(); + prepare(new File("src/test/resources/deploy/hosted-app"), params); + assertEquals(List.of(), readContainerEndpoints(applicationId)); } @Test @@ -311,16 +327,10 @@ public class SessionPreparerTest { } private void prepare(File app, PrepareParams params) throws IOException { - preparer.prepare(getContext(getApplicationPackage(app)), getLogger(), params, Optional.empty(), tenantPath, Instant.now()); - } - - private SessionContext getContext(FilesApplicationPackage app) throws IOException { - return new SessionContext(app, - new SessionZooKeeperClient(curator, sessionsPath), - app.getAppDir(), - TenantApplications.create(componentRegistry, new MockReloadHandler(), TenantName.from("tenant")), - new HostRegistry<>(), - flagSource); + FilesApplicationPackage applicationPackage = getApplicationPackage(app); + preparer.prepare(new HostRegistry<>(), getLogger(), params, + Optional.empty(), tenantPath, Instant.now(), applicationPackage.getAppDir(), + applicationPackage, new SessionZooKeeperClient(curator, sessionsPath)); } private FilesApplicationPackage getApplicationPackage(File testFile) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepoTest.java deleted file mode 100644 index b8a3d0bc401..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepoTest.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Test; - -import com.yahoo.config.provision.TenantName; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; - -/** - * @author hmusum - */ -public class SessionRepoTest { - @Test - public void require_that_sessionrepo_is_initialized() { - SessionRepo<TestSession> sessionRepo = new SessionRepo<>(); - assertNull(sessionRepo.getSession(1L)); - sessionRepo.addSession(new TestSession(1)); - assertThat(sessionRepo.getSession(1L).getSessionId(), is(1L)); - } - - @Test(expected = IllegalArgumentException.class) - public void require_that_adding_existing_session_fails() { - SessionRepo<TestSession> sessionRepo = new SessionRepo<>(); - final TestSession session = new TestSession(1); - sessionRepo.addSession(session); - sessionRepo.addSession(session); - } - - private class TestSession extends Session { - TestSession(long sessionId) { - super(TenantName.defaultName(), - sessionId, - new MockSessionZKClient(new MockCurator(), TenantName.defaultName(), sessionId)); - } - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java new file mode 100644 index 00000000000..b9e872261c5 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java @@ -0,0 +1,217 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.OrchestratorMock; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.function.LongPredicate; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; + +/** + * @author Ulf Lilleengen + */ +public class SessionRepositoryTest { + + private static final TenantName tenantName = TenantName.defaultName(); + private static final ApplicationId applicationId = ApplicationId.from(tenantName.value(), "testApp", "default"); + private static final File testApp = new File("src/test/apps/app"); + + private MockCurator curator; + private TenantRepository tenantRepository; + private ApplicationRepository applicationRepository; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + public void setup() throws Exception { + setup(new InMemoryFlagSource()); + } + + private void setup(FlagSource flagSource) throws Exception { + curator = new MockCurator(); + File configserverDbDir = temporaryFolder.newFolder().getAbsoluteFile(); + GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .configServerDBDir(configserverDbDir.getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .sessionLifetime(5) + .build()) + .flagSource(flagSource) + .build(); + tenantRepository = new TenantRepository(globalComponentRegistry, false); + tenantRepository.addTenant(SessionRepositoryTest.tenantName); + applicationRepository = new ApplicationRepository(tenantRepository, + new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), + Clock.systemUTC()); + } + + @Test + public void require_that_local_sessions_are_created_and_deleted() throws Exception { + setup(); + long firstSessionId = deploy(); + long secondSessionId = deploy(); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + assertNotNull(sessionRepository.getLocalSession(firstSessionId)); + assertNotNull(sessionRepository.getLocalSession(secondSessionId)); + assertNull(sessionRepository.getLocalSession(secondSessionId + 1)); + + sessionRepository.close(); + // All created sessions are deleted + assertNull(sessionRepository.getLocalSession(firstSessionId)); + assertNull(sessionRepository.getLocalSession(secondSessionId)); + } + + @Test + public void require_that_local_sessions_belong_to_a_tenant() throws Exception { + setup(); + // tenant is "default" + + long firstSessionId = deploy(); + long secondSessionId = deploy(); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + assertNotNull(sessionRepository.getLocalSession(firstSessionId)); + assertNotNull(sessionRepository.getLocalSession(secondSessionId)); + assertNull(sessionRepository.getLocalSession(secondSessionId + 1)); + + // tenant is "newTenant" + TenantName newTenant = TenantName.from("newTenant"); + tenantRepository.addTenant(newTenant); + long sessionId = deploy(ApplicationId.from(newTenant.value(), "testapp", "default")); + SessionRepository sessionRepository2 = tenantRepository.getTenant(newTenant).getSessionRepository(); + assertNotNull(sessionRepository2.getLocalSession(sessionId)); + } + + @Test + public void testInitialize() throws Exception { + setup(); + createSession(10L, false); + createSession(11L, false); + assertRemoteSessionExists(10L); + assertRemoteSessionExists(11L); + } + + @Test + public void testSessionStateChange() throws Exception { + setup(); + long sessionId = 3L; + createSession(sessionId, true); + assertRemoteSessionStatus(sessionId, Session.Status.NEW); + assertStatusChange(sessionId, Session.Status.PREPARE); + assertStatusChange(sessionId, Session.Status.ACTIVATE); + + com.yahoo.path.Path session = TenantRepository.getSessionsPath(tenantName).append("" + sessionId); + curator.delete(session); + assertSessionRemoved(sessionId); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + assertNull(sessionRepository.getRemoteSession(sessionId)); + } + + // If reading a session throws an exception it should be handled and not prevent other applications + // from loading. In this test we just show that we end up with one session in remote session + // repo even if it had bad data (by making getSessionIdForApplication() in FailingTenantApplications + // throw an exception). + @Test + public void testBadApplicationRepoOnActivate() throws Exception { + setup(); + long sessionId = 3L; + TenantName mytenant = TenantName.from("mytenant"); + curator.set(TenantRepository.getApplicationsPath(mytenant).append("mytenant:appX:default"), new byte[0]); // Invalid data + tenantRepository.addTenant(mytenant); + Tenant tenant = tenantRepository.getTenant(mytenant); + curator.create(TenantRepository.getSessionsPath(mytenant)); + SessionRepository sessionRepository = tenant.getSessionRepo(); + assertThat(sessionRepository.getRemoteSessions().size(), is(0)); + createSession(sessionId, true, mytenant); + assertThat(sessionRepository.getRemoteSessions().size(), is(1)); + } + + private void createSession(long sessionId, boolean wait) { + createSession(sessionId, wait, tenantName); + } + + private void createSession(long sessionId, boolean wait, TenantName tenantName) { + com.yahoo.path.Path sessionsPath = TenantRepository.getSessionsPath(tenantName); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, sessionsPath.append(String.valueOf(sessionId))); + zkc.createNewSession(Instant.now()); + if (wait) { + Curator.CompletionWaiter waiter = zkc.getUploadWaiter(); + waiter.awaitCompletion(Duration.ofSeconds(120)); + } + } + + private void assertStatusChange(long sessionId, Session.Status status) throws Exception { + com.yahoo.path.Path statePath = TenantRepository.getSessionsPath(tenantName).append("" + sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); + curator.create(statePath); + curator.framework().setData().forPath(statePath.getAbsolute(), Utf8.toBytes(status.toString())); + assertRemoteSessionStatus(sessionId, status); + } + + private void assertSessionRemoved(long sessionId) { + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + waitFor(p -> sessionRepository.getRemoteSession(sessionId) == null, sessionId); + assertNull(sessionRepository.getRemoteSession(sessionId)); + } + + private void assertRemoteSessionExists(long sessionId) { + assertRemoteSessionStatus(sessionId, Session.Status.NEW); + } + + private void assertRemoteSessionStatus(long sessionId, Session.Status status) { + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + waitFor(p -> sessionRepository.getRemoteSession(sessionId) != null && + sessionRepository.getRemoteSession(sessionId).getStatus() == status, sessionId); + assertNotNull(sessionRepository.getRemoteSession(sessionId)); + assertThat(sessionRepository.getRemoteSession(sessionId).getStatus(), is(status)); + } + + private void waitFor(LongPredicate predicate, long sessionId) { + long endTime = System.currentTimeMillis() + 60_000; + boolean ok; + do { + ok = predicate.test(sessionId); + try { + Thread.sleep(10); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } while (System.currentTimeMillis() < endTime && !ok); + } + + private long deploy() { + return deploy(applicationId); + } + + private long deploy(ApplicationId applicationId) { + applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId).build()); + return applicationRepository.getActiveSession(applicationId).getSessionId(); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java index b2ad0af8f9a..5ae5910d827 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java @@ -1,19 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.curator.mock.MockCurator; +import java.io.File; import java.time.Instant; import java.util.ArrayList; import java.util.Optional; /** * @author Ulf Lilleengen - * @since 5.1 */ public class SessionTest { @@ -25,7 +28,10 @@ public class SessionTest { } @Override - public ConfigChangeActions prepare(SessionContext context, DeployLogger logger, PrepareParams params, Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, Instant now) { + public ConfigChangeActions prepare(HostValidator<ApplicationId> hostValidator, DeployLogger logger, PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, + Instant now, File serverDbSessionDir, ApplicationPackage applicationPackage, + SessionZooKeeperClient sessionZooKeeperClient) { isPrepared = true; return new ConfigChangeActions(new ArrayList<>()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java index 27e04bd7422..5633ec2c5f8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java @@ -1,8 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.path.Path; +import com.yahoo.config.FileReference; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; @@ -10,7 +11,7 @@ import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; import org.junit.Test; -import java.util.concurrent.TimeUnit; +import java.time.Instant; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; @@ -92,18 +93,19 @@ public class SessionZooKeeperClientTest { public void require_that_create_time_can_be_written_and_read() { SessionZooKeeperClient zkc = createSessionZKClient("3"); curator.delete(Path.fromString("3")); - assertThat(zkc.readCreateTime(), is(0L)); - zkc.createNewSession(123456L, TimeUnit.SECONDS); - assertThat(zkc.readCreateTime(), is(123456L)); + assertThat(zkc.readCreateTime(), is(Instant.EPOCH)); + Instant now = Instant.now(); + zkc.createNewSession(now); + // resolution is in seconds, so need to go back use that when comparing + assertThat(zkc.readCreateTime(), is(Instant.ofEpochSecond(now.getEpochSecond()))); } @Test - public void require_that_create_time_has_correct_unit() { + public void require_that_application_package_file_reference_can_be_written_and_read() { + final FileReference testRef = new FileReference("test-ref"); SessionZooKeeperClient zkc = createSessionZKClient("3"); - curator.delete(Path.fromString("3")); - assertThat(zkc.readCreateTime(), is(0L)); - zkc.createNewSession(60, TimeUnit.MINUTES); - assertThat(zkc.readCreateTime(), is(3600L)); + zkc.writeApplicationPackageReference(testRef); + assertThat(zkc.readApplicationPackageReference(), is(testRef)); } private void assertApplicationIdParse(String sessionId, String idString, String expectedIdString) { @@ -116,7 +118,7 @@ public class SessionZooKeeperClientTest { private SessionZooKeeperClient createSessionZKClient(String sessionId) { SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, Path.fromString(sessionId)); - zkc.createNewSession(100, TimeUnit.MILLISECONDS); + zkc.createNewSession(Instant.now()); return zkc; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ApplicationRolesStoreTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ApplicationRolesStoreTest.java new file mode 100644 index 00000000000..a03ea61ec54 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ApplicationRolesStoreTest.java @@ -0,0 +1,39 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.ApplicationRoles; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author mortent + */ +public class ApplicationRolesStoreTest { + @Test + public void persists_entry_correctly() { + // Persist + var applicationRolesStore = new ApplicationRolesStore(new MockCurator(), Path.createRoot()); + var roles = new ApplicationRoles("hostRole", "containerRole"); + applicationRolesStore.writeApplicationRoles(ApplicationId.defaultId(), roles); + + // Read + Optional<ApplicationRoles> deserialized = applicationRolesStore.readApplicationRoles(ApplicationId.defaultId()); + assertTrue(deserialized.isPresent()); + assertEquals(roles, deserialized.get()); + } + + @Test + public void read_non_existent() { + var applicationRolesStore = new ApplicationRolesStore(new MockCurator(), Path.createRoot()); + Optional<ApplicationRoles> applicationRoles = applicationRolesStore.readApplicationRoles(ApplicationId.defaultId()); + assertTrue(applicationRoles.isEmpty()); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 9bd5c5f1614..51b0a36d8f4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -2,19 +2,20 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.TenantApplicationsTest; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -45,7 +46,7 @@ public class TenantRepositoryTest { private TenantRepository tenantRepository; private TestComponentRegistry globalComponentRegistry; - private TenantRequestHandlerTest.MockReloadListener listener; + private TenantApplicationsTest.MockReloadListener listener; private MockTenantListener tenantListener; private Curator curator; @@ -59,7 +60,7 @@ public class TenantRepositoryTest { public void setupSessions() { curator = new MockCurator(); globalComponentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - listener = (TenantRequestHandlerTest.MockReloadListener)globalComponentRegistry.getReloadListener(); + listener = (TenantApplicationsTest.MockReloadListener)globalComponentRegistry.getReloadListener(); tenantListener = (MockTenantListener)globalComponentRegistry.getTenantListener(); assertFalse(tenantListener.tenantsLoaded); tenantRepository = new TenantRepository(globalComponentRegistry); @@ -205,8 +206,8 @@ public class TenantRepositoryTest { } @Override - protected void createTenant(TenantBuilder builder) { - throw new RuntimeException("Failed to create: " + builder.getTenantName()); + public void createTenant(TenantName tenantName) { + throw new RuntimeException("Failed to create: " + tenantName); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java deleted file mode 100644 index 8b9028fee4a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java +++ /dev/null @@ -1,374 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.component.Version; -import com.yahoo.config.ConfigInstance; -import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.DeployData; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.GetConfigRequest; -import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.protocol.DefContent; -import com.yahoo.vespa.config.protocol.VespaVersion; -import com.yahoo.vespa.config.server.ReloadListener; -import com.yahoo.vespa.config.server.ServerCache; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; -import com.yahoo.vespa.config.server.model.TestModelFactory; -import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; -import com.yahoo.vespa.config.server.session.RemoteSession; -import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.VespaModelFactory; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.xml.sax.SAXException; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author Ulf Lilleengen - */ -public class TenantRequestHandlerTest { - - private static final Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); - private TenantRequestHandler server; - private MockReloadListener listener = new MockReloadListener(); - private File app1 = new File("src/test/apps/cs1"); - private File app2 = new File("src/test/apps/cs2"); - private TenantName tenant = TenantName.from("mytenant"); - private TestComponentRegistry componentRegistry; - private Curator curator; - - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); - - private ApplicationId defaultApp() { - return new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(); - } - - @Before - public void setUp() throws IOException { - curator = new MockCurator(); - - feedApp(app1, 1, defaultApp(), false); - Metrics sh = Metrics.createTestMetrics(); - List<ReloadListener> listeners = new ArrayList<>(); - listeners.add(listener); - componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) - .modelFactoryRegistry(createRegistry()) - .build(); - server = new TenantRequestHandler(sh, tenant, listeners, new UncompressedConfigResponseFactory(), componentRegistry); - } - - private void feedApp(File appDir, long sessionId, ApplicationId appId, boolean internalRedeploy) throws IOException { - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); - zkc.writeApplicationId(appId); - File app = tempFolder.newFolder(); - IOUtils.copyDirectory(appDir, app); - ZooKeeperDeployer deployer = zkc.createDeployer(new BaseDeployLogger()); - DeployData deployData = new DeployData("user", - appDir.toString(), - appId, - 0L, - internalRedeploy, - 0L, - 0L); - ApplicationPackage appPackage = FilesApplicationPackage.fromFileWithDeployData(appDir, deployData); - deployer.deploy(appPackage, - Collections.singletonMap(vespaVersion, new MockFileRegistry()), - AllocatedHosts.withHosts(Collections.emptySet())); - } - - private ApplicationSet reloadConfig(long sessionId) { - return reloadConfig(sessionId, "default"); - } - - private ApplicationSet reloadConfig(long sessionId, String application) { - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); - zkc.writeApplicationId(new ApplicationId.Builder().tenant(tenant).applicationName(application).build()); - RemoteSession session = new RemoteSession(tenant, sessionId, componentRegistry, zkc); - return session.ensureApplicationLoaded(); - } - - private ModelFactoryRegistry createRegistry() { - return new ModelFactoryRegistry(Arrays.asList(new TestModelFactory(vespaVersion), - new TestModelFactory(new Version(3, 2, 1)))); - } - - private <T extends ConfigInstance> T resolve(Class<T> clazz, - TenantRequestHandler tenantRequestHandler, - ApplicationId appId, - Version vespaVersion, - String configId) { - ConfigResponse response = getConfigResponse(clazz, tenantRequestHandler, appId, vespaVersion, configId); - return ConfigPayload.fromUtf8Array(response.getPayload()).toInstance(clazz, configId); - } - - private <T extends ConfigInstance> ConfigResponse getConfigResponse(Class<T> clazz, - TenantRequestHandler tenantRequestHandler, - ApplicationId appId, - Version vespaVersion, - String configId) { - return tenantRequestHandler.resolveConfig(appId, new GetConfigRequest() { - @Override - public ConfigKey<T> getConfigKey() { - return new ConfigKey<>(clazz, configId); - } - - @Override - public DefContent getDefContent() { - return DefContent.fromClass(clazz); - } - - @Override - public Optional<VespaVersion> getVespaVersion() { - return Optional.of(VespaVersion.fromString(vespaVersion.toFullString())); - } - - @Override - public boolean noCache() { - return false; - } - }, Optional.empty()); - } - - @Test - public void testReloadConfig() throws IOException { - ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(); - - server.applications().createApplication(applicationId); - server.applications().createPutTransaction(applicationId, 1).commit(); - server.reloadConfig(reloadConfig(1)); - assertThat(listener.reloaded.get(), is(1)); - // Using only payload list for this simple test - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertThat(config.intval(), is(1337)); - assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(1L)); - - server.reloadConfig(reloadConfig(1L)); - ConfigResponse configResponse = getConfigResponse(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertFalse(configResponse.isInternalRedeploy()); - config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertThat(config.intval(), is(1337)); - assertThat(listener.reloaded.get(), is(2)); - assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(1L)); - assertThat(listener.tenantHosts.size(), is(1)); - assertThat(server.resolveApplicationId("mytesthost"), is(applicationId)); - - listener.reloaded.set(0); - feedApp(app2, 2, defaultApp(), true); - server.applications().createPutTransaction(applicationId, 2).commit(); - server.reloadConfig(reloadConfig(2L)); - configResponse = getConfigResponse(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertTrue(configResponse.isInternalRedeploy()); - config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion,""); - assertThat(config.intval(), is(1330)); - assertThat(listener.reloaded.get(), is(1)); - assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(2L)); - } - - @Test - public void testRemoveApplication() { - ApplicationId appId = ApplicationId.from(tenant.value(), "default", "default"); - server.reloadConfig(reloadConfig(1)); - assertThat(listener.reloaded.get(), is(0)); - - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - server.reloadConfig(reloadConfig(1)); - assertThat(listener.reloaded.get(), is(1)); - - assertThat(listener.removed.get(), is(0)); - - server.removeApplication(appId); - assertThat(listener.removed.get(), is(0)); - - server.applications().createDeleteTransaction(appId).commit(); - server.removeApplication(appId); - assertThat(listener.removed.get(), is(1)); - } - - @Test - public void testResolveForAppId() { - long id = 1L; - ApplicationId appId = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp").instanceName("myinst").build(); - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(id))); - zkc.writeApplicationId(appId); - RemoteSession session = new RemoteSession(appId.tenant(), id, componentRegistry, zkc); - server.reloadConfig(session.ensureApplicationLoaded()); - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId, vespaVersion, ""); - assertThat(config.intval(), is(1337)); - } - - @Test - public void testResolveMultipleApps() throws IOException { - ApplicationId appId1 = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp1").instanceName("myinst1").build(); - ApplicationId appId2 = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp2").instanceName("myinst2").build(); - feedAndReloadApp(app1, 1, appId1); - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId1, vespaVersion, ""); - assertThat(config.intval(), is(1337)); - - feedAndReloadApp(app2, 2, appId2); - config = resolve(SimpletypesConfig.class, server, appId2, vespaVersion, ""); - assertThat(config.intval(), is(1330)); - } - - @Test - public void testResolveMultipleVersions() throws IOException { - ApplicationId appId = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp1").instanceName("myinst1").build(); - feedAndReloadApp(app1, 1, appId); - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId, vespaVersion, ""); - assertThat(config.intval(), is(1337)); - config = resolve(SimpletypesConfig.class, server, appId, new Version(3, 2, 1), ""); - assertThat(config.intval(), is(1337)); - } - - private void feedAndReloadApp(File appDir, long sessionId, ApplicationId appId) throws IOException { - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, sessionId).commit(); - feedApp(appDir, sessionId, appId, false); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); - zkc.writeApplicationId(appId); - RemoteSession session = new RemoteSession(tenant, sessionId, componentRegistry, zkc); - server.reloadConfig(session.ensureApplicationLoaded()); - } - - public static class MockReloadListener implements ReloadListener { - AtomicInteger reloaded = new AtomicInteger(0); - AtomicInteger removed = new AtomicInteger(0); - Map<String, Collection<String>> tenantHosts = new LinkedHashMap<>(); - - @Override - public void configActivated(ApplicationSet application) { - reloaded.incrementAndGet(); - } - - @Override - public void hostsUpdated(TenantName tenant, Collection<String> newHosts) { - tenantHosts.put(tenant.value(), newHosts); - } - - @Override - public void verifyHostsAreAvailable(TenantName tenant, Collection<String> newHosts) { - } - - @Override - public void applicationRemoved(ApplicationId applicationId) { - removed.incrementAndGet(); - } - } - - @Test - public void testHasApplication() { - assertdefaultAppNotFound(); - ApplicationId appId = ApplicationId.from(tenant.value(), "default", "default"); - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - server.reloadConfig(reloadConfig(1)); - assertTrue(server.hasApplication(appId, Optional.of(vespaVersion))); - } - - private void assertdefaultAppNotFound() { - assertFalse(server.hasApplication(ApplicationId.defaultId(), Optional.of(vespaVersion))); - } - - @Test - public void testMultipleApplicationsReload() { - ApplicationId appId = ApplicationId.from(tenant.value(), "foo", "default"); - assertdefaultAppNotFound(); - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - server.reloadConfig(reloadConfig(1, "foo")); - assertdefaultAppNotFound(); - assertTrue(server.hasApplication(appId, - Optional.of(vespaVersion))); - assertThat(server.resolveApplicationId("doesnotexist"), is(ApplicationId.defaultId())); - assertThat(server.resolveApplicationId("mytesthost"), is(new ApplicationId.Builder() - .tenant(tenant) - .applicationName("foo").build())); // Host set in application package. - } - - @Test - public void testListConfigs() throws IOException, SAXException { - assertdefaultAppNotFound(); - - VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); - server.applications().createApplication(ApplicationId.defaultId()); - server.applications().createPutTransaction(ApplicationId.defaultId(), 1).commit(); - server.reloadConfig(ApplicationSet.fromSingle(new Application(model, - new ServerCache(), - 1, - false, - vespaVersion, - MetricUpdater.createTestUpdater(), - ApplicationId.defaultId()))); - Set<ConfigKey<?>> configNames = server.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), false); - assertTrue(configNames.contains(new ConfigKey<>("sentinel", "hosts", "cloud.config"))); - - configNames = server.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), true); - System.out.println(configNames); - assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); - assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "", "document.config"))); - assertTrue(configNames.contains(new ConfigKey<>("documenttypes", "", "document"))); - assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); - assertTrue(configNames.contains(new ConfigKey<>("health-monitor", "container", "container.jdisc.config"))); - assertTrue(configNames.contains(new ConfigKey<>("specific", "container", "project"))); - } - - @Test - public void testAppendIdsInNonRecursiveListing() { - assertEquals(server.appendOneLevelOfId("search/music", "search/music/qrservers/default/qr.0"), "search/music/qrservers"); - assertEquals(server.appendOneLevelOfId("search", "search/music/qrservers/default/qr.0"), "search/music"); - assertEquals(server.appendOneLevelOfId("search/music/qrservers/default/qr.0", "search/music/qrservers/default/qr.0"), "search/music/qrservers/default/qr.0"); - assertEquals(server.appendOneLevelOfId("", "search/music/qrservers/default/qr.0"), "search"); - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java index 5b8f5299c01..74833da6d66 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java @@ -3,9 +3,7 @@ package com.yahoo.vespa.config.server.tenant; import com.google.common.testing.EqualsTester; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.TenantApplications; import org.junit.Before; import org.junit.Test; @@ -36,9 +34,7 @@ public class TenantTest { private Tenant createTenant(String name) { TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); TenantName tenantName = TenantName.from(name); - TenantBuilder tenantBuilder = TenantBuilder.create(componentRegistry, tenantName) - .withApplicationRepo(TenantApplications.create(componentRegistry, new MockReloadHandler(), tenantName)); - tenantRepository.addTenant(tenantBuilder); + tenantRepository.addTenant(tenantName); return tenantRepository.getTenant(tenantName); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg deleted file mode 100644 index 0bc17bae65e..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg +++ /dev/null @@ -1,18 +0,0 @@ -asyncfetchocc 10 -e 4 -search[2].feeder[1] "bazfeeder" -search[1].feeder[0] "barfeeder1_1" -search[1].feeder[3] "barfeeder2_1" -onlyindef 45 - -speciallog[0].filehandler.rotation "0 1 ..." - -rulebase[4] -rulebase[0].name "cjk" -rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# ä½³:\u4f73\n# 能:\u80fd\n# ç´¢:\u7d22\n# å°¼:\u5c3c\n# æƒ :\u60e0\n# æ™®:\u666e\n\n@default\n\naç´¢ -> ç´¢a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,æƒ æ™®,佳能;\n" -rulebase[1].name "common" -rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" -rulebase[2].name "egyik" -rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" -rulebase[3].name "masik" -rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg deleted file mode 100644 index 88b50384058..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg +++ /dev/null @@ -1 +0,0 @@ -usercfgwithid 86 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg deleted file mode 100644 index b34c4ed311e..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg +++ /dev/null @@ -1 +0,0 @@ -foo "test" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg deleted file mode 100644 index 12c5b53de7d..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg +++ /dev/null @@ -1 +0,0 @@ -theint 34 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg deleted file mode 100644 index 73ed41667f9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg +++ /dev/null @@ -1 +0,0 @@ -keepsuccess true diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java index 0b028b96eab..a34c17dc909 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java @@ -1,11 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.zookeeper; -import com.google.common.io.Files; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationFileTest; import com.yahoo.path.Path; import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -14,21 +15,23 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.1 */ public class ZKApplicationFileTest extends ApplicationFileTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private void feed(ConfigCurator zk, File dirToFeed) { assertTrue(dirToFeed.isDirectory()); String appPath = "/0"; - zk.feedZooKeeper(dirToFeed, appPath + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); + ZKApplicationPackageTest.feedZooKeeper(zk, dirToFeed, appPath + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); zk.putData(appPath, ZKApplicationPackage.fileRegistryNode, "dummyfiles"); } @Override public ApplicationFile getApplicationFile(Path path) throws IOException{ ConfigCurator configCurator = ConfigCurator.create(new MockCurator()); - File tmp = Files.createTempDir(); + File tmp = temporaryFolder.newFolder(); writeAppTo(tmp); feed(configCurator, tmp); return new ZKApplicationFile(path, new ZKApplication(configCurator, Path.fromString("/0"))); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 9c7da7134e6..4b4605cce7d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -5,10 +5,12 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -20,9 +22,12 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.io.Reader; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.regex.Pattern; @@ -35,13 +40,19 @@ import static org.junit.Assert.assertTrue; public class ZKApplicationPackageTest { + private static final FilenameFilter acceptsAllFileNameFilter = (dir, name) -> true; private static final String APP = "src/test/apps/zkapp"; private static final String TEST_FLAVOR_NAME = "test-flavor"; private static final Optional<Flavor> TEST_FLAVOR = new MockNodeFlavors().getFlavor(TEST_FLAVOR_NAME); private static final AllocatedHosts ALLOCATED_HOSTS = AllocatedHosts.withHosts( - Collections.singleton(new HostSpec("foo.yahoo.com", Collections.emptyList(), TEST_FLAVOR, Optional.empty(), + Collections.singleton(new HostSpec("foo.yahoo.com", + TEST_FLAVOR.get().resources(), + TEST_FLAVOR.get().resources(), + NodeResources.unspecified(), + ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), + Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))), Optional.of(Version.fromString("6.0.1")), Optional.empty(), - Optional.empty(), Optional.of(DockerImage.fromString("docker repo"))))); + Optional.of(DockerImage.fromString("docker repo"))))); private ConfigCurator configCurator; @@ -79,8 +90,8 @@ public class ZKApplicationPackageTest { assertFalse(zkApp.getFileRegistries().containsKey(new Version(0, 0, 0))); assertThat(zkApp.getFileRegistries().get(goodVersion).fileSourceHost(), is("dummyfiles")); AllocatedHosts readInfo = zkApp.getAllocatedHosts().get(); - assertThat(Utf8.toString(toJson(readInfo)), is(Utf8.toString(toJson(ALLOCATED_HOSTS)))); - assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR)); + assertEquals(Utf8.toString(toJson(ALLOCATED_HOSTS)), Utf8.toString(toJson(readInfo))); + assertEquals(TEST_FLAVOR.get().resources(), readInfo.getHosts().iterator().next().advertisedResources()); assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); // TODO: Enable when dockerImageRepo is written to zk //assertEquals("docker repo", readInfo.getHosts().iterator().next().dockerImageRepo().get()); @@ -90,7 +101,7 @@ public class ZKApplicationPackageTest { private void feed(ConfigCurator zk, File dirToFeed) throws IOException { assertTrue(dirToFeed.isDirectory()); - zk.feedZooKeeper(dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); + feedZooKeeper(zk, dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"id\":\"foo:foo:default\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData); zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles"); @@ -108,4 +119,63 @@ public class ZKApplicationPackageTest { } } + /** + * Takes for instance the dir /app and puts the contents into the given ZK path. Ignores files starting with dot, + * and dirs called CVS. + * + * @param dir directory which holds the summary class part files + * @param path zookeeper path + * @param filenameFilter A FilenameFilter which decides which files in dir are fed to zookeeper + * @param recurse recurse subdirectories + */ + static void feedZooKeeper(ConfigCurator zk, File dir, String path, FilenameFilter filenameFilter, boolean recurse) { + try { + if (filenameFilter == null) { + filenameFilter = acceptsAllFileNameFilter; + } + if (!dir.isDirectory()) { + throw new IllegalArgumentException(dir + " is not a directory"); + } + for (File file : listFiles(dir, filenameFilter)) { + if (file.getName().startsWith(".")) continue; //.svn , .git ... + if ("CVS".equals(file.getName())) continue; + if (file.isFile()) { + String contents = IOUtils.readFile(file); + zk.putData(path, file.getName(), contents); + } else if (recurse && file.isDirectory()) { + zk.createNode(path, file.getName()); + feedZooKeeper(zk, file, path + '/' + file.getName(), filenameFilter, recurse); + } + } + } + catch (IOException e) { + throw new RuntimeException("Exception feeding ZooKeeper at path " + path, e); + } + } + + /** + * Same as normal listFiles, but use the filter only for normal files + * + * @param dir directory to list files in + * @param filter A FilenameFilter which decides which files in dir are listed + * @return an array of Files + */ + protected static File[] listFiles(File dir, FilenameFilter filter) { + File[] rawList = dir.listFiles(); + List<File> ret = new ArrayList<>(); + if (rawList != null) { + for (File f : rawList) { + if (f.isDirectory()) { + ret.add(f); + } else { + if (filter.accept(dir, f.getName())) { + ret.add(f); + } + } + } + } + return ret.toArray(new File[0]); + } + + } diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 82d34d88b99..dac33d2d431 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -171,9 +171,11 @@ "public void <init>()", "public void <init>(com.yahoo.container.handler.ThreadpoolConfig)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxthreads(int)", + "public com.yahoo.container.handler.ThreadpoolConfig$Builder corePoolSize(int)", + "public com.yahoo.container.handler.ThreadpoolConfig$Builder keepAliveTime(double)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder queueSize(int)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxThreadExecutionTimeSeconds(int)", - "public com.yahoo.container.handler.ThreadpoolConfig$Builder softStartSeconds(double)", + "public com.yahoo.container.handler.ThreadpoolConfig$Builder name(java.lang.String)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", "public final java.lang.String getDefMd5()", "public final java.lang.String getDefName()", @@ -211,9 +213,11 @@ "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.container.handler.ThreadpoolConfig$Builder)", "public int maxthreads()", + "public int corePoolSize()", + "public double keepAliveTime()", "public int queueSize()", "public int maxThreadExecutionTimeSeconds()", - "public double softStartSeconds()" + "public java.lang.String name()" ], "fields": [ "public static final java.lang.String CONFIG_DEF_MD5", @@ -876,55 +880,5 @@ "public bridge synthetic java.lang.Object clone()" ], "fields": [] - }, - "ai.vespa.cloud.Environment": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static ai.vespa.cloud.Environment[] values()", - "public static ai.vespa.cloud.Environment valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum ai.vespa.cloud.Environment dev", - "public static final enum ai.vespa.cloud.Environment perf", - "public static final enum ai.vespa.cloud.Environment test", - "public static final enum ai.vespa.cloud.Environment staging", - "public static final enum ai.vespa.cloud.Environment prod" - ] - }, - "ai.vespa.cloud.SystemInfo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.cloud.config.ConfigserverConfig)", - "public void <init>(ai.vespa.cloud.Zone)", - "public ai.vespa.cloud.Zone zone()" - ], - "fields": [] - }, - "ai.vespa.cloud.Zone": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(ai.vespa.cloud.Environment, java.lang.String)", - "public ai.vespa.cloud.Environment environment()", - "public java.lang.String region()", - "public java.lang.String toString()", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public static ai.vespa.cloud.Zone from(java.lang.String)" - ], - "fields": [] } }
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 0b42b3a481b..1d6e1a0893d 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -35,9 +35,6 @@ public class LogHandler extends ThreadedHttpRequestHandler { .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MAX); return new HttpResponse(200) { - { - headers().add("Content-Encoding", "gzip"); - } @Override public void render(OutputStream outputStream) { logReader.writeLogs(outputStream, from, to); diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index e3fef4e0e44..3cf849a6835 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -1,12 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; +import com.google.common.collect.Iterators; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.yolean.Exceptions; import java.io.BufferedReader; import java.io.BufferedWriter; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -19,18 +19,21 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Comparator; +import java.util.Iterator; import java.util.List; -import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Comparator.comparing; /** * @author olaaun @@ -39,6 +42,9 @@ import static java.util.Comparator.comparing; */ class LogReader { + static final Pattern logArchivePathPattern = Pattern.compile("(\\d{4})/(\\d{2})/(\\d{2})/(\\d{2})-\\d+(.gz)?"); + static final Pattern vespaLogPathPattern = Pattern.compile("vespa\\.log(?:-(\\d{4})-(\\d{2})-(\\d{2})\\.(\\d{2})-(\\d{2})-(\\d{2})(?:.gz)?)?"); + private final Path logDirectory; private final Pattern logFilePattern; @@ -51,61 +57,110 @@ class LogReader { this.logFilePattern = logFilePattern; } - void writeLogs(OutputStream outputStream, Instant from, Instant to) { + void writeLogs(OutputStream out, Instant from, Instant to) { + double fromSeconds = from.getEpochSecond() + from.getNano() / 1e9; + double toSeconds = to.getEpochSecond() + to.getNano() / 1e9; + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out)); try { - List<Path> logs = getMatchingFiles(from, to); - for (int i = 0; i < logs.size(); i++) { - Path log = logs.get(i); - boolean zipped = log.toString().endsWith(".gz"); - try (InputStream in = Files.newInputStream(log)) { - InputStream inProxy; - - // If the log needs filtering, possibly unzip (and rezip) it, and filter its lines on timestamp. - if (i == 0 || i == logs.size() - 1) { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(in) : in, UTF_8)); - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(zipped ? new GZIPOutputStream(buffer) : buffer, UTF_8))) { - for (String line; (line = reader.readLine()) != null; ) { - String[] parts = line.split("\t"); - if (parts.length != 7) - continue; - - Instant at = Instant.EPOCH.plus((long) (Double.parseDouble(parts[0]) * 1_000_000), ChronoUnit.MICROS); - if (at.isAfter(from) && ! at.isAfter(to)) { - writer.write(line); - writer.newLine(); - } - } - } - inProxy = new ByteArrayInputStream(buffer.toByteArray()); + for (List<Path> logs : getMatchingFiles(from, to)) { + List<LogLineIterator> logLineIterators = new ArrayList<>(); + try { + // Logs in each sub-list contain entries covering the same time interval, so do a merge sort while reading + for (Path log : logs) + logLineIterators.add(new LogLineIterator(log, fromSeconds, toSeconds)); + + Iterator<LineWithTimestamp> lines = Iterators.mergeSorted(logLineIterators, + Comparator.comparingDouble(LineWithTimestamp::timestamp)); + while (lines.hasNext()) { + writer.write(lines.next().line()); + writer.newLine(); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + finally { + for (LogLineIterator ll : logLineIterators) { + try { ll.close(); } catch (IOException ignored) { } } - else - inProxy = in; - - // At the point when logs switch to un-zipped, replace the output stream with a zipping proxy. - if ( ! zipped && ! (outputStream instanceof GZIPOutputStream)) - outputStream = new GZIPOutputStream(outputStream); - - inProxy.transferTo(outputStream); } } } - catch (IOException e) { - throw new UncheckedIOException(e); - } finally { + Exceptions.uncheck(writer::flush); + } + } + + private static class LogLineIterator implements Iterator<LineWithTimestamp>, AutoCloseable { + + private final BufferedReader reader; + private final double from; + private final double to; + private LineWithTimestamp next; + + private LogLineIterator(Path log, double from, double to) throws IOException { + boolean zipped = log.toString().endsWith(".gz"); + InputStream in = Files.newInputStream(log); + this.reader = new BufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(in) : in, UTF_8)); + this.from = from; + this.to = to; + this.next = readNext(); + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public LineWithTimestamp next() { + LineWithTimestamp current = next; + next = readNext(); + return current; + } + + @Override + public void close() throws IOException { + reader.close(); + } + + private LineWithTimestamp readNext() { try { - outputStream.close(); + for (String line; (line = reader.readLine()) != null; ) { + String[] parts = line.split("\t"); + if (parts.length != 7) + continue; + + double timestamp = Double.parseDouble(parts[0]); + if (timestamp > to) + return null; + + if (timestamp >= from) + return new LineWithTimestamp(line, timestamp); + } + return null; } catch (IOException e) { throw new UncheckedIOException(e); } } + } - /** Returns log files which may have relevant entries, sorted by modification time — the first and last must be filtered. */ - private List<Path> getMatchingFiles(Instant from, Instant to) { - Map<Path, Instant> paths = new HashMap<>(); + private static class LineWithTimestamp { + final String line; + final double timestamp; + LineWithTimestamp(String line, double timestamp) { + this.line = line; + this.timestamp = timestamp; + } + String line() { return line; } + double timestamp() { return timestamp; } + } + + /** Returns log files which may have relevant entries, grouped and sorted by {@link #extractTimestamp(Path)} — the first and last group must be filtered. */ + private List<List<Path>> getMatchingFiles(Instant from, Instant to) { + List<Path> paths = new ArrayList<>(); try { Files.walkFileTree(logDirectory, new SimpleFileVisitor<>() { @@ -117,7 +172,7 @@ class LogReader { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { if (logFilePattern.matcher(file.getFileName().toString()).matches()) - paths.put(file, attrs.lastModifiedTime().toInstant()); + paths.add(file); return FileVisitResult.CONTINUE; } @@ -132,15 +187,54 @@ class LogReader { throw new UncheckedIOException(e); } - List<Path> sorted = new ArrayList<>(); - for (var entries = paths.entrySet().stream().sorted(comparing(Map.Entry::getValue)).iterator(); entries.hasNext(); ) { - var entry = entries.next(); - if (entry.getValue().isAfter(from)) - sorted.add(entry.getKey()); - if (entry.getValue().isAfter(to)) + var logsByTimestamp = paths.stream() + .collect(Collectors.groupingBy(this::extractTimestamp, + TreeMap::new, + Collectors.toList())); + + List<List<Path>> sorted = new ArrayList<>(); + for (var entry : logsByTimestamp.entrySet()) { + if (entry.getKey().isAfter(from)) + sorted.add(entry.getValue()); + if (entry.getKey().isAfter(to)) break; } return sorted; } + /** Extracts a timestamp after all entries in the log file with the given path. */ + Instant extractTimestamp(Path path) { + String relativePath = logDirectory.relativize(path).toString(); + Matcher matcher = logArchivePathPattern.matcher(relativePath); + if (matcher.matches()) { + return ZonedDateTime.of(Integer.parseInt(matcher.group(1)), + Integer.parseInt(matcher.group(2)), + Integer.parseInt(matcher.group(3)), + Integer.parseInt(matcher.group(4)), + 0, + 0, + 0, + ZoneId.of("UTC")) + .toInstant() + .plus(Duration.ofHours(1)); + } + matcher = vespaLogPathPattern.matcher(relativePath); + if (matcher.matches()) { + if (matcher.group(1) == null) + return Instant.MAX; + + return ZonedDateTime.of(Integer.parseInt(matcher.group(1)), + Integer.parseInt(matcher.group(2)), + Integer.parseInt(matcher.group(3)), + Integer.parseInt(matcher.group(4)), + Integer.parseInt(matcher.group(5)), + Integer.parseInt(matcher.group(6)), + 0, + ZoneId.of("UTC")) + .toInstant() + .plus(Duration.ofSeconds(1)); + } + throw new IllegalArgumentException("Unrecognized file pattern for file at '" + path + "'"); + } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java index 0e786cfbc8f..425387039ff 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java @@ -1,26 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; -import com.google.common.util.concurrent.ForwardingExecutorService; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.container.handler.threadpool.ContainerThreadPool; import com.yahoo.container.protect.ProcessTerminator; import com.yahoo.jdisc.Metric; -import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; /** * A configurable thread pool provider. This provides the worker threads used for normal request processing. @@ -32,40 +20,15 @@ import java.util.concurrent.atomic.AtomicLong; */ public class ThreadPoolProvider extends AbstractComponent implements Provider<Executor> { - private final ExecutorServiceWrapper threadpool; + private final ContainerThreadPool threadpool; - private static BlockingQueue<Runnable> createQ(int queueSize, int maxThreads) { - return (queueSize == 0) - ? new SynchronousQueue<>(false) - : (queueSize < 0) - ? new ArrayBlockingQueue<>(maxThreads*4) - : new ArrayBlockingQueue<>(queueSize); - } - - private static int computeThreadPoolSize(int maxNumThreads) { - return (maxNumThreads <= 0) - ? Runtime.getRuntime().availableProcessors() * 4 - : maxNumThreads; - } @Inject public ThreadPoolProvider(ThreadpoolConfig threadpoolConfig, Metric metric) { - this(threadpoolConfig, metric, new ProcessTerminator()); + this.threadpool = new ContainerThreadPool(threadpoolConfig, metric); } public ThreadPoolProvider(ThreadpoolConfig threadpoolConfig, Metric metric, ProcessTerminator processTerminator) { - int maxNumThreads = computeThreadPoolSize(threadpoolConfig.maxthreads()); - WorkerCompletionTimingThreadPoolExecutor executor = - new WorkerCompletionTimingThreadPoolExecutor(maxNumThreads, maxNumThreads, - 0L, TimeUnit.SECONDS, - createQ(threadpoolConfig.queueSize(), maxNumThreads), - ThreadFactoryFactory.getThreadFactory("threadpool"), - metric); - // Prestart needed, if not all threads will be created by the fist N tasks and hence they might also - // get the dreaded thread locals initialized even if they will never run. - // That counters what we we want to achieve with the Q that will prefer thread locality. - executor.prestartAllCoreThreads(); - threadpool = new ExecutorServiceWrapper(executor, metric, processTerminator, - threadpoolConfig.maxThreadExecutionTimeSeconds() * 1000L); + this.threadpool = new ContainerThreadPool(threadpoolConfig, metric, processTerminator); } /** @@ -75,7 +38,7 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex * @return a possibly shared executor */ @Override - public Executor get() { return threadpool; } + public Executor get() { return threadpool.executor(); } /** * Shutdown the thread pool, give a grace period of 1 second before forcibly @@ -83,142 +46,7 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex */ @Override public void deconstruct() { - boolean terminated; - - super.deconstruct(); - threadpool.shutdown(); - try { - terminated = threadpool.awaitTermination(1, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - if (!terminated) { - threadpool.shutdownNow(); - } - } - - /** - * A service executor wrapper which emits metrics and - * shuts down the vm when no workers are available for too long to avoid containers lingering in a blocked state. - * Package private for testing - */ - final static class ExecutorServiceWrapper extends ForwardingExecutorService { - - private final WorkerCompletionTimingThreadPoolExecutor wrapped; - private final Metric metric; - private final ProcessTerminator processTerminator; - private final long maxThreadExecutionTimeMillis; - private final Thread metricReporter; - private final AtomicBoolean closed = new AtomicBoolean(false); - - private ExecutorServiceWrapper(WorkerCompletionTimingThreadPoolExecutor wrapped, - Metric metric, ProcessTerminator processTerminator, - long maxThreadExecutionTimeMillis) { - this.wrapped = wrapped; - this.metric = metric; - this.processTerminator = processTerminator; - this.maxThreadExecutionTimeMillis = maxThreadExecutionTimeMillis; - - metric.set(MetricNames.THREAD_POOL_SIZE, wrapped.getPoolSize(), null); - metric.set(MetricNames.ACTIVE_THREADS, wrapped.getActiveCount(), null); - metric.add(MetricNames.REJECTED_REQUEST, 0, null); - metricReporter = new Thread(this::reportMetrics); - metricReporter.setDaemon(true); - metricReporter.start(); - } - - private final void reportMetrics() { - try { - while (!closed.get()) { - metric.set(MetricNames.THREAD_POOL_SIZE, wrapped.getPoolSize(), null); - metric.set(MetricNames.ACTIVE_THREADS, wrapped.getActiveCount(), null); - Thread.sleep(100); - } - } catch (InterruptedException e) { } - } - - @Override - public void shutdown() { - super.shutdown(); - closed.set(true); - } - - /** - * Tracks all instances of RejectedExecutionException. - * ThreadPoolProvider returns an executor, so external uses will not - * have access to the methods declared by ExecutorService. - * (execute(Runnable) is declared by Executor.) - */ - @Override - public void execute(Runnable command) { - try { - super.execute(command); - } catch (RejectedExecutionException e) { - metric.add(MetricNames.REJECTED_REQUEST, 1, null); - long timeSinceLastReturnedThreadMillis = System.currentTimeMillis() - wrapped.lastThreadAssignmentTimeMillis; - if (timeSinceLastReturnedThreadMillis > maxThreadExecutionTimeMillis) - processTerminator.logAndDie("No worker threads have been available for " + - timeSinceLastReturnedThreadMillis + " ms. Shutting down.", true); - throw e; - } - } - - @Override - protected ExecutorService delegate() { return wrapped; } - - private static final class MetricNames { - private static final String REJECTED_REQUEST = "serverRejectedRequests"; - private static final String THREAD_POOL_SIZE = "serverThreadPoolSize"; - private static final String ACTIVE_THREADS = "serverActiveThreads"; - } - - } - - /** - * A thread pool executor which maintains the last time a worker completed - * package private for testing - **/ - final static class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor { - - private static final String UNHANDLED_EXCEPTIONS_METRIC = "jdisc.thread_pool.unhandled_exceptions"; - - volatile long lastThreadAssignmentTimeMillis = System.currentTimeMillis(); - private final AtomicLong startedCount = new AtomicLong(0); - private final AtomicLong completedCount = new AtomicLong(0); - private final Metric metric; - - public WorkerCompletionTimingThreadPoolExecutor(int corePoolSize, - int maximumPoolSize, - long keepAliveTime, - TimeUnit unit, - BlockingQueue<Runnable> workQueue, - ThreadFactory threadFactory, - Metric metric) { - super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); - this.metric = metric; - } - - @Override - protected void beforeExecute(Thread t, Runnable r) { - super.beforeExecute(t, r); - lastThreadAssignmentTimeMillis = System.currentTimeMillis(); - startedCount.incrementAndGet(); - } - - @Override - protected void afterExecute(Runnable r, Throwable t) { - super.afterExecute(r, t); - completedCount.incrementAndGet(); - if (t != null) { - metric.add(UNHANDLED_EXCEPTIONS_METRIC, 1L, metric.createContext(Map.of("exception", t.getClass().getSimpleName()))); - } - } - - @Override - public int getActiveCount() { - return (int)(startedCount.get() - completedCount.get()); - } + threadpool.close(); } } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadPool.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadPool.java new file mode 100644 index 00000000000..6fc9da298a8 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadPool.java @@ -0,0 +1,89 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.threadpool; + +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.container.protect.ProcessTerminator; +import com.yahoo.jdisc.Metric; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.TimeUnit; + +/** + * A configurable thread pool. This provides the worker threads used for normal request processing. + * + * @author Steinar Knutsen + * @author baldersheim + * @author bratseth + * @author bjorncs + */ +public class ContainerThreadPool implements AutoCloseable { + + private final ExecutorServiceWrapper threadpool; + + public ContainerThreadPool(ThreadpoolConfig config, Metric metric) { + this(config, metric, new ProcessTerminator()); + } + + public ContainerThreadPool(ThreadpoolConfig threadpoolConfig, Metric metric, ProcessTerminator processTerminator) { + ThreadPoolMetric threadPoolMetric = new ThreadPoolMetric(metric, threadpoolConfig.name()); + int maxNumThreads = computeMaximumThreadPoolSize(threadpoolConfig.maxthreads()); + int coreNumThreads = computeCoreThreadPoolSize(threadpoolConfig.corePoolSize(), maxNumThreads); + WorkerCompletionTimingThreadPoolExecutor executor = + new WorkerCompletionTimingThreadPoolExecutor(coreNumThreads, maxNumThreads, + (int)threadpoolConfig.keepAliveTime() * 1000, TimeUnit.MILLISECONDS, + createQ(threadpoolConfig.queueSize(), maxNumThreads), + ThreadFactoryFactory.getThreadFactory(threadpoolConfig.name()), + threadPoolMetric); + // Prestart needed, if not all threads will be created by the fist N tasks and hence they might also + // get the dreaded thread locals initialized even if they will never run. + // That counters what we we want to achieve with the Q that will prefer thread locality. + executor.prestartAllCoreThreads(); + threadpool = new ExecutorServiceWrapper(executor, threadPoolMetric, processTerminator, + threadpoolConfig.maxThreadExecutionTimeSeconds() * 1000L); + } + + public Executor executor() { return threadpool; } + @Override public void close() { closeInternal(); } + + /** + * Shutdown the thread pool, give a grace period of 1 second before forcibly + * shutting down all worker threads. + */ + private void closeInternal() { + boolean terminated; + + threadpool.shutdown(); + try { + terminated = threadpool.awaitTermination(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + if (!terminated) { + threadpool.shutdownNow(); + } + } + + private static BlockingQueue<Runnable> createQ(int queueSize, int maxThreads) { + return (queueSize == 0) + ? new SynchronousQueue<>(false) + : (queueSize < 0) + ? new ArrayBlockingQueue<>(maxThreads*4) + : new ArrayBlockingQueue<>(queueSize); + } + + private static int computeMaximumThreadPoolSize(int maxNumThreads) { + return (maxNumThreads <= 0) + ? Runtime.getRuntime().availableProcessors() * 4 + : maxNumThreads; + } + + private static int computeCoreThreadPoolSize(int corePoolSize, int maxNumThreads) { + return Math.min(corePoolSize, maxNumThreads); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java new file mode 100644 index 00000000000..1144d1ebbf6 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java @@ -0,0 +1,86 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.threadpool; + +import com.google.common.util.concurrent.ForwardingExecutorService; +import com.yahoo.container.protect.ProcessTerminator; + +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A service executor wrapper which emits metrics and + * shuts down the vm when no workers are available for too long to avoid containers lingering in a blocked state. + * Package private for testing + * + * @author Steinar Knutsen + * @author baldersheim + * @author bratseth + */ +class ExecutorServiceWrapper extends ForwardingExecutorService { + + private final WorkerCompletionTimingThreadPoolExecutor wrapped; + private final ThreadPoolMetric metric; + private final ProcessTerminator processTerminator; + private final long maxThreadExecutionTimeMillis; + private final Thread metricReporter; + private final AtomicBoolean closed = new AtomicBoolean(false); + + ExecutorServiceWrapper( + WorkerCompletionTimingThreadPoolExecutor wrapped, + ThreadPoolMetric metric, ProcessTerminator processTerminator, + long maxThreadExecutionTimeMillis) { + this.wrapped = wrapped; + this.metric = metric; + this.processTerminator = processTerminator; + this.maxThreadExecutionTimeMillis = maxThreadExecutionTimeMillis; + + metric.reportThreadPoolSize(wrapped.getPoolSize()); + metric.reportActiveThreads(wrapped.getActiveCount()); + metricReporter = new Thread(this::reportMetrics); + metricReporter.setDaemon(true); + metricReporter.start(); + } + + private final void reportMetrics() { + try { + while (!closed.get()) { + metric.reportThreadPoolSize(wrapped.getPoolSize()); + metric.reportActiveThreads(wrapped.getActiveCount()); + Thread.sleep(100); + } + } catch (InterruptedException e) { } + } + + @Override + public void shutdown() { + super.shutdown(); + closed.set(true); + } + + /** + * Tracks all instances of {@link RejectedExecutionException}. + * {@link ContainerThreadPool} returns an executor, so external uses will not + * have access to the methods declared by {@link ExecutorService}. + * ({@link Executor#execute(Runnable)} is declared by {@link Executor}.) + */ + @Override + public void execute(Runnable command) { + try { + super.execute(command); + } catch (RejectedExecutionException e) { + metric.reportRejectRequest(); + long timeSinceLastReturnedThreadMillis = System.currentTimeMillis() - wrapped.lastThreadAssignmentTimeMillis; + if (timeSinceLastReturnedThreadMillis > maxThreadExecutionTimeMillis) + processTerminator.logAndDie("No worker threads have been available for " + + timeSinceLastReturnedThreadMillis + " ms. Shutting down.", true); + throw e; + } + } + + @Override + protected ExecutorService delegate() { return wrapped; } + +} + diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java new file mode 100644 index 00000000000..d9ab020bcb8 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.threadpool; + +import com.yahoo.jdisc.Metric; + +import java.util.Map; + +/** + * @author bjorncs + */ +class ThreadPoolMetric { + + private static final String THREAD_POOL_NAME_DIMENSION = "threadpool"; + + private final Metric metric; + private final Metric.Context defaultContext; + private final String threadPoolName; + + ThreadPoolMetric(Metric metric, String threadPoolName) { + this.metric = metric; + this.threadPoolName = threadPoolName; + this.defaultContext = metric.createContext(Map.of(THREAD_POOL_NAME_DIMENSION, threadPoolName)); + } + + void reportRejectRequest() { metric.add("serverRejectedRequests", 1L, defaultContext); } + void reportThreadPoolSize(long size) { metric.set("serverThreadPoolSize", size, defaultContext); } + void reportActiveThreads(long threads) { metric.set("serverActiveThreads", threads, defaultContext); } + void reportUnhandledException(Throwable t) { + Metric.Context ctx = metric.createContext(Map.of( + THREAD_POOL_NAME_DIMENSION, threadPoolName, + "exception", t.getClass().getSimpleName())); + metric.set("jdisc.thread_pool.unhandled_exceptions", 1L, ctx); + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java new file mode 100644 index 00000000000..56f8319c110 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java @@ -0,0 +1,60 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.threadpool; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A thread pool executor which maintains the last time a worker completed + * package private for testing + * + * @author Steinar Knutsen + * @author baldersheim + * @author bratseth + */ +class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor { + + + + volatile long lastThreadAssignmentTimeMillis = System.currentTimeMillis(); + private final AtomicLong startedCount = new AtomicLong(0); + private final AtomicLong completedCount = new AtomicLong(0); + private final ThreadPoolMetric metric; + + WorkerCompletionTimingThreadPoolExecutor( + int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue<Runnable> workQueue, + ThreadFactory threadFactory, + ThreadPoolMetric metric) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); + this.metric = metric; + } + + @Override + protected void beforeExecute(Thread t, Runnable r) { + super.beforeExecute(t, r); + lastThreadAssignmentTimeMillis = System.currentTimeMillis(); + startedCount.incrementAndGet(); + } + + @Override + protected void afterExecute(Runnable r, Throwable t) { + super.afterExecute(r, t); + completedCount.incrementAndGet(); + if (t != null) { + metric.reportUnhandledException(t); + } + } + + @Override + public int getActiveCount() { + return (int)(startedCount.get() - completedCount.get()); + } +} + diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/package-info.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/package-info.java new file mode 100644 index 00000000000..6a94cea49da --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.container.handler.threadpool; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java b/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java index 38f5b72336b..16cf741813c 100644 --- a/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java +++ b/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java @@ -5,7 +5,7 @@ import com.yahoo.protect.Process; /** * An injectable terminator of the Java vm. - * Components that encounters conditions where the vm should be terminator should + * Components that encounters conditions where the vm should be terminated should * request an instance of this injected. That makes termination testable * as tests can create subclasses of this which register the termination request * rather than terminating. diff --git a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java index 72a5fbdce55..6ab71c9a9dc 100644 --- a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java +++ b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java @@ -164,8 +164,7 @@ public abstract class AbstractProcessingHandler<COMPONENT extends Processor> ext private static Renderer selectRenderer(com.yahoo.processing.Request processingRequest, ComponentRegistry<Renderer> renderers, Renderer defaultRenderer) { Renderer renderer = null; - // TODO: Support setting a particular renderer instead of just selecting - // by name? + // TODO: Support setting a particular renderer instead of just selecting by name? String rendererId = processingRequest.properties().getString("format"); if (rendererId != null && !"".equals(rendererId)) { renderer = renderers.getComponent(ComponentSpecification.fromString(rendererId)); diff --git a/container-core/src/main/java/com/yahoo/processing/handler/ProcessingHandler.java b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingHandler.java index 1504a0e861b..d1ccc88d34f 100644 --- a/container-core/src/main/java/com/yahoo/processing/handler/ProcessingHandler.java +++ b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingHandler.java @@ -13,12 +13,12 @@ import com.yahoo.processing.rendering.Renderer; import java.util.concurrent.Executor; /** - * A jDisc request handler which invokes a processing chain to produce the response. + * A request handler which invokes a processing chain to produce the response. * * @author Tony Vaagenes - * @since 5.1.7 */ public class ProcessingHandler extends AbstractProcessingHandler<Processor> { + public ProcessingHandler(ChainRegistry<Processor> chainRegistry, ComponentRegistry<Renderer> renderers, Executor executor, @@ -51,4 +51,5 @@ public class ProcessingHandler extends AbstractProcessingHandler<Processor> { Metric metric) { super(processingChainsConfig, chainedComponents, renderers, executor, accessLog, metric); } + } diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java index ba246f26639..fdc3b63fc92 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java @@ -29,7 +29,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.logging.Level; import java.util.logging.Logger; /** diff --git a/container-core/src/main/resources/configdefinitions/threadpool.def b/container-core/src/main/resources/configdefinitions/threadpool.def index abc60f9f06d..d966738ea9f 100644 --- a/container-core/src/main/resources/configdefinitions/threadpool.def +++ b/container-core/src/main/resources/configdefinitions/threadpool.def @@ -2,10 +2,16 @@ namespace=container.handler -## Num ber of thread in the thread pool +## Maximum number of thread in the thread pool ## Setting it to 0 or negative number will cause it to be set to #cores * 4 maxthreads int default=500 +# The number of threads to keep in the pool, even if they are idle +corePoolSize int default=500 + +# The number of seconds that excess idle threads will wait for new tasks before terminating +keepAliveTime double default=5.0 + ## max queue size ## There can be queueSize + maxthreads requests inflight concurrently ## The container will start replying 503 @@ -17,7 +23,5 @@ queueSize int default=0 # time of each request when in a state of overload, i.e about "worst case execution time*2" maxThreadExecutionTimeSeconds int default=190 -# Length of period for soft start -# During this period number of availble threads will be gradually increased. -# Currently used to avoid feeding overload in container during cold start. -softStartSeconds double default=0 +# Prefix for the name of the threads +name string default="default-pool" diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index 01dcb885a97..ab0d0d54675 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -47,12 +47,12 @@ public class LogHandlerTest { } @Override - protected void writeLogs(OutputStream outputStream, Instant from, Instant to) { + protected void writeLogs(OutputStream out, Instant from, Instant to) { try { if (to.isAfter(Instant.ofEpochMilli(1000))) { - outputStream.write("newer log".getBytes()); + out.write("newer log".getBytes()); } else { - outputStream.write("older log".getBytes()); + out.write("older log".getBytes()); } } catch (Exception e) {} } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index c68facf4f01..3f7a78e13be 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -12,7 +12,7 @@ import java.io.OutputStream; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.FileTime; +import java.time.Duration; import java.time.Instant; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; @@ -26,47 +26,56 @@ public class LogReaderTest { private final FileSystem fileSystem = TestFileSystem.create(); private final Path logDirectory = fileSystem.getPath("/opt/vespa/logs"); - private static final String log1 = "0.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"; - private static final String log2 = "0.2\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)\n"; + private static final String logv11 = "3600.2\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tfourth\n"; + private static final String logv = "90000.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tlast\n"; + private static final String log100 = "0.2\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tsecond\n"; + private static final String log101 = "0.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"; + private static final String log110 = "3600.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tthird\n"; + private static final String log200 = "86400.1\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)\n"; @Before public void setup() throws IOException { - Files.createDirectories(logDirectory.resolve("subfolder")); - - Files.setLastModifiedTime( - Files.write(logDirectory.resolve("log1.log.gz"), compress(log1)), - FileTime.from(Instant.ofEpochMilli(123))); - Files.setLastModifiedTime( - Files.write(logDirectory.resolve("subfolder/log2.log"), log2.getBytes(UTF_8)), - FileTime.from(Instant.ofEpochMilli(234))); - + // Log archive paths and file names indicate what hour they contain logs for, with the start of that hour. + // Multiple entries may exist for each hour. + Files.createDirectories(logDirectory.resolve("1970/01/01")); + Files.write(logDirectory.resolve("1970/01/01/00-0.gz"), compress(log100)); + Files.write(logDirectory.resolve("1970/01/01/00-1"), log101.getBytes(UTF_8)); + Files.write(logDirectory.resolve("1970/01/01/01-0.gz"), compress(log110)); + + Files.createDirectories(logDirectory.resolve("1970/01/02")); + Files.write(logDirectory.resolve("1970/01/02/00-0"), log200.getBytes(UTF_8)); + + // Vespa log file names are the second-truncated timestamp of the last entry. + // The current log file has no timestamp suffix. + Files.write(logDirectory.resolve("vespa.log-1970-01-01.01-00-00"), logv11.getBytes(UTF_8)); + Files.write(logDirectory.resolve("vespa.log"), logv.getBytes(UTF_8)); } @Test public void testThatLogsOutsideRangeAreExcluded() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(160)); + logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050)); - assertEquals("", decompress(baos.toByteArray())); + assertEquals(log100 + logv11 + log110, baos.toString(UTF_8)); } @Test public void testThatLogsNotMatchingRegexAreExcluded() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*2\\.log")); - logReader.writeLogs(baos, Instant.ofEpochMilli(0), Instant.ofEpochMilli(300)); + LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*-1.*")); + logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2))); - assertEquals(log2, decompress(baos.toByteArray())); + assertEquals(log101 + logv11, baos.toString(UTF_8)); } @Test public void testZippedStreaming() throws IOException { ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - logReader.writeLogs(zippedBaos, Instant.ofEpochMilli(0), Instant.ofEpochMilli(300)); + logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2))); - assertEquals(log1 + log2, decompress(zippedBaos.toByteArray())); + assertEquals(log101 + log100 + logv11 + log110 + log200 + logv, zippedBaos.toString(UTF_8)); } private byte[] compress(String input) throws IOException { @@ -77,10 +86,4 @@ public class LogReaderTest { return baos.toByteArray(); } - private String decompress(byte[] input) throws IOException { - if (input.length == 0) return ""; - byte[] decompressed = new GZIPInputStream(new ByteArrayInputStream(input)).readAllBytes(); - return new String(decompressed); - } - } diff --git a/container-core/src/test/java/com/yahoo/container/handler/ThreadPoolProviderTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolTest.java index 761ed40763c..f6a3aebd7ff 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/ThreadPoolProviderTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolTest.java @@ -1,37 +1,33 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.handler; - -import static org.junit.Assert.fail; - -import java.util.concurrent.Executor; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadPoolExecutor; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.handler.threadpool; +import com.yahoo.collections.Tuple2; +import com.yahoo.concurrent.Receiver; +import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.protect.ProcessTerminator; +import com.yahoo.jdisc.Metric; import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; -import com.yahoo.concurrent.Receiver; -import com.yahoo.concurrent.Receiver.MessageState; -import com.yahoo.collections.Tuple2; -import com.yahoo.jdisc.Metric; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** - * Check threadpool provider accepts tasks and shuts down properly. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @author bjorncs */ -public class ThreadPoolProviderTestCase { - +public class ContainerThreadPoolTest { @Test - public final void testThreadPoolProvider() throws InterruptedException { + public final void testThreadPool() throws InterruptedException { ThreadpoolConfig config = new ThreadpoolConfig(new ThreadpoolConfig.Builder().maxthreads(1)); - ThreadPoolProvider provider = new ThreadPoolProvider(config, Mockito.mock(Metric.class)); - Executor exec = provider.get(); - Tuple2<MessageState, Boolean> reply; + ContainerThreadPool threadPool = new ContainerThreadPool(config, Mockito.mock(Metric.class)); + Executor exec = threadPool.executor(); + Tuple2<Receiver.MessageState, Boolean> reply; FlipIt command = new FlipIt(); for (boolean done = false; !done;) { try { @@ -42,13 +38,13 @@ public class ThreadPoolProviderTestCase { } } reply = command.didItRun.get(5 * 60 * 1000); - if (reply.first != MessageState.VALID) { + if (reply.first != Receiver.MessageState.VALID) { fail("Executor task probably timed out, five minutes should be enough to flip a boolean."); } if (reply.second != Boolean.TRUE) { fail("Executor task seemed to run, but did not get correct value."); } - provider.deconstruct(); + threadPool.close(); command = new FlipIt(); try { exec.execute(command); @@ -61,9 +57,9 @@ public class ThreadPoolProviderTestCase { private ThreadPoolExecutor createPool(int maxThreads, int queueSize) { ThreadpoolConfig config = new ThreadpoolConfig(new ThreadpoolConfig.Builder().maxthreads(maxThreads).queueSize(queueSize)); - ThreadPoolProvider provider = new ThreadPoolProvider(config, Mockito.mock(Metric.class)); - ThreadPoolProvider.ExecutorServiceWrapper wrapper = (ThreadPoolProvider.ExecutorServiceWrapper) provider.get(); - ThreadPoolProvider.WorkerCompletionTimingThreadPoolExecutor executor = (ThreadPoolProvider.WorkerCompletionTimingThreadPoolExecutor)wrapper.delegate(); + ContainerThreadPool threadPool = new ContainerThreadPool(config, Mockito.mock(Metric.class)); + ExecutorServiceWrapper wrapper = (ExecutorServiceWrapper) threadPool.executor(); + WorkerCompletionTimingThreadPoolExecutor executor = (WorkerCompletionTimingThreadPoolExecutor)wrapper.delegate(); return executor; } @@ -103,37 +99,37 @@ public class ThreadPoolProviderTestCase { @Test @Ignore // Ignored because it depends on the system time and so is unstable on factory - public void testThreadPoolProviderTerminationOnBreakdown() throws InterruptedException { + public void testThreadPoolTerminationOnBreakdown() throws InterruptedException { ThreadpoolConfig config = new ThreadpoolConfig(new ThreadpoolConfig.Builder().maxthreads(2) - .maxThreadExecutionTimeSeconds(1)); + .maxThreadExecutionTimeSeconds(1)); MockProcessTerminator terminator = new MockProcessTerminator(); - ThreadPoolProvider provider = new ThreadPoolProvider(config, Mockito.mock(Metric.class), terminator); + ContainerThreadPool threadPool = new ContainerThreadPool(config, Mockito.mock(Metric.class), terminator); // No dying when threads hang shorter than max thread execution time - provider.get().execute(new Hang(500)); - provider.get().execute(new Hang(500)); + threadPool.executor().execute(new Hang(500)); + threadPool.executor().execute(new Hang(500)); assertEquals(0, terminator.dieRequests); - assertRejected(provider, new Hang(500)); // no more threads + assertRejected(threadPool, new Hang(500)); // no more threads assertEquals(0, terminator.dieRequests); // ... but not for long enough yet try { Thread.sleep(1500); } catch (InterruptedException e) {} - provider.get().execute(new Hang(1)); + threadPool.executor().execute(new Hang(1)); assertEquals(0, terminator.dieRequests); try { Thread.sleep(50); } catch (InterruptedException e) {} // Make sure both threads are available // Dying when hanging both thread pool threads for longer than max thread execution time - provider.get().execute(new Hang(2000)); - provider.get().execute(new Hang(2000)); + threadPool.executor().execute(new Hang(2000)); + threadPool.executor().execute(new Hang(2000)); assertEquals(0, terminator.dieRequests); - assertRejected(provider, new Hang(2000)); // no more threads + assertRejected(threadPool, new Hang(2000)); // no more threads assertEquals(0, terminator.dieRequests); // ... but not for long enough yet try { Thread.sleep(1500); } catch (InterruptedException e) {} - assertRejected(provider, new Hang(2000)); // no more threads + assertRejected(threadPool, new Hang(2000)); // no more threads assertEquals(1, terminator.dieRequests); // ... for longer than maxThreadExecutionTime } - private void assertRejected(ThreadPoolProvider provider, Runnable task) { + private void assertRejected(ContainerThreadPool threadPool, Runnable task) { try { - provider.get().execute(task); + threadPool.executor().execute(task); fail("Expected execution rejected"); } catch (final RejectedExecutionException expected) { } @@ -165,4 +161,4 @@ public class ThreadPoolProviderTestCase { } -} +}
\ No newline at end of file diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index b06734b3f87..1c4c9e9521b 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -446,7 +446,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.27.v20200227</jetty.version> + <jetty.version>9.4.30.v20200611</jetty.version> <org.lz4.version>1.7.1</org.lz4.version> <org.json.version>20090211</org.json.version> <slf4j.version>1.7.5</slf4j.version> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 1bb06ab9694..dd2d9ceb188 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -187,6 +187,11 @@ <artifactId>config-bundle</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-zone-api</artifactId> + <version>${project.version}</version> + </dependency> <!-- NOTE: Dependencies below are added explicitly to exclude transitive deps that are not provided runtime by the container, and hence make them invisible to user projects' build classpath. diff --git a/container-di/src/main/java/com/yahoo/container/di/Container.java b/container-di/src/main/java/com/yahoo/container/di/Container.java index d8fec35633b..dcdd1767c9c 100644 --- a/container-di/src/main/java/com/yahoo/container/di/Container.java +++ b/container-di/src/main/java/com/yahoo/container/di/Container.java @@ -140,7 +140,7 @@ public class Container { snapshot = configurer.getConfigs(graph.configKeys(), leastGeneration, restartOnRedeploy); log.log(FINE, String.format("createNewGraph:\n" + "graph.configKeys = %s\n" + "graph.generation = %s\n" + "snapshot = %s\n", - graph.configKeys(), graph.generation(), snapshot)); + graph.configKeys(), graph.generation(), snapshot)); if (snapshot instanceof BootstrapConfigs) { if (getBootstrapGeneration() <= previousConfigGeneration) { @@ -182,7 +182,7 @@ public class Container { } private ComponentGraph createAndConfigureComponentsGraph(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> componentsConfigs, - Injector fallbackInjector) { + Injector fallbackInjector) { ComponentGraph componentGraph = createComponentsGraph(componentsConfigs, getComponentsGeneration(), fallbackInjector); componentGraph.setAvailableConfigs(componentsConfigs); return componentGraph; diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java index fdaaf4b698d..2e4db01fb1b 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java @@ -142,8 +142,7 @@ public class ComponentGraph { } public void setAvailableConfigs(Map<ConfigKey<? extends ConfigInstance>, ConfigInstance> configs) { - Map<ConfigKey<ConfigInstance>, ConfigInstance> invariantMap = Keys.invariantCopy(configs); - componentNodes().forEach(node -> node.setAvailableConfigs(invariantMap)); + componentNodes().forEach(node -> node.setAvailableConfigs(Keys.invariantCopy(configs))); } public void reuseNodes(ComponentGraph old) { diff --git a/container-disc/pom.xml b/container-disc/pom.xml index 15b8cd08808..48872d0665b 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -106,6 +106,12 @@ <artifactId>vespalog</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-zone-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <!-- WARNING: These are only here to make bundlification work --> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -184,6 +190,7 @@ container-search-and-docproc-jar-with-dependencies.jar, container-search-gui-jar-with-dependencies.jar, docprocs-jar-with-dependencies.jar, + hosted-zone-api-jar-with-dependencies.jar, jdisc-security-filters-jar-with-dependencies.jar, jdisc_http_service-jar-with-dependencies.jar, model-evaluation-jar-with-dependencies.jar, diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 2a72d0b3442..d600b9d3e04 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -9,6 +9,7 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.ConfigInterruptedException; +import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.container.Container; import com.yahoo.container.QrConfig; import com.yahoo.container.core.ChainsConfig; @@ -216,8 +217,7 @@ public final class ConfiguredApplication implements Application { } private <T extends ConfigInstance> T getConfig(Class<T> configClass) { - Subscriber subscriber = subscriberFactory.getSubscriber( - Collections.singleton(new ConfigKey<>(configClass, configId))); + Subscriber subscriber = subscriberFactory.getSubscriber(Collections.singleton(new ConfigKey<>(configClass, configId))); try { subscriber.waitNextGeneration(); return configClass.cast(first(subscriber.config().values())); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java new file mode 100644 index 00000000000..0bb3832ddf5 --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java @@ -0,0 +1,27 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import ai.vespa.cloud.Environment; +import ai.vespa.cloud.SystemInfo; +import ai.vespa.cloud.Zone; +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.container.di.componentgraph.Provider; + +/** + * Provides information about the system in which this container is running. + * This is available and can be injected when running in a cloud environment. + * + * @author bratseth + */ +public class SystemInfoProvider extends AbstractComponent implements Provider<SystemInfo> { + + private final SystemInfo instance; + + @Inject public SystemInfoProvider(ConfigserverConfig config) { + this.instance = new SystemInfo(new Zone(Environment.valueOf(config.environment()), config.region())); + } + + @Override public SystemInfo get() { return instance; } +} diff --git a/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html b/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html index 0ae9e72a19b..01c1835c5e3 100644 --- a/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html +++ b/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html @@ -145,7 +145,7 @@ - <li class="collapseable"><a href="/documentation/build-vespa.html">Build Vespa</a></li> + <li class="collapseable"><a href="/documentation/build-install-vespa.html">Build Vespa</a></li> diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 51fee99a743..2b4424654a2 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -4191,6 +4191,7 @@ "public" ], "methods": [ + "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.search.searchchain.ExecutionFactory, java.util.Optional)", "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.statistics.Statistics, com.yahoo.language.Linguistics, com.yahoo.jdisc.Metric, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.container.core.ContainerHttpConfig)", @@ -6063,8 +6064,9 @@ ], "methods": [ "public void <init>()", + "public void <init>(com.yahoo.search.query.profile.config.QueryProfilesConfig)", "public void <init>(com.yahoo.search.query.profile.types.QueryProfileTypeRegistry)", - "public void register(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", + "public final void register(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", "public com.yahoo.search.query.profile.types.QueryProfileTypeRegistry getTypeRegistry()", "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile findQueryProfile(java.lang.String)" ], diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 338add37213..56dfc700ca7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -190,7 +190,7 @@ public class FastHit extends Hit { /** * Returns values for the features listed in - * <a href="https://docs.vespa.ai/documentation/reference/search-definitions-reference.html#summary-features">summary-features</a> + * <a href="https://docs.vespa.ai/documentation/reference/schema-reference.html#summary-features">summary-features</a> * in the rank profile specified in the query producing this. */ public FeatureData features() { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index ea65bc7d7d2..475a80f7ae0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -32,7 +32,7 @@ public abstract class Item implements Cloneable { /** * The definitions in Item.ItemType must match the ones in - * searchlib/src/searchlib/parsequery/parse.h + * searchlib/src/vespa/searchlib/parsequery/parse.h */ public static enum ItemType { OR(0), diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java index e8fa70afd1b..52ef6c40a6a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java @@ -82,7 +82,7 @@ public class NearestNeighborItem extends SimpleTaggableItem { buffer.append("{field=").append(field); buffer.append(",queryTensorName=").append(queryTensorName); buffer.append(",hnsw.exploreAdditionalHits=").append(hnswExploreAdditionalHits); - buffer.append(",approximate=").append(String.valueOf(approximate)); + buffer.append(",approximate=").append(approximate); buffer.append(",targetNumHits=").append(targetNumHits).append("}"); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java index 97c68ee3da8..2c017410109 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java @@ -12,8 +12,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.compress.IntegerCompressor; /** - * A set words with differing exactness scores to be used for literal boost - * ranking. + * A set of words with differing exactness scores to be used for literal boost ranking. * * @author Steinar Knutsen */ @@ -145,17 +144,14 @@ public class WordAlternativesItem extends TermItem { } /** - * Return an immutable snapshot of the contained terms. This list will not - * reflect later changes to the item. + * Return an immutable snapshot of the contained terms. This list will not reflect later changes to the item. * - * @return an immutable list of word alternatives and their respective - * scores + * @return an immutable list of word alternatives and their respective scores */ public List<Alternative> getAlternatives() { return alternatives; } - @Override public void encodeThis(ByteBuffer target) { super.encodeThis(target); @@ -172,17 +168,14 @@ public class WordAlternativesItem extends TermItem { * equal or higher exactness score. If the term string is present with a * lower exactness score, the new, higher score will take precedence. * - * @param term - * one of several string interpretations of the input word - * @param exactness - * how close the term string matches what the user input + * @param term one of several string interpretations of the input word + * @param exactness how close the term string matches what the user input */ public void addTerm(String term, double exactness) { // do note, Item is Cloneable, and overwriting the reference is what // saves us from overriding the method - if (alternatives.stream().anyMatch((a) -> a.word.equals(term) && a.exactness >= exactness )) { - return; - } + if (alternatives.stream().anyMatch((a) -> a.word.equals(term) && a.exactness >= exactness )) return; + List<Alternative> newTerms = new ArrayList<>(alternatives.size() + 1); newTerms.addAll(alternatives); newTerms.add(new Alternative(term, exactness)); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java index 8003d9c6744..d48e337b0c1 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java @@ -12,16 +12,23 @@ public class TopKEstimator { private final TDistribution studentT; private final double defaultP; private final boolean estimate; + private final double skewFactor; private static boolean needEstimate(double p) { return (0.0 < p) && (p < 1.0); } - public TopKEstimator(double freedom, double defaultProbability) { + TopKEstimator(double freedom, double defaultProbability) { + this(freedom, defaultProbability, 0.0); + } + public TopKEstimator(double freedom, double defaultProbability, double skewFactor) { this.studentT = new TDistribution(null, freedom); defaultP = defaultProbability; estimate = needEstimate(defaultP); + this.skewFactor = skewFactor; } double estimateExactK(double k, double n, double p) { + double p_max = (1 + skewFactor)/n; + n = Math.max(1, 1/p_max); double variance = k * 1/n * (1 - 1/n); double p_inverse = 1 - (1 - p)/n; return k/n + studentT.inverseCumulativeProbability(p_inverse) * Math.sqrt(variance); @@ -31,12 +38,12 @@ public class TopKEstimator { } public int estimateK(int k, int n) { return (estimate && n > 1) - ? (int)Math.ceil(estimateExactK(k, n, defaultP)) + ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, defaultP))) : k; } public int estimateK(int k, int n, double p) { return (needEstimate(p) && (n > 1)) - ? (int)Math.ceil(estimateExactK(k, n, p)) + ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, p))) : k; } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 7dfc03fd2d7..2f62b07ac04 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -41,6 +41,7 @@ public class SearchCluster implements NodeManager<Node> { private final PingFactory pingFactory; private final TopKEstimator hitEstimator; private long nextLogTime = 0; + private static final double SKEW_FACTOR = 0.05; /** * A search node on this local machine having the entire corpus, which we therefore @@ -78,7 +79,7 @@ public class SearchCluster implements NodeManager<Node> { for (Node node : nodes) nodesByHostBuilder.put(node.hostname(), node); this.nodesByHost = nodesByHostBuilder.build(); - hitEstimator = new TopKEstimator(30.0, dispatchConfig.topKProbability()); + hitEstimator = new TopKEstimator(30.0, dispatchConfig.topKProbability(), SKEW_FACTOR); this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), size, diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index c8386f3c75c..c658d404adb 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -121,6 +121,28 @@ public class SearchHandler extends LoggingRequestHandler { Metric metric, Executor executor, AccessLog accessLog, + CompiledQueryProfileRegistry queryProfileRegistry, + ContainerHttpConfig containerHttpConfig, + ExecutionFactory executionFactory) { + this(statistics, + metric, + executor, + accessLog, + queryProfileRegistry, + executionFactory, + containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), + containerHttpConfig.hostResponseHeaderKey().equals("") ? + Optional.empty() : Optional.of(containerHttpConfig.hostResponseHeaderKey())); + } + + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 + public SearchHandler(Statistics statistics, + Metric metric, + Executor executor, + AccessLog accessLog, QueryProfilesConfig queryProfileConfig, ContainerHttpConfig containerHttpConfig, ExecutionFactory executionFactory) { @@ -132,7 +154,7 @@ public class SearchHandler extends LoggingRequestHandler { executionFactory, containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), containerHttpConfig.hostResponseHeaderKey().equals("") ? - Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey())); + Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey())); } public SearchHandler(Statistics statistics, diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java index 7ab05d0cd1e..744c6eb6933 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java @@ -1,9 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.compiled; +import com.google.inject.Inject; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileCompiler; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; /** @@ -18,6 +23,15 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer private final QueryProfileTypeRegistry typeRegistry; + @Inject + public CompiledQueryProfileRegistry(QueryProfilesConfig config) { + QueryProfileRegistry registry = QueryProfileConfigurer.createFromConfig(config); + typeRegistry = registry.getTypeRegistry(); + for (QueryProfile inputProfile : registry.allComponents()) { + register(QueryProfileCompiler.compile(inputProfile, this)); + } + } + /** Creates a compiled query profile registry with no types */ public CompiledQueryProfileRegistry() { this(QueryProfileTypeRegistry.emptyFrozen()); @@ -28,7 +42,7 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer } /** Registers a type by its id */ - public void register(CompiledQueryProfile profile) { + public final void register(CompiledQueryProfile profile) { super.register(profile.getId(), profile); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index e16f09a58ab..2bfa778a2ba 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -228,7 +228,7 @@ public class InterleavedSearchInvokerTest { @Test public void requireThatTopKProbabilityOverrideTakesEffect() throws IOException { validateThatTopKProbabilityOverrideTakesEffect(null, 8); - validateThatTopKProbabilityOverrideTakesEffect(0.8, 6); + validateThatTopKProbabilityOverrideTakesEffect(0.8, 7); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index cafba58662e..4afb6186c60 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -120,6 +120,7 @@ public class MockSearchCluster extends SearchCluster { builder.minGroupCoverage(99.0); builder.maxNodesDownPerGroup(0); builder.minSearchCoverage(minSearchCoverage); + builder.distributionPolicy(DispatchConfig.DistributionPolicy.Enum.ROUNDROBIN); if (minSearchCoverage < 100.0) { builder.minWaitAfterCoverageFactor(0); builder.maxWaitAfterCoverageFactor(0.5); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java index c14e4f984f1..271e145087b 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java @@ -25,4 +25,142 @@ public class TopKEstimatorTest { assertEquals(44.909040374464155, estimator.estimateExactK(200, 10, 0.99999), 0.0); assertEquals(45, estimator.estimateK(200, 10, 0.99999)); } + @Test + public void requireHitsAreEstimatedAccordingToPartitionsAndProbabilityForVaryingN_K200() { + TopKEstimator estimator = new TopKEstimator(30, 0.99999); + assertEquals(200, estimator.estimateExactK(200, 1), 0.0); + assertEquals(200, estimator.estimateK(200, 1)); + assertEquals(137.4727798056239, estimator.estimateExactK(200, 2), 0.0); + assertEquals(102.95409291533568, estimator.estimateExactK(200, 3), 0.0); + assertEquals(44.909040374464155, estimator.estimateExactK(200, 10), 0.0); + assertEquals(28.86025772029091, estimator.estimateExactK(200, 20), 0.0); + } + + @Test + public void requireHitsAreEstimatedAccordingToPartitionsAndProbabilityForVaryingN_K20() { + TopKEstimator estimator = new TopKEstimator(30, 0.99999); + assertEquals(20, estimator.estimateExactK(20, 1), 0.0); + assertEquals(20, estimator.estimateK(20, 1)); + assertEquals(21.849933444373328, estimator.estimateExactK(20, 2), 0.0); + assertEquals(18.14175840378403, estimator.estimateExactK(20, 3), 0.0); + assertEquals(9.87693019124002, estimator.estimateExactK(20, 10), 0.0); + assertEquals(6.964137165389415, estimator.estimateExactK(20, 20), 0.0); + } + + @Test + public void requireHitsAreEstimatedAccordingToPartitionsAndProbabilityForVaryingN_K10_Five9() { + TopKEstimator estimator = new TopKEstimator(30, 0.99999); + assertEquals(10, estimator.estimateExactK(10, 1), 0.0); + assertEquals(10, estimator.estimateK(10, 1)); + assertEquals(13.379168295125641, estimator.estimateExactK(10, 2), 0.0); + assertEquals(11.447448515386741, estimator.estimateExactK(10, 3), 0.0); + assertEquals(6.569830753158866, estimator.estimateExactK(10, 10), 0.0); + assertEquals(4.717281833573569, estimator.estimateExactK(10, 20), 0.0); + } + + @Test + public void requireHitsAreEstimatedAccordingToPartitionsAndProbabilityForVaryingN_K10_Four9() { + TopKEstimator estimator = new TopKEstimator(30, 0.9999); + assertEquals(10, estimator.estimateExactK(10, 1), 0.0); + assertEquals(10, estimator.estimateK(10, 1)); + assertEquals(12.087323848369289, estimator.estimateExactK(10, 2), 0.0); + assertEquals(10.230749855131009, estimator.estimateExactK(10, 3), 0.0); + assertEquals(5.794676146031378, estimator.estimateExactK(10, 10), 0.0); + assertEquals(4.152394782937266, estimator.estimateExactK(10, 20), 0.0); + } + + @Test + public void requireEstimatesAreRoundeUp() { + TopKEstimator estimator = new TopKEstimator(30, 0.9999); + assertEquals(5.794676146031378, estimator.estimateExactK(10, 10), 0.0); + assertEquals(6, estimator.estimateK(10, 10)); + } + + @Test + public void requireEstimatesAreCappedAtInputK() { + TopKEstimator estimator = new TopKEstimator(30, 0.9999); + assertEquals(12.087323848369289, estimator.estimateExactK(10, 2), 0.0); + assertEquals(10, estimator.estimateK(10, 2)); + } + + @Test + public void requireThatLargeKAreSane() { + System.out.println(dumpProbability(10, 0.05)); + TopKEstimator idealEstimator = new TopKEstimator(30, 0.9999); + TopKEstimator skewedEstimator = new TopKEstimator(30, 0.9999, 0.05); + int [] K = {10, 20, 40, 80, 100, 200, 400, 800, 1000, 2000, 4000, 8000, 10000, 20000, 40000, 80000, 100000}; + int [] expecedWithZeroSkew = {6, 9, 14, 22, 26, 42, 71, 123, 148, 268, 496, 936, 1152, 2215, 4304, 8429, 10480}; + int [] expecedWith5pSkew = {6, 10, 14, 23, 26, 43, 73, 128, 154, 280, 518, 979, 1205, 2319, 4509, 8837, 10989}; + for (int i = 0; i < K.length; i++) { + assertEquals(expecedWithZeroSkew[i], idealEstimator.estimateK(K[i], 10)); + assertEquals(expecedWith5pSkew[i], skewedEstimator.estimateK(K[i], 10)); + } + + String expected = + "Prob/Hits: 1.0000000000 0.9999000000 0.9999900000 0.9999990000 0.9999999000 0.9999999900 0.9999999990 0.9999999999\n" + + " 10: 10.000 6.000 7.000 8.000 9.000 10.000 10.000 10.000\n" + + " 20: 10.000 4.500 5.000 5.500 6.500 7.000 7.500 8.000\n" + + " 40: 10.000 3.500 4.000 4.250 4.750 5.250 5.500 6.000\n" + + " 80: 10.000 2.750 3.000 3.250 3.625 3.875 4.250 4.500\n" + + " 100: 10.000 2.600 2.800 3.100 3.300 3.600 3.900 4.200\n" + + " 200: 10.000 2.100 2.250 2.450 2.650 2.800 3.000 3.200\n" + + " 400: 10.000 1.775 1.900 2.025 2.150 2.275 2.425 2.575\n" + + " 800: 10.000 1.538 1.625 1.713 1.813 1.900 2.000 2.100\n" + + " 1000: 10.000 1.480 1.560 1.640 1.720 1.810 1.890 1.990\n" + + " 2000: 10.000 1.340 1.395 1.450 1.510 1.570 1.630 1.695\n" + + " 4000: 10.000 1.240 1.280 1.320 1.360 1.403 1.445 1.493\n" + + " 8000: 10.000 1.170 1.198 1.225 1.254 1.284 1.315 1.348\n" + + " 10000: 10.000 1.152 1.177 1.202 1.227 1.254 1.282 1.311\n" + + " 20000: 10.000 1.108 1.125 1.143 1.161 1.180 1.199 1.220\n" + + " 40000: 10.000 1.076 1.088 1.101 1.114 1.127 1.141 1.156\n" + + " 80000: 10.000 1.054 1.062 1.071 1.080 1.090 1.100 1.110\n" + + " 100000: 10.000 1.048 1.056 1.064 1.072 1.080 1.089 1.098\n"; + assertEquals(expected, dumpProbability(10, 0.0)); + String expectedSkew = + "Prob/Hits: 1.0000000000 0.9999000000 0.9999900000 0.9999990000 0.9999999000 0.9999999900 0.9999999990 0.9999999999\n" + + " 10: 10.000 6.000 7.000 8.000 9.000 10.000 10.000 10.000\n" + + " 20: 10.000 5.000 5.500 6.000 6.500 7.000 7.500 8.500\n" + + " 40: 10.000 3.500 4.000 4.500 4.750 5.250 5.750 6.250\n" + + " 80: 10.000 2.875 3.125 3.375 3.750 4.000 4.375 4.625\n" + + " 100: 10.000 2.600 2.900 3.100 3.400 3.700 4.000 4.300\n" + + " 200: 10.000 2.150 2.350 2.500 2.700 2.900 3.100 3.300\n" + + " 400: 10.000 1.825 1.950 2.075 2.225 2.350 2.500 2.650\n" + + " 800: 10.000 1.600 1.688 1.775 1.875 1.975 2.075 2.175\n" + + " 1000: 10.000 1.540 1.620 1.700 1.790 1.870 1.960 2.060\n" + + " 2000: 10.000 1.400 1.455 1.510 1.570 1.630 1.695 1.760\n" + + " 4000: 10.000 1.295 1.335 1.375 1.418 1.460 1.505 1.553\n" + + " 8000: 10.000 1.224 1.251 1.280 1.309 1.340 1.371 1.405\n" + + " 10000: 10.000 1.205 1.230 1.255 1.282 1.309 1.337 1.367\n" + + " 20000: 10.000 1.160 1.177 1.195 1.214 1.233 1.253 1.275\n" + + " 40000: 10.000 1.127 1.140 1.153 1.166 1.179 1.194 1.209\n" + + " 80000: 10.000 1.105 1.114 1.123 1.132 1.141 1.152 1.162\n" + + " 100000: 10.000 1.099 1.107 1.115 1.123 1.132 1.141 1.150\n"; + assertEquals(expectedSkew, dumpProbability(10, 0.05)); + } + + /** + * This make a table showing how many more hits will be fetched as a factor of hits requested. + * It shows how it varies with probability and hits requested for a given number of partitions. + */ + private String dumpProbability(int numPartitions, double skewFactor) { + TopKEstimator estimator = new TopKEstimator(30, 0.9999, skewFactor); + int [] K = {10, 20, 40, 80, 100, 200, 400, 800, 1000, 2000, 4000, 8000, 10000, 20000, 40000, 80000, 100000}; + double [] P = {1.0, 0.9999, 0.99999, 0.999999, 0.9999999, 0.99999999, 0.999999999, 0.9999999999}; + int n = numPartitions; + StringBuilder sb = new StringBuilder(); + sb.append(String.format("Prob/Hits:")); + for (double p : P) { + sb.append(String.format(" %1.10f", p)); + } + sb.append("\n"); + for (int k : K) { + sb.append(String.format("%9d:", k)); + for (double p : P) { + sb.append(String.format("%13.3f", (double)(estimator.estimateK(k, n, p)*n) / k)); + } + sb.append("\n"); + } + return sb.toString(); + } + } diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg index 96843d78aae..915da8dc037 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg @@ -1,4 +1,4 @@ -handler[7] +handler[8] handler[0].id com.yahoo.search.handler.SearchHandler handler[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningHandler handler[2].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningAsyncHandler @@ -6,3 +6,4 @@ handler[3].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingHandle handler[4].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingAsyncHandler handler[5].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingHandler handler[6].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingAsyncHandler +handler[7].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java new file mode 100644 index 00000000000..39d4fec2716 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java @@ -0,0 +1,28 @@ +package com.yahoo.search.query.profile.compiled; + +import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author gjoranv + */ +public class CompiledQueryProfileRegistryTest { + + @Test + public void registry_can_be_created_from_config() { + var config = new QueryProfilesConfig.Builder() + .queryprofile(new QueryProfilesConfig.Queryprofile.Builder() + .id("profile1") + .property(new QueryProfilesConfig.Queryprofile.Property.Builder() + .name("hits") + .value("5"))) + .build(); + + var registry = new CompiledQueryProfileRegistry(config); + var profile1 = registry.findQueryProfile("profile1"); + assertEquals("5", profile1.get("hits")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java index 819cd3cdfd4..a1fba8e07f1 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java @@ -1,27 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.config.test; -import com.yahoo.config.subscription.ConfigInstanceUtil; -import com.yahoo.io.IOUtils; import com.yahoo.search.Query; import com.yahoo.search.query.profile.QueryProfile; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.search.query.profile.config.QueryProfilesConfig.Queryprofile; import com.yahoo.search.test.QueryTestCase; -import com.yahoo.vespa.config.ConfigPayload; -import org.junit.Ignore; import org.junit.Test; -import static org.junit.Assert.*; -import java.io.File; -import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; + /** * @author bratseth */ diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg index a047ae1cb73..04dcbb22d5d 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg @@ -10,3 +10,4 @@ components[3].classId com.yahoo.search.query.profile.config.test.QueryProfileInt components[4].id com.yahoo.search.handler.SearchHandler components[5].id com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack components[6].id com.yahoo.search.searchchain.ExecutionFactory +components[7].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg index ef9d4490a77..b5bc450ec17 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg @@ -10,3 +10,4 @@ components[3].classId com.yahoo.search.query.profile.config.test.QueryProfileInt components[4].id com.yahoo.search.handler.SearchHandler components[5].id com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack components[6].id com.yahoo.search.searchchain.ExecutionFactory +components[7].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg index ad20005e7ad..53811bdf536 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg @@ -1,2 +1,3 @@ -handler[1] +handler[2] handler[0].id com.yahoo.search.handler.SearchHandler +handler[1].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg index ad20005e7ad..53811bdf536 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg @@ -1,2 +1,3 @@ -handler[1] +handler[2] handler[0].id com.yahoo.search.handler.SearchHandler +handler[1].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg index 8a985f92d10..e1d5c418c6a 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg @@ -22,3 +22,4 @@ components[8].classId com.yahoo.search.searchchain.config.test.twosearchers.Mult components[8].bundle twosearchers components[9].id com.yahoo.search.handler.SearchHandler components[10].id com.yahoo.container.handler.config.HandlersConfigurerDi$RegistriesHack +components[11].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index 0afe9347341..a74d1f34da0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.hosted.controller.api.integration.aws.ApplicationRoles; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -28,12 +29,14 @@ public class DeploymentData { private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private final Optional<DockerImage> dockerImageRepo; private final Optional<AthenzDomain> athenzDomain; + private final Optional<ApplicationRoles> applicationRoles; public DeploymentData(ApplicationId instance, ZoneId zone, byte[] applicationPackage, Version platform, Set<ContainerEndpoint> containerEndpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepo, - Optional<AthenzDomain> athenzDomain) { + Optional<AthenzDomain> athenzDomain, + Optional<ApplicationRoles> applicationRoles) { this.instance = requireNonNull(instance); this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); @@ -42,6 +45,7 @@ public class DeploymentData { this.endpointCertificateMetadata = requireNonNull(endpointCertificateMetadata); this.dockerImageRepo = requireNonNull(dockerImageRepo); this.athenzDomain = athenzDomain; + this.applicationRoles = applicationRoles; } public ApplicationId instance() { @@ -75,4 +79,8 @@ public class DeploymentData { public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } + + public Optional<ApplicationRoles> applicationRoles() { + return applicationRoles; + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index ca939023245..0b5f2538892 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -1,8 +1,10 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration; +import com.yahoo.vespa.hosted.controller.api.integration.aws.ApplicationRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanController; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; @@ -10,15 +12,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepo import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandler; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; +import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; -import com.yahoo.vespa.hosted.controller.api.integration.resource.TenantCost; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; @@ -58,8 +59,6 @@ public interface ServiceRegistry { CostReportConsumer costReportConsumer(); - Billing billingService(); - AwsEventFetcher eventFetcherService(); ArtifactRepository artifactRepository(); @@ -70,10 +69,14 @@ public interface ServiceRegistry { RunDataStore runDataStore(); - TenantCost tenantCost(); - ZoneRegistry zoneRegistry(); ResourceTagger resourceTagger(); + ApplicationRoleService applicationRoleService(); + + SystemMonitor systemMonitor(); + + PlanController planController(); + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ApplicationRoleService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ApplicationRoleService.java new file mode 100644 index 00000000000..e72ba5823d8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ApplicationRoleService.java @@ -0,0 +1,13 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.aws; + +import com.yahoo.config.provision.ApplicationId; + +import java.util.Optional; + +/** + * @author mortent + */ +public interface ApplicationRoleService { + Optional<ApplicationRoles> createApplicationRoles(ApplicationId applicationId); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ApplicationRoles.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ApplicationRoles.java new file mode 100644 index 00000000000..de3e84ac0c3 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ApplicationRoles.java @@ -0,0 +1,23 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.aws; + +/** + * @author mortent + */ +public class ApplicationRoles { + private final String hostRole; + private final String containerRole; + + public ApplicationRoles(String hostRole, String containerRole) { + this.hostRole = hostRole; + this.containerRole = containerRole; + } + + public String hostRole() { + return hostRole; + } + + public String containerRole() { + return containerRole; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopApplicationRoleService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopApplicationRoleService.java new file mode 100644 index 00000000000..4842389bccb --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/NoopApplicationRoleService.java @@ -0,0 +1,17 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.aws; + +import com.yahoo.config.provision.ApplicationId; + +import java.util.Optional; + +/** + * @author mortent + */ +public class NoopApplicationRoleService implements ApplicationRoleService { + + @Override + public Optional<ApplicationRoles> createApplicationRoles(ApplicationId applicationId) { + return Optional.empty(); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CostCalculator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CostCalculator.java new file mode 100644 index 00000000000..628beec8450 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CostCalculator.java @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.controller.api.integration.resource.CostInfo; + + +/** + * @author ogronnesby + */ +public interface CostCalculator { + + /** Calculate the cost for the given usage */ + CostInfo calculate(ResourceUsage usage); + + /** Estimate the cost for the given resources */ + double calculate(NodeResources resources); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Plan.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Plan.java new file mode 100644 index 00000000000..75a88136c45 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Plan.java @@ -0,0 +1,23 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +/** + * A Plan decides two different things: + * + * - How to map from usage to a sum of money that is owed. + * - Limits on how much resources can be used. + * + * @author ogronnesby + */ +public interface Plan { + + /** The ID of the plan as used in APIs and storage systems */ + String id(); + + /** The calculator used to calculate a bill for usage */ + CostCalculator calculator(); + + /** The quota limits associated with the plan */ + Object quota(); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanController.java new file mode 100644 index 00000000000..f13c251d212 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanController.java @@ -0,0 +1,10 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import com.yahoo.config.provision.TenantName; + +public interface PlanController { + + Plan getPlan(TenantName tenant); + +}
\ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/ResourceUsage.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/ResourceUsage.java new file mode 100644 index 00000000000..cbfd2b6ff50 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/ResourceUsage.java @@ -0,0 +1,54 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; + +import java.math.BigDecimal; + +/** + * @author olaa + */ +public class ResourceUsage { + + private final ApplicationId applicationId; + private final ZoneId zoneId; + private final Plan plan; + private final BigDecimal cpuMillis; + private final BigDecimal memoryMillis; + private final BigDecimal diskMillis; + + public ResourceUsage(ApplicationId applicationId, ZoneId zoneId, Plan plan, + BigDecimal cpuMillis, BigDecimal memoryMillis, BigDecimal diskMillis) { + this.applicationId = applicationId; + this.zoneId = zoneId; + this.cpuMillis = cpuMillis; + this.memoryMillis = memoryMillis; + this.diskMillis = diskMillis; + this.plan = plan; + } + + public ApplicationId getApplicationId() { + return applicationId; + } + + public ZoneId getZoneId() { + return zoneId; + } + + public BigDecimal getCpuMillis() { + return cpuMillis; + } + + public BigDecimal getMemoryMillis() { + return memoryMillis; + } + + public BigDecimal getDiskMillis() { + return diskMillis; + } + + public Plan getPlan() { + return plan; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java index 23799f48f91..b0d45b0d7bb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java @@ -25,7 +25,7 @@ public class EndpointCertificateMock implements EndpointCertificateProvider { this.dnsNames.put(applicationId, dnsNames); String endpointCertificatePrefix = String.format("vespa.tls.%s.%s.%s", applicationId.tenant(), applicationId.application(), applicationId.instance()); - return new EndpointCertificateMetadata(endpointCertificatePrefix + "-key", endpointCertificatePrefix + "-cert", 0); + return new EndpointCertificateMetadata(endpointCertificatePrefix + "-key", endpointCertificatePrefix + "-cert", 0, Optional.of("mock-id-string"), Optional.of(dnsNames), Optional.of("mockCa")); } @Override diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index b5d49df7e9c..aebfab7cbff 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMemb import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.List; @@ -76,7 +77,7 @@ public interface NodeRepository { void upgrade(ZoneId zone, NodeType type, Version version); /** Upgrade OS for all nodes of given type to a new version */ - void upgradeOs(ZoneId zone, NodeType type, Version version); + void upgradeOs(ZoneId zone, NodeType type, Version version, Optional<Duration> upgradeBudget); /** Get target versions for upgrades in given zone */ TargetVersions targetVersionsOf(ZoneId zone); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index b54446c071e..59324079e6f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -5,6 +5,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.dns; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; @@ -22,11 +23,11 @@ public class MemoryNameService implements NameService { return Collections.unmodifiableSet(records); } - private void add(Record record) { - if (records.stream().anyMatch(r -> r.type().equals(record.type()) && - r.name().equals(record.name()) && - r.data().equals(record.data()))) { - throw new IllegalArgumentException("Record already exists: " + record); + public void add(Record record) { + Optional<Record> conflict = records.stream().filter(r -> conflicts(r, record)).findFirst(); + if (conflict.isPresent()) { + throw new AssertionError("'" + record + "' conflicts with existing record '" + + conflict.get() + "'"); } records.add(record); } @@ -45,8 +46,9 @@ public class MemoryNameService implements NameService { .map(target -> new Record(Record.Type.ALIAS, name, target.asData())) .collect(Collectors.toList()); // Satisfy idempotency contract of interface - removeRecords(records); - records.forEach(this::add); + records.stream() + .filter(r -> !this.records.contains(r)) + .forEach(this::add); return records; } @@ -108,4 +110,15 @@ public class MemoryNameService implements NameService { this.records.removeAll(records); } + /** + * Returns whether record r1 and r2 can co-exist in a name service. This attempts to enforce the same constraints as + * most real name services. + */ + private static boolean conflicts(Record r1, Record r2) { + if (!r1.name().equals(r2.name())) return false; // Distinct names never conflict + if (r1.type() == Record.Type.ALIAS && r1.type() == r2.type()) // ALIAS records only require distinct data + return r1.data().equals(r2.data()); + return true; // Anything else is considered a conflict + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java index a7b9cdc5e66..c45c2c258f2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java @@ -36,14 +36,16 @@ public class NodeHistory { operator, application, system, - NodeFailer, - Rebalancer, DirtyExpirer, + DynamicProvisioningMaintainer, FailedExpirer, InactiveExpirer, + NodeFailer, ProvisionedExpirer, + Rebalancer, ReservationExpirer, - DynamicProvisioningMaintainer + RetiringUpgrader, + SpareCapacityMaintainer } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeUpgrade.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeUpgrade.java index 21c1d23ba3a..b8e2a626c72 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeUpgrade.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeUpgrade.java @@ -22,12 +22,16 @@ public class NodeUpgrade { @JsonProperty("force") private final boolean force; + @JsonProperty("upgradeBudget") + private final String upgradeBudget; + @JsonCreator public NodeUpgrade(@JsonProperty("version") String version, @JsonProperty("osVersion") String osVersion, - @JsonProperty("force") boolean force) { + @JsonProperty("force") boolean force, @JsonProperty("upgradeBudget") String upgradeBudget) { this.version = version; this.osVersion = osVersion; this.force = force; + this.upgradeBudget = upgradeBudget; } public String getVersion() { @@ -42,4 +46,8 @@ public class NodeUpgrade { return force; } + public String getUpgradeBudget() { + return upgradeBudget; + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java deleted file mode 100644 index 1e76917ebd5..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.organization; - -import com.yahoo.config.provision.ApplicationId; - -/** - * @author olaa - */ -public interface Billing { - - void handleBilling(ApplicationId applicationId, String customerId); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java deleted file mode 100644 index 84156cd9d2a..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.organization; - -import com.yahoo.config.provision.ApplicationId; - -/** - * @author olaa - */ -public class MockBilling implements Billing { - - @Override - public void handleBilling(ApplicationId applicationId, String customerId) {} - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/SystemMonitor.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/SystemMonitor.java new file mode 100644 index 00000000000..b8b5400f7b4 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/SystemMonitor.java @@ -0,0 +1,19 @@ +package com.yahoo.vespa.hosted.controller.api.integration.organization; + +import com.yahoo.component.Version; + +/** + * Montitors a Vespa controller and its system. + * + * @author jonmv + */ +public interface SystemMonitor { + + /** Notifies the monitor of the current system version and its confidence. */ + void reportSystemVersion(Version systemVersion, Confidence confidence); + + enum Confidence { + broken, low, normal, high; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MockTenantCost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MockTenantCost.java deleted file mode 100644 index fa3d28fe50c..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MockTenantCost.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.resource; - -import com.yahoo.config.provision.TenantName; - -import java.time.YearMonth; -import java.util.Collections; -import java.util.List; -import java.util.Set; - -/** - * @author olaa - */ -public class MockTenantCost implements TenantCost { - - private Set<YearMonth> monthsOfMetering = Collections.emptySet(); - private List<CostInfo> costInfoList = Collections.emptyList(); - - @Override - public Set<YearMonth> monthsWithMetering(TenantName tenantName) { - return monthsOfMetering; - } - - @Override - public List<CostInfo> getTenantCostOfPeriod(TenantName tenantName, long startTimestamp, long endTimestamp) { - return costInfoList; - } - - public void setMonthsWithMetering(Set<YearMonth> monthsOfMetering) { - this.monthsOfMetering = monthsOfMetering; - } - - public void setCostInfoList(List<CostInfo> costInfoList) { - this.costInfoList = costInfoList; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/TenantCost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/TenantCost.java deleted file mode 100644 index a2a91454366..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/TenantCost.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.resource; - -import com.yahoo.config.provision.TenantName; - -import java.time.LocalDate; -import java.time.YearMonth; -import java.time.temporal.ChronoUnit; -import java.util.Collections; -import java.util.List; -import java.util.Set; - -/** - * @author olaa - */ -public interface TenantCost { - - Set<YearMonth> monthsWithMetering(TenantName tenantName); - - List<CostInfo> getTenantCostOfPeriod(TenantName tenantName, long startTimestamp, long endTimestamp); - - default List<CostInfo> getTenantCostOfMonth(TenantName tenantName, YearMonth month) { - return getTenantCostOfPeriod(tenantName, getMonthStartTimeStamp(month), getMonthEndTimeStamp(month)); - } - - static TenantCost empty() { - return new TenantCost() { - @Override - public Set<YearMonth> monthsWithMetering(TenantName tenantName) { - return Collections.emptySet(); - } - - @Override - public List<CostInfo> getTenantCostOfPeriod(TenantName tenantName, long startTime, long endTime) { - return Collections.emptyList(); - } - }; - } - - private long getMonthStartTimeStamp(YearMonth month) { - LocalDate startOfMonth = LocalDate.of(month.getYear(), month.getMonth(), 1); - return startOfMonth.atStartOfDay(java.time.ZoneId.of("UTC")) - .toInstant() - .toEpochMilli(); - } - private long getMonthEndTimeStamp(YearMonth month) { - LocalDate startOfMonth = LocalDate.of(month.getYear(), month.getMonth(), 1); - return startOfMonth.plus(1, ChronoUnit.MONTHS) - .atStartOfDay(java.time.ZoneId.of("UTC")) - .toInstant() - .toEpochMilli(); - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/DummySystemMonitor.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/DummySystemMonitor.java new file mode 100644 index 00000000000..e2e5fe9e155 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/DummySystemMonitor.java @@ -0,0 +1,14 @@ +package com.yahoo.vespa.hosted.controller.api.integration.stubs; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor; + +/** + * @author jonmv + */ +public class DummySystemMonitor implements SystemMonitor { + + @Override + public void reportSystemVersion(Version systemVersion, Confidence confidence) { } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java index 295f8e8fd98..dfdd273b6f5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java @@ -75,4 +75,8 @@ public class MockUserManagement implements UserManagement { return List.copyOf(get(role)); } + @Override + public List<Role> listRoles(UserId userId) { + return List.of(); + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java index 578f516f01e..a0c73fa7ff8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java @@ -36,6 +36,7 @@ public class Roles { String[] parts = value.split("\\."); if (parts.length == 1 && parts[0].equals("hostedOperator")) return Role.hostedOperator(); if (parts.length == 1 && parts[0].equals("hostedSupporter")) return Role.hostedSupporter(); + if (parts.length == 1 && parts[0].equals("hostedAccountant")) return Role.hostedAccountant(); if (parts.length == 2) return toRole(TenantName.from(parts[0]), parts[1]); if (parts.length == 3) return toRole(TenantName.from(parts[0]), ApplicationName.from(parts[1]), parts[2]); throw new IllegalArgumentException("Malformed or illegal role value '" + value + "'."); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/User.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/User.java index 7ad8a8a4197..dcce25bda95 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/User.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/User.java @@ -8,7 +8,7 @@ import java.util.Objects; */ public class User { - public static final String ATTRIBUTE_NAME = User.class.getName(); + public static final String ATTRIBUTE_NAME = "vespa.user.attributes"; private final String email; private final String name; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java index 8a549b505c7..bfb617a75b6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java @@ -34,4 +34,6 @@ public interface UserManagement { /** Returns all users in the given role, or throws if the role does not exist. */ List<User> listUsers(Role role); + /** Returns all roles of which the given user is part, or throws if the user does not exist */ + List<Role> listRoles(UserId user); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 2ed4c057dfd..68dff26529f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -22,7 +22,8 @@ enum PathGroup { /** Paths exclusive to operators (including read), used for system management. */ classifiedOperator(PathPrefix.api, - "/configserver/v1/{*}"), + "/configserver/v1/{*}", + "/deployment/v1/{*}"), /** Paths used for system management by operators. */ operator(PathPrefix.none, @@ -56,8 +57,6 @@ enum PathGroup { tenantInfo(Matcher.tenant, PathPrefix.api, "/application/v4/tenant/{tenant}/application/", - "/application/v4/tenant/{tenant}/cost", - "/application/v4/tenant/{tenant}/cost/{date}", "/routing/v1/status/tenant/{tenant}/{*}"), tenantKeys(Matcher.tenant, @@ -201,15 +200,11 @@ enum PathGroup { "/", "/d/{*}"), - /** Same as classifiedInfo, but with optional /api prefix */ - classifiedApiInfo(PathPrefix.api, - "/deployment/v1/{*}", - "/user/v1/user"), - /** Paths providing public information. */ publicInfo(PathPrefix.api, - "/badge/v1/{*}", - "/zone/v1/{*}"), + "/user/v1/user", // Information about who you are. + "/badge/v1/{*}", // Badges for deployment jobs. + "/zone/v1/{*}"), // Lists environment and regions. /** Paths used for deploying system-wide feature flags. */ systemFlagsDeploy(PathPrefix.none, "/system-flags/v1/deploy"), @@ -221,8 +216,10 @@ enum PathGroup { /** Paths used for receiving payment callbacks */ paymentProcessor(PathPrefix.none, "/payment/notification"), - /** Invoice management */ - invoiceManagement(PathPrefix.none, "/billing/v1/invoice"); + /** Paths used for invoice management */ + hostedAccountant(PathPrefix.api, + "/billing/v1/invoice/{*}", + "/billing/v1/billing"); final List<String> pathSpecs; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index 0afa0668a00..9a5a0ad0e77 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -22,8 +22,11 @@ enum Policy { /** Full access to everything. */ operator(Privilege.grant(Action.all()) - .on(PathGroup.all()) - .in(SystemName.all())), + .on(PathGroup.allExcept(PathGroup.hostedAccountant)) + .in(SystemName.all()), + Privilege.grant(Action.read) + .on(PathGroup.hostedAccountant) + .in(SystemName.PublicCd)), /** Full access to everything. */ supporter(Privilege.grant(Action.read) @@ -45,15 +48,10 @@ enum Policy { .on(PathGroup.user) .in(SystemName.main, SystemName.cd, SystemName.dev)), - /** Access to create a tenant in select systems. */ + /** Access to create a tenant. */ tenantCreate(Privilege.grant(Action.create) .on(PathGroup.tenant) - .in(SystemName.main, SystemName.cd, SystemName.dev)), // TODO SystemName.all() - - /** Access to create a tenant in public */ - tenantCreatePublic(Privilege.grant(Action.create) - .on(PathGroup.tenant) - .in(SystemName.PublicCd, SystemName.Public)), + .in(SystemName.all())), /** Full access to tenant information and settings. */ tenantDelete(Privilege.grant(Action.delete) @@ -120,10 +118,6 @@ enum Policy { .on(PathGroup.allExcept(PathGroup.classifiedOperator)) .in(SystemName.main, SystemName.cd, SystemName.dev)), - classifiedApiRead(Privilege.grant(Action.read) - .on(PathGroup.classifiedApiInfo) - .in(SystemName.all())), - /** Read access to public info. */ publicRead(Privilege.grant(Action.read) .on(PathGroup.publicInfo) @@ -167,6 +161,11 @@ enum Policy { /** Read the generated bills */ billingInformationRead(Privilege.grant(Action.read) .on(PathGroup.billingList) + .in(SystemName.PublicCd)), + + /** Invoice management */ + hostedAccountant(Privilege.grant(Action.all()) + .on(PathGroup.hostedAccountant) .in(SystemName.PublicCd)); private final Set<Privilege> privileges; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java index d3c5e412215..90350de5dbd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java @@ -76,6 +76,9 @@ public abstract class Role { /** Returns the role of the payment processor */ public static UnboundRole paymentProcessor() { return new UnboundRole(RoleDefinition.paymentProcessor); } + /** Returns the role of the invoice manager */ + public static UnboundRole hostedAccountant() { return new UnboundRole(RoleDefinition.hostedAccountant); } + /** Returns the role definition of this bound role. */ public RoleDefinition definition() { return roleDefinition; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index 438e79bcc4f..b9d534019db 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -22,12 +22,10 @@ public enum RoleDefinition { hostedOperator(Policy.operator), /** Machina autem exspiravit. */ - hostedSupporter(Policy.supporter, - Policy.tenantCreatePublic), + hostedSupporter(Policy.supporter), /** Base role which every user is part of. */ everyone(Policy.classifiedRead, - Policy.classifiedApiRead, Policy.publicRead, Policy.user, Policy.tenantCreate), @@ -63,6 +61,7 @@ public enum RoleDefinition { /** Admin — the administrative function for user management etc. */ administrator(Policy.tenantUpdate, Policy.tenantManager, + Policy.tenantDelete, Policy.applicationManager, Policy.paymentInstrumentRead, Policy.paymentInstrumentUpdate, @@ -89,7 +88,9 @@ public enum RoleDefinition { systemFlagsDryrunner(Policy.systemFlagsDryrun), - paymentProcessor(Policy.paymentProcessor); + paymentProcessor(Policy.paymentProcessor), + + hostedAccountant(Policy.hostedAccountant); private final Set<RoleDefinition> parents; private final Set<Policy> policies; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index 1e42efdd256..e6310cc6432 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -206,6 +206,9 @@ public class SystemFlagsDataArchive { // Throws exception if not recognized NodeType.valueOf(nodeTypeString); }); + } else if (dimension.isEqualTo(DimensionHelper.toWire(FetchVector.Dimension.CONSOLE_USER_EMAIL))) { + condition.get("values").forEachArrayElement(conditionValue -> conditionValue.asString() + .orElseThrow(() -> new IllegalArgumentException("Non-string email address: " + conditionValue))); } })); } diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index 2da93c5ceca..10d4732984c 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import org.junit.Test; -import java.awt.event.AdjustmentEvent; import java.net.URI; import java.util.List; import java.util.stream.Stream; @@ -59,8 +58,9 @@ public class RoleTest { assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/application/v4/tenant/t2/application/a2"))); assertFalse(mainEnforcer.allows(role, Action.delete, URI.create("/application/v4/tenant/t8/application/a6/instance/i1/environment/dev/region/r1"))); - // Check that we are allowed to create tenants in public - assertTrue(publicEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t1"))); + // Check that we are allowed to create tenants in public. + // hostedSupporter isn't actually allowed to create tenants - but any logged in user will be a member of the "everyone" role. + assertTrue(publicEnforcer.allows(Role.everyone(), Action.create, URI.create("/application/v4/tenant/t1"))); } @Test diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index 35f6794e28d..fad1e944e36 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -200,6 +200,30 @@ public class SystemFlagsDataArchiveTest { } } + @Test + public void normalize_json_fail_on_invalid_email() { + try { + SystemFlagsDataArchive.normalizeJson("{\n" + + " \"id\": \"foo\",\n" + + " \"rules\": [\n" + + " {\n" + + " \"conditions\": [\n" + + " {\n" + + " \"type\": \"whitelist\",\n" + + " \"dimension\": \"console-user-email\",\n" + + " \"values\": [ 123 ]\n" + + " }\n" + + " ],\n" + + " \"value\": true\n" + + " }\n" + + " ]\n" + + "}\n"); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Non-string email address: 123", e.getMessage()); + } + } + private static void assertArchiveReturnsCorrectTestFlagDataForTarget(SystemFlagsDataArchive archive) { assertFlagDataHasValue(archive, MY_TEST_FLAG, mainControllerTarget, "main.controller"); assertFlagDataHasValue(archive, MY_TEST_FLAG, prodUsWestCfgTarget, "main.prod.us-west-1"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e02c1b9f5dd..d644cf21638 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; @@ -30,6 +31,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; +import com.yahoo.vespa.hosted.controller.api.integration.aws.ApplicationRoles; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; @@ -51,11 +53,11 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; +import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificateManager; import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; -import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificateManager; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; @@ -113,6 +115,7 @@ public class ApplicationController { private final ApplicationPackageValidator applicationPackageValidator; private final EndpointCertificateManager endpointCertificateManager; private final StringFlag dockerImageRepoFlag; + private final BooleanFlag provisionApplicationRoles; ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, Clock clock, SecretStore secretStore, FlagSource flagSource) { @@ -125,6 +128,7 @@ public class ApplicationController { this.artifactRepository = controller.serviceRegistry().artifactRepository(); this.applicationStore = controller.serviceRegistry().applicationStore(); this.dockerImageRepoFlag = Flags.DOCKER_IMAGE_REPO.bindTo(flagSource); + this.provisionApplicationRoles = Flags.PROVISION_APPLICATION_ROLES.bindTo(flagSource); deploymentTrigger = new DeploymentTrigger(controller, clock); applicationPackageValidator = new ApplicationPackageValidator(controller); @@ -298,6 +302,7 @@ public class ApplicationController { try (Lock deploymentLock = lockForDeployment(job.application(), zone)) { Set<ContainerEndpoint> endpoints; Optional<EndpointCertificateMetadata> endpointCertificateMetadata; + Optional<ApplicationRoles> applicationRoles = Optional.empty(); Run run = controller.jobController().last(job) .orElseThrow(() -> new IllegalStateException("No known run of '" + job + "'")); @@ -325,13 +330,23 @@ public class ApplicationController { && run.testerCertificate().isPresent()) applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get()); - endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, zone); + endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, zone, applicationPackage.deploymentSpec().instance(instance.name())); endpoints = controller.routing().registerEndpointsInDns(application.get(), job.application().instance(), zone); + + // Provision application roles if enabled for the zone + if (provisionApplicationRoles.with(FetchVector.Dimension.ZONE_ID, zone.value()).value()) { + try { + applicationRoles = controller.serviceRegistry().applicationRoleService().createApplicationRoles(instance.id()); + } catch (Exception e) { + log.log(Level.SEVERE, "Exception creating application roles for application: " + instance.id(), e); + throw new RuntimeException("Unable to provision iam roles for application"); + } + } } // Release application lock while doing the deployment, which is a lengthy task. // Carry out deployment without holding the application lock. - ActivateResult result = deploy(job.application(), applicationPackage, zone, platform, endpoints, endpointCertificateMetadata); + ActivateResult result = deploy(job.application(), applicationPackage, zone, platform, endpoints, endpointCertificateMetadata, applicationRoles); lockApplicationOrThrow(applicationId, application -> store(application.with(job.application().instance(), @@ -398,14 +413,15 @@ public class ApplicationController { validateRun(application.get().require(instance), zone, platformVersion, applicationVersion); } - endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(application.get().require(instance), zone); + endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata( + application.get().require(instance), zone, applicationPackage.deploymentSpec().instance(instance)); endpoints = controller.routing().registerEndpointsInDns(application.get(), instance, zone); } // Release application lock while doing the deployment, which is a lengthy task. // Carry out deployment without holding the application lock. ActivateResult result = deploy(instanceId, applicationPackage, zone, platformVersion, - endpoints, endpointCertificateMetadata); + endpoints, endpointCertificateMetadata, Optional.empty()); lockApplicationOrThrow(applicationId, application -> store(application.with(instanceId.instance(), @@ -477,7 +493,7 @@ public class ApplicationController { ApplicationPackage applicationPackage = new ApplicationPackage( artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); - return deploy(application.id(), applicationPackage, zone, version, Set.of(), /* No application cert */ Optional.empty()); + return deploy(application.id(), applicationPackage, zone, version, Set.of(), /* No application cert */ Optional.empty(), Optional.empty()); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -485,12 +501,13 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, Version platform) { - return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), /* No application cert for tester*/ Optional.empty()); + return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), /* No application cert for tester*/ Optional.empty(), Optional.empty()); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, - Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { + Optional<EndpointCertificateMetadata> endpointCertificateMetadata, + Optional<ApplicationRoles> applicationRoles) { try { Optional<DockerImage> dockerImageRepo = Optional.ofNullable( dockerImageRepoFlag @@ -506,7 +523,7 @@ public class ApplicationController { ConfigServer.PreparedApplication preparedApplication = configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, - endpoints, endpointCertificateMetadata, dockerImageRepo, domain)); + endpoints, endpointCertificateMetadata, dockerImageRepo, domain, applicationRoles)); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 297f3fcf218..15ab14e3241 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -26,12 +26,14 @@ import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; +import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.time.Clock; +import java.time.Duration; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -188,33 +190,37 @@ public class Controller extends AbstractComponent { /** Returns the target OS version for infrastructure in this system. The controller will drive infrastructure OS * upgrades to this version */ - public Optional<OsVersion> osVersion(CloudName cloud) { - return osVersions().stream().filter(osVersion -> osVersion.cloud().equals(cloud)).findFirst(); + public Optional<OsVersionTarget> osVersionTarget(CloudName cloud) { + return osVersionTargets().stream().filter(target -> target.osVersion().cloud().equals(cloud)).findFirst(); } /** Returns all target OS versions in this system */ - public Set<OsVersion> osVersions() { - return curator.readOsVersions(); + public Set<OsVersionTarget> osVersionTargets() { + return curator.readOsVersionTargets(); } /** Set the target OS version for infrastructure on cloud in this system */ - public void upgradeOsIn(CloudName cloud, Version version, boolean force) { + public void upgradeOsIn(CloudName cloudName, Version version, Optional<Duration> upgradeBudget, boolean force) { if (version.isEmpty()) { throw new IllegalArgumentException("Invalid version '" + version.toFullString() + "'"); } - if (!clouds().contains(cloud)) { - throw new IllegalArgumentException("Cloud '" + cloud.value() + "' does not exist in this system"); + Set<CloudName> clouds = clouds(); + if (!clouds.contains(cloudName)) { + throw new IllegalArgumentException("Cloud '" + cloudName + "' does not exist in this system"); + } + if (!zoneRegistry.zones().ofCloud(cloudName).reprovisionToUpgradeOs().ids().isEmpty() && upgradeBudget.isEmpty()) { + throw new IllegalArgumentException("Cloud '" + cloudName.value() + "' requires a time budget for OS upgrades"); } try (Lock lock = curator.lockOsVersions()) { - Set<OsVersion> versions = new TreeSet<>(curator.readOsVersions()); - if (!force && versions.stream().anyMatch(osVersion -> osVersion.cloud().equals(cloud) && - osVersion.version().isAfter(version))) { - throw new IllegalArgumentException("Cannot downgrade cloud '" + cloud.value() + "' to version " + + Set<OsVersionTarget> targets = new TreeSet<>(curator.readOsVersionTargets()); + if (!force && targets.stream().anyMatch(target -> target.osVersion().cloud().equals(cloudName) && + target.osVersion().version().isAfter(version))) { + throw new IllegalArgumentException("Cannot downgrade cloud '" + cloudName.value() + "' to version " + version.toFullString()); } - versions.removeIf(osVersion -> osVersion.cloud().equals(cloud)); // Only allow a single target per cloud - versions.add(new OsVersion(version, cloud)); - curator.writeOsVersions(versions); + targets.removeIf(target -> target.osVersion().cloud().equals(cloudName)); // Only allow a single target per cloud + targets.add(new OsVersionTarget(new OsVersion(version, cloudName), upgradeBudget)); + curator.writeOsVersionTargets(targets); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 463ffb58460..c441188b1be 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointList; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; @@ -84,13 +85,14 @@ public class RoutingController { /** Returns zone-scoped endpoints for given deployment */ public EndpointList endpointsOf(DeploymentId deployment) { var endpoints = new LinkedHashSet<Endpoint>(); + boolean isSystemApplication = SystemApplication.matching(deployment.applicationId()).isPresent(); // Avoid reading application more than once per call to this var application = Suppliers.memoize(() -> controller.applications().requireApplication(TenantAndApplicationId.from(deployment.applicationId()))); for (var policy : routingPolicies.get(deployment).values()) { if (!policy.status().isActive()) continue; for (var routingMethod : controller.zoneRegistry().routingMethods(policy.id().zone())) { - if (routingMethod.isDirect() && !canRouteDirectlyTo(deployment, application.get())) continue; - endpoints.add(policy.endpointIn(controller.system(), routingMethod)); + if (routingMethod.isDirect() && !isSystemApplication && !canRouteDirectlyTo(deployment, application.get())) continue; + endpoints.add(policy.endpointIn(controller.system(), routingMethod, controller.zoneRegistry())); } } return EndpointList.copyOf(endpoints); @@ -98,6 +100,7 @@ public class RoutingController { /** Returns global-scoped endpoints for given instance */ public EndpointList endpointsOf(ApplicationId instance) { + if (SystemApplication.matching(instance).isPresent()) return EndpointList.copyOf(List.of()); return endpointsOf(controller.applications().requireApplication(TenantAndApplicationId.from(instance)), instance.instance()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index ed5add8b98a..f8982c96637 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -36,20 +36,18 @@ public class Endpoint { private final RoutingMethod routingMethod; private final boolean tls; - private Endpoint(String name, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system, - Port port, boolean legacy, RoutingMethod routingMethod) { + private Endpoint(String name, URI url, List<ZoneId> zones, Scope scope, Port port, boolean legacy, + RoutingMethod routingMethod) { Objects.requireNonNull(name, "name must be non-null"); - Objects.requireNonNull(application, "application must be non-null"); Objects.requireNonNull(zones, "zones must be non-null"); Objects.requireNonNull(scope, "scope must be non-null"); - Objects.requireNonNull(system, "system must be non-null"); Objects.requireNonNull(port, "port must be non-null"); Objects.requireNonNull(routingMethod, "routingMethod must be non-null"); if (scope == Scope.zone && zones.size() != 1) { throw new IllegalArgumentException("A single zone must be given for zone-scoped endpoints"); } this.name = name; - this.url = createUrl(name, application, zones, scope, system, port, legacy, routingMethod); + this.url = url; this.zones = List.copyOf(zones); this.scope = scope; this.legacy = legacy; @@ -57,6 +55,20 @@ public class Endpoint { this.tls = port.tls; } + private Endpoint(String name, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system, + Port port, boolean legacy, RoutingMethod routingMethod) { + this(name, + createUrl(name, + Objects.requireNonNull(application, "application must be non-null"), + zones, + scope, + Objects.requireNonNull(system, "system must be non-null"), + port, + legacy, + routingMethod), + zones, scope, port, legacy, routingMethod); + } + /** * Returns the name of this endpoint (the first component of the DNS name). Depending on the endpoint type, this * can be one of the following: @@ -286,6 +298,14 @@ public class Endpoint { return new EndpointBuilder(application); } + /** Create an endpoint for given system application */ + public static Endpoint of(SystemApplication systemApplication, ZoneId zone, URI url) { + if (!systemApplication.hasEndpoint()) throw new IllegalArgumentException(systemApplication + " has no endpoint"); + RoutingMethod routingMethod = RoutingMethod.exclusive; + Port port = url.getPort() == -1 ? Port.tls() : Port.tls(url.getPort()); // System application endpoints are always TLS + return new Endpoint("", url, List.of(zone), Scope.zone, port, false, routingMethod); + } + public static class EndpointBuilder { private final ApplicationId application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index cc72074295f..c6046751696 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -21,13 +21,16 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapaci import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; +/** + * @author jonmv + */ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceList> { - private final Map<ApplicationId, DeploymentStatus> statuses; + private final Map<ApplicationId, DeploymentStatus> instances; - private InstanceList(Collection<? extends ApplicationId> items, boolean negate, Map<ApplicationId, DeploymentStatus> statuses) { - super(items, negate, (i, n) -> new InstanceList(i, n, statuses)); - this.statuses = statuses; + private InstanceList(Collection<? extends ApplicationId> items, boolean negate, Map<ApplicationId, DeploymentStatus> instances) { + super(items, negate, (i, n) -> new InstanceList(i, n, instances)); + this.instances = instances; } public static InstanceList from(DeploymentStatusList statuses) { @@ -53,9 +56,9 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances that are allowed to upgrade to the given version at the given time */ public InstanceList canUpgradeAt(Version version, Instant instant) { - return matching(id -> statuses.get(id).instanceSteps().get(id.instance()) - .readyAt(Change.of(version)) - .map(readyAt -> ! readyAt.isAfter(instant)).orElse(false)); + return matching(id -> instances.get(id).instanceSteps().get(id.instance()) + .readyAt(Change.of(version)) + .map(readyAt -> ! readyAt.isAfter(instant)).orElse(false)); } /** Returns the subset of instances which have at least one productiog deployment */ @@ -63,15 +66,22 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL return matching(id -> instance(id).productionDeployments().size() > 0); } - /** Returns the subset of instances which have at least one deployment on a lower version than the given one */ + /** Returns the subset of instances which contain declared jobs */ + public InstanceList withDeclaredJobs() { + return matching(id -> instances.get(id).jobSteps().values().stream() + .anyMatch(job -> job.isDeclared() && job.job().get().application().equals(id))); + } + + /** Returns the subset of instances which have at least one deployment on a lower version than the given one, or which have no production deployments */ public InstanceList onLowerVersionThan(Version version) { - return matching(id -> instance(id).productionDeployments().values().stream() - .anyMatch(deployment -> deployment.version().isBefore(version))); + return matching(id -> instance(id).productionDeployments().isEmpty() + || instance(id).productionDeployments().values().stream() + .anyMatch(deployment -> deployment.version().isBefore(version))); } /** Returns the subset of instances which have changes left to deploy; blocked, or deploying */ public InstanceList withChanges() { - return matching(id -> instance(id).change().hasTargets() || statuses.get(id).outstandingChange(id.instance()).hasTargets()); + return matching(id -> instance(id).change().hasTargets() || instances.get(id).outstandingChange(id.instance()).hasTargets()); } /** Returns the subset of instances which are currently deploying a change */ @@ -81,9 +91,9 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which currently have failing jobs on the given version */ public InstanceList failingOn(Version version) { - return matching(id -> ! statuses.get(id).instanceJobs().get(id).failing() - .not().withStatus(outOfCapacity) - .lastCompleted().on(version).isEmpty()); + return matching(id -> ! instances.get(id).instanceJobs().get(id).failing() + .not().withStatus(outOfCapacity) + .lastCompleted().on(version).isEmpty()); } /** Returns the subset of instances which are not pinned to a certain Vespa version. */ @@ -93,12 +103,12 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which are not currently failing any jobs. */ public InstanceList failing() { - return matching(id -> ! statuses.get(id).instanceJobs().get(id).failing().not().withStatus(outOfCapacity).isEmpty()); + return matching(id -> ! instances.get(id).instanceJobs().get(id).failing().not().withStatus(outOfCapacity).isEmpty()); } /** Returns the subset of instances which are currently failing an upgrade. */ public InstanceList failingUpgrade() { - return matching(id -> ! statuses.get(id).instanceJobs().get(id).failing().not().failingApplicationChange().isEmpty()); + return matching(id -> ! instances.get(id).instanceJobs().get(id).failing().not().failingApplicationChange().isEmpty()); } /** Returns the subset of instances which are upgrading (to any version), not considering block windows. */ @@ -123,7 +133,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which started failing on the given version */ public InstanceList startedFailingOn(Version version) { - return matching(id -> ! statuses.get(id).instanceJobs().get(id).firstFailing().on(version).isEmpty()); + return matching(id -> ! instances.get(id).instanceJobs().get(id).firstFailing().on(version).isEmpty()); } /** Returns this list sorted by increasing oldest production deployment version. Applications without any deployments are ordered first. */ @@ -135,7 +145,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL } private Application application(ApplicationId id) { - return statuses.get(id).application(); + return instances.get(id).application(); } private Instance instance(ApplicationId id) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 66015c76b06..e1acf867744 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -8,7 +8,9 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -63,16 +65,35 @@ public enum SystemApplication { .orElse(false); } - /** Returns whether this should receive OS upgrades */ - public boolean shouldUpgradeOs() { + /** Returns whether this should receive OS upgrades in given cloud */ + public boolean shouldUpgradeOsIn(ZoneId zone, Controller controller) { + if (controller.zoneRegistry().zones().reprovisionToUpgradeOs().ids().contains(zone)) { + return nodeType == NodeType.host; // TODO(mpolden): Remove once all node types are supported + } return nodeType.isDockerHost(); } + /** Returns whether this has an endpoint */ + public boolean hasEndpoint() { + return this == configServer; + } + + /** Returns the endpoint of this, if any */ + public Optional<Endpoint> endpointIn(ZoneId zone, ZoneRegistry zoneRegistry) { + if (!hasEndpoint()) return Optional.empty(); + return Optional.of(Endpoint.of(this, zone, zoneRegistry.getConfigServerVipUri(zone))); + } + /** All known system applications */ public static List<SystemApplication> all() { return List.of(values()); } + /** Returns the system application matching given id, if any */ + public static Optional<SystemApplication> matching(ApplicationId id) { + return Arrays.stream(values()).filter(app -> app.id().equals(id)).findFirst(); + } + @Override public String toString() { return String.format("system application %s of type %s", id, nodeType); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index ae9d35e6790..c8cce94d479 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -64,12 +64,12 @@ public class AthenzFacade implements AccessControl { this.service = factory.getControllerIdentity(); this.userDomains = factory.cacheLookups() ? CacheBuilder.newBuilder() - .expireAfterWrite(10, TimeUnit.MINUTES) + .expireAfterWrite(10, TimeUnit.SECONDS) .build(CacheLoader.from(this::getUserDomains))::getUnchecked : this::getUserDomains; this.accessRights = factory.cacheLookups() ? CacheBuilder.newBuilder() - .expireAfterWrite(10, TimeUnit.MINUTES) + .expireAfterWrite(10, TimeUnit.SECONDS) .build(CacheLoader.from(this::lookupAccess))::getUnchecked : this::lookupAccess; } @@ -247,6 +247,10 @@ public class AthenzFacade implements AccessControl { return hasAccess("callback", new AthenzResourceName(service.getDomain().getName(), "payment-notification-resource").toResourceNameString(), identity); } + public boolean hasAccountingAccess(AthenzIdentity identity) { + return hasAccess("modify", new AthenzResourceName(service.getDomain().getName(), "hosted-accounting-resource").toResourceNameString(), identity); + } + /** * Used when creating tenancies. As there are no tenancy policies at this point, * we cannot use {@link #hasTenantAdminAccess(AthenzIdentity, AthenzDomain)} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index 7e0fec4ba66..64549825b04 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -3,15 +3,20 @@ package com.yahoo.vespa.hosted.controller.certificate; import com.google.common.collect.Sets; import com.google.common.hash.Hashing; import com.google.common.io.BaseEncoding; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; + +import java.util.LinkedHashSet; import java.util.logging.Level; + import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.flags.BooleanFlag; @@ -91,16 +96,16 @@ public class EndpointCertificateManager { }, 1, 10, TimeUnit.MINUTES); } - public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone) { + public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone, Optional<DeploymentInstanceSpec> instanceSpec) { var t0 = Instant.now(); - Optional<EndpointCertificateMetadata> metadata = getOrProvision(instance, zone); + Optional<EndpointCertificateMetadata> metadata = getOrProvision(instance, zone, instanceSpec); Duration duration = Duration.between(t0, Instant.now()); if (duration.toSeconds() > 30) log.log(Level.INFO, String.format("Getting endpoint certificate metadata for %s took %d seconds!", instance.id().serializedForm(), duration.toSeconds())); return metadata; } @NotNull - private Optional<EndpointCertificateMetadata> getOrProvision(Instance instance, ZoneId zone) { + private Optional<EndpointCertificateMetadata> getOrProvision(Instance instance, ZoneId zone, Optional<DeploymentInstanceSpec> instanceSpec) { boolean endpointCertInSharedRouting = this.endpointCertInSharedRouting.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value(); if (!zoneRegistry.zones().directlyRouted().ids().contains(zone) && !endpointCertInSharedRouting) return Optional.empty(); @@ -108,7 +113,7 @@ public class EndpointCertificateManager { final var currentCertificateMetadata = curator.readEndpointCertificateMetadata(instance.id()); if (currentCertificateMetadata.isEmpty()) { - var provisionedCertificateMetadata = provisionEndpointCertificate(instance, Optional.empty()); + var provisionedCertificateMetadata = provisionEndpointCertificate(instance, Optional.empty(), zone, instanceSpec); // We do not verify the certificate if one has never existed before - because we do not want to // wait for it to be available before we deploy. This allows the config server to start // provisioning nodes ASAP, and the risk is small for a new deployment. @@ -118,9 +123,9 @@ public class EndpointCertificateManager { // Reprovision certificate if it is missing SANs for the zone we are deploying to var sansInCertificate = currentCertificateMetadata.get().requestedDnsSans(); - var requiredSansForZone = dnsNamesOf(instance.id(), List.of(zone)); + var requiredSansForZone = dnsNamesOf(instance.id(), zone); if (sansInCertificate.isPresent() && !sansInCertificate.get().containsAll(requiredSansForZone)) { - var reprovisionedCertificateMetadata = provisionEndpointCertificate(instance, currentCertificateMetadata); + var reprovisionedCertificateMetadata = provisionEndpointCertificate(instance, currentCertificateMetadata, zone, instanceSpec); curator.writeEndpointCertificateMetadata(instance.id(), reprovisionedCertificateMetadata); // Verification is unlikely to succeed in this case, as certificate must be available first - controller will retry validateEndpointCertificate(reprovisionedCertificateMetadata, instance, zone); @@ -206,9 +211,41 @@ public class EndpointCertificateManager { } } - private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance, Optional<EndpointCertificateMetadata> currentMetadata) { - List<ZoneId> zones = zoneRegistry.zones().controllerUpgraded().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList()); - return endpointCertificateProvider.requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones), currentMetadata); + private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance, Optional<EndpointCertificateMetadata> currentMetadata, ZoneId deploymentZone, Optional<DeploymentInstanceSpec> instanceSpec) { + + List<String> currentlyPresentNames = currentMetadata.isPresent() ? + currentMetadata.get().requestedDnsSans().orElseThrow(() -> new RuntimeException("Certificate metadata exists but SANs are not present!")) + : Collections.emptyList(); + + var requiredZones = new LinkedHashSet<>(Set.of(deploymentZone)); + + var zoneCandidateList = zoneRegistry.zones().controllerUpgraded().zones().stream().map(ZoneApi::getId).collect(Collectors.toList()); + + // If not deploying to a dev or perf zone, require all prod zones in deployment spec + test and staging + if (!deploymentZone.environment().isManuallyDeployed()) { + zoneCandidateList.stream() + .filter(z -> z.environment().isTest() || instanceSpec.isPresent() && instanceSpec.get().deploysTo(z.environment(), z.region())) + .forEach(requiredZones::add); + } + + var requiredNames = requiredZones.stream() + .flatMap(zone -> dnsNamesOf(instance.id(), zone).stream()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + + // Make sure all currently present names will remain present. + // Instead of just adding "currently present names", we regenerate them in case the names for a zone have changed. + zoneCandidateList.stream() + .map(zone -> dnsNamesOf(instance.id(), zone)) + .filter(zoneNames -> zoneNames.stream().anyMatch(currentlyPresentNames::contains)) + .filter(currentlyPresentNames::containsAll) + .forEach(requiredNames::addAll); + + // This check must be relaxed if we ever remove from the set of names generated. + if (!requiredNames.containsAll(currentlyPresentNames)) + throw new RuntimeException("SANs to be requested do not cover all existing names! Missing names: " + + currentlyPresentNames.stream().filter(s -> !requiredNames.contains(s)).collect(Collectors.joining(", "))); + + return endpointCertificateProvider.requestCaSignedCertificate(instance.id(), List.copyOf(requiredNames), currentMetadata); } private void validateEndpointCertificate(EndpointCertificateMetadata endpointCertificateMetadata, Instance instance, ZoneId zone) { @@ -243,7 +280,7 @@ public class EndpointCertificateManager { .filter(san -> san.getType().equals(SubjectAlternativeName.Type.DNS_NAME)) .map(SubjectAlternativeName::getValue).collect(Collectors.toSet()); - var dnsNamesOfZone = dnsNamesOf(instance.id(), List.of(zone)); + var dnsNamesOfZone = dnsNamesOf(instance.id(), zone); if (!subjectAlternativeNames.containsAll(dnsNamesOfZone)) throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate is missing required SANs for zone " + zone.value()); @@ -259,22 +296,24 @@ public class EndpointCertificateManager { } } - private List<String> dnsNamesOf(ApplicationId applicationId, List<ZoneId> zones) { + private List<String> dnsNamesOf(ApplicationId applicationId, ZoneId zone) { List<String> endpointDnsNames = new ArrayList<>(); // We add first an endpoint name based on a hash of the applicationId, // as the certificate provider requires the first CN to be < 64 characters long. endpointDnsNames.add(commonNameHashOf(applicationId, zoneRegistry.system())); - var globalDefaultEndpoint = Endpoint.of(applicationId).named(EndpointId.defaultId()); - var rotationEndpoints = Endpoint.of(applicationId).wildcard(); + List<Endpoint.EndpointBuilder> endpoints = new ArrayList<>(); + + if(zone.environment().isProduction()) { + endpoints.add(Endpoint.of(applicationId).named(EndpointId.defaultId())); + endpoints.add(Endpoint.of(applicationId).wildcard()); + } - var zoneLocalEndpoints = zones.stream().flatMap(zone -> Stream.of( - Endpoint.of(applicationId).target(ClusterSpec.Id.from("default"), zone), - Endpoint.of(applicationId).wildcard(zone) - )); + endpoints.add(Endpoint.of(applicationId).target(ClusterSpec.Id.from("default"), zone)); + endpoints.add(Endpoint.of(applicationId).wildcard(zone)); - Stream.concat(Stream.of(globalDefaultEndpoint, rotationEndpoints), zoneLocalEndpoints) + endpoints.stream() .map(endpoint -> endpoint.routingMethod(RoutingMethod.exclusive)) .map(endpoint -> endpoint.on(Endpoint.Port.tls())) .map(endpointBuilder -> endpointBuilder.in(zoneRegistry.system())) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 3d5a181b987..2cf6c7e8670 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -151,13 +151,31 @@ public class DeploymentStatus { ImmutableMap::copyOf)); } - /** The set of jobs that need to run for the given change to be considered complete. */ + /** The set of jobs that need to run for the given changes to be considered complete. */ public Map<JobId, List<Versions>> jobsToRun(Map<InstanceName, Change> changes) { Map<JobId, Versions> productionJobs = new LinkedHashMap<>(); changes.forEach((instance, change) -> productionJobs.putAll(productionJobs(instance, change))); Map<JobId, List<Versions>> testJobs = testJobs(productionJobs); Map<JobId, List<Versions>> jobs = new LinkedHashMap<>(testJobs); productionJobs.forEach((job, versions) -> jobs.put(job, List.of(versions))); + // Add runs for idle, declared test jobs if they have no successes on their instance's change's versions. + jobSteps.forEach((job, step) -> { + if ( ! step.isDeclared() || jobs.containsKey(job)) + return; + + Change change = changes.get(job.application().instance()); + if (change == null || ! change.hasTargets()) + return; + + Optional<JobId> firstProductionJobWithDeployment = jobSteps.keySet().stream() + .filter(jobId -> jobId.type().isProduction() && jobId.type().isDeployment()) + .filter(jobId -> deploymentFor(jobId).isPresent()) + .findFirst(); + + Versions versions = Versions.from(change, application, firstProductionJobWithDeployment.flatMap(this::deploymentFor), systemVersion); + if (step.completedAt(change, firstProductionJobWithDeployment).isEmpty()) + jobs.merge(job, List.of(versions), DeploymentStatus::union); + }); return ImmutableMap.copyOf(jobs); } @@ -237,6 +255,7 @@ public class DeploymentStatus { && testJobs.get(test).contains(versions))) testJobs.merge(firstDeclaredOrElseImplicitTest(testType), List.of(versions), DeploymentStatus::union); }); + // Add runs for declared tests in instances without production jobs, if no successes exist for given change. } return ImmutableMap.copyOf(testJobs); } @@ -312,7 +331,7 @@ public class DeploymentStatus { if (step instanceof DeploymentInstanceSpec) { DeploymentInstanceSpec spec = ((DeploymentInstanceSpec) step); - StepStatus instanceStatus = new InstanceStatus(spec, previous, now, application.require(spec.name())); + StepStatus instanceStatus = new InstanceStatus(spec, previous, now, application.require(spec.name()), this); instance = spec.name(); allSteps.add(instanceStatus); previous = List.of(instanceStatus); @@ -458,20 +477,26 @@ public class DeploymentStatus { private final DeploymentInstanceSpec spec; private final Instant now; private final Instance instance; + private final DeploymentStatus status; private InstanceStatus(DeploymentInstanceSpec spec, List<StepStatus> dependencies, Instant now, - Instance instance) { + Instance instance, DeploymentStatus status) { super(StepType.instance, spec, dependencies, spec.name()); this.spec = spec; this.now = now; this.instance = instance; + this.status = status; } - /** Time of completion of its dependencies, if all parts of the given change are contained in the change for this instance. */ + /** + * Time of completion of its dependencies, if all parts of the given change are contained in the change + * for this instance, or if no more jobs should run for this instance for the given change. + */ @Override public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { - return (change.platform().isEmpty() || change.platform().equals(instance.change().platform())) - && (change.application().isEmpty() || change.application().equals(instance.change().application())) + return ( (change.platform().isEmpty() || change.platform().equals(instance.change().platform())) + && (change.application().isEmpty() || change.application().equals(instance.change().application())) + || status.jobsToRun(Map.of(instance.name(), change)).isEmpty()) ? dependenciesCompletedAt(change, dependent) : Optional.empty(); } @@ -535,7 +560,8 @@ public class DeploymentStatus { return firstFailing.equals(lastCompleted) ? Optional.of(lastCompleted) : Optional.of(lastCompleted.plus(Duration.ofMinutes(10)) .plus(Duration.between(firstFailing, lastCompleted) - .dividedBy(2))); + .dividedBy(2))) + .filter(status.now::isBefore); } private static JobStepStatus ofProductionDeployment(DeclaredZone step, List<StepStatus> dependencies, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index e2fb74ec0b4..092143255d4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -167,10 +167,11 @@ public class DeploymentTrigger { public JobId reTrigger(ApplicationId applicationId, JobType jobType) { Application application = applications().requireApplication(TenantAndApplicationId.from(applicationId)); Instance instance = application.require(applicationId.instance()); - DeploymentStatus status = jobs.deploymentStatus(application); JobId job = new JobId(instance.id(), jobType); - JobStatus jobStatus = status.jobs().get(new JobId(applicationId, jobType)).get(); - Versions versions = jobStatus.lastTriggered().get().versions(); + JobStatus jobStatus = jobs.jobStatus(new JobId(applicationId, jobType)); + Versions versions = jobStatus.lastTriggered() + .orElseThrow(() -> new IllegalArgumentException(job + " has never been triggered")) + .versions(); trigger(deploymentJob(instance, versions, jobType, jobStatus, clock.instant())); return job; } @@ -420,8 +421,10 @@ public class DeploymentTrigger { @Override public String toString() { - return jobType + " for " + instanceId + " on (" + versions.targetPlatform() + ", " + - versions.targetApplication().id() + "), ready since " + availableSince; + return jobType + " for " + instanceId + + " on (" + versions.targetPlatform() + versions.sourcePlatform().map(version -> " <-- " + version).orElse("") + + ", " + versions.targetApplication().id() + versions.sourceApplication().map(version -> " <-- " + version.id()).orElse("") + + "), ready since " + availableSince; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index bc206011bb1..f08cce57dcb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -194,7 +194,7 @@ public class InternalStepRunner implements StepRunner { } private Optional<RunStatus> deployTester(RunId id, DualLogger logger) { - Version platform = controller.systemVersion(); + Version platform = testerPlatformVersion(id); logger.log("Deploying the tester container on platform " + platform + " ..."); return deploy(id.tester().id(), id.type(), @@ -409,9 +409,15 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } + private Version testerPlatformVersion(RunId id) { + return application(id.application()).change().isPinned() + ? controller.jobController().run(id).get().versions().targetPlatform() + : controller.systemVersion(); + } + private Optional<RunStatus> installTester(RunId id, DualLogger logger) { Run run = controller.jobController().run(id).get(); - Version platform = controller.systemVersion(); + Version platform = testerPlatformVersion(id); ZoneId zone = id.type().zone(controller.system()); ApplicationId testerId = id.tester().id(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java index ee5d50414c1..786547d4a67 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java @@ -1,7 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; -import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import java.util.ArrayList; @@ -10,6 +9,8 @@ import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.concurrent.LinkedBlockingDeque; +import java.util.function.UnaryOperator; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -39,12 +40,14 @@ public class NameServiceQueue { return Collections.unmodifiableCollection(requests); } - /** Returns a copy of this containing only the n most recent requests */ + /** Returns a copy of this containing the last n requests */ public NameServiceQueue last(int n) { - requireNonNegative(n); - if (requests.size() <= n) return this; - List<NameServiceRequest> requests = new ArrayList<>(this.requests); - return new NameServiceQueue(requests.subList(requests.size() - n, requests.size())); + return resize(n, (requests) -> requests.subList(requests.size() - n, requests.size())); + } + + /** Returns a copy of this containing the first n requests */ + public NameServiceQueue first(int n) { + return resize(n, (requests) -> requests.subList(0, n)); } /** Returns a copy of this with given request queued according to priority */ @@ -58,6 +61,7 @@ public class NameServiceQueue { return queue; } + /** Returns a copy of this with given request added */ public NameServiceQueue with(NameServiceRequest request) { return with(request, Priority.normal); } @@ -91,6 +95,13 @@ public class NameServiceQueue { return requests.toString(); } + private NameServiceQueue resize(int n, UnaryOperator<List<NameServiceRequest>> resizer) { + requireNonNegative(n); + if (requests.size() <= n) return this; + List<NameServiceRequest> requests = new ArrayList<>(this.requests); + return new NameServiceQueue(resizer.apply(requests)); + } + private static void requireNonNegative(int n) { if (n < 0) throw new IllegalArgumentException("n must be >= 0, got " + n); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java deleted file mode 100644 index 7427fac93d2..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; - -import java.time.Duration; -import java.util.EnumSet; - -/** - * @author olaa - */ -public class BillingMaintainer extends ControllerMaintainer { - - private final Billing billing; - - public BillingMaintainer(Controller controller, Duration interval) { - super(controller, interval, BillingMaintainer.class.getSimpleName(), EnumSet.of(SystemName.cd)); - this.billing = controller.serviceRegistry().billingService(); - } - - @Override - public void maintain() { - controller().tenants().asList() - .stream() - .filter(tenant -> tenant instanceof CloudTenant) - .map(tenant -> (CloudTenant) tenant) - .forEach(cloudTenant -> controller().applications().asList(cloudTenant.name()) - .forEach(application -> { - billing.handleBilling(application.id().defaultInstance(), cloudTenant.billingInfo().customerId()); - }) - ); - } - -} - - diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index b92da2ce740..ca695a2d234 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -40,10 +40,10 @@ public class ControllerMaintenance extends AbstractComponent { private final CostReportMaintainer costReportMaintainer; private final ResourceMeterMaintainer resourceMeterMaintainer; private final NameServiceDispatcher nameServiceDispatcher; - private final BillingMaintainer billingMaintainer; private final CloudEventReporter cloudEventReporter; private final RotationStatusUpdater rotationStatusUpdater; private final ResourceTagMaintainer resourceTagMaintainer; + private final SystemRoutingPolicyMaintainer systemRoutingPolicyMaintainer; @Inject @SuppressWarnings("unused") // instantiated by Dependency Injection @@ -69,10 +69,10 @@ public class ControllerMaintenance extends AbstractComponent { nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10)); costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), controller.serviceRegistry().costReportConsumer()); resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(1), metric, controller.serviceRegistry().meteringService()); - billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3)); cloudEventReporter = new CloudEventReporter(controller, Duration.ofDays(1)); rotationStatusUpdater = new RotationStatusUpdater(controller, maintenanceInterval); resourceTagMaintainer = new ResourceTagMaintainer(controller, Duration.ofMinutes(30), controller.serviceRegistry().resourceTagger()); + systemRoutingPolicyMaintainer = new SystemRoutingPolicyMaintainer(controller, Duration.ofMinutes(10)); } public Upgrader upgrader() { return upgrader; } @@ -96,10 +96,10 @@ public class ControllerMaintenance extends AbstractComponent { costReportMaintainer.close(); resourceMeterMaintainer.close(); nameServiceDispatcher.close(); - billingMaintainer.close(); cloudEventReporter.close(); rotationStatusUpdater.close(); resourceTagMaintainer.close(); + systemRoutingPolicyMaintainer.close(); } /** Create one OS upgrader per cloud found in the zone registry of controller */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 40072df48b5..7006458538d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -23,11 +23,11 @@ import java.util.logging.Logger; * * @author mpolden */ -public abstract class InfrastructureUpgrader extends ControllerMaintainer { +public abstract class InfrastructureUpgrader<VERSION> extends ControllerMaintainer { private static final Logger log = Logger.getLogger(InfrastructureUpgrader.class.getName()); - private final UpgradePolicy upgradePolicy; + protected final UpgradePolicy upgradePolicy; public InfrastructureUpgrader(Controller controller, Duration interval, UpgradePolicy upgradePolicy, String name) { super(controller, interval, name, EnumSet.allOf(SystemName.class)); @@ -40,7 +40,7 @@ public abstract class InfrastructureUpgrader extends ControllerMaintainer { } /** Deploy a list of system applications until they converge on the given version */ - private void upgradeAll(Version target, List<SystemApplication> applications) { + private void upgradeAll(VERSION target, List<SystemApplication> applications) { for (List<ZoneApi> zones : upgradePolicy.asList()) { boolean converged = true; for (ZoneApi zone : zones) { @@ -63,11 +63,11 @@ public abstract class InfrastructureUpgrader extends ControllerMaintainer { } /** Returns whether all applications have converged to the target version in zone */ - private boolean upgradeAll(Version target, List<SystemApplication> applications, ZoneApi zone) { + private boolean upgradeAll(VERSION target, List<SystemApplication> applications, ZoneApi zone) { boolean converged = true; for (SystemApplication application : applications) { if (convergedOn(target, application.dependencies(), zone)) { - if (shouldUpgrade(target, application, zone)) { + if (changeTargetTo(target, application, zone)) { upgrade(target, application, zone); } converged &= convergedOn(target, application, zone); @@ -76,24 +76,24 @@ public abstract class InfrastructureUpgrader extends ControllerMaintainer { return converged; } - private boolean convergedOn(Version target, List<SystemApplication> applications, ZoneApi zone) { + private boolean convergedOn(VERSION target, List<SystemApplication> applications, ZoneApi zone) { return applications.stream().allMatch(application -> convergedOn(target, application, zone)); } - /** Returns whether application in zone should be told to upgrade to given target */ - protected abstract boolean shouldUpgrade(Version target, SystemApplication application, ZoneApi zone); + /** Returns whether target version for application in zone should be changed */ + protected abstract boolean changeTargetTo(VERSION target, SystemApplication application, ZoneApi zone); /** Upgrade component to target version. Implementation should be idempotent */ - protected abstract void upgrade(Version target, SystemApplication application, ZoneApi zone); + protected abstract void upgrade(VERSION target, SystemApplication application, ZoneApi zone); /** Returns whether application has converged to target version in zone */ - protected abstract boolean convergedOn(Version target, SystemApplication application, ZoneApi zone); + protected abstract boolean convergedOn(VERSION target, SystemApplication application, ZoneApi zone); /** Returns the target version for the component upgraded by this, if any */ - protected abstract Optional<Version> targetVersion(); + protected abstract Optional<VERSION> targetVersion(); - /** Returns whether the upgrader should require given node to upgrade */ - protected abstract boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone); + /** Returns whether the upgrader should expect given node to upgrade */ + protected abstract boolean expectUpgradeOf(Node node, SystemApplication application, ZoneApi zone); /** Find the minimum value of a version field in a zone by comparing all nodes */ protected final Optional<Version> minVersion(ZoneApi zone, SystemApplication application, Function<Node, Version> versionField) { @@ -102,7 +102,7 @@ public abstract class InfrastructureUpgrader extends ControllerMaintainer { .nodeRepository() .list(zone.getId(), application.id()) .stream() - .filter(node -> requireUpgradeOf(node, application, zone)) + .filter(node -> expectUpgradeOf(node, application, zone)) .map(versionField) .min(Comparator.naturalOrder()); } catch (Exception e) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 15b4da7703a..cc4a8c628eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import com.yahoo.vespa.hosted.controller.deployment.JobList; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; @@ -25,9 +26,9 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; @@ -48,14 +49,17 @@ public class MetricsReporter extends ControllerMaintainer { public static final String DEPLOYMENT_WARNINGS = "deployment.warnings"; public static final String OS_CHANGE_DURATION = "deployment.osChangeDuration"; public static final String PLATFORM_CHANGE_DURATION = "deployment.platformChangeDuration"; + public static final String OS_NODE_COUNT = "deployment.nodeCountByOsVersion"; + public static final String PLATFORM_NODE_COUNT = "deployment.nodeCountByPlatformVersion"; public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; + public static final String OPERATION_PREFIX = "operation."; private final Metric metric; private final Clock clock; - // Tracks hosts and versions for which we have reported change duration metrics - private final ConcurrentHashMap<HostName, Map<String, Set<Version>>> reportedVersions = new ConcurrentHashMap<>(); + // Keep track of reported node counts for each version + private final ConcurrentHashMap<NodeCountKey, Long> nodeCounts = new ConcurrentHashMap<>(); public MetricsReporter(Controller controller, Metric metric) { super(controller, Duration.ofMinutes(1)); // use fixed rate for metrics @@ -68,8 +72,49 @@ public class MetricsReporter extends ControllerMaintainer { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); - reportChangeDurations(osChangeDurations(), OS_CHANGE_DURATION); - reportChangeDurations(platformChangeDurations(), PLATFORM_CHANGE_DURATION); + reportInfrastructureUpgradeMetrics(); + reportAuditLog(); + } + + private void reportAuditLog() { + AuditLog log = controller().auditLogger().readLog(); + HashMap<String, HashMap<String, Integer>> metricCounts = new HashMap<>(); + + for (AuditLog.Entry entry : log.entries()) { + String[] resource = entry.resource().split("/"); + if((resource.length > 1) && (resource[1] != null)) { + String api = resource[1]; + String operationMetric = OPERATION_PREFIX + api; + HashMap<String, Integer> dimension = metricCounts.get(operationMetric); + if (dimension != null) { + Integer count = dimension.get(entry.principal()); + if (count != null) { + dimension.replace(entry.principal(), ++count); + } else { + dimension.put(entry.principal(), 1); + } + + } else { + dimension = new HashMap<>(); + dimension.put(entry.principal(),1); + metricCounts.put(operationMetric, dimension); + } + } + } + for (String operationMetric : metricCounts.keySet()) { + for (String userDimension : metricCounts.get(operationMetric).keySet()) { + metric.set(operationMetric, (metricCounts.get(operationMetric)).get(userDimension), metric.createContext(Map.of("operator", userDimension))); + } + } + } + + private void reportInfrastructureUpgradeMetrics() { + Map<NodeVersion, Duration> osChangeDurations = osChangeDurations(); + Map<NodeVersion, Duration> platformChangeDurations = platformChangeDurations(); + reportChangeDurations(osChangeDurations, OS_CHANGE_DURATION); + reportChangeDurations(platformChangeDurations, PLATFORM_CHANGE_DURATION); + reportNodeCount(osChangeDurations.keySet(), OS_NODE_COUNT); + reportNodeCount(platformChangeDurations.keySet(), PLATFORM_NODE_COUNT); } private void reportRemainingRotations() { @@ -111,26 +156,28 @@ public class MetricsReporter extends ControllerMaintainer { metric.createContext(Map.of())); } - private void reportChangeDurations(Map<NodeVersion, Duration> changeDurations, String metricName) { - changeDurations.forEach((nodeVersion, duration) -> { - // Zero the metric for past versions because our metrics framework remembers the last value for each unique - // dimension set for each metric. - reportVersion(metricName, nodeVersion.hostname(), nodeVersion.currentVersion()); - for (var reportedVersion : reportedVersions.getOrDefault(nodeVersion.hostname(), Map.of()).get(metricName)) { - if (reportedVersion.equals(nodeVersion.currentVersion())) continue; - metric.set(metricName, 0, metric.createContext(dimensions(nodeVersion.hostname(), nodeVersion.zone(), reportedVersion))); + private void reportNodeCount(Set<NodeVersion> nodeVersions, String metricName) { + Map<NodeCountKey, Long> newNodeCounts = nodeVersions.stream() + .collect(Collectors.groupingBy(nodeVersion -> { + return new NodeCountKey(metricName, + nodeVersion.currentVersion(), + nodeVersion.zone()); + }, Collectors.counting())); + nodeCounts.putAll(newNodeCounts); + nodeCounts.forEach((key, count) -> { + if (newNodeCounts.containsKey(key)) { + // Version is still present: Update the metric. + metric.set(metricName, count, metric.createContext(dimensions(key.zone, key.version))); + } else if (key.metricName.equals(metricName)) { + // Version is no longer present, but has been previously reported: Set it to zero. + metric.set(metricName, 0, metric.createContext(dimensions(key.zone, key.version))); } - metric.set(metricName, duration.toSeconds(), metric.createContext(dimensions(nodeVersion.hostname(), nodeVersion.zone(), nodeVersion.currentVersion()))); }); } - private void reportVersion(String metricName, HostName hostname, Version version) { - reportedVersions.compute(hostname, (ignored, values) -> { - if (values == null) { - values = new HashMap<>(); - } - values.computeIfAbsent(metricName, k -> new HashSet<>()).add(version); - return values; + private void reportChangeDurations(Map<NodeVersion, Duration> changeDurations, String metricName) { + changeDurations.forEach((nodeVersion, duration) -> { + metric.set(metricName, duration.toSeconds(), metric.createContext(dimensions(nodeVersion.hostname(), nodeVersion.zone()))); }); } @@ -207,10 +254,42 @@ public class MetricsReporter extends ControllerMaintainer { "app",application.application().value() + "." + application.instance().value()); } - private static Map<String, String> dimensions(HostName hostname, ZoneId zone, Version currentVersion) { + private static Map<String, String> dimensions(HostName hostname, ZoneId zone) { return Map.of("host", hostname.value(), - "zone", zone.value(), + "zone", zone.value()); + } + + private static Map<String, String> dimensions(ZoneId zone, Version currentVersion) { + return Map.of("zone", zone.value(), "currentVersion", currentVersion.toFullString()); } + private static class NodeCountKey { + + private final String metricName; + private final Version version; + private final ZoneId zone; + + public NodeCountKey(String metricName, Version version, ZoneId zone) { + this.metricName = metricName; + this.version = version; + this.zone = zone; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeCountKey that = (NodeCountKey) o; + return metricName.equals(that.metricName) && + version.equals(that.version) && + zone.equals(that.zone); + } + + @Override + public int hashCode() { + return Objects.hash(metricName, version, zone); + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java index 73daef7c2b0..e7eaf083a57 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java @@ -45,7 +45,7 @@ public class NameServiceDispatcher extends ControllerMaintainer { var remaining = queue.dispatchTo(nameService, requestCount); if (queue == remaining) return; // Queue unchanged - var dispatched = queue.last(requestCount); + var dispatched = queue.first(requestCount); if (!dispatched.requests().isEmpty()) { log.log(Level.INFO, "Dispatched name service request(s) in " + Duration.between(instant, clock.instant()) + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 8a97f8f1a9d..66fb20e8a71 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -3,11 +3,12 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.versions.OsVersion; +import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import java.time.Duration; import java.util.Optional; @@ -19,7 +20,7 @@ import java.util.logging.Logger; * * @author mpolden */ -public class OsUpgrader extends InfrastructureUpgrader { +public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { private static final Logger log = Logger.getLogger(OsUpgrader.class.getName()); @@ -37,48 +38,65 @@ public class OsUpgrader extends InfrastructureUpgrader { } @Override - protected void upgrade(Version target, SystemApplication application, ZoneApi zone) { - log.info(String.format("Upgrading OS of %s to version %s in %s in cloud %s", application.id(), target, zone.getId(), zone.getCloudName())); - controller().serviceRegistry().configServer().nodeRepository().upgradeOs(zone.getId(), application.nodeType(), target); + protected void upgrade(OsVersionTarget target, SystemApplication application, ZoneApi zone) { + Optional<Duration> zoneUpgradeBudget = target.upgradeBudget() + .map(totalBudget -> zoneBudgetOf(totalBudget, zone)); + log.info(String.format("Upgrading OS of %s to version %s in %s in cloud %s%s", application.id(), + target.osVersion().version().toFullString(), + zone.getId(), zone.getCloudName(), + zoneUpgradeBudget.map(d -> " with time budget " + d).orElse(""))); + controller().serviceRegistry().configServer().nodeRepository().upgradeOs(zone.getId(), application.nodeType(), + target.osVersion().version(), + zoneUpgradeBudget); } @Override - protected boolean convergedOn(Version target, SystemApplication application, ZoneApi zone) { - return currentVersion(zone, application, target).equals(target); + protected boolean convergedOn(OsVersionTarget target, SystemApplication application, ZoneApi zone) { + return currentVersion(zone, application, target.osVersion().version()).equals(target.osVersion().version()); } @Override - protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { - return cloud.equals(zone.getCloudName()) && eligibleForUpgrade(node, application); + protected boolean expectUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { + return cloud.equals(zone.getCloudName()) && // Cloud is managed by this upgrader + application.shouldUpgradeOsIn(zone.getId(), controller()) && // Application should upgrade in this cloud + canUpgrade(node); // Node is in an upgradable state } @Override - protected Optional<Version> targetVersion() { + protected Optional<OsVersionTarget> targetVersion() { // Return target if we have nodes in this cloud on a lower version - return controller().osVersion(cloud) + return controller().osVersionTarget(cloud) .filter(target -> controller().osVersionStatus().nodesIn(cloud).stream() - .anyMatch(node -> node.currentVersion().isBefore(target.version()))) - .map(OsVersion::version); + .anyMatch(node -> node.currentVersion().isBefore(target.osVersion().version()))); } @Override - protected boolean shouldUpgrade(Version target, SystemApplication application, ZoneApi zone) { - if (!application.shouldUpgradeOs()) return false; // Never upgrade + protected boolean changeTargetTo(OsVersionTarget target, SystemApplication application, ZoneApi zone) { + if (!application.shouldUpgradeOsIn(zone.getId(), controller())) return false; return controller().serviceRegistry().configServer().nodeRepository() .targetVersionsOf(zone.getId()) .osVersion(application.nodeType()) - .map(target::isAfter) // Upgrade if target is after current - .orElse(true); // Upgrade if target is unset + .map(currentTarget -> target.osVersion().version().isAfter(currentTarget)) + .orElse(true); } private Version currentVersion(ZoneApi zone, SystemApplication application, Version defaultVersion) { return minVersion(zone, application, Node::currentOsVersion).orElse(defaultVersion); } - /** Returns whether node in application should be upgraded by this */ - public static boolean eligibleForUpgrade(Node node, SystemApplication application) { - return upgradableNodeStates.contains(node.state()) && - application.shouldUpgradeOs(); + /** Returns the available upgrade budget for given zone */ + private Duration zoneBudgetOf(Duration totalBudget, ZoneApi zone) { + if (!zone.getEnvironment().isProduction()) return Duration.ZERO; + long consecutiveProductionZones = upgradePolicy.asList().stream() + .filter(parallelZones -> parallelZones.stream().map(ZoneApi::getEnvironment) + .anyMatch(Environment::isProduction)) + .count(); + return totalBudget.dividedBy(consecutiveProductionZones); + } + + /** Returns whether node is in a state where it can be upgraded */ + public static boolean canUpgrade(Node node) { + return upgradableNodeStates.contains(node.state()); } private static String name(CloudName cloud) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java new file mode 100644 index 00000000000..6c271ed0470 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java @@ -0,0 +1,40 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; + +import java.time.Duration; + +/** + * This maintains {@link RoutingPolicy}'s for {@link SystemApplication}s. In contrast to regular applications, this + * refreshes policies at an interval, not on deployment. + * + * @author mpolden + */ +public class SystemRoutingPolicyMaintainer extends ControllerMaintainer { + + private final BooleanFlag featureFlag; + + public SystemRoutingPolicyMaintainer(Controller controller, Duration interval) { + super(controller, interval); + this.featureFlag = Flags.CONFIGSERVER_PROVISION_LB.bindTo(controller.flagSource()); + } + + @Override + protected void maintain() { + for (var zone : controller().zoneRegistry().zones().all().ids()) { + for (var application : SystemApplication.values()) { + if (!application.hasEndpoint()) continue; + if (!featureFlag.with(FetchVector.Dimension.ZONE_ID, zone.value()).value()) continue; + controller().routing().policies().refresh(application.id(), DeploymentSpec.empty, zone); + } + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index be87b5f9223..bf44c796f34 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; -import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -19,7 +18,7 @@ import java.util.logging.Logger; * * @author mpolden */ -public class SystemUpgrader extends InfrastructureUpgrader { +public class SystemUpgrader extends InfrastructureUpgrader<Version> { private static final Logger log = Logger.getLogger(SystemUpgrader.class.getName()); @@ -46,7 +45,7 @@ public class SystemUpgrader extends InfrastructureUpgrader { } @Override - protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { + protected boolean expectUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { return eligibleForUpgrade(node); } @@ -59,7 +58,7 @@ public class SystemUpgrader extends InfrastructureUpgrader { } @Override - protected boolean shouldUpgrade(Version target, SystemApplication application, ZoneApi zone) { + protected boolean changeTargetTo(Version target, SystemApplication application, ZoneApi zone) { if (application.hasApplicationPackage()) { // For applications with package we do not have a zone-wide version target. This means that we must check // the wanted version of each node. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 6df467c8790..5f0f2e4ba4e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; -import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.curator.Lock; @@ -20,6 +19,7 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Random; import java.util.function.BinaryOperator; import java.util.function.Function; import java.util.logging.Logger; @@ -39,10 +39,12 @@ public class Upgrader extends ControllerMaintainer { private static final Logger log = Logger.getLogger(Upgrader.class.getName()); private final CuratorDb curator; + private final Random random; public Upgrader(Controller controller, Duration interval, CuratorDb curator) { super(controller, interval); this.curator = Objects.requireNonNull(curator, "curator cannot be null"); + this.random = new Random(controller.clock().instant().toEpochMilli()); // Seed with clock for test determinism } /** @@ -105,7 +107,7 @@ public class Upgrader extends ControllerMaintainer { /** Returns a list of all production application instances, except those which are pinned, which we should not manipulate here. */ private InstanceList instances() { return InstanceList.from(controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().readable()))) - .withProductionDeployment() + .withDeclaredJobs() .unpinned(); } @@ -115,6 +117,7 @@ public class Upgrader extends ControllerMaintainer { .not().deploying() .onLowerVersionThan(version) .canUpgradeAt(version, controller().clock().instant()) + .shuffle(random) // Shuffle so we do not always upgrade instances in the same order .byIncreasingDeployedVersion() .first(numberToUpgrade).asList() .forEach(instance -> controller().applications().deploymentTrigger().triggerChange(instance, Change.of(version))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index 4f1f453637e..d8b74a4ae99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -1,17 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.util.logging.Level; +import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.broken; +import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.high; +import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.low; +import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.normal; + /** * This maintenance job periodically updates the version status. - * Since the version status is expensive to compute and do not need to be perfectly up to date, + * Since the version status is expensive to compute and does not need to be perfectly up to date, * we do not want to recompute it each time it is accessed. * * @author bratseth @@ -27,10 +33,24 @@ public class VersionStatusUpdater extends ControllerMaintainer { try { VersionStatus newStatus = VersionStatus.compute(controller()); controller().updateVersionStatus(newStatus); + newStatus.systemVersion().ifPresent(version -> { + controller().serviceRegistry().systemMonitor().reportSystemVersion(version.versionNumber(), + convert(version.confidence())); + }); } catch (Exception e) { log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } } + static SystemMonitor.Confidence convert(VespaVersion.Confidence confidence) { + switch (confidence) { + case broken: return broken; + case low: return low; + case normal: return normal; + case high: return high; + default: throw new IllegalArgumentException("Unexpected confidence '" + confidence + "'"); + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index ebd4921ccd7..3058037ccc0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -31,8 +31,8 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; -import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; +import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -98,6 +98,7 @@ public class CuratorDb implements JobControl.Db { private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); private final RunSerializer runSerializer = new RunSerializer(); private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); + private final OsVersionTargetSerializer osVersionTargetSerializer = new OsVersionTargetSerializer(osVersionSerializer); private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer); private final RoutingPolicySerializer routingPolicySerializer = new RoutingPolicySerializer(); private final ZoneRoutingPolicySerializer zoneRoutingPolicySerializer = new ZoneRoutingPolicySerializer(routingPolicySerializer); @@ -297,12 +298,12 @@ public class CuratorDb implements JobControl.Db { // Infrastructure upgrades - public void writeOsVersions(Set<OsVersion> versions) { - curator.set(osTargetVersionPath(), asJson(osVersionSerializer.toSlime(versions))); + public void writeOsVersionTargets(Set<OsVersionTarget> versions) { + curator.set(osVersionTargetsPath(), asJson(osVersionTargetSerializer.toSlime(versions))); } - public Set<OsVersion> readOsVersions() { - return readSlime(osTargetVersionPath()).map(osVersionSerializer::fromSlime).orElseGet(Collections::emptySet); + public Set<OsVersionTarget> readOsVersionTargets() { + return readSlime(osVersionTargetsPath()).map(osVersionTargetSerializer::fromSlime).orElseGet(Collections::emptySet); } public void writeOsVersionStatus(OsVersionStatus status) { @@ -598,7 +599,7 @@ public class CuratorDb implements JobControl.Db { return root.append("upgrader").append("confidenceOverrides"); } - private static Path osTargetVersionPath() { + private static Path osVersionTargetsPath() { return root.append("osUpgrader").append("targetVersion"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java new file mode 100644 index 00000000000..627ad9f709c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.versions.OsVersion; +import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; + +import java.time.Duration; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; + +/** + * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.OsVersionTarget}. + * + * @author mpolden + */ +public class OsVersionTargetSerializer { + + private final OsVersionSerializer osVersionSerializer; + + private static final String versionsField = "versions"; + private static final String upgradeBudgetField = "upgradeBudget"; + + public OsVersionTargetSerializer(OsVersionSerializer osVersionSerializer) { + this.osVersionSerializer = osVersionSerializer; + } + + public Slime toSlime(Set<OsVersionTarget> osVersionTargets) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor array = root.setArray(versionsField); + osVersionTargets.forEach(target -> toSlime(target, array.addObject())); + return slime; + } + + public Set<OsVersionTarget> fromSlime(Slime slime) { + Inspector array = slime.get().field(versionsField); + Set<OsVersionTarget> osVersionTargets = new TreeSet<>(); + array.traverse((ArrayTraverser) (i, inspector) -> { + OsVersion osVersion = osVersionSerializer.fromSlime(inspector); + Optional<Duration> upgradeBudget = Serializers.optionalDuration(inspector.field(upgradeBudgetField)); + osVersionTargets.add(new OsVersionTarget(osVersion, upgradeBudget)); + }); + return Collections.unmodifiableSet(osVersionTargets); + } + + private void toSlime(OsVersionTarget target, Cursor object) { + osVersionSerializer.toSlime(target.osVersion(), object); + target.upgradeBudget().ifPresent(d -> object.setLong(upgradeBudgetField, d.toMillis())); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java index e5adccc850c..b254732f324 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.slime.Inspector; import com.yahoo.slime.SlimeUtils; +import java.time.Duration; import java.time.Instant; import java.util.Optional; import java.util.OptionalDouble; @@ -40,4 +41,9 @@ public class Serializers { return value.isPresent() ? Optional.of(Instant.ofEpochMilli(value.getAsLong())) : Optional.empty(); } + public static Optional<Duration> optionalDuration(Inspector field) { + var value = optionalLong(field); + return value.isPresent() ? Optional.of(Duration.ofMillis(value.getAsLong())) : Optional.empty(); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java index 529acc48cbe..110d994c179 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java @@ -10,5 +10,7 @@ import com.yahoo.container.jdisc.HttpResponse; * @author Haakon Dybdahl */ public interface ConfigServerRestExecutor { - HttpResponse handle(ProxyRequest proxyRequest) throws ProxyException; + + HttpResponse handle(ProxyRequest request); + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index d10c4dd226b..f5dcae9c961 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -1,15 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; +import ai.vespa.util.http.retry.Sleeper; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.HttpRequest.Method; -import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import org.apache.http.Header; +import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; @@ -19,11 +20,15 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.entity.InputStreamEntity; +import org.apache.http.impl.DefaultConnectionReuseStrategy; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpCoreContext; import org.apache.http.util.EntityUtils; import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSession; import java.io.IOException; import java.io.InputStream; @@ -37,6 +42,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -50,75 +56,87 @@ import static com.yahoo.yolean.Exceptions.uncheck; @SuppressWarnings("unused") // Injected public class ConfigServerRestExecutorImpl extends AbstractComponent implements ConfigServerRestExecutor { - private static final Logger log = Logger.getLogger(ConfigServerRestExecutorImpl.class.getName()); - + private static final Logger LOG = Logger.getLogger(ConfigServerRestExecutorImpl.class.getName()); private static final Duration PROXY_REQUEST_TIMEOUT = Duration.ofSeconds(10); + private static final Duration PING_REQUEST_TIMEOUT = Duration.ofMillis(500); + private static final Duration SINGLE_TARGET_WAIT = Duration.ofSeconds(2); + private static final int SINGLE_TARGET_RETRIES = 3; private static final Set<String> HEADERS_TO_COPY = Set.of("X-HTTP-Method-Override", "Content-Type"); private final CloseableHttpClient client; + private final Sleeper sleeper; @Inject public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ServiceIdentityProvider sslContextProvider) { - RequestConfig config = RequestConfig.custom() - .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) - .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) - .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); + this(zoneRegistry, sslContextProvider.getIdentitySslContext(), new Sleeper.Default(), + new ConnectionReuseStrategy(zoneRegistry)); + } - this.client = createHttpClient(config, sslContextProvider, - new ControllerOrConfigserverHostnameVerifier(zoneRegistry)); + ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, SSLContext sslContext, + Sleeper sleeper, ConnectionReuseStrategy connectionReuseStrategy) { + this.client = createHttpClient(sslContext, + new ControllerOrConfigserverHostnameVerifier(zoneRegistry), + connectionReuseStrategy); + this.sleeper = sleeper; } @Override - public ProxyResponse handle(ProxyRequest proxyRequest) throws ProxyException { - // Make a local copy of the list as we want to manipulate it in case of ping problems. - List<URI> allServers = new ArrayList<>(proxyRequest.getTargets()); + public ProxyResponse handle(ProxyRequest request) { + List<URI> targets = new ArrayList<>(request.getTargets()); StringBuilder errorBuilder = new StringBuilder(); - if (queueFirstServerIfDown(allServers)) { + boolean singleTarget = targets.size() == 1; + if (singleTarget) { + for (int i = 0; i < SINGLE_TARGET_RETRIES - 1; i++) { + targets.add(targets.get(0)); + } + } else if (queueFirstServerIfDown(targets)) { errorBuilder.append("Change ordering due to failed ping."); } - for (URI uri : allServers) { - Optional<ProxyResponse> proxyResponse = proxyCall(uri, proxyRequest, errorBuilder); + + for (URI url : targets) { + Optional<ProxyResponse> proxyResponse = proxy(request, url, errorBuilder); if (proxyResponse.isPresent()) { return proxyResponse.get(); } + if (singleTarget) { + sleeper.sleep(SINGLE_TARGET_WAIT); + } } - // TODO Add logging, for now, experimental and we want to not add more noise. - throw new ProxyException(ErrorResponse.internalServerError("Failed talking to config servers: " - + errorBuilder.toString())); + + throw new RuntimeException("Failed talking to config servers: " + errorBuilder.toString()); } - private Optional<ProxyResponse> proxyCall(URI uri, ProxyRequest proxyRequest, StringBuilder errorBuilder) - throws ProxyException { - final HttpRequestBase requestBase = createHttpBaseRequest( - proxyRequest.getMethod(), proxyRequest.createConfigServerRequestUri(uri), proxyRequest.getData()); + private Optional<ProxyResponse> proxy(ProxyRequest request, URI url, StringBuilder errorBuilder) { + HttpRequestBase requestBase = createHttpBaseRequest( + request.getMethod(), request.createConfigServerRequestUri(url), request.getData()); // Empty list of headers to copy for now, add headers when needed, or rewrite logic. - copyHeaders(proxyRequest.getHeaders(), requestBase); + copyHeaders(request.getHeaders(), requestBase); try (CloseableHttpResponse response = client.execute(requestBase)) { String content = getContent(response); int status = response.getStatusLine().getStatusCode(); if (status / 100 == 5) { - errorBuilder.append("Talking to server ").append(uri.getHost()); + errorBuilder.append("Talking to server ").append(url.getHost()); errorBuilder.append(", got ").append(status).append(" ") .append(content).append("\n"); - log.log(Level.FINE, () -> String.format("Got response from %s with status code %d and content:\n %s", - uri.getHost(), status, content)); + LOG.log(Level.FINE, () -> String.format("Got response from %s with status code %d and content:\n %s", + url.getHost(), status, content)); return Optional.empty(); } - final Header contentHeader = response.getLastHeader("Content-Type"); - final String contentType; + Header contentHeader = response.getLastHeader("Content-Type"); + String contentType; if (contentHeader != null && contentHeader.getValue() != null && ! contentHeader.getValue().isEmpty()) { contentType = contentHeader.getValue().replace("; charset=UTF-8",""); } else { contentType = "application/json"; } // Send response back - return Optional.of(new ProxyResponse(proxyRequest, content, status, uri, contentType)); + return Optional.of(new ProxyResponse(request, content, status, url, contentType)); } catch (Exception e) { - errorBuilder.append("Talking to server ").append(uri.getHost()); + errorBuilder.append("Talking to server ").append(url.getHost()); errorBuilder.append(" got exception ").append(e.getMessage()); - log.log(Level.FINE, e, () -> "Got exception while sending request to " + uri.getHost()); + LOG.log(Level.FINE, e, () -> "Got exception while sending request to " + url.getHost()); return Optional.empty(); } } @@ -129,33 +147,32 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C .orElse(""); } - private static HttpRequestBase createHttpBaseRequest(Method method, URI uri, InputStream data) throws ProxyException { + private static HttpRequestBase createHttpBaseRequest(Method method, URI url, InputStream data) { switch (method) { case GET: - return new HttpGet(uri); + return new HttpGet(url); case POST: - HttpPost post = new HttpPost(uri); + HttpPost post = new HttpPost(url); if (data != null) { post.setEntity(new InputStreamEntity(data)); } return post; case PUT: - HttpPut put = new HttpPut(uri); + HttpPut put = new HttpPut(url); if (data != null) { put.setEntity(new InputStreamEntity(data)); } return put; case DELETE: - return new HttpDelete(uri); + return new HttpDelete(url); case PATCH: - HttpPatch patch = new HttpPatch(uri); + HttpPatch patch = new HttpPatch(url); if (data != null) { patch.setEntity(new InputStreamEntity(data)); } return patch; - default: - throw new ProxyException(ErrorResponse.methodNotAllowed("Will not proxy such calls.")); } + throw new IllegalArgumentException("Refusing to proxy " + method + " " + url + ": Unsupported method"); } private static void copyHeaders(Map<String, List<String>> headers, HttpRequestBase toRequest) { @@ -169,7 +186,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C } /** - * During upgrade, one server can be down, this is normal. Therefor we do a quick ping on the first server, + * During upgrade, one server can be down, this is normal. Therefore we do a quick ping on the first server, * if it is not responding, we try the other servers first. False positive/negatives are not critical, * but will increase latency to some extent. */ @@ -178,15 +195,15 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C return false; } URI uri = allServers.get(0); - HttpGet httpget = new HttpGet(uri); + HttpGet httpGet = new HttpGet(uri); - int timeout = 500; RequestConfig config = RequestConfig.custom() - .setConnectTimeout(timeout) - .setConnectionRequestTimeout(timeout) - .setSocketTimeout(timeout).build(); - httpget.setConfig(config); - try (CloseableHttpResponse response = client.execute(httpget)) { + .setConnectTimeout((int) PING_REQUEST_TIMEOUT.toMillis()) + .setConnectionRequestTimeout((int) PING_REQUEST_TIMEOUT.toMillis()) + .setSocketTimeout((int) PING_REQUEST_TIMEOUT.toMillis()).build(); + httpGet.setConfig(config); + + try (CloseableHttpResponse response = client.execute(httpGet)) { if (response.getStatusLine().getStatusCode() == 200) { return false; } @@ -208,18 +225,25 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C } } - private static CloseableHttpClient createHttpClient(RequestConfig config, - ServiceIdentityProvider sslContextProvider, - HostnameVerifier hostnameVerifier) { + private static CloseableHttpClient createHttpClient(SSLContext sslContext, + HostnameVerifier hostnameVerifier, + org.apache.http.ConnectionReuseStrategy connectionReuseStrategy) { + + RequestConfig config = RequestConfig.custom() + .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) + .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) + .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); return HttpClientBuilder.create() - .setUserAgent("config-server-proxy-client") - .setSSLContext(sslContextProvider.getIdentitySslContext()) - .setSSLHostnameVerifier(hostnameVerifier) - .setDefaultRequestConfig(config) - .setMaxConnPerRoute(10) - .setMaxConnTotal(500) - .setConnectionTimeToLive(1, TimeUnit.MINUTES) - .build(); + .setUserAgent("config-server-proxy-client") + .setSSLContext(sslContext) + .setSSLHostnameVerifier(hostnameVerifier) + .setDefaultRequestConfig(config) + .setMaxConnPerRoute(10) + .setMaxConnTotal(500) + .setConnectionReuseStrategy(connectionReuseStrategy) + .setConnectionTimeToLive(1, TimeUnit.MINUTES) + .build(); + } private static class ControllerOrConfigserverHostnameVerifier implements HostnameVerifier { @@ -242,4 +266,36 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C return "localhost".equals(hostname) || configserverVerifier.verify(hostname, session); } } + + /** + * A connection reuse strategy which avoids reusing connections to VIPs. Since VIPs are TCP-level load balancers, + * a reconnect is needed to (potentially) switch real server. + */ + public static class ConnectionReuseStrategy extends DefaultConnectionReuseStrategy { + + private final Set<String> vips; + + public ConnectionReuseStrategy(ZoneRegistry zoneRegistry) { + this(zoneRegistry.zones().all().ids().stream() + .map(zoneRegistry::getConfigServerVipUri) + .map(URI::getHost) + .collect(Collectors.toUnmodifiableSet())); + } + + public ConnectionReuseStrategy(Set<String> vips) { + this.vips = Set.copyOf(vips); + } + + @Override + public boolean keepAlive(HttpResponse response, HttpContext context) { + HttpCoreContext coreContext = HttpCoreContext.adapt(context); + String host = coreContext.getTargetHost().getHostName(); + if (vips.contains(host)) { + return false; + } + return super.keepAlive(response, context); + } + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ErrorResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ErrorResponse.java deleted file mode 100644 index 3673c0227a3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ErrorResponse.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.proxy; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; - -import java.io.IOException; -import java.io.OutputStream; - -import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; -import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; -import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; -import static com.yahoo.jdisc.Response.Status.NOT_FOUND; - -/** - * Class for generating error responses. - * - * @author Haakon Dybdahl - */ -public class ErrorResponse extends HttpResponse { - - private final Slime slime = new Slime(); - public final String message; - - public ErrorResponse(int code, String errorType, String message) { - super(code); - this.message = message; - Cursor root = slime.setObject(); - root.setString("error-code", errorType); - root.setString("message", message); - } - - public enum errorCodes { - NOT_FOUND, - BAD_REQUEST, - METHOD_NOT_ALLOWED, - INTERNAL_SERVER_ERROR, - - } - - public static ErrorResponse notFoundError(String message) { - return new ErrorResponse(NOT_FOUND, errorCodes.NOT_FOUND.name(), message); - } - - public static ErrorResponse internalServerError(String message) { - return new ErrorResponse(INTERNAL_SERVER_ERROR, errorCodes.INTERNAL_SERVER_ERROR.name(), message); - } - - public static ErrorResponse badRequest(String message) { - return new ErrorResponse(BAD_REQUEST, errorCodes.BAD_REQUEST.name(), message); - } - - public static ErrorResponse methodNotAllowed(String message) { - return new ErrorResponse(METHOD_NOT_ALLOWED, errorCodes.METHOD_NOT_ALLOWED.name(), message); - } - - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { return "application/json"; } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyException.java deleted file mode 100644 index aa828bc0c83..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyException.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.proxy; - -/** - * Exceptions related to proxying calls to config servers. - * - * @author Haakon Dybdahl - */ -public class ProxyException extends Exception { - public final ErrorResponse errorResponse; - - public ProxyException(ErrorResponse errorResponse) { - super(errorResponse.message); - this.errorResponse = errorResponse; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java index f398683567b..a6314df9581 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java @@ -28,33 +28,21 @@ public class ProxyRequest { private final List<URI> targets; private final String targetPath; - /** - * The constructor calls exception if the request is invalid. - * - * @param request the request from the jdisc framework. - * @param targets list of targets this request should be proxied to (targets are tried once in order until a response is returned). - * @param targetPath the path to proxy to. - * @throws ProxyException on errors - */ - public ProxyRequest(HttpRequest request, List<URI> targets, String targetPath) throws ProxyException { - this(request.getMethod(), request.getUri(), request.getJDiscRequest().headers(), request.getData(), - targets, targetPath); - } - - ProxyRequest(Method method, URI requestUri, Map<String, List<String>> headers, InputStream body, - List<URI> targets, String targetPath) throws ProxyException { - Objects.requireNonNull(requestUri, "Request must be non-null"); - if (!requestUri.getPath().endsWith(targetPath)) - throw new ProxyException(ErrorResponse.badRequest(String.format( - "Request path '%s' does not end with proxy path '%s'", requestUri.getPath(), targetPath))); - + ProxyRequest(Method method, URI url, Map<String, List<String>> headers, InputStream body, List<URI> targets, + String path) { + Objects.requireNonNull(url); + if (!url.getPath().endsWith(path)) { + throw new IllegalArgumentException(String.format("Request path '%s' does not end with proxy path '%s'", url.getPath(), path)); + } + if (targets.isEmpty()) { + throw new IllegalArgumentException("targets must be non-empty"); + } this.method = Objects.requireNonNull(method); - this.requestUri = Objects.requireNonNull(requestUri); + this.requestUri = Objects.requireNonNull(url); this.headers = Objects.requireNonNull(headers); this.requestData = body; - this.targets = List.copyOf(targets); - this.targetPath = targetPath.startsWith("/") ? targetPath : "/" + targetPath; + this.targetPath = path.startsWith("/") ? path : "/" + path; } @@ -100,4 +88,16 @@ public class ProxyRequest { return "[targets: " + targets + " request: " + targetPath + "]"; } + /** Create a proxy request that tries all given targets in order */ + public static ProxyRequest tryAll(List<URI> targets, String path, HttpRequest request) { + return new ProxyRequest(request.getMethod(), request.getUri(), request.getJDiscRequest().headers(), + request.getData(), targets, path); + } + + /** Create a proxy request that repeatedly tries a single target */ + public static ProxyRequest tryOne(URI target, String path, HttpRequest request) { + return new ProxyRequest(request.getMethod(), request.getUri(), request.getJDiscRequest().headers(), + request.getData(), List.of(target), path); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 5115ecaf196..54004685cf1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -54,7 +54,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.api.integration.resource.CostInfo; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; @@ -207,8 +206,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/")) return root(request); if (path.matches("/application/v4/tenant")) return tenants(request); if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/cost")) return tenantCost(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/cost/{month}")) return tenantCost(path.get("tenant"), path.get("month"), request); if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), Optional.empty(), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/compile-version")) return compileVersion(path.get("tenant"), path.get("application")); @@ -348,63 +345,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse tenantCost(String tenantName, HttpRequest request) { - return controller.tenants().get(TenantName.from(tenantName)) - .map(tenant -> tenantCost(tenant, request)) - .orElseGet(() -> ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist")); - } - - private HttpResponse tenantCost(Tenant tenant, HttpRequest request) { - Set<YearMonth> months = controller.serviceRegistry().tenantCost().monthsWithMetering(tenant.name()); - - var slime = new Slime(); - var objectCursor = slime.setObject(); - var monthsCursor = objectCursor.setArray("months"); - - months.forEach(month -> monthsCursor.addString(month.toString())); - return new SlimeJsonResponse(slime); - } - - private HttpResponse tenantCost(String tenantName, String dateString, HttpRequest request) { - return controller.tenants().get(TenantName.from(tenantName)) - .map(tenant -> tenantCost(tenant, tenantCostParseDate(dateString), request)) - .orElseGet(() -> ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist")); - } - - private YearMonth tenantCostParseDate(String dateString) { - try { - return YearMonth.parse(dateString); - } catch (DateTimeParseException e){ - throw new IllegalArgumentException("Could not parse year-month '" + dateString + "'"); - } - } - - private HttpResponse tenantCost(Tenant tenant, YearMonth month, HttpRequest request) { - var slime = new Slime(); - Cursor cursor = slime.setObject(); - cursor.setString("month", month.toString()); - List<CostInfo> costInfos = controller.serviceRegistry().tenantCost() - .getTenantCostOfMonth(tenant.name(), month); - Cursor array = cursor.setArray("items"); - - costInfos.forEach(costInfo -> { - Cursor costObject = array.addObject(); - costObject.setString("applicationId", costInfo.getApplicationId().serializedForm()); - costObject.setString("zoneId", costInfo.getZoneId().value()); - Cursor cpu = costObject.setObject("cpu"); - cpu.setDouble("usage", costInfo.getCpuHours().setScale(1, RoundingMode.HALF_UP).doubleValue()); - cpu.setLong("charge", costInfo.getCpuCost()); - Cursor memory = costObject.setObject("memory"); - memory.setDouble("usage", costInfo.getMemoryHours().setScale(1, RoundingMode.HALF_UP).doubleValue()); - memory.setLong("charge", costInfo.getMemoryCost()); - Cursor disk = costObject.setObject("disk"); - disk.setDouble("usage", costInfo.getDiskHours().setScale(1, RoundingMode.HALF_UP).doubleValue()); - disk.setLong("charge", costInfo.getDiskCost()); - }); - - return new SlimeJsonResponse(slime); - } - private HttpResponse applications(String tenantName, Optional<String> applicationName, HttpRequest request) { TenantName tenant = TenantName.from(tenantName); if (controller.tenants().get(tenantName).isEmpty()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java index 6ffdea93a1c..f65f7534476 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; -import com.yahoo.vespa.hosted.controller.proxy.ProxyException; import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; import com.yahoo.yolean.Exceptions; @@ -95,11 +94,7 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { "' through /configserver/v1, following APIs are permitted: " + String.join(", ", WHITELISTED_APIS)); } - try { - return proxy.handle(new ProxyRequest(request, List.of(getEndpoint(zoneId)), cfgPath)); - } catch (ProxyException e) { - throw new RuntimeException(e); - } + return proxy.handle(ProxyRequest.tryOne(getEndpoint(zoneId), cfgPath, request)); } private HttpResponse root(HttpRequest request) { @@ -126,4 +121,5 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { private URI getEndpoint(ZoneId zoneId) { return CONTROLLER_ZONE.equals(zoneId) ? CONTROLLER_URI : zoneRegistry.getConfigServerVipUri(zoneId); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 25ee95e6d80..b9cf5ca4f4d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -125,6 +125,11 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { roleMemberships.add(Role.paymentProcessor()); })); + futures.add(executor.submit(() -> { + if (athenz.hasAccountingAccess(identity)) + roleMemberships.add(Role.hostedAccountant()); + })); + // Run last request in handler thread to avoid creating extra thread. if (athenz.hasSystemFlagsAccess(identity, /*dryrun*/true)) roleMemberships.add(Role.systemFlagsDryrunner()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index 7d8c5922c3f..ac63f0cbcaf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -6,28 +6,30 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.vespa.hosted.controller.versions.OsVersion; +import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.yolean.Exceptions; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.time.Duration; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.StringJoiner; import java.util.logging.Level; @@ -133,6 +135,7 @@ public class OsApiHandler extends AuditLoggingRequestHandler { Inspector root = requestData.get(); Inspector versionField = root.field("version"); Inspector cloudField = root.field("cloud"); + Inspector upgradeBudgetField = root.field("upgradeBudget"); boolean force = root.field("force").asBool(); if (!versionField.valid() || !cloudField.valid()) { throw new IllegalArgumentException("Fields 'version' and 'cloud' are required"); @@ -145,25 +148,37 @@ public class OsApiHandler extends AuditLoggingRequestHandler { } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Invalid version '" + versionField.asString() + "'", e); } - - controller.upgradeOsIn(cloud, target, force); + Optional<Duration> upgradeBudget = Optional.of(upgradeBudgetField) + .filter(Inspector::valid) + .map(Inspector::asString).map(s -> { + try { + return Duration.parse(s); + } catch (Exception e2) { + throw new IllegalArgumentException("Invalid duration '" + s + "'", e2); + } + }); + + controller.upgradeOsIn(cloud, target, upgradeBudget, force); Slime response = new Slime(); Cursor cursor = response.setObject(); cursor.setString("message", "Set target OS version for cloud '" + cloud.value() + "' to " + - target.toFullString()); + target.toFullString() + upgradeBudget.map(d -> " with upgrade budget " + d) + .orElse("")); return response; } private Slime osVersions() { Slime slime = new Slime(); Cursor root = slime.setObject(); - Set<OsVersion> osVersions = controller.osVersions(); + Set<OsVersionTarget> targets = controller.osVersionTargets(); Cursor versions = root.setArray("versions"); controller.osVersionStatus().versions().forEach((osVersion, nodeVersions) -> { Cursor currentVersionObject = versions.addObject(); currentVersionObject.setString("version", osVersion.version().toFullString()); - currentVersionObject.setBool("targetVersion", osVersions.contains(osVersion)); + Optional<OsVersionTarget> target = targets.stream().filter(t -> t.osVersion().equals(osVersion)).findFirst(); + currentVersionObject.setBool("targetVersion", target.isPresent()); + target.flatMap(OsVersionTarget::upgradeBudget).ifPresent(budget -> currentVersionObject.setString("upgradeBudget", budget.toString())); currentVersionObject.setString("cloud", osVersion.cloud().value()); Cursor nodesArray = currentVersionObject.setArray("nodes"); nodeVersions.asMap().values().forEach(nodeVersion -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 0e3295b1143..1b3c216c1f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -18,6 +18,10 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeStream; import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; @@ -60,12 +64,14 @@ public class UserApiHandler extends LoggingRequestHandler { private final UserManagement users; private final Controller controller; + private final BooleanFlag enable_public_signup_flow; @Inject - public UserApiHandler(Context parentCtx, UserManagement users, Controller controller) { + public UserApiHandler(Context parentCtx, UserManagement users, Controller controller, FlagSource flagSource) { super(parentCtx); this.users = users; this.controller = controller; + this.enable_public_signup_flow = Flags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource); } @Override @@ -120,8 +126,18 @@ public class UserApiHandler extends LoggingRequestHandler { return response; } + private static final Set<RoleDefinition> hostedOperators = Set.of( + RoleDefinition.hostedOperator, + RoleDefinition.hostedSupporter, + RoleDefinition.hostedAccountant); + private HttpResponse userMetadata(HttpRequest request) { - User user = getAttribute(request, User.ATTRIBUTE_NAME, User.class); + @SuppressWarnings("unchecked") + Map<String, String> userAttributes = (Map<String, String>) getAttribute(request, User.ATTRIBUTE_NAME, Map.class); + User user = new User(userAttributes.get("email"), + userAttributes.get("name"), + userAttributes.get("nickname"), + userAttributes.get("picture")); Set<Role> roles = getAttribute(request, SecurityContext.ATTRIBUTE_NAME, SecurityContext.class).roles(); Map<TenantName, List<TenantRole>> tenantRolesByTenantName = roles.stream() @@ -130,10 +146,9 @@ public class UserApiHandler extends LoggingRequestHandler { .sorted(Comparator.comparing(Role::definition).reversed()) .collect(Collectors.groupingBy(TenantRole::tenant, Collectors.toList())); - // List of operator roles, currently only one available, but possible to extend + // List of operator roles as defined in `hostedOperators` above List<Role> operatorRoles = roles.stream() - .filter(role -> role.definition().equals(RoleDefinition.hostedOperator) || - role.definition().equals(RoleDefinition.hostedSupporter)) + .filter(role -> hostedOperators.contains(role.definition())) .sorted(Comparator.comparing(Role::definition)) .collect(Collectors.toList()); @@ -141,6 +156,9 @@ public class UserApiHandler extends LoggingRequestHandler { Cursor root = slime.setObject(); root.setBool("isPublic", controller.system().isPublic()); + root.setBool("isCd", controller.system().isCd()); + root.setBool(enable_public_signup_flow.id().toString(), + enable_public_signup_flow.with(FetchVector.Dimension.CONSOLE_USER_EMAIL, user.email()).value()); toSlime(root.setObject("user"), user); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index a127a44efb2..7c44ae6d1a5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -16,12 +16,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; -import com.yahoo.vespa.hosted.controller.proxy.ProxyException; import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; import com.yahoo.yolean.Exceptions; -import java.net.URI; -import java.util.List; import java.util.logging.Level; /** @@ -84,11 +81,7 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { if ( ! zoneRegistry.hasZone(zoneId)) { throw new IllegalArgumentException("No such zone: " + zoneId.value()); } - try { - return proxy.handle(new ProxyRequest(request, getConfigserverEndpoints(zoneId), path.getRest())); - } catch (ProxyException e) { - throw new RuntimeException(e); - } + return proxy.handle(proxyRequest(zoneId, path.getRest(), request)); } private HttpResponse root(HttpRequest request) { @@ -114,13 +107,12 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private List<URI> getConfigserverEndpoints(ZoneId zoneId) { + private ProxyRequest proxyRequest(ZoneId zoneId, String path, HttpRequest request) { // TODO: Use config server VIP for all zones that have one if (zoneId.region().value().startsWith("aws-") || zoneId.region().value().contains("-aws-")) { - return List.of(zoneRegistry.getConfigServerVipUri(zoneId)); - } else { - return zoneRegistry.getConfigServerUris(zoneId); + return ProxyRequest.tryOne(zoneRegistry.getConfigServerVipUri(zoneId), path, request); } + return ProxyRequest.tryAll(zoneRegistry.getConfigServerUris(zoneId), path, request); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 033539f64c4..a429c444e0b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -74,15 +75,15 @@ public class RoutingPolicies { * load balancers for given application have changed. */ public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { - var loadBalancers = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer() - .getLoadBalancers(application, zone), - deploymentSpec); + var allocation = new LoadBalancerAllocation(application, zone, controller.serviceRegistry().configServer() + .getLoadBalancers(application, zone), + deploymentSpec); var inactiveZones = inactiveZones(application, deploymentSpec); try (var lock = db.lockRoutingPolicies()) { - removeGlobalDnsUnreferencedBy(loadBalancers, lock); - storePoliciesOf(loadBalancers, lock); - removePoliciesUnreferencedBy(loadBalancers, lock); - updateGlobalDnsOf(get(loadBalancers.deployment.applicationId()).values(), inactiveZones, lock); + removeGlobalDnsUnreferencedBy(allocation, lock); + storePoliciesOf(allocation, lock); + removePoliciesUnreferencedBy(allocation, lock); + updateGlobalDnsOf(get(allocation.deployment.applicationId()).values(), inactiveZones, lock); } } @@ -155,13 +156,13 @@ public class RoutingPolicies { } /** Store routing policies for given load balancers */ - private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var policies = new LinkedHashMap<>(get(loadBalancers.deployment.applicationId())); - for (LoadBalancer loadBalancer : loadBalancers.list) { - var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), loadBalancers.deployment.zoneId()); + private void storePoliciesOf(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) { + var policies = new LinkedHashMap<>(get(allocation.deployment.applicationId())); + for (LoadBalancer loadBalancer : allocation.loadBalancers) { + var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), allocation.deployment.zoneId()); var existingPolicy = policies.get(policyId); var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.dnsZone(), - loadBalancers.endpointIdsOf(loadBalancer), + allocation.endpointIdsOf(loadBalancer), new Status(isActive(loadBalancer), GlobalRouting.DEFAULT_STATUS)); // Preserve global routing status for existing policy if (existingPolicy != null) { @@ -170,53 +171,62 @@ public class RoutingPolicies { updateZoneDnsOf(newPolicy); policies.put(newPolicy.id(), newPolicy); } - db.writeRoutingPolicies(loadBalancers.deployment.applicationId(), policies); + db.writeRoutingPolicies(allocation.deployment.applicationId(), policies); } /** Update zone DNS record for given policy */ private void updateZoneDnsOf(RoutingPolicy policy) { - var name = RecordName.from(policy.endpointIn(controller.system(), RoutingMethod.exclusive).dnsName()); + var name = RecordName.from(policy.endpointIn(controller.system(), RoutingMethod.exclusive, controller.zoneRegistry()) + .dnsName()); var data = RecordData.fqdn(policy.canonicalName().value()); - nameUpdaterIn(policy.id().zone()).createCname(name, data); + NameUpdater nameUpdater = nameUpdaterIn(policy.id().zone()); + if (policy.id().owner().equals(SystemApplication.configServer.id())) { + // TODO(mpolden): Remove this after transition is complete. Before automatic provisioning of config server + // load balancers, the DNS records for the config server LB were of type A. It's not possible + // to change the type of an existing record, we therefore remove the A record before creating + // a CNAME. + nameUpdater.removeRecords(Record.Type.A, name); + } + nameUpdater.createCname(name, data); } /** Remove policies and zone DNS records unreferenced by given load balancers */ - private void removePoliciesUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var policies = get(loadBalancers.deployment.applicationId()); + private void removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) { + var policies = get(allocation.deployment.applicationId()); var newPolicies = new LinkedHashMap<>(policies); - var activeLoadBalancers = loadBalancers.list.stream().map(LoadBalancer::hostname).collect(Collectors.toSet()); + var activeIds = allocation.asPolicyIds(); for (var policy : policies.values()) { // Leave active load balancers and irrelevant zones alone - if (activeLoadBalancers.contains(policy.canonicalName()) || - !policy.id().zone().equals(loadBalancers.deployment.zoneId())) continue; + if (activeIds.contains(policy.id()) || + !policy.id().zone().equals(allocation.deployment.zoneId())) continue; - var dnsName = policy.endpointIn(controller.system(), RoutingMethod.exclusive).dnsName(); - nameUpdaterIn(loadBalancers.deployment.zoneId()).removeRecords(Record.Type.CNAME, RecordName.from(dnsName)); + var dnsName = policy.endpointIn(controller.system(), RoutingMethod.exclusive, controller.zoneRegistry()).dnsName(); + nameUpdaterIn(allocation.deployment.zoneId()).removeRecords(Record.Type.CNAME, RecordName.from(dnsName)); newPolicies.remove(policy.id()); } - db.writeRoutingPolicies(loadBalancers.deployment.applicationId(), newPolicies); + db.writeRoutingPolicies(allocation.deployment.applicationId(), newPolicies); } /** Remove unreferenced global endpoints from DNS */ - private void removeGlobalDnsUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var zonePolicies = get(loadBalancers.deployment).values(); + private void removeGlobalDnsUnreferencedBy(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) { + var zonePolicies = get(allocation.deployment).values(); var removalCandidates = new HashSet<>(routingTableFrom(zonePolicies).keySet()); - var activeRoutingIds = routingIdsFrom(loadBalancers); + var activeRoutingIds = routingIdsFrom(allocation); removalCandidates.removeAll(activeRoutingIds); for (var id : removalCandidates) { var endpoints = controller.routing().endpointsOf(id.application()) .not().requiresRotation() .named(id.endpointId()); - var nameUpdater = nameUpdaterIn(loadBalancers.deployment.zoneId()); + var nameUpdater = nameUpdaterIn(allocation.deployment.zoneId()); endpoints.forEach(endpoint -> nameUpdater.removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()))); } } /** Compute routing IDs from given load balancers */ - private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) { + private static Set<RoutingId> routingIdsFrom(LoadBalancerAllocation allocation) { Set<RoutingId> routingIds = new LinkedHashSet<>(); - for (var loadBalancer : loadBalancers.list) { - for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) { + for (var loadBalancer : allocation.loadBalancers) { + for (var endpointId : allocation.endpointIdsOf(loadBalancer)) { routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); } } @@ -257,19 +267,28 @@ public class RoutingPolicies { } /** Load balancers allocated to a deployment */ - private static class AllocatedLoadBalancers { + private static class LoadBalancerAllocation { private final DeploymentId deployment; - private final List<LoadBalancer> list; + private final List<LoadBalancer> loadBalancers; private final DeploymentSpec deploymentSpec; - private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, + private LoadBalancerAllocation(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, DeploymentSpec deploymentSpec) { this.deployment = new DeploymentId(application, zone); - this.list = List.copyOf(loadBalancers); + this.loadBalancers = List.copyOf(loadBalancers); this.deploymentSpec = deploymentSpec; } + /** Returns the policy IDs of the load balancers contained in this */ + private Set<RoutingPolicyId> asPolicyIds() { + return loadBalancers.stream() + .map(lb -> new RoutingPolicyId(lb.application(), + lb.cluster(), + deployment.zoneId())) + .collect(Collectors.toUnmodifiableSet()); + } + /** Compute all endpoint IDs for given load balancer */ private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { if (!deployment.zoneId().environment().isProduction()) { // Only production deployments have configurable endpoints diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 37027e8a8c9..c56a5f0bd66 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -5,9 +5,11 @@ import com.google.common.collect.ImmutableSortedSet; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import java.util.Objects; import java.util.Optional; @@ -68,7 +70,10 @@ public class RoutingPolicy { } /** Returns the endpoint of this */ - public Endpoint endpointIn(SystemName system, RoutingMethod routingMethod) { + public Endpoint endpointIn(SystemName system, RoutingMethod routingMethod, ZoneRegistry zoneRegistry) { + Optional<Endpoint> infraEndpoint = SystemApplication.matching(id.owner()) + .flatMap(app -> app.endpointIn(id.zone(), zoneRegistry)); + if (infraEndpoint.isPresent()) return infraEndpoint.get(); return Endpoint.of(id.owner()) .target(id.cluster(), id.zone()) .on(Port.fromRoutingMethod(routingMethod)) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java new file mode 100644 index 00000000000..a908b341039 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/Auth0Credentials.java @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.security; + +import com.yahoo.vespa.hosted.controller.api.role.Role; + +import java.security.Principal; +import java.util.Collections; +import java.util.Set; + +/** + * Like {@link Credentials}, but we know the principal is authenticated by Auth0. + * Also includes the set of roles for which the principal is a member. + * + * @author andreer + */ +public class Auth0Credentials extends Credentials { + + private final Set<Role> roles; + + public Auth0Credentials(Principal user, Set<Role> roles) { + super(user); + this.roles = Collections.unmodifiableSet(roles); + } + + /** The set of roles set in the auth0 cookie, extracted by CloudAccessControlRequests. */ + public Set<Role> getRolesFromCookie() { + return roles; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index f34c9c67baa..bd0143ef879 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -3,7 +3,13 @@ package com.yahoo.vespa.hosted.controller.security; import com.google.inject.Inject; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanController; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; @@ -15,24 +21,34 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; +import javax.ws.rs.ForbiddenException; import java.util.List; +import static com.yahoo.vespa.hosted.controller.api.role.RoleDefinition.*; + /** * @author jonmv + * @author andreer */ public class CloudAccessControl implements AccessControl { private static final BillingInfo defaultBillingInfo = new BillingInfo("customer", "Vespa"); private final UserManagement userManagement; + private final BooleanFlag enablePublicSignup; + private final PlanController planController; @Inject - public CloudAccessControl(UserManagement userManagement) { + public CloudAccessControl(UserManagement userManagement, FlagSource flagSource, ServiceRegistry serviceRegistry) { this.userManagement = userManagement; + this.enablePublicSignup = Flags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource); + planController = serviceRegistry.planController(); } @Override public CloudTenant createTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing) { + requireTenantCreationAllowed((Auth0Credentials) credentials); + CloudTenantSpec spec = (CloudTenantSpec) tenantSpec; CloudTenant tenant = CloudTenant.create(spec.tenant(), defaultBillingInfo); @@ -48,6 +64,36 @@ public class CloudAccessControl implements AccessControl { return tenant; } + private void requireTenantCreationAllowed(Auth0Credentials auth0Credentials) { + if (allowedByPrivilegedRole(auth0Credentials)) return; + + if (!allowedByFeatureFlag(auth0Credentials)) { + throw new ForbiddenException("You are not currently permitted to create tenants. Please contact the Vespa team to request access."); + } + + if(administeredTenants(auth0Credentials) >= 3) { + throw new ForbiddenException("You are already administering 3 tenants. If you need more, please contact the Vespa team."); + } + } + + private boolean allowedByPrivilegedRole(Auth0Credentials auth0Credentials) { + return auth0Credentials.getRolesFromCookie().stream() + .map(Role::definition) + .anyMatch(rd -> rd == hostedOperator || rd == hostedSupporter); + } + + private boolean allowedByFeatureFlag(Auth0Credentials auth0Credentials) { + return enablePublicSignup.with(FetchVector.Dimension.CONSOLE_USER_EMAIL, auth0Credentials.user().getName()).value(); + } + + private long administeredTenants(Auth0Credentials auth0Credentials) { + // We have to verify the roles with auth0 to ensure the user is not using an "old" cookie to make too many tenants. + return userManagement.listRoles(new UserId(auth0Credentials.user().getName())).stream() + .map(Role::definition) + .filter(rd -> rd == administrator) + .count(); + } + @Override public Tenant updateTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing, List<Application> applications) { throw new UnsupportedOperationException("Update is not supported here, as it would entail changing the tenant name."); @@ -55,12 +101,17 @@ public class CloudAccessControl implements AccessControl { @Override public void deleteTenant(TenantName tenant, Credentials credentials) { - // Probably terminate customer subscription? + if(!(allowedByPrivilegedRole((Auth0Credentials) credentials) || isTrial(tenant))) + throw new ForbiddenException("Please contact the Vespa team for assistance in deleting non-trial tenants"); for (TenantRole role : Roles.tenantRoles(tenant)) userManagement.deleteRole(role); } + private boolean isTrial(TenantName tenant) { + return planController.getPlan(tenant).id().equals("trial"); + } + @Override public void createApplication(TenantAndApplicationId id, Credentials credentials) { for (Role role : Roles.applicationRoles(id.tenant(), id.application())) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java index c0f8f585216..cc26ed1427a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControlRequests.java @@ -4,25 +4,37 @@ package com.yahoo.vespa.hosted.controller.security; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.slime.Inspector; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import java.util.Optional; +import java.util.Set; + /** * Extracts access control data for {@link CloudTenant}s from HTTP requests. * * @author jonmv + * @author andreer */ public class CloudAccessControlRequests implements AccessControlRequests { @Override public CloudTenantSpec specification(TenantName tenant, Inspector requestObject) { - // TODO extract marketplace token. - return new CloudTenantSpec(tenant, "token"); + return new CloudTenantSpec(tenant, "token"); // TODO: remove token } @Override public Credentials credentials(TenantName tenant, Inspector requestObject, HttpRequest request) { - // TODO Include roles, if this is to be used for displaying accessible data. - return new Credentials(request.getUserPrincipal()); + return new Auth0Credentials(request.getUserPrincipal(), getUserRoles(request)); + } + + private static Set<Role> getUserRoles(HttpRequest request) { + var securityContext = Optional.ofNullable(request.context().get(SecurityContext.ATTRIBUTE_NAME)) + .filter(SecurityContext.class::isInstance) + .map(SecurityContext.class::cast) + .orElseThrow(() -> new IllegalArgumentException("Attribute '" + SecurityContext.ATTRIBUTE_NAME + "' was not set on request")); + return securityContext.roles(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java index b91ed3734f1..99cf7542d53 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -88,9 +88,9 @@ public class DeploymentStatistics { // Add all unsuccessful runs for failing jobs as any run may have resulted in an incomplete deployment // where a subset of nodes have upgraded. - // TODO jonmv: canary-pipeline.custom on 7.188.11, but not really, in staging ... failing.not().failingApplicationChange() .not().withStatus(RunStatus.outOfCapacity) + .not().withStatus(RunStatus.aborted) .mapToList(JobStatus::runs) .forEach(runs -> runs.descendingMap().values().stream() .dropWhile(run -> ! run.hasEnded()) @@ -103,6 +103,7 @@ public class DeploymentStatistics { failing.failingApplicationChange() .concat(failing.withStatus(RunStatus.outOfCapacity)) + .concat(failing.withStatus(RunStatus.aborted)) .lastCompleted().asList() .forEach(run -> { otherFailing.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java index 4a00add411f..7aab759f676 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java @@ -66,24 +66,6 @@ public class NodeVersion { return suspendedAt; } - /** Returns a copy of this with current version set to given version */ - public NodeVersion withCurrentVersion(Version version) { - if (currentVersion.equals(version)) return this; - return new NodeVersion(hostname, zone, version, wantedVersion, suspendedAt); - } - - /** Returns a copy of this with wanted version set to given version */ - public NodeVersion withWantedVersion(Version version) { - if (wantedVersion.equals(version)) return this; - return new NodeVersion(hostname, zone, currentVersion, version, suspendedAt); - } - - /** Returns a copy of this with wanted version set to given version */ - public NodeVersion withSuspendedAt(Optional<Instant> suspendedAt) { - if (suspendedAt.equals(this.suspendedAt)) return this; - return new NodeVersion(hostname, zone, currentVersion, wantedVersion, suspendedAt); - } - @Override public String toString() { return hostname + ": " + currentVersion + " -> " + wantedVersion + " [zone=" + zone + ", suspendedAt=" + suspendedAt.map(Instant::toString).orElse("<not suspended>") + "]"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index 1dffd1383bd..c505dbfe1c6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -59,18 +59,18 @@ public class OsVersionStatus { /** Compute the current OS versions in this system. This is expensive and should be called infrequently */ public static OsVersionStatus compute(Controller controller) { var osVersions = new HashMap<OsVersion, List<NodeVersion>>(); - controller.osVersions().forEach(osVersion -> osVersions.put(osVersion, new ArrayList<>())); + controller.osVersionTargets().forEach(target -> osVersions.put(target.osVersion(), new ArrayList<>())); for (var application : SystemApplication.all()) { - if (!application.shouldUpgradeOs()) continue; for (var zone : zonesToUpgrade(controller)) { + if (!application.shouldUpgradeOsIn(zone.getId(), controller)) continue; var targetOsVersion = controller.serviceRegistry().configServer().nodeRepository() .targetVersionsOf(zone.getId()) .osVersion(application.nodeType()) .orElse(Version.emptyVersion); for (var node : controller.serviceRegistry().configServer().nodeRepository().list(zone.getId(), application.id())) { - if (!OsUpgrader.eligibleForUpgrade(node, application)) continue; + if (!OsUpgrader.canUpgrade(node)) continue; var suspendedAt = node.suspendedSince(); var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(), targetOsVersion, suspendedAt); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java new file mode 100644 index 00000000000..bacd2ada298 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java @@ -0,0 +1,68 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.versions; + +import org.jetbrains.annotations.NotNull; + +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + +/** + * An {@link OsVersion} and its upgrade budget. + * + * @author mpolden + */ +public class OsVersionTarget implements Comparable<OsVersionTarget> { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private final OsVersion osVersion; + private final Optional<Duration> upgradeBudget; + + public OsVersionTarget(OsVersion osVersion, Optional<Duration> upgradeBudget) { + this.osVersion = Objects.requireNonNull(osVersion); + this.upgradeBudget = requireNotNegative(upgradeBudget); + } + + /** The OS version contained in this target */ + public OsVersion osVersion() { + return osVersion; + } + + /** The total time budget across all zones for applying target, if any */ + public Optional<Duration> upgradeBudget() { + return upgradeBudget; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OsVersionTarget that = (OsVersionTarget) o; + return osVersion.equals(that.osVersion) && + upgradeBudget.equals(that.upgradeBudget); + } + + @Override + public int hashCode() { + return Objects.hash(osVersion, upgradeBudget); + } + + private static Optional<Duration> requireNotNegative(Optional<Duration> duration) { + Objects.requireNonNull(duration); + if (duration.isEmpty()) return duration; + if (duration.get().isNegative()) throw new IllegalArgumentException("Duration cannot be negative"); + return duration; + } + + @Override + public int compareTo(@NotNull OsVersionTarget o) { + return osVersion.compareTo(o.osVersion); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index 7122b7dcc40..593b1438856 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -46,9 +46,9 @@ public class VespaVersion implements Comparable<VespaVersion> { } public static Confidence confidenceFrom(DeploymentStatistics statistics, Controller controller) { - InstanceList all = InstanceList.from(controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()))) - .withProductionDeployment(); - // 'production on this': All deployment jobs upgrading to this version have completed without failure + InstanceList all = InstanceList.from(controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()) + .withProductionDeployment())); + // 'production on this': All production deployment jobs upgrading to this version have completed without failure InstanceList productionOnThis = all.matching(instance -> statistics.productionSuccesses().stream().anyMatch(run -> run.id().application().equals(instance))) .not().failingUpgrade() .not().upgradingTo(statistics.version()); @@ -63,12 +63,12 @@ public class VespaVersion implements Comparable<VespaVersion> { return Confidence.broken; // 'low' unless all canary applications are upgraded - if (productionOnThis.with(UpgradePolicy.canary).size() < all.with(UpgradePolicy.canary).size()) + if (productionOnThis.with(UpgradePolicy.canary).size() < all.withProductionDeployment().with(UpgradePolicy.canary).size()) return Confidence.low; // 'high' if 90% of all default upgrade applications upgraded if (productionOnThis.with(UpgradePolicy.defaultPolicy).size() >= - all.with(UpgradePolicy.defaultPolicy).size() * 0.9) + all.withProductionDeployment().with(UpgradePolicy.defaultPolicy).size() * 0.9) return Confidence.high; return Confidence.normal; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 0ff14e01874..dfca2d69a5e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -700,6 +700,8 @@ public class ControllerTest { // Create app1 var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); var prodZone = ZoneId.from("prod", "us-west-1"); + var stagingZone = ZoneId.from("staging", "us-east-3"); + var testZone = ZoneId.from("test", "us-east-1"); tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(prodZone)); var applicationPackage = new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION) @@ -713,7 +715,7 @@ public class ControllerTest { assertEquals(Stream.concat(Stream.of("vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud", "app1.tenant1.global.vespa.oath.cloud", "*.app1.tenant1.global.vespa.oath.cloud"), - tester.controller().zoneRegistry().zones().controllerUpgraded().ids().stream() + Stream.of(prodZone, testZone, stagingZone) .flatMap(zone -> Stream.of("", "*.") .map(prefix -> prefix + "app1.tenant1." + zone.region().value() + (zone.environment() == Environment.prod ? "" : "." + zone.environment().value()) + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 189615ad763..35093c22f42 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -41,6 +41,7 @@ import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; +import com.yahoo.vespa.hosted.controller.security.Auth0Credentials; import com.yahoo.vespa.hosted.controller.security.CloudTenantSpec; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; @@ -54,6 +55,7 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.OptionalLong; @@ -302,7 +304,7 @@ public final class ControllerTester { private TenantName createCloudTenant(String tenantName) { TenantName tenant = TenantName.from(tenantName); TenantSpec spec = new CloudTenantSpec(tenant, "token"); - controller().tenants().create(spec, new Credentials(new SimplePrincipal("dev"))); + controller().tenants().create(spec, new Auth0Credentials(new SimplePrincipal("dev"), Collections.emptySet())); return tenant; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java index d7bc73adf37..d0e87056821 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java @@ -1,6 +1,9 @@ package com.yahoo.vespa.hosted.controller.certificate; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyAlgorithm; @@ -25,8 +28,10 @@ import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -45,20 +50,49 @@ public class EndpointCertificateManagerTest { private final EndpointCertificateManager endpointCertificateManager = new EndpointCertificateManager(zoneRegistryMock, mockCuratorDb, secretStore, endpointCertificateMock, clock, inMemoryFlagSource); + private static List<String> expectedSans = List.of( + "vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", + "default.default.global.vespa.oath.cloud", + "*.default.default.global.vespa.oath.cloud", + "default.default.aws-us-east-1a.vespa.oath.cloud", + "*.default.default.aws-us-east-1a.vespa.oath.cloud", + "default.default.us-east-1.test.vespa.oath.cloud", + "*.default.default.us-east-1.test.vespa.oath.cloud", + "default.default.us-east-3.staging.vespa.oath.cloud", + "*.default.default.us-east-3.staging.vespa.oath.cloud" + ); + + private static List<String> expectedAdditionalSans = List.of( + "default.default.ap-northeast-1.vespa.oath.cloud", + "*.default.default.ap-northeast-1.vespa.oath.cloud" + ); + + private static List<String> expectedCombinedSans = new ArrayList<>() {{ + addAll(expectedSans); + addAll(expectedAdditionalSans); + }}; + + private static List<String> expectedDevSans = List.of( + "vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", + "default.default.us-east-1.dev.vespa.oath.cloud", + "*.default.default.us-east-1.dev.vespa.oath.cloud" + ); + private static final KeyPair testKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 192); - private static final X509Certificate testCertificate = X509CertificateBuilder - .fromKeypair( - testKeyPair, - new X500Principal("CN=test"), - Instant.now(), Instant.now().plus(5, ChronoUnit.MINUTES), - SignatureAlgorithm.SHA256_WITH_ECDSA, - X509CertificateBuilder.generateRandomSerialNumber()) - .addSubjectAlternativeName("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud") - .addSubjectAlternativeName("default.default.global.vespa.oath.cloud") - .addSubjectAlternativeName("*.default.default.global.vespa.oath.cloud") - .addSubjectAlternativeName("default.default.us-east-1.test.vespa.oath.cloud") - .addSubjectAlternativeName("*.default.default.us-east-1.test.vespa.oath.cloud") - .build(); + private static final X509Certificate testCertificate = makeTestCert(expectedSans); + private static final X509Certificate testCertificate2 = makeTestCert(expectedCombinedSans); + + private static X509Certificate makeTestCert(List<String> sans) { + X509CertificateBuilder x509CertificateBuilder = X509CertificateBuilder + .fromKeypair( + testKeyPair, + new X500Principal("CN=test"), + Instant.now(), Instant.now().plus(5, ChronoUnit.MINUTES), + SignatureAlgorithm.SHA256_WITH_ECDSA, + X509CertificateBuilder.generateRandomSerialNumber()); + for (String san : sans) x509CertificateBuilder = x509CertificateBuilder.addSubjectAlternativeName(san); + return x509CertificateBuilder.build(); + } private final Instance testInstance = new Instance(ApplicationId.defaultId()); private final String testKeyName = "testKeyName"; @@ -68,25 +102,37 @@ public class EndpointCertificateManagerTest { @Before public void setUp() { zoneRegistryMock.exclusiveRoutingIn(zoneRegistryMock.zones().all().zones()); - testZone = zoneRegistryMock.zones().directlyRouted().zones().stream().findFirst().orElseThrow().getId(); + testZone = zoneRegistryMock.zones().directlyRouted().in(Environment.prod).zones().stream().findFirst().orElseThrow().getId(); inMemoryFlagSource.withBooleanFlag(Flags.VALIDATE_ENDPOINT_CERTIFICATES.id(), true); } @Test - public void provisions_new_certificate() { - Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + public void provisions_new_certificate_in_dev() { + ZoneId testZone = zoneRegistryMock.zones().directlyRouted().in(Environment.dev).zones().stream().findFirst().orElseThrow().getId(); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); + assertTrue(endpointCertificateMetadata.isPresent()); + assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, endpointCertificateMetadata.get().version()); + assertEquals(expectedDevSans, endpointCertificateMetadata.get().requestedDnsSans().orElseThrow()); + } + + @Test + public void provisions_new_certificate_in_prod() { + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); assertTrue(endpointCertificateMetadata.isPresent()); assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); assertEquals(0, endpointCertificateMetadata.get().version()); + assertEquals(expectedSans, endpointCertificateMetadata.get().requestedDnsSans().orElseThrow()); } @Test public void reuses_stored_certificate_metadata() { mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7)); secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 7); - secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 7); - Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 7); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); assertTrue(endpointCertificateMetadata.isPresent()); assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); assertEquals(testCertName, endpointCertificateMetadata.get().certName()); @@ -101,9 +147,9 @@ public class EndpointCertificateManagerTest { secretStore.setSecret(testCertName, "cert", 7); secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8); secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9); - secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 8); + secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 8); mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7)); - Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); assertTrue(endpointCertificateMetadata.isPresent()); assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); assertEquals(testCertName, endpointCertificateMetadata.get().certName()); @@ -114,11 +160,50 @@ public class EndpointCertificateManagerTest { public void reprovisions_certificate_when_necessary() { mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, Optional.of("uuid"), Optional.of(List.of()), Optional.empty())); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); - secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 0); - Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 0); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); assertTrue(endpointCertificateMetadata.isPresent()); assertEquals(0, endpointCertificateMetadata.get().version()); assertEquals(endpointCertificateMetadata, mockCuratorDb.readEndpointCertificateMetadata(testInstance.id())); } + @Test + public void reprovisions_certificate_with_added_sans_when_deploying_to_new_zone() { + ZoneId testZone = zoneRegistryMock.zones().directlyRouted().in(Environment.prod).zones().stream().skip(1).findFirst().orElseThrow().getId(); + + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, Optional.of("uuid"), Optional.of(expectedSans), Optional.of("mockCa"))); + secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), -1); + secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), -1); + + secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); + secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate2) + X509CertificateUtils.toPem(testCertificate2), 0); + + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); + assertTrue(endpointCertificateMetadata.isPresent()); + assertEquals(0, endpointCertificateMetadata.get().version()); + assertEquals(endpointCertificateMetadata, mockCuratorDb.readEndpointCertificateMetadata(testInstance.id())); + assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans().orElseThrow())); + } + + @Test + public void includes_zones_in_deployment_spec_when_deploying_to_staging() { + + DeploymentSpec deploymentSpec = new DeploymentSpecXmlReader(true).read( + "<deployment version=\"1.0\">\n" + + " <instance id=\"default\">\n" + + " <prod>\n" + + " <region active=\"true\">aws-us-east-1a</region>\n" + + " <region active=\"true\">ap-northeast-1</region>\n" + + " </prod>\n" + + " </instance>\n" + + "</deployment>\n"); + + ZoneId testZone = zoneRegistryMock.zones().controllerUpgraded().in(Environment.staging).zones().stream().findFirst().orElseThrow().getId(); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.of(deploymentSpec.requireInstance("default"))); + assertTrue(endpointCertificateMetadata.isPresent()); + assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, endpointCertificateMetadata.get().version()); + assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans().orElseThrow())); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index dd2267be3a4..acb8cf1a2a9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -447,7 +447,7 @@ public class DeploymentContext { if (job.type() == JobType.stagingTest) { // Do the initial deployment and installation of the real application. assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); Versions versions = currentRun(job).versions(); - tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), versions.sourcePlatform().orElse(versions.targetPlatform())); + tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), tester.configServer().application(job.application(), zone).get().version().get()); configServer().convergeServices(id.application(), zone); runner.advance(currentRun(job)); assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); @@ -471,7 +471,7 @@ public class DeploymentContext { DeploymentId deployment = new DeploymentId(job.application(), zone); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installReal)); - configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), currentRun(job).versions().targetPlatform()); + configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), tester.configServer().application(job.application(), zone).get().version().get()); runner.advance(currentRun(job)); } @@ -507,7 +507,7 @@ public class DeploymentContext { ZoneId zone = zone(job); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); - configServer().nodeRepository().doUpgrade(new DeploymentId(TesterId.of(job.application()).id(), zone), Optional.empty(), tester.controller().systemVersion()); + configServer().nodeRepository().doUpgrade(new DeploymentId(TesterId.of(job.application()).id(), zone), Optional.empty(), tester.configServer().application(id.tester().id(), zone).get().version().get()); runner.advance(currentRun(job)); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); configServer().convergeServices(TesterId.of(id.application()).id(), zone); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 2b5e8b3e91c..6417119618b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -1128,4 +1128,48 @@ public class DeploymentTriggerTest { .runJob(productionCdAwsUsEast1a); } + @Test + public void testsInSeparateInstance() { + String deploymentSpec = + "<deployment version='1.0'>\n" + + " <instance id='canary'>\n" + + " <upgrade policy='canary' />\n" + + " <test />\n" + + " <staging />\n" + + " </instance>\n" + + " <instance id='default'>\n" + + " <prod>\n" + + " <region active='true'>eu-west-1</region>\n" + + " <test>eu-west-1</test>\n" + + " </prod>\n" + + " </instance>\n" + + "</deployment>\n"; + + ApplicationPackage applicationPackage = ApplicationPackageBuilder.fromDeploymentXml(deploymentSpec); + var canary = tester.newDeploymentContext("t", "a", "canary").submit(applicationPackage); + var conservative = tester.newDeploymentContext("t", "a", "default"); + + canary.runJob(systemTest) + .runJob(stagingTest); + conservative.runJob(productionEuWest1) + .runJob(testEuWest1); + + canary.submit(applicationPackage) + .runJob(systemTest) + .runJob(stagingTest); + tester.outstandingChangeDeployer().run(); + conservative.runJob(productionEuWest1) + .runJob(testEuWest1); + + tester.controllerTester().upgradeSystem(new Version("7.7.7")); + tester.upgrader().maintain(); + + canary.runJob(systemTest) + .runJob(stagingTest); + tester.upgrader().maintain(); + conservative.runJob(productionEuWest1) + .runJob(testEuWest1); + + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java index d0362ae98b8..30ed9b5432c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueueTest.java @@ -74,13 +74,21 @@ public class NameServiceQueueTest { assertTrue(queue.requests().isEmpty()); assertTrue("Removed " + r1, nameService.findRecords(Record.Type.CNAME, r1.name()).isEmpty()); - // Keep n most recent requests + // Keep n last requests queue = queue.with(req1).with(req2).with(req3).with(req4).with(req6) .last(2); assertEquals(List.of(req4, req6), List.copyOf(queue.requests())); assertSame(queue, queue.last(2)); assertSame(queue, queue.last(10)); assertTrue(queue.last(0).requests().isEmpty()); + + // Keep n first requests + queue = NameServiceQueue.EMPTY.with(req1).with(req2).with(req3).with(req4).with(req6) + .first(3); + assertEquals(List.of(req1, req2, req3), List.copyOf(queue.requests())); + assertSame(queue, queue.first(3)); + assertSame(queue, queue.first(10)); + assertTrue(queue.first(0).requests().isEmpty()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java index d6e1af07938..d481aaa2c77 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java @@ -20,10 +20,10 @@ public class ConfigServerProxyMock extends AbstractComponent implements ConfigSe private volatile String requestBody = null; @Override - public HttpResponse handle(ProxyRequest proxyRequest) { - lastReceived = proxyRequest; + public HttpResponse handle(ProxyRequest request) { + lastReceived = request; // Copy request body as the input stream is drained once the request completes - requestBody = asString(proxyRequest.getData()); + requestBody = asString(request.getData()); return new StringResponse("ok"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 76a30c289b8..90276b6b590 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import java.time.Duration; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -37,6 +38,7 @@ public class NodeRepositoryMock implements NodeRepository { private final Map<ZoneId, Map<HostName, Node>> nodeRepository = new HashMap<>(); private final Map<ZoneId, Map<ApplicationId, Application>> applications = new HashMap<>(); private final Map<ZoneId, TargetVersions> targetVersions = new HashMap<>(); + private final Map<Integer, Duration> osUpgradeBudgets = new HashMap<>(); /** Add or update given nodes in zone */ public void putNodes(ZoneId zone, List<Node> nodes) { @@ -190,7 +192,8 @@ public class NodeRepositoryMock implements NodeRepository { } @Override - public void upgradeOs(ZoneId zone, NodeType type, Version version) { + public void upgradeOs(ZoneId zone, NodeType type, Version version, Optional<Duration> upgradeBudget) { + upgradeBudget.ifPresent(d -> this.osUpgradeBudgets.put(Objects.hash(zone, type, version), d)); this.targetVersions.compute(zone, (ignored, targetVersions) -> { if (targetVersions == null) { targetVersions = TargetVersions.EMPTY; @@ -223,6 +226,10 @@ public class NodeRepositoryMock implements NodeRepository { nodeRepository.get(zoneId).remove(HostName.from(hostName)); } + public Optional<Duration> osUpgradeBudget(ZoneId zone, NodeType type, Version version) { + return Optional.ofNullable(osUpgradeBudgets.get(Objects.hash(zone, type, version))); + } + public void doUpgrade(DeploymentId deployment, Optional<HostName> hostName, Version version) { modifyNodes(deployment, hostName, node -> { assert node.wantedVersion().equals(version); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 98178f2a19f..b7e7c9814e3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -7,21 +7,24 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.SystemName; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; +import com.yahoo.vespa.hosted.controller.api.integration.aws.ApplicationRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; +import com.yahoo.vespa.hosted.controller.api.integration.aws.NoopApplicationRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanController; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; -import com.yahoo.vespa.hosted.controller.api.integration.organization.MockBilling; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; +import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MockTenantCost; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummySystemMonitor; import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; @@ -48,15 +51,16 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final DummyOwnershipIssues dummyOwnershipIssues = new DummyOwnershipIssues(); private final LoggingDeploymentIssues loggingDeploymentIssues = new LoggingDeploymentIssues(); private final MemoryEntityService memoryEntityService = new MemoryEntityService(); + private final DummySystemMonitor systemMonitor = new DummySystemMonitor(); private final CostReportConsumerMock costReportConsumerMock = new CostReportConsumerMock(); - private final MockBilling mockBilling = new MockBilling(); private final MockAwsEventFetcher mockAwsEventFetcher = new MockAwsEventFetcher(); private final ArtifactRepositoryMock artifactRepositoryMock = new ArtifactRepositoryMock(); private final MockTesterCloud mockTesterCloud; private final ApplicationStoreMock applicationStoreMock = new ApplicationStoreMock(); private final MockRunDataStore mockRunDataStore = new MockRunDataStore(); - private final MockTenantCost mockTenantCost = new MockTenantCost(); private final MockResourceTagger mockResourceTagger = new MockResourceTagger(); + private final ApplicationRoleService applicationRoleService = new NoopApplicationRoleService(); + private final PlanController planController = (tenantName) -> null; public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); @@ -134,11 +138,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public MockBilling billingService() { - return mockBilling; - } - - @Override public MockAwsEventFetcher eventFetcherService() { return mockAwsEventFetcher; } @@ -169,9 +168,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public MockTenantCost tenantCost() { return mockTenantCost;} - - @Override public ZoneRegistryMock zoneRegistry() { return zoneRegistryMock; } @@ -181,6 +177,16 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return mockResourceTagger; } + @Override + public ApplicationRoleService applicationRoleService() { + return applicationRoleService; + } + + @Override + public DummySystemMonitor systemMonitor() { + return systemMonitor; + } + public ConfigServerMock configServerMock() { return configServerMock; } @@ -197,4 +203,9 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return endpointCertificateMock; } + @Override + public PlanController planController() { + return planController; + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index 23f59683f4b..841624cb011 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -26,21 +26,23 @@ public class ZoneFilterMock implements ZoneList { private final List<ZoneApi> zones; private final Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods; + private final Set<ZoneApi> reprovisionToUpgradeOs; private final boolean negate; - private ZoneFilterMock(List<ZoneApi> zones, Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods, boolean negate) { + private ZoneFilterMock(List<ZoneApi> zones, Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods, Set<ZoneApi> reprovisionToUpgradeOs, boolean negate) { this.zones = zones; this.zoneRoutingMethods = zoneRoutingMethods; + this.reprovisionToUpgradeOs = reprovisionToUpgradeOs; this.negate = negate; } - public static ZoneFilter from(Collection<? extends ZoneApi> zones, Map<ZoneApi, List<RoutingMethod>> routingMethods) { - return new ZoneFilterMock(List.copyOf(zones), Map.copyOf(routingMethods), false); + public static ZoneFilter from(Collection<? extends ZoneApi> zones, Map<ZoneApi, List<RoutingMethod>> routingMethods, Set<ZoneApi> reprovisionToUpgradeOs) { + return new ZoneFilterMock(List.copyOf(zones), Map.copyOf(routingMethods), reprovisionToUpgradeOs, false); } @Override public ZoneList not() { - return new ZoneFilterMock(zones, zoneRoutingMethods, ! negate); + return new ZoneFilterMock(zones, zoneRoutingMethods, reprovisionToUpgradeOs, ! negate); } @Override @@ -69,6 +71,11 @@ public class ZoneFilterMock implements ZoneList { } @Override + public ZoneList reprovisionToUpgradeOs() { + return filter(reprovisionToUpgradeOs::contains); + } + + @Override public ZoneList in(Environment... environments) { return filter(zone -> Set.of(environments).contains(zone.getEnvironment())); } @@ -100,7 +107,7 @@ public class ZoneFilterMock implements ZoneList { condition.negate().test(zone) : condition.test(zone)) .collect(Collectors.toList()), - zoneRoutingMethods, false); + zoneRoutingMethods, reprovisionToUpgradeOs, false); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index efc875b06f5..8fcd9527517 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -23,9 +23,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; import java.time.Duration; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; /** * @author mpolden @@ -36,6 +38,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>(); private final Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); private final Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods = new HashMap<>(); + private final Set<ZoneApi> reprovisionToUpgradeOs = new HashSet<>(); private List<? extends ZoneApi> zones; private SystemName system; @@ -123,6 +126,15 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return this; } + public ZoneRegistryMock reprovisionToUpgradeOsIn(ZoneApi... zones) { + return reprovisionToUpgradeOsIn(List.of(zones)); + } + + public ZoneRegistryMock reprovisionToUpgradeOsIn(List<ZoneApi> zones) { + this.reprovisionToUpgradeOs.addAll(zones); + return this; + } + @Override public SystemName system() { return system; @@ -130,7 +142,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public ZoneFilter zones() { - return ZoneFilterMock.from(zones, zoneRoutingMethods); + return ZoneFilterMock.from(zones, zoneRoutingMethods, reprovisionToUpgradeOs); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 8e7f04f6ecc..ab274dc5ef8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -22,6 +23,7 @@ import org.junit.Test; import java.time.Duration; import java.util.Comparator; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -42,6 +44,64 @@ public class MetricsReporterTest { private final MetricsMock metrics = new MetricsMock(); @Test + public void audit_log_metric() { + var tester = new ControllerTester(); + + MetricsReporter metricsReporter = createReporter(tester.controller()); + // Log some operator actions + HttpRequest req1 = HttpRequest.createTestRequest( + "http://localhost:8080/zone/v2/prod/some_region/nodes/v2/state/dirty/hostname", + com.yahoo.jdisc.http.HttpRequest.Method.PUT + ); + req1.getJDiscRequest().setUserPrincipal(() -> "user.janedoe"); + tester.controller().auditLogger().log((req1)); + HttpRequest req2 = HttpRequest.createTestRequest( + "http://localhost:8080/routing/v1/inactive/tenant/some_tenant/application/some_app/instance/default/environment/prod/region/some-region", + com.yahoo.jdisc.http.HttpRequest.Method.POST + ); + req2.getJDiscRequest().setUserPrincipal(() -> "user.johndoe"); + tester.controller().auditLogger().log((req2)); + + // Report metrics + metricsReporter.maintain(); + assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.janedoe")); + assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.johndoe")); + + // Log some more operator actions + HttpRequest req3 = HttpRequest.createTestRequest( + "http://localhost:8080/zone/v2/prod/us-northeast-1/nodes/v2/state/dirty/le04614.ostk.bm2.prod.ca1.yahoo.com", + com.yahoo.jdisc.http.HttpRequest.Method.PUT + ); + req3.getJDiscRequest().setUserPrincipal(() -> "user.janedoe"); + tester.controller().auditLogger().log((req3)); + HttpRequest req4 = HttpRequest.createTestRequest( + "http://localhost:8080/routing/v1/inactive/tenant/some_publishing/application/someindexing/instance/default/environment/prod/region/us-northeast-1", + com.yahoo.jdisc.http.HttpRequest.Method.POST + ); + req4.getJDiscRequest().setUserPrincipal(() -> "user.johndoe"); + tester.controller().auditLogger().log((req4)); + HttpRequest req5 = HttpRequest.createTestRequest( + "http://localhost:8080/zone/v2/prod/us-northeast-1/nodes/v2/state/dirty/le04614.ostk.bm2.prod.ca1.yahoo.com", + com.yahoo.jdisc.http.HttpRequest.Method.PUT + ); + req5.getJDiscRequest().setUserPrincipal(() -> "user.johndoe"); + tester.controller().auditLogger().log((req5)); + HttpRequest req6 = HttpRequest.createTestRequest( + "http://localhost:8080/routing/v1/inactive/tenant/some_publishing/application/someindexing/instance/default/environment/prod/region/us-northeast-1", + com.yahoo.jdisc.http.HttpRequest.Method.POST + ); + req6.getJDiscRequest().setUserPrincipal(() -> "user.janedoe"); + tester.controller().auditLogger().log((req6)); + + // Report metrics + metricsReporter.maintain(); + assertEquals(2, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.janedoe")); + assertEquals(2, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.johndoe")); + assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "zone", "user.johndoe")); + assertEquals(1, getMetric(MetricsReporter.OPERATION_PREFIX + "routing", "user.janedoe")); + } + + @Test public void deployment_fail_ratio() { var tester = new DeploymentTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -242,7 +302,7 @@ public class MetricsReporterTest { tester.upgradeSystem(version0); reporter.maintain(); var hosts = tester.configServer().nodeRepository().list(zone, SystemApplication.configServer.id()); - assertPlatformChangeDuration(Duration.ZERO, hosts, version0); + assertPlatformChangeDuration(Duration.ZERO, hosts); var targets = List.of(Version.fromString("7.1"), Version.fromString("7.2")); for (int i = 0; i < targets.size(); i++) { @@ -251,13 +311,13 @@ public class MetricsReporterTest { // System starts upgrading to next version tester.upgradeController(version); reporter.maintain(); - assertPlatformChangeDuration(Duration.ZERO, hosts, currentVersion); + assertPlatformChangeDuration(Duration.ZERO, hosts); systemUpgrader.maintain(); // 30 minutes pass and nothing happens tester.clock().advance(Duration.ofMinutes(30)); runAll(tester::computeVersionStatus, reporter); - assertPlatformChangeDuration(Duration.ZERO, hosts, currentVersion); + assertPlatformChangeDuration(Duration.ZERO, hosts); // 1/3 nodes upgrade within timeout assertEquals("Wanted version is raised for all nodes", version, @@ -272,14 +332,15 @@ public class MetricsReporterTest { // 2/3 spend their budget and are reported as failures tester.clock().advance(Duration.ofHours(1)); runAll(tester::computeVersionStatus, reporter); - assertPlatformChangeDuration(Duration.ZERO, List.of(firstHost), version); - assertPlatformChangeDuration(Duration.ofHours(1), hosts.subList(1, hosts.size()), currentVersion); + assertPlatformChangeDuration(Duration.ZERO, List.of(firstHost)); + assertPlatformChangeDuration(Duration.ofHours(1), hosts.subList(1, hosts.size())); // Remaining nodes eventually upgrade upgradeTo(version, hosts.subList(1, hosts.size()), zone, tester); runAll(tester::computeVersionStatus, reporter); - assertPlatformChangeDuration(Duration.ZERO, hosts, version); + assertPlatformChangeDuration(Duration.ZERO, hosts); assertEquals(version, tester.controller().systemVersion()); + assertPlatformNodeCount(hosts.size(), version); } } @@ -290,39 +351,39 @@ public class MetricsReporterTest { var zone = ZoneId.from("prod.eu-west-1"); var cloud = CloudName.defaultName(); tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(ZoneApiMock.from(zone))); - var osUpgrader = new OsUpgrader(tester.controller(), Duration.ofDays(1), - CloudName.defaultName()); + var osUpgrader = new OsUpgrader(tester.controller(), Duration.ofDays(1), CloudName.defaultName()); var statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1) ); tester.configServer().bootstrap(List.of(zone), SystemApplication.configServerHost, SystemApplication.tenantHost); // All nodes upgrade to initial OS version var version0 = Version.fromString("8.0"); - tester.controller().upgradeOsIn(cloud, version0, false); + tester.controller().upgradeOsIn(cloud, version0, Optional.empty(), false); osUpgrader.maintain(); tester.configServer().setOsVersion(version0, SystemApplication.tenantHost.id(), zone); tester.configServer().setOsVersion(version0, SystemApplication.configServerHost.id(), zone); runAll(statusUpdater, reporter); List<Node> hosts = tester.configServer().nodeRepository().list(zone); - assertOsChangeDuration(Duration.ZERO, hosts, version0); + assertOsChangeDuration(Duration.ZERO, hosts); var targets = List.of(Version.fromString("8.1"), Version.fromString("8.2")); var allVersions = Stream.concat(Stream.of(version0), targets.stream()).collect(Collectors.toSet()); for (int i = 0; i < targets.size(); i++) { var currentVersion = i == 0 ? version0 : targets.get(i - 1); - var version = targets.get(i); + var nextVersion = targets.get(i); // System starts upgrading to next OS version - tester.controller().upgradeOsIn(cloud, version, false); + tester.controller().upgradeOsIn(cloud, nextVersion, Optional.empty(), false); runAll(osUpgrader, statusUpdater, reporter); - assertOsChangeDuration(Duration.ZERO, hosts, currentVersion); + assertOsChangeDuration(Duration.ZERO, hosts); + assertOsNodeCount(hosts.size(), currentVersion); // Over 30 minutes pass and nothing happens tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); runAll(statusUpdater, reporter); - assertOsChangeDuration(Duration.ZERO, hosts, currentVersion); + assertOsChangeDuration(Duration.ZERO, hosts); // Nodes are told to upgrade, but do not suspend yet - assertEquals("Wanted OS version is raised for all nodes", version, + assertEquals("Wanted OS version is raised for all nodes", nextVersion, tester.configServer().nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream() .map(Node::wantedOsVersion).min(Comparator.naturalOrder()).get()); assertTrue("No nodes are suspended", tester.controller().serviceRegistry().configServer() @@ -332,57 +393,75 @@ public class MetricsReporterTest { // Another 30 minutes pass tester.clock().advance(Duration.ofMinutes(30)); runAll(statusUpdater, reporter); - assertOsChangeDuration(Duration.ZERO, hosts, currentVersion); + assertOsChangeDuration(Duration.ZERO, hosts); // 3/6 hosts suspend var suspendedHosts = hosts.subList(0, 3); suspend(suspendedHosts, zone, tester); runAll(statusUpdater, reporter); - assertOsChangeDuration(Duration.ZERO, hosts, currentVersion); + assertOsChangeDuration(Duration.ZERO, hosts); // Two hosts spend 20 minutes upgrading var hostsUpgraded = suspendedHosts.subList(0, 2); tester.clock().advance(Duration.ofMinutes(20)); runAll(statusUpdater, reporter); - assertOsChangeDuration(Duration.ofMinutes(20), hostsUpgraded, currentVersion); - upgradeOsTo(version, hostsUpgraded, zone, tester); + assertOsChangeDuration(Duration.ofMinutes(20), hostsUpgraded); + upgradeOsTo(nextVersion, hostsUpgraded, zone, tester); runAll(statusUpdater, reporter); - assertOsChangeDuration(Duration.ZERO, hostsUpgraded, version); + assertOsChangeDuration(Duration.ZERO, hostsUpgraded); + assertOsNodeCount(hostsUpgraded.size(), nextVersion); // One host consumes budget without upgrading var brokenHost = suspendedHosts.get(2); tester.clock().advance(Duration.ofMinutes(15)); runAll(statusUpdater, reporter); - assertOsChangeDuration(Duration.ofMinutes(35), List.of(brokenHost), currentVersion); + assertOsChangeDuration(Duration.ofMinutes(35), List.of(brokenHost)); // Host eventually upgrades and is no longer reported - upgradeOsTo(version, List.of(brokenHost), zone, tester); + upgradeOsTo(nextVersion, List.of(brokenHost), zone, tester); runAll(statusUpdater, reporter); - assertOsChangeDuration(Duration.ZERO, List.of(brokenHost), version); + assertOsChangeDuration(Duration.ZERO, List.of(brokenHost)); + assertOsNodeCount(hostsUpgraded.size() + 1, nextVersion); // Remaining hosts suspend and upgrade successfully var remainingHosts = hosts.subList(3, hosts.size()); suspend(remainingHosts, zone, tester); - upgradeOsTo(version, remainingHosts, zone, tester); + upgradeOsTo(nextVersion, remainingHosts, zone, tester); runAll(statusUpdater, reporter); - assertOsChangeDuration(Duration.ZERO, hosts, version); - - // Dimensions used for OS metric are only known OS versions - for (var host : hosts) { - Set<Version> versionDimensions = metrics.getMetrics((dimensions) -> host.hostname().value().equals(dimensions.get("host"))) - .entrySet() - .stream() - .filter(kv -> kv.getValue().containsKey(MetricsReporter.OS_CHANGE_DURATION)) - .map(kv -> kv.getKey().getDimensions()) - .map(dimensions -> dimensions.get("currentVersion")) - .map(Version::fromString) - .collect(Collectors.toSet()); - assertTrue("Reports only OS versions", allVersions.containsAll(versionDimensions)); - } - + assertOsChangeDuration(Duration.ZERO, hosts); + assertOsNodeCount(hosts.size(), nextVersion); + assertOsNodeCount(0, currentVersion); + + // Dimensions used for node count metric are only known OS versions + Set<Version> versionDimensions = metrics.getMetrics((dimensions) -> true) + .entrySet() + .stream() + .filter(kv -> kv.getValue().containsKey(MetricsReporter.OS_NODE_COUNT)) + .map(kv -> kv.getKey().getDimensions()) + .map(dimensions -> dimensions.get("currentVersion")) + .map(Version::fromString) + .collect(Collectors.toSet()); + assertTrue("Reports only OS versions", allVersions.containsAll(versionDimensions)); } } + private void assertNodeCount(String metric, int n, Version version) { + long nodeCount = metrics.getMetric((dimensions) -> version.toFullString().equals(dimensions.get("currentVersion")), metric) + .stream() + .map(Number::longValue) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Expected to find metric for version " + version)); + assertEquals("Expected number of nodes are on " + version.toFullString(), n, nodeCount); + } + + private void assertPlatformNodeCount(int n, Version version) { + assertNodeCount(MetricsReporter.PLATFORM_NODE_COUNT, n, version); + } + + private void assertOsNodeCount(int n, Version version) { + assertNodeCount(MetricsReporter.OS_NODE_COUNT, n, version); + } + private void runAll(Runnable... runnables) { for (var r : runnables) r.run(); } @@ -409,9 +488,9 @@ public class MetricsReporterTest { } private List<Node> getNodes(ZoneId zone, List<Node> nodes, ControllerTester tester) { - return tester.configServer().nodeRepository().list(zone, nodes.stream() - .map(Node::hostname) - .collect(Collectors.toList())); + return tester.configServer().nodeRepository().list(zone, nodes.stream() + .map(Node::hostname) + .collect(Collectors.toList())); } private void updateNodes(List<Node> nodes, UnaryOperator<Node.Builder> builderOps, ZoneId zone, @@ -435,28 +514,23 @@ public class MetricsReporterTest { return getMetric(MetricsReporter.DEPLOYMENT_WARNINGS, id).intValue(); } - private Duration getChangeDuration(String metric, HostName hostname, Version currentVersion) { - return metrics.getMetrics((dimensions) -> hostname.value().equals(dimensions.get("host")) && - currentVersion.toFullString().equals(dimensions.get("currentVersion"))) - .values() - .stream() - .map(metrics -> metrics.get(metric)) + private Duration getChangeDuration(String metric, HostName hostname) { + return metrics.getMetric((dimensions) -> hostname.value().equals(dimensions.get("host")), metric) .map(n -> Duration.ofSeconds(n.longValue())) - .findFirst() .orElseThrow(() -> new IllegalArgumentException("Expected to find metric for " + hostname)); } - private void assertPlatformChangeDuration(Duration duration, List<Node> nodes, Version currentVersion) { + private void assertPlatformChangeDuration(Duration duration, List<Node> nodes) { for (var node : nodes) { - assertEquals("Platform change duration of " + node.hostname() + " on version " + currentVersion, - duration, getChangeDuration(MetricsReporter.PLATFORM_CHANGE_DURATION, node.hostname(), currentVersion)); + assertEquals("Platform change duration of " + node.hostname(), + duration, getChangeDuration(MetricsReporter.PLATFORM_CHANGE_DURATION, node.hostname())); } } - private void assertOsChangeDuration(Duration duration, List<Node> nodes, Version currentVersion) { + private void assertOsChangeDuration(Duration duration, List<Node> nodes) { for (var node : nodes) { - assertEquals("OS change duration of " + node.hostname() + " on version " + currentVersion, - duration, getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, node.hostname(), currentVersion)); + assertEquals("OS change duration of " + node.hostname(), + duration, getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, node.hostname())); } } @@ -467,6 +541,12 @@ public class MetricsReporterTest { .orElseThrow(() -> new RuntimeException("Expected metric to exist for " + id)); } + private Number getMetric(String name, String operator) { + return metrics.getMetric((dimensions) -> operator.equals(dimensions.get("operator")), + name) + .orElseThrow(() -> new RuntimeException("Expected metric to exist for " + operator)); + } + private MetricsReporter createReporter(Controller controller) { return new MetricsReporter(controller, metrics); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index a5a7304398b..cb906d61a3b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -16,43 +16,45 @@ import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import org.junit.Test; import java.time.Duration; +import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author mpolden */ public class OsUpgraderTest { - private static final ZoneApi zone1 = ZoneApiMock.newBuilder().withId("prod.eu-west-1").build(); - private static final ZoneApi zone2 = ZoneApiMock.newBuilder().withId("prod.us-west-1").build(); - private static final ZoneApi zone3 = ZoneApiMock.newBuilder().withId("prod.us-central-1").build(); - private static final ZoneApi zone4 = ZoneApiMock.newBuilder().withId("prod.us-east-3").build(); - private static final ZoneApi zone5 = ZoneApiMock.newBuilder().withId("prod.us-north-1").withCloud("other").build(); - private final ControllerTester tester = new ControllerTester(); - private final OsVersionStatusUpdater statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1) - ); + private final OsVersionStatusUpdater statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1)); + @Test public void upgrade_os() { - OsUpgrader osUpgrader = osUpgrader( - UpgradePolicy.create() - .upgrade(zone1) - .upgradeInParallel(zone2, zone3) - .upgrade(zone5) // Belongs to a different cloud and is ignored by this upgrader - .upgrade(zone4), - SystemName.cd - ); + CloudName cloud1 = CloudName.from("c1"); + CloudName cloud2 = CloudName.from("c2"); + ZoneApi zone1 = zone("prod.eu-west-1", cloud1); + ZoneApi zone2 = zone("prod.us-west-1", cloud1); + ZoneApi zone3 = zone("prod.us-central-1", cloud1); + ZoneApi zone4 = zone("prod.us-east-3", cloud1); + ZoneApi zone5 = zone("prod.us-north-1", cloud2); + UpgradePolicy upgradePolicy = UpgradePolicy.create() + .upgrade(zone1) + .upgradeInParallel(zone2, zone3) + .upgrade(zone5) // Belongs to a different cloud and is ignored by this upgrader + .upgrade(zone4); + OsUpgrader osUpgrader = osUpgrader(upgradePolicy, SystemName.cd, cloud1, false); // Bootstrap system tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId(), zone5.getId()), List.of(SystemApplication.tenantHost)); - // Add system applications that exist in a real system, but are currently not upgraded + // Add system applications that exist in a real system, but isn't upgraded tester.configServer().addNodes(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId(), zone5.getId()), List.of(SystemApplication.configServer)); @@ -62,10 +64,9 @@ public class OsUpgraderTest { // New OS version released Version version1 = Version.fromString("7.1"); - CloudName cloud = CloudName.defaultName(); - tester.controller().upgradeOsIn(cloud, Version.fromString("7.0"), false); - tester.controller().upgradeOsIn(cloud, version1, false); - assertEquals(1, tester.controller().osVersions().size()); // Only allows one version per cloud + tester.controller().upgradeOsIn(cloud1, Version.fromString("7.0"), Optional.empty(), false); + tester.controller().upgradeOsIn(cloud1, version1, Optional.empty(), false); + assertEquals(1, tester.controller().osVersionTargets().size()); // Only allows one version per cloud statusUpdater.maintain(); // zone 1: begins upgrading @@ -102,10 +103,74 @@ public class OsUpgraderTest { osUpgrader.maintain(); assertWanted(version1, SystemApplication.tenantHost, zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()); statusUpdater.maintain(); - assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodesIn(cloud).stream() + assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodesIn(cloud1).stream() .allMatch(node -> node.currentVersion().equals(version1))); } + @Test + public void upgrade_os_with_budget() { + CloudName cloud = CloudName.from("cloud"); + ZoneApi zone1 = zone("dev.us-east-1", cloud); + ZoneApi zone2 = zone("prod.us-west-1", cloud); + ZoneApi zone3 = zone("prod.us-central-1", cloud); + ZoneApi zone4 = zone("prod.eu-west-1", cloud); + UpgradePolicy upgradePolicy = UpgradePolicy.create() + .upgrade(zone1) + .upgradeInParallel(zone2, zone3) + .upgrade(zone4); + OsUpgrader osUpgrader = osUpgrader(upgradePolicy, SystemName.cd, cloud, true); + + // Bootstrap system + tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()), + List.of(SystemApplication.tenantHost)); + tester.configServer().addNodes(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()), + List.of(SystemApplication.configServerHost)); // Not supported yet + + // Upgrade without budget fails + Version version = Version.fromString("7.1"); + try { + tester.controller().upgradeOsIn(cloud, version, Optional.empty(), false); + fail("Expected exception"); + } catch (IllegalArgumentException ignored) {} + + // Upgrade with budget + tester.controller().upgradeOsIn(cloud, version, Optional.of(Duration.ofHours(12)), false); + assertEquals(Duration.ofHours(12), tester.controller().osVersionTarget(cloud).get().upgradeBudget().get()); + statusUpdater.maintain(); + osUpgrader.maintain(); + + // First zone upgrades + assertWanted(Version.emptyVersion, SystemApplication.configServerHost, zone1.getId()); + assertEquals("Dev zone gets a zero budget", Duration.ZERO, upgradeBudget(zone1.getId(), SystemApplication.tenantHost, version)); + completeUpgrade(version, SystemApplication.tenantHost, zone1.getId()); + + // Next set of zones upgrade + osUpgrader.maintain(); + for (var zone : List.of(zone2.getId(), zone3.getId())) { + assertEquals("Parallel prod zones share the budget of a single zone", Duration.ofHours(6), + upgradeBudget(zone, SystemApplication.tenantHost, version)); + completeUpgrade(version, SystemApplication.tenantHost, zone); + } + + // Last zone upgrades + osUpgrader.maintain(); + assertEquals("Last prod zone gets the budget of a single zone", Duration.ofHours(6), + upgradeBudget(zone4.getId(), SystemApplication.tenantHost, version)); + completeUpgrade(version, SystemApplication.tenantHost, zone4.getId()); + + // All host applications upgraded + statusUpdater.maintain(); + assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodesIn(cloud).stream() + .allMatch(node -> node.currentVersion().equals(version))); + } + + private Duration upgradeBudget(ZoneId zone, SystemApplication application, Version version) { + var upgradeBudget = tester.configServer().nodeRepository().osUpgradeBudget(zone, application.nodeType(), version); + assertTrue("Expected budget for upgrade to " + version + " of " + application.id() + " in " + zone, + upgradeBudget.isPresent()); + return upgradeBudget.get(); + } + private List<NodeVersion> nodesOn(Version version) { return tester.controller().osVersionStatus().versions().entrySet().stream() .filter(entry -> entry.getKey().version().equals(version)) @@ -136,7 +201,7 @@ public class OsUpgraderTest { private List<Node> nodesRequiredToUpgrade(ZoneId zone, SystemApplication application) { return nodeRepository().list(zone, application.id()) .stream() - .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) + .filter(OsUpgrader::canUpgrade) .collect(Collectors.toList()); } @@ -164,13 +229,20 @@ public class OsUpgraderTest { return tester.configServer().nodeRepository(); } - private OsUpgrader osUpgrader(UpgradePolicy upgradePolicy, SystemName system) { + private OsUpgrader osUpgrader(UpgradePolicy upgradePolicy, SystemName system, CloudName cloud, boolean reprovisionToUpgradeOs) { + var zones = upgradePolicy.asList().stream().flatMap(Collection::stream).collect(Collectors.toList()); tester.zoneRegistry() - .setZones(zone1, zone2, zone3, zone4, zone5) + .setZones(zones) .setSystemName(system) - .setOsUpgradePolicy(CloudName.defaultName(), upgradePolicy); - return new OsUpgrader(tester.controller(), Duration.ofDays(1), - CloudName.defaultName()); + .setOsUpgradePolicy(cloud, upgradePolicy); + if (reprovisionToUpgradeOs) { + tester.zoneRegistry().reprovisionToUpgradeOsIn(zones); + } + return new OsUpgrader(tester.controller(), Duration.ofDays(1), cloud); + } + + private static ZoneApi zone(String id, CloudName cloud) { + return ZoneApiMock.newBuilder().withId(id).with(cloud).build(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java index 5ddd2064c32..e9a1adcfe88 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import org.junit.Test; import java.time.Duration; +import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -40,7 +41,7 @@ public class OsVersionStatusUpdaterTest { // Setting a new target adds it to current status Version version1 = Version.fromString("7.1"); CloudName cloud = CloudName.defaultName(); - tester.controller().upgradeOsIn(cloud, version1, false); + tester.controller().upgradeOsIn(cloud, version1, Optional.empty(), false); statusUpdater.maintain(); var osVersions = tester.controller().osVersionStatus().versions(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java new file mode 100644 index 00000000000..8d6316d447f --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java @@ -0,0 +1,61 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +/** + * @author mpolden + */ +public class SystemRoutingPolicyMaintainerTest { + + @Test + public void maintain() { + var tester = new ControllerTester(); + var updater = new SystemRoutingPolicyMaintainer(tester.controller(), Duration.ofDays(1)); + var dispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofDays(1), Integer.MAX_VALUE); + + var zone = ZoneId.from("prod", "us-west-1"); + tester.zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(zone)); + tester.configServer().putLoadBalancers(zone, List.of(new LoadBalancer("lb1", + SystemApplication.configServer.id(), + ClusterSpec.Id.from("config"), + HostName.from("lb1.example.com"), + LoadBalancer.State.active, + Optional.of("dns-zone-1")))); + + // Nothing happens without feature flag + updater.run(); + dispatcher.run(); + assertEquals(Set.of(), tester.nameService().records()); + + // Record is created + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.CONFIGSERVER_PROVISION_LB.id(), true); + updater.run(); + dispatcher.run(); + Set<Record> records = tester.nameService().records(); + assertEquals(1, records.size()); + Record record = records.iterator().next(); + assertSame(Record.Type.CNAME, record.type()); + assertEquals("cfg.prod.us-west-1.test.vip", record.name().asString()); + assertEquals("lb1.example.com.", record.data().asString()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index b49686237d8..cc92c6cd271 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -2,21 +2,30 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.OptionalInt; +import java.util.Set; +import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; @@ -1063,6 +1072,49 @@ public class UpgraderTest { assertEquals(174, upgrades); } + @Test + public void testUpgradeShuffling() { + // Deploy applications on initial version + var default0 = createAndDeploy("default0", "default"); + var default1 = createAndDeploy("default1", "default"); + var default2 = createAndDeploy("default2", "default"); + var applications = Map.of(default0.instanceId(), default0, + default1.instanceId(), default1, + default2.instanceId(), default2); + + // Throttle upgrades per run + ((ManualClock) tester.controller().clock()).setInstant(Instant.ofEpochMilli(1589787109000L)); // Fixed random seed + Upgrader upgrader = new Upgrader(tester.controller(), Duration.ofMinutes(10), + tester.controllerTester().curator()); + upgrader.setUpgradesPerMinute(0.1); + + // Trigger some upgrades + List<Version> versions = List.of(Version.fromString("6.2"), Version.fromString("6.3")); + Set<List<ApplicationId>> upgradeOrders = new HashSet<>(versions.size()); + for (var version : versions) { + // Upgrade system + tester.controllerTester().upgradeSystem(version); + List<ApplicationId> upgraderOrder = new ArrayList<>(applications.size()); + + // Upgrade all applications + for (int i = 0; i < applications.size(); i++) { + upgrader.maintain(); + tester.triggerJobs(); + Set<ApplicationId> triggered = tester.jobs().active().stream() + .map(Run::id) + .map(RunId::application) + .collect(Collectors.toSet()); + assertEquals("Expected number of applications is triggered", 1, triggered.size()); + ApplicationId application = triggered.iterator().next(); + upgraderOrder.add(application); + applications.get(application).completeRollout(); + tester.clock().advance(Duration.ofMinutes(1)); + } + upgradeOrders.add(upgraderOrder); + } + assertEquals("Upgrade orders are distinct", versions.size(), upgradeOrders.size()); + } + private ApplicationPackage applicationPackage(String upgradePolicy) { return new ApplicationPackageBuilder().upgradePolicy(upgradePolicy) .region("us-west-1") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java index d3efac55a1a..d287c025b42 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java @@ -2,12 +2,16 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; import java.time.Duration; import java.util.Collections; +import java.util.List; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -29,5 +33,14 @@ public class VersionStatusUpdaterTest { updater.maintain(); assertTrue(tester.controller().versionStatus().systemVersion().isPresent()); } + + @Test + public void testConfidenceConversion() { + List.of(VespaVersion.Confidence.values()).forEach(VersionStatusUpdater::convert); + assertEquals(SystemMonitor.Confidence.broken, VersionStatusUpdater.convert(VespaVersion.Confidence.broken)); + assertEquals(SystemMonitor.Confidence.low, VersionStatusUpdater.convert(VespaVersion.Confidence.low)); + assertEquals(SystemMonitor.Confidence.normal, VersionStatusUpdater.convert(VespaVersion.Confidence.normal)); + assertEquals(SystemMonitor.Confidence.high, VersionStatusUpdater.convert(VespaVersion.Confidence.high)); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java new file mode 100644 index 00000000000..dea09998952 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializerTest.java @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.component.Version; +import com.yahoo.config.provision.CloudName; +import com.yahoo.vespa.hosted.controller.versions.OsVersion; +import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; +import org.junit.Test; + +import java.time.Duration; +import java.util.Optional; +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class OsVersionTargetSerializerTest { + + @Test + public void serialization() { + OsVersionTargetSerializer serializer = new OsVersionTargetSerializer(new OsVersionSerializer()); + Set<OsVersionTarget> targets = ImmutableSet.of( + new OsVersionTarget(new OsVersion(Version.fromString("7.1"), CloudName.defaultName()), Optional.empty()), + new OsVersionTarget(new OsVersion(Version.fromString("7.1"), CloudName.from("foo")), Optional.of(Duration.ofDays(1))) + ); + Set<OsVersionTarget> serialized = serializer.fromSlime(serializer.toSlime(targets)); + assertEquals(targets, serialized); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java new file mode 100644 index 00000000000..1fce7ba5695 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java @@ -0,0 +1,123 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.yahoo.config.provision.SystemName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpCoreContext; +import org.junit.Rule; +import org.junit.Test; + +import javax.net.ssl.SSLContext; +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class ConfigServerRestExecutorImplTest { + + @Rule + public final WireMockRule wireMock = new WireMockRule(options().dynamicPort(), true); + + @Test + public void proxy_with_retries() throws Exception { + var connectionReuseStrategy = new CountingConnectionReuseStrategy(Set.of("127.0.0.1")); + var proxy = new ConfigServerRestExecutorImpl(new ZoneRegistryMock(SystemName.cd), SSLContext.getDefault(), + (duration) -> {}, connectionReuseStrategy); + + URI url = url(); + String path = url.getPath(); + stubRequests(path); + + HttpRequest request = HttpRequest.createTestRequest(url.toString(), com.yahoo.jdisc.http.HttpRequest.Method.GET); + ProxyRequest proxyRequest = ProxyRequest.tryOne(url, path, request); + + // Request is retried + HttpResponse response = proxy.handle(proxyRequest); + wireMock.verify(3, getRequestedFor(urlEqualTo(path))); + assertEquals(200, response.getStatus()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + response.render(out); + assertEquals("OK", out.toString()); + + // No connections are reused as host is a VIP + assertEquals(0, connectionReuseStrategy.reusedConnections.get(url.getHost()).intValue()); + } + + @Test + public void proxy_without_connection_reuse() throws Exception { + var connectionReuseStrategy = new CountingConnectionReuseStrategy(Set.of()); + var proxy = new ConfigServerRestExecutorImpl(new ZoneRegistryMock(SystemName.cd), SSLContext.getDefault(), + (duration) -> {}, connectionReuseStrategy); + + URI url = url(); + String path = url.getPath(); + stubRequests(path); + + HttpRequest request = HttpRequest.createTestRequest(url.toString(), com.yahoo.jdisc.http.HttpRequest.Method.GET); + ProxyRequest proxyRequest = ProxyRequest.tryOne(url, path, request); + + // Connections are reused + assertEquals(200, proxy.handle(proxyRequest).getStatus()); + assertEquals(3, connectionReuseStrategy.reusedConnections.get(url.getHost()).intValue()); + } + + private URI url() { + return URI.create("http://127.0.0.1:" + wireMock.port() + "/"); + } + + private void stubRequests(String path) { + String retryScenario = "Retry scenario"; + String retryRequest = "Retry request 1"; + String retryRequestAgain = "Retry request 2"; + + wireMock.stubFor(get(urlEqualTo(path)).inScenario(retryScenario) + .whenScenarioStateIs(Scenario.STARTED) + .willSetStateTo(retryRequest) + .willReturn(aResponse().withStatus(500))); + + wireMock.stubFor(get(urlEqualTo(path)).inScenario(retryScenario) + .whenScenarioStateIs(retryRequest) + .willSetStateTo(retryRequestAgain) + .willReturn(aResponse().withStatus(500))); + + wireMock.stubFor(get(urlEqualTo(path)).inScenario(retryScenario) + .whenScenarioStateIs(retryRequestAgain) + .willReturn(aResponse().withBody("OK"))); + } + + private static class CountingConnectionReuseStrategy extends ConfigServerRestExecutorImpl.ConnectionReuseStrategy { + + private final Map<String, Integer> reusedConnections = new HashMap<>(); + + public CountingConnectionReuseStrategy(Set<String> vips) { + super(vips); + } + + @Override + public boolean keepAlive(org.apache.http.HttpResponse response, HttpContext context) { + boolean keepAlive = super.keepAlive(response, context); + String host = HttpCoreContext.adapt(context).getTargetHost().getHostName(); + reusedConnections.putIfAbsent(host, 0); + if (keepAlive) reusedConnections.compute(host, (ignored, count) -> ++count); + return keepAlive; + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java index d8373cb8928..15ec138354d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java @@ -21,12 +21,6 @@ public class ProxyRequestTest { public final ExpectedException exception = ExpectedException.none(); @Test - public void testEmpty() throws Exception { - exception.expectMessage("Request must be non-null"); - new ProxyRequest(HttpRequest.Method.GET, null, Map.of(), null, List.of(), "/zone/v2"); - } - - @Test public void testBadUri() throws Exception { exception.expectMessage("Request path '/path' does not end with proxy path '/zone/v2/'"); testRequest("http://domain.tld/path", "/zone/v2/"); @@ -67,8 +61,9 @@ public class ProxyRequestTest { } } - private static ProxyRequest testRequest(String url, String pathPrefix) throws ProxyException { - return new ProxyRequest( - HttpRequest.Method.GET, URI.create(url), Map.of(), null, List.of(), pathPrefix); + private static ProxyRequest testRequest(String url, String pathPrefix) { + return new ProxyRequest(HttpRequest.Method.GET, URI.create(url), Map.of(), null, + List.of(URI.create("http://example.com")), pathPrefix); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java index 0aac59321b5..539d4a6dd75 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java @@ -20,7 +20,7 @@ public class ProxyResponseTest { @Test public void testRewriteUrl() throws Exception { ProxyRequest request = new ProxyRequest(HttpRequest.Method.GET, URI.create("http://domain.tld/zone/v2/dev/us-north-1/configserver"), - Map.of(), null, List.of(), "configserver"); + Map.of(), null, List.of(URI.create("http://example.com")), "configserver"); ProxyResponse proxyResponse = new ProxyResponse( request, "response link is http://configserver:1234/bla/bla/", @@ -38,7 +38,7 @@ public class ProxyResponseTest { @Test public void testRewriteSecureUrl() throws Exception { ProxyRequest request = new ProxyRequest(HttpRequest.Method.GET, URI.create("https://domain.tld/zone/v2/prod/eu-south-3/configserver"), - Map.of(), null, List.of(), "configserver"); + Map.of(), null, List.of(URI.create("http://example.com")), "configserver"); ProxyResponse proxyResponse = new ProxyResponse( request, "response link is http://configserver:1234/bla/bla/", @@ -52,4 +52,5 @@ public class ProxyResponseTest { String document = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); assertEquals("response link is https://domain.tld/zone/v2/prod/eu-south-3/bla/bla/", document); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java index 8dd1f9ad10a..b935f8cbbe4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -12,6 +12,8 @@ import com.yahoo.yolean.Exceptions; import java.nio.charset.StandardCharsets; import java.security.Principal; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import java.util.function.Supplier; @@ -91,7 +93,17 @@ public class ControllerContainerCloudTest extends ControllerContainerTest { public Request get() { Request request = new Request("http://localhost:8080" + path, data, method, principal); request.getAttributes().put(SecurityContext.ATTRIBUTE_NAME, new SecurityContext(principal, roles)); - if (user != null) request.getAttributes().put(User.ATTRIBUTE_NAME, user); + if (user != null) { + Map<String, String> userAttributes = new HashMap<>(); + userAttributes.put("email", user.email()); + if (user.name() != null) + userAttributes.put("name", user.name()); + if (user.nickname() != null) + userAttributes.put("nickname", user.nickname()); + if (user.picture() != null) + userAttributes.put("picture", user.picture()); + request.getAttributes().put(User.ATTRIBUTE_NAME, Map.copyOf(userAttributes)); + } request.getHeaders().put("Content-Type", contentType); return request; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 5c5dc4b5fe6..a1b06262241 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -12,11 +14,13 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; +import com.yahoo.vespa.hosted.controller.security.Auth0Credentials; import com.yahoo.vespa.hosted.controller.security.CloudTenantSpec; import com.yahoo.vespa.hosted.controller.security.Credentials; import org.junit.Before; import org.junit.Test; +import java.util.Collections; import java.util.Set; import static com.yahoo.application.container.handler.Request.Method.POST; @@ -36,6 +40,8 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { @Before public void before() { tester = new ContainerTester(container, responseFiles); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withBooleanFlag(Flags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); deploymentTester = new DeploymentTester(new ControllerTester(tester)); deploymentTester.controllerTester().computeVersionStatus(); } @@ -73,6 +79,6 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { } private static Credentials credentials(String name) { - return new Credentials(() -> name); + return new Auth0Credentials(() -> name, Collections.emptySet()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 7c703735dbf..388ca65dc40 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -40,9 +40,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.api.integration.resource.CostInfo; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MockTenantCost; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; @@ -176,14 +174,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); - // GET list of months for a tenant - tester.assertResponse(request("/application/v4/tenant/tenant1/cost", GET).userIdentity(USER_ID).oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - "{\"months\":[]}"); - - // GET cost for a month for a tenant - tester.assertResponse(request("/application/v4/tenant/tenant1/cost/2018-01", GET).userIdentity(USER_ID).oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - "{\"month\":\"2018-01\",\"items\":[]}"); - // Add another Athens domain, so we can try to create more tenants createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN_2, USER_ID); // New domain to test tenant w/property ID // Add property info for that property id, as well, in the mock organization. @@ -985,44 +975,6 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void testTenantCostResponse() { - ApplicationId applicationId = createTenantAndApplication(); - MockTenantCost mockTenantCost = deploymentTester.controllerTester().serviceRegistry().tenantCost(); - - mockTenantCost.setMonthsWithMetering( - new TreeSet<>(Set.of( - YearMonth.of(2019, 10), - YearMonth.of(2019, 9) - )) - ); - - tester.assertResponse(request("/application/v4/tenant/" + applicationId.tenant().value() + "/cost", GET) - .userIdentity(USER_ID) - .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - "{\"months\":[\"2019-09\",\"2019-10\"]}"); - - CostInfo costInfo1 = new CostInfo(applicationId, ZoneId.from("prod", "us-south-1"), - new BigDecimal("7.0"), - new BigDecimal("600.0"), - new BigDecimal("1000.0"), - 35, 23, 10); - CostInfo costInfo2 = new CostInfo(applicationId, ZoneId.from("prod", "us-north-1"), - new BigDecimal("2.0"), - new BigDecimal("3.0"), - new BigDecimal("4.0"), - 10, 20, 30); - - mockTenantCost.setCostInfoList( - List.of(costInfo1, costInfo2) - ); - - tester.assertResponse(request("/application/v4/tenant/" + applicationId.tenant().value() + "/cost/2019-09", GET) - .userIdentity(USER_ID) - .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - new File("cost-report.json")); - } - - @Test public void testErrorResponses() throws Exception { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); @@ -1179,17 +1131,6 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete instance 'tenant1.application1.instance1': Instance not found\"}", 404); - // GET cost of unknown tenant - tester.assertResponse(request("/application/v4/tenant/no-such-tenant/cost", GET).userIdentity(USER_ID).oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'no-such-tenant' does not exist\"}", 404); - - tester.assertResponse(request("/application/v4/tenant/no-such-tenant/cost/2018-01-01", GET).userIdentity(USER_ID).oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'no-such-tenant' does not exist\"}", 404); - - // GET cost with invalid date string - tester.assertResponse(request("/application/v4/tenant/tenant1/cost/not-a-valid-date", GET).userIdentity(USER_ID).oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not parse year-month 'not-a-valid-date'\"}", 400); - // DELETE tenant tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE) .userIdentity(USER_ID) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json index 49d82a09775..b2e634e5d6b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json @@ -12,7 +12,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 11.8 + "cost": 0.12 }, "max": { "nodes": 2, @@ -25,7 +25,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 47.5 + "cost": 0.47 }, "current": { "nodes": 2, @@ -38,7 +38,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 23.9 + "cost": 0.24 }, "target": { "nodes": 2, @@ -51,7 +51,7 @@ "diskSpeed": "slow", "storageType": "remote" }, - "cost": 31.9 + "cost": 0.32 } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index fbdf8caaed7..acd542b001c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -4,9 +4,6 @@ "name": "ApplicationOwnershipConfirmer" }, { - "name": "BillingMaintainer" - }, - { "name": "CloudEventReporter" }, { @@ -55,6 +52,9 @@ "name": "RotationStatusUpdater" }, { + "name": "SystemRoutingPolicyMaintainer" + }, + { "name": "SystemUpgrader" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 32a77137182..f3c24458e6e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -73,8 +73,8 @@ public class DeploymentApiTest extends ControllerContainerTest { productionApp.submit(multiInstancePackage).failDeployment(JobType.systemTest); tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()))); - tester.assertResponse(authenticatedRequest("http://localhost:8080/deployment/v1/"), new File("root.json")); - tester.assertResponse(authenticatedRequest("http://localhost:8080/api/deployment/v1/"), new File("root.json")); + tester.assertResponse(operatorRequest("http://localhost:8080/deployment/v1/"), new File("root.json")); + tester.assertResponse(operatorRequest("http://localhost:8080/api/deployment/v1/"), new File("root.json")); } private VersionStatus censorConfigServers(VersionStatus versionStatus) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java index 5e50e80b7a7..3da662ee373 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java @@ -70,13 +70,13 @@ public class AthenzRoleFilterTest { public void testTranslations() throws Exception { // Hosted operators are always members of the hostedOperator role. - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedAccountant(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, NO_CONTEXT_PATH)); - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedAccountant(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, TENANT_CONTEXT_PATH)); - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedAccountant(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, APPLICATION_CONTEXT_PATH)); // Tenant admins are members of the athenzTenantAdmin role within their tenant subtree. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 66493e6e226..857530df1fe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -51,6 +51,7 @@ public class OsApiTest extends ControllerContainerTest { addUserToHostedOperatorRole(operator); zoneRegistryMock().setSystemName(SystemName.cd) .setZones(zone1, zone2, zone3) + .reprovisionToUpgradeOsIn(zone3) .setOsUpgradePolicy(cloud1, UpgradePolicy.create().upgrade(zone1).upgrade(zone2)) .setOsUpgradePolicy(cloud2, UpgradePolicy.create().upgrade(zone3)); osUpgraders = List.of( @@ -70,8 +71,8 @@ public class OsApiTest extends ControllerContainerTest { // Upgrade OS to a different version in each cloud assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.5.2\", \"cloud\": \"cloud1\"}", Request.Method.PATCH), "{\"message\":\"Set target OS version for cloud 'cloud1' to 7.5.2\"}", 200); - assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"8.2.1\", \"cloud\": \"cloud2\"}", Request.Method.PATCH), - "{\"message\":\"Set target OS version for cloud 'cloud2' to 8.2.1\"}", 200); + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"8.2.1\", \"cloud\": \"cloud2\", \"upgradeBudget\": \"PT24H\"}", Request.Method.PATCH), + "{\"message\":\"Set target OS version for cloud 'cloud2' to 8.2.1 with upgrade budget PT24H\"}", 200); // Status is updated after some zones are upgraded upgradeAndUpdateStatus(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json index 01af1bd70dd..5834e4cef4a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json @@ -100,39 +100,10 @@ { "version": "8.2.1", "targetVersion": true, + "upgradeBudget": "PT24H", "cloud": "cloud2", "nodes": [ { - "hostname": "node-1-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-1-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { "hostname": "node-1-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json index 2b907c1156c..c8833fea100 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json @@ -110,36 +110,6 @@ "cloud": "cloud2", "nodes": [ { - "hostname": "node-1-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-1-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { "hostname": "node-1-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" @@ -159,6 +129,7 @@ { "version": "8.2.1", "targetVersion": true, + "upgradeBudget": "PT24H", "cloud": "cloud2", "nodes": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java index 343a1f07a21..3a63caf52cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java @@ -12,6 +12,8 @@ import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.junit.Test; import java.io.File; +import java.util.HashMap; +import java.util.Map; import java.util.stream.Stream; /** @@ -61,7 +63,15 @@ public class UserApiOnPremTest extends ControllerContainerTest { private Request createUserRequest(User user, AthenzIdentity identity) { Request request = new Request("http://localhost:8080/api/user/v1/user"); - request.getAttributes().put(User.ATTRIBUTE_NAME, user); + Map<String, String> userAttributes = new HashMap<>(); + userAttributes.put("email", user.email()); + if (user.name() != null) + userAttributes.put("name", user.name()); + if (user.nickname() != null) + userAttributes.put("nickname", user.nickname()); + if (user.picture() != null) + userAttributes.put("picture", user.picture()); + request.getAttributes().put(User.ATTRIBUTE_NAME, Map.copyOf(userAttributes)); return addIdentityToRequest(request, identity); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 51466e5b1e2..d9ad30020db 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.controller.restapi.user; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.api.role.Role; @@ -62,7 +64,7 @@ public class UserApiTest extends ControllerContainerCloudTest { // POST a tenant is not available to everyone. tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) .data("{\"token\":\"hello\"}"), - accessDenied, 403); + "{\"error-code\":\"FORBIDDEN\",\"message\":\"You are not currently permitted to create tenants. Please contact the Vespa team to request access.\"}", 403); // POST a tenant is available to operators. tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) @@ -200,8 +202,10 @@ public class UserApiTest extends ControllerContainerCloudTest { @Test public void userMetadataTest() { ContainerTester tester = new ContainerTester(container, responseFiles); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withBooleanFlag(Flags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); ControllerTester controller = new ControllerTester(tester); - Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter()); + Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter(), Role.hostedAccountant()); User user = new User("dev@domail", "Joe Developer", "dev", null); tester.assertResponse(request("/api/user/v1/user") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json index 56108dce94f..36918c743fa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json @@ -1,5 +1,7 @@ { "isPublic": false, + "isCd": false, + "enable-public-signup-flow": (ignore), "user": { "name": "Joe Developer", "email": "dev@domail", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json index ea76aa977ce..27398352e53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json @@ -1,5 +1,7 @@ { "isPublic": true, +"isCd": false, +"enable-public-signup-flow": (ignore), "user": { "name": "Joe Developer", "email": "dev@domail", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json index 400fe8d4d9b..b62a70d1871 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json @@ -1,5 +1,7 @@ { "isPublic": (ignore), + "isCd": (ignore), + "enable-public-signup-flow": (ignore), "user": { "name": "Joe Developer", "email": "dev@domail", @@ -8,6 +10,7 @@ "tenants": {}, "operator": [ "hostedOperator", - "hostedSupporter" + "hostedSupporter", + "hostedAccountant" ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index e44a1364185..760d80d230c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; @@ -138,13 +139,18 @@ public class RoutingPoliciesTest { @Test public void zone_routing_policies() { + zone_routing_policies(false); + zone_routing_policies(true); + } + + private void zone_routing_policies(boolean sharedRoutingLayer) { var tester = new RoutingPoliciesTester(); var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); // Deploy application int clustersPerZone = 2; - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Deployment creates records and policies for all clusters in all zones @@ -163,7 +169,7 @@ public class RoutingPoliciesTest { assertEquals(4, tester.policiesOf(context1.instanceId()).size()); // Add 1 cluster in each zone and deploy - tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), zone1, zone2); + tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), sharedRoutingLayer, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -177,7 +183,7 @@ public class RoutingPoliciesTest { assertEquals(6, tester.policiesOf(context1.instanceId()).size()); // Deploy another application - tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), zone1, zone2); + tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), sharedRoutingLayer, zone1, zone2); context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -195,7 +201,7 @@ public class RoutingPoliciesTest { assertEquals(4, tester.policiesOf(context2.instanceId()).size()); // Deploy removes cluster from app1 - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -574,6 +580,23 @@ public class RoutingPoliciesTest { tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); } + @Test + public void config_server_routing_policy() { + var tester = new RoutingPoliciesTester(); + var app = SystemApplication.configServer.id(); + RecordName name = RecordName.from("cfg.prod.us-west-1.test.vip"); + tester.controllerTester().nameService().add(new Record(Record.Type.A, name, RecordData.from("192.0.2.1"))); + + tester.provisionLoadBalancers(1, app, zone1); + tester.routingPolicies().refresh(app, DeploymentSpec.empty, zone1); + new NameServiceDispatcher(tester.tester.controller(), Duration.ofDays(1), Integer.MAX_VALUE).run(); + + List<Record> records = tester.controllerTester().nameService().findRecords(Record.Type.CNAME, name); + assertEquals(1, records.size()); + assertEquals(RecordData.from("lb-0--hosted-vespa:zone-config-servers:default--prod.us-west-1."), + records.get(0).data()); + } + /** Returns an application package builder that satisfies requirements for a directly routed endpoint */ private static ApplicationPackageBuilder applicationPackageBuilder() { return new ApplicationPackageBuilder() @@ -581,15 +604,21 @@ public class RoutingPoliciesTest { .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION); } - private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count) { + private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, boolean shared, int count) { List<LoadBalancer> loadBalancers = new ArrayList<>(); for (int i = 0; i < count; i++) { + HostName lbHostname; + if (shared) { + lbHostname = HostName.from("shared-lb--" + zone.value()); + } else { + lbHostname = HostName.from("lb-" + i + "--" + application.serializedForm() + + "--" + zone.value()); + } loadBalancers.add( new LoadBalancer("LB-" + i + "-Z-" + zone.value(), application, ClusterSpec.Id.from("c" + i), - HostName.from("lb-" + i + "--" + application.serializedForm() + - "--" + zone.value()), + lbHostname, LoadBalancer.State.active, Optional.of("dns-zone-1"))); } @@ -626,13 +655,17 @@ public class RoutingPoliciesTest { tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones()); } - private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { + private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, boolean shared, ZoneId... zones) { for (ZoneId zone : zones) { tester.configServer().removeLoadBalancers(application, zone); - tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone)); + tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, shared, clustersPerZone)); } } + private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { + provisionLoadBalancers(clustersPerZone, application, false, zones); + } + private Collection<RoutingPolicy> policiesOf(ApplicationId instance) { return tester.controller().curator().readRoutingPolicies(instance).values(); } diff --git a/default_build_settings.cmake b/default_build_settings.cmake index c9908380f9c..5ba7494d1e1 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -18,7 +18,7 @@ endfunction() function(setup_vespa_default_build_settings_rhel_8) message("-- Setting up default build settings for rhel 8") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_centos_7) @@ -31,7 +31,7 @@ endfunction() function(setup_vespa_default_build_settings_centos_8) message("-- Setting up default build settings for centos 8") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_darwin) @@ -159,10 +159,13 @@ function(vespa_use_default_build_settings) endif() if(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 6.10") setup_vespa_default_build_settings_rhel_6_10() - elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 7.7" OR - VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 7.8") + elseif(VESPA_OS_DISTRO STREQUAL "rhel" AND + VESPA_OS_DISTRO_VERSION VERSION_GREATER_EQUAL "7" AND + VESPA_OS_DISTRO_VERSION VERSION_LESS "8") setup_vespa_default_build_settings_rhel_7() - elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.1") + elseif(VESPA_OS_DISTRO STREQUAL "rhel" AND + VESPA_OS_DISTRO_VERSION VERSION_GREATER_EQUAL "8" AND + VESPA_OS_DISTRO_VERSION VERSION_LESS "9") setup_vespa_default_build_settings_rhel_8() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 7") setup_vespa_default_build_settings_centos_7() diff --git a/dist/vespa.spec b/dist/vespa.spec index 89348dabb88..1e1658d3d55 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -62,7 +62,7 @@ BuildRequires: vespa-icu-devel >= 65.1.0-1 %endif %if 0%{?el8} BuildRequires: cmake >= 3.11.4-3 -BuildRequires: llvm-devel >= 8.0.1 +BuildRequires: llvm-devel >= 9.0.1 BuildRequires: boost-devel >= 1.66 BuildRequires: openssl-devel BuildRequires: vespa-gtest >= 1.8.1-1 @@ -72,13 +72,6 @@ BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: cmake >= 3.9.1 BuildRequires: maven BuildRequires: openssl-devel -%if 0%{?fc30} -BuildRequires: vespa-protobuf-devel >= 3.7.0-4 -BuildRequires: llvm-devel >= 8.0.0 -BuildRequires: boost-devel >= 1.69 -BuildRequires: gtest-devel -BuildRequires: gmock-devel -%endif %if 0%{?fc31} BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: llvm-devel >= 9.0.0 @@ -139,7 +132,9 @@ Requires: perl-LWP-Protocol-https Requires: perl-Net-INET6Glue Requires: perl-Pod-Usage Requires: perl-URI +%if ! 0%{?el7} Requires: valgrind +%endif Requires: Judy Requires: xxhash Requires: xxhash-libs >= 0.7.3 @@ -164,25 +159,21 @@ Requires: vespa-openssl >= 1.1.1g-1 Requires: vespa-icu >= 65.1.0-1 Requires: vespa-protobuf >= 3.7.0-4 Requires: vespa-telegraf >= 1.1.1-1 +Requires: vespa-valgrind >= 3.16.0-1 %define _vespa_llvm_version 7 %define _extra_link_directory /usr/lib64/llvm7.0/lib;%{_vespa_deps_prefix}/lib64 %define _extra_include_directory /usr/include/llvm7.0;%{_vespa_deps_prefix}/include;/usr/include/openblas %endif %if 0%{?el8} -Requires: llvm-libs >= 8.0.1 +Requires: llvm-libs >= 9.0.1 +%define _vespa_llvm_version 9 Requires: vespa-protobuf >= 3.7.0-4 Requires: openssl-libs -%define _vespa_llvm_version 8 %define _extra_link_directory %{_vespa_deps_prefix}/lib64 %define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas %endif %if 0%{?fedora} Requires: openssl-libs -%if 0%{?fc30} -Requires: vespa-protobuf >= 3.7.0-4 -Requires: llvm-libs >= 8.0.0 -%define _vespa_llvm_version 8 -%endif %if 0%{?fc31} Requires: vespa-protobuf >= 3.7.0-4 Requires: llvm-libs >= 9.0.0 @@ -599,6 +590,7 @@ fi %{_prefix}/lib/jars/docprocs-jar-with-dependencies.jar %{_prefix}/lib/jars/flags-jar-with-dependencies.jar %{_prefix}/lib/jars/hk2-*.jar +%{_prefix}/lib/jars/hosted-zone-api-jar-with-dependencies.jar %{_prefix}/lib/jars/jackson-*.jar %{_prefix}/lib/jars/javassist-*.jar %{_prefix}/lib/jars/javax.*.jar diff --git a/docproc/src/main/java/com/yahoo/docproc/Processing.java b/docproc/src/main/java/com/yahoo/docproc/Processing.java index e157ad0b09b..eb918e3ea0c 100644 --- a/docproc/src/main/java/com/yahoo/docproc/Processing.java +++ b/docproc/src/main/java/com/yahoo/docproc/Processing.java @@ -26,7 +26,7 @@ public class Processing { private CallStack callStack = null; /** The collection of documents or document updates processed by this. This is never null */ - private List<DocumentOperation> documentOperations; + private final List<DocumentOperation> documentOperations; /** * Documents or document updates which should be added to <code>documents</code> before diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index 335fec8c5c0..edf773919f8 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -10,7 +10,6 @@ import com.yahoo.config.docproc.DocprocConfig; import com.yahoo.config.docproc.SchemamappingConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.document.ContainerDocumentConfig; -import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.docproc.AbstractConcreteDocumentFactory; import com.yahoo.docproc.CallStack; import com.yahoo.docproc.DocprocService; @@ -50,17 +49,17 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { private final ChainRegistry<DocumentProcessor> chainRegistry = new ChainRegistry<>(); private final ScheduledThreadPoolExecutor laterExecutor = new ScheduledThreadPoolExecutor(2, new DaemonThreadFactory("docproc-later-")); - private ContainerDocumentConfig containerDocConfig; + private final ContainerDocumentConfig containerDocConfig; private final DocumentTypeManager documentTypeManager; - public DocumentProcessingHandler(ComponentRegistry<DocprocService> docprocServiceRegistry, - ComponentRegistry<DocumentProcessor> documentProcessorComponentRegistry, - ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, - int numThreads, - DocumentTypeManager documentTypeManager, - ChainsModel chainsModel, SchemaMap schemaMap, Statistics statistics, - Metric metric, - ContainerDocumentConfig containerDocConfig) { + private DocumentProcessingHandler(ComponentRegistry<DocprocService> docprocServiceRegistry, + ComponentRegistry<DocumentProcessor> documentProcessorComponentRegistry, + ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, + int numThreads, + DocumentTypeManager documentTypeManager, + ChainsModel chainsModel, SchemaMap schemaMap, Statistics statistics, + Metric metric, + ContainerDocumentConfig containerDocConfig) { this.docprocServiceRegistry = docprocServiceRegistry; this.docFactoryRegistry = docFactoryRegistry; this.containerDocConfig = containerDocConfig; @@ -79,17 +78,16 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { docprocServiceRegistry.register(service.getId(), service); } } - } private static int computeNumThreads(int maxThreads) { return (maxThreads > 0) ? maxThreads : Runtime.getRuntime().availableProcessors(); } - public DocumentProcessingHandler(ComponentRegistry<DocprocService> docprocServiceRegistry, - ComponentRegistry<DocumentProcessor> documentProcessorComponentRegistry, - ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, - DocumentProcessingHandlerParameters params) { + DocumentProcessingHandler(ComponentRegistry<DocprocService> docprocServiceRegistry, + ComponentRegistry<DocumentProcessor> documentProcessorComponentRegistry, + ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, + DocumentProcessingHandlerParameters params) { this(docprocServiceRegistry, documentProcessorComponentRegistry, docFactoryRegistry, params.getMaxNumThreads(), params.getDocumentTypeManager(), params.getChainsModel(), params.getSchemaMap(), @@ -105,23 +103,25 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { SchemamappingConfig mappingConfig, DocumentmanagerConfig docManConfig, DocprocConfig docprocConfig, - ContainerMbusConfig containerMbusConfig, ContainerDocumentConfig containerDocConfig, Statistics manager, Metric metric) { this(new ComponentRegistry<>(), - documentProcessorComponentRegistry, docFactoryRegistry, new DocumentProcessingHandlerParameters().setMaxNumThreads - (docprocConfig.numthreads()) + documentProcessorComponentRegistry, docFactoryRegistry, + new DocumentProcessingHandlerParameters() + .setMaxNumThreads(docprocConfig.numthreads()) .setDocumentTypeManager(new DocumentTypeManager(docManConfig)) .setChainsModel(buildFromConfig(chainsConfig)).setSchemaMap(configureMapping(mappingConfig)) .setStatisticsManager(manager) .setMetric(metric) .setContainerDocumentConfig(containerDocConfig)); + docprocServiceRegistry.freeze(); } @Override protected void destroy() { - //threadPoolMap.values().forEach( pool -> pool.shutdown()); //calling shutdownNow() seems like a bit of an overkill + laterExecutor.shutdown(); + docprocServiceRegistry.allComponents().forEach(docprocService -> docprocService.deconstruct()); } public ComponentRegistry<DocprocService> getDocprocServiceRegistry() { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java index 2da494e19cf..a9bd63d96c3 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java @@ -75,9 +75,8 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { @Override public void skip() { - if (deserialized.get()) { - throw new IllegalStateException("Can not skip processing after deserialization."); - } + if (deserialized.get()) + throw new IllegalStateException("Can not skip processing after deserialization"); dispatchRequest(requestMsg, request.getUri().getPath(), responseHandler); } @@ -91,10 +90,7 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { } } } - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "Forwarding " + messages.size() + " messages from " + processings.size() + - " processings."); - } + log.log(Level.FINE, () ->"Forwarding " + messages.size() + " messages from " + processings.size() + " processings."); if (messages.isEmpty()) { dispatchResponse(Response.Status.OK); return; @@ -102,10 +98,10 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { long inputSequenceId = requestMsg.getSequenceId(); ResponseMerger responseHandler = new ResponseMerger(requestMsg, messages.size(), this); for (Message message : messages) { - // See comment for internalNoThrottledSource. - dispatchRequest(message, (inputSequenceId == message.getSequenceId()) - ? getUri().getPath() - : "/" + internalNoThrottledSource, + // See comment for internalNoThrottledSource + dispatchRequest(message, + message.getSequenceId() == inputSequenceId ? getUri().getPath() + : "/" + internalNoThrottledSource, responseHandler); } } @@ -177,7 +173,7 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { ResponseDispatch.newInstance(new MbusResponse(status, requestMsg.createReply())).dispatch(this); } - private void dispatchRequest(final Message msg, final String uriPath, final ResponseHandler handler) { + private void dispatchRequest(Message msg, String uriPath, ResponseHandler handler) { try { new RequestDispatch() { @@ -197,15 +193,10 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { } } - private static MessageFactory newMessageFactory(final DocumentMessage msg) { - if (msg == null) { - return null; - } - final Route route = msg.getRoute(); - if (route == null || !route.hasHops()) { - return null; - } - return new MessageFactory(msg); + private static MessageFactory newMessageFactory(DocumentMessage message) { + if (message == null) return null; + if (message.getRoute() == null || ! message.getRoute().hasHops()) return null; + return new MessageFactory(message); } private static URI resolveUri(String path) { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java index 92f4952b5b5..1365ed955fc 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java @@ -42,14 +42,12 @@ class MessageFactory { message.setTimeReceivedNow(); message.setTimeRemaining(requestMsg.getTimeRemainingNow()); message.getTrace().setLevel(requestMsg.getTrace().getLevel()); - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "Created '" + message.getClass().getName() + - "', route = '" + message.getRoute() + - "', priority = '" + message.getPriority().name() + - "', load type = '" + message.getLoadType() + - "', trace level = '" + message.getTrace().getLevel() + - "', time remaining = '" + message.getTimeRemaining() + "'."); - } + log.log(Level.FINE, () -> "Created '" + message.getClass().getName() + + "', route = '" + message.getRoute() + + "', priority = '" + message.getPriority().name() + + "', load type = '" + message.getLoadType() + + "', trace level = '" + message.getTrace().getLevel() + + "', time remaining = '" + message.getTimeRemaining() + "'."); return message; } diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java index e825db4e21d..e1482dead8d 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java @@ -239,7 +239,7 @@ public class ProxyDocument extends Document implements DocumentOperationWrapper @Override @SuppressWarnings("deprecation") public Struct getBody() { - return doc.getBody(); + return null; } @Override diff --git a/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java index 96ea96b9e78..617cd01e6e1 100644 --- a/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java +++ b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java @@ -306,7 +306,6 @@ public class SchemaMappingAndAccessesTest { assertEquals(mapped.getId().toString(), "id:map:album::2"); assertEquals(doc.getId().toString(), "id:map:album::2"); assertEquals(doc.getHeader(), mapped.getHeader()); - assertEquals(doc.getBody(), mapped.getBody()); assertEquals(doc.getSerializedSize(), mapped.getSerializedSize()); ByteArrayOutputStream bos = new ByteArrayOutputStream(); ByteArrayOutputStream bos2 = new ByteArrayOutputStream(); diff --git a/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg b/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg index 3347c3127b5..65ce1b56811 100644 --- a/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg +++ b/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg @@ -4,16 +4,15 @@ datatype[0].id -1407012075 datatype[0].structtype[1] datatype[0].structtype[0].name "outerdoc.body" datatype[0].structtype[0].version 0 -datatype[0].structtype[0].field[1] -datatype[0].structtype[0].field[0].datatype -2035324352 -datatype[0].structtype[0].field[0].name "innerdocuments" datatype[1].id -1686125086 datatype[1].structtype[1] datatype[1].structtype[0].name "docindoc.header" datatype[1].structtype[0].version 0 -datatype[1].structtype[0].field[1] +datatype[1].structtype[0].field[2] datatype[1].structtype[0].field[0].datatype 2 datatype[1].structtype[0].field[0].name "name" +datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].name "content" datatype[2].id -2035324352 datatype[2].arraytype[1] datatype[2].arraytype[0].datatype 1447635645 @@ -21,6 +20,9 @@ datatype[3].id -2040625920 datatype[3].structtype[1] datatype[3].structtype[0].name "outerdoc.header" datatype[3].structtype[0].version 0 +datatype[3].structtype[0].field[1] +datatype[3].structtype[0].field[0].datatype -2035324352 +datatype[3].structtype[0].field[0].name "innerdocuments" datatype[4].id 1447635645 datatype[4].documenttype[1] datatype[4].documenttype[0].bodystruct 2030224503 @@ -37,6 +39,3 @@ datatype[6].id 2030224503 datatype[6].structtype[1] datatype[6].structtype[0].name "docindoc.body" datatype[6].structtype[0].version 0 -datatype[6].structtype[0].field[1] -datatype[6].structtype[0].field[0].datatype 2 -datatype[6].structtype[0].field[0].name "content" diff --git a/document/abi-spec.json b/document/abi-spec.json index 3764015b917..7a0637db1aa 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -426,7 +426,9 @@ ], "methods": [ "public void <init>(java.lang.String)", + "public void <init>(java.lang.String, com.yahoo.document.StructDataType)", "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType)", + "public void <init>(java.lang.String, com.yahoo.document.StructDataType, java.util.Set)", "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType, java.util.Set)", "public void <init>(java.lang.String, java.util.Set)", "public com.yahoo.document.DocumentType clone()", @@ -435,7 +437,6 @@ "public boolean isValueCompatible(com.yahoo.document.datatypes.FieldValue)", "public com.yahoo.document.StructDataType contentStruct()", "public com.yahoo.document.StructDataType getHeaderType()", - "public com.yahoo.document.StructDataType getBodyType()", "protected void register(com.yahoo.document.DocumentTypeManager, java.util.List)", "public boolean isA(java.lang.String)", "public void addField(com.yahoo.document.Field)", diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java index 375d8d962a5..568fe9265d5 100644 --- a/document/src/main/java/com/yahoo/document/Document.java +++ b/document/src/main/java/com/yahoo/document/Document.java @@ -44,7 +44,6 @@ public class Document extends StructuredFieldValue { public static final short SERIALIZED_VERSION = 8; private DocumentId docId; private Struct header; - private Struct body; private Long lastModified = null; /** @@ -75,7 +74,6 @@ public class Document extends StructuredFieldValue { public Document(Document doc) { this(doc.getDataType(), doc.getId()); header = doc.header; - body = doc.body; lastModified = doc.lastModified; } @@ -104,7 +102,7 @@ public class Document extends StructuredFieldValue { /** @deprecated do not use: Use getField(), getFieldValue() or iterator() instead */ @Deprecated // TODO: Remove on Vespa 8 - public Struct getBody() { return body; } + public Struct getBody() { return null; } @Override public void assign(Object o) { @@ -116,14 +114,11 @@ public class Document extends StructuredFieldValue { Document doc = (Document) super.clone(); doc.docId = docId.clone(); doc.header = header.clone(); - doc.body = body.clone(); return doc; } - @SuppressWarnings("deprecation") private void setNewType(DocumentType type) { header = type.contentStruct().createFieldValue(); - body = type.getBodyType().createFieldValue(); } public void setDataType(DataType type) { @@ -178,9 +173,6 @@ public class Document extends StructuredFieldValue { public Field getField(String fieldName) { Field field = header.getField(fieldName); if (field == null) { - field = body.getField(fieldName); - } - if (field == null) { for(DocumentType parent : getDataType().getInheritedTypes()) { field = parent.getField(fieldName); if (field != null) { @@ -193,68 +185,27 @@ public class Document extends StructuredFieldValue { @Override public FieldValue getFieldValue(Field field) { - FieldValue fv = header.getFieldValue(field); - if (fv == null) { - fv = body.getFieldValue(field); - } - return fv; + return header.getFieldValue(field); } @Override - @SuppressWarnings("deprecation") protected void doSetFieldValue(Field field, FieldValue value) { - if (field.isHeader()) { - header.setFieldValue(field, value); - } else { - body.setFieldValue(field, value); - } + header.setFieldValue(field, value); } @Override public FieldValue removeFieldValue(Field field) { - FieldValue removed = header.removeFieldValue(field); - if (removed == null) { - removed = body.removeFieldValue(field); - } - return removed; + return header.removeFieldValue(field); } @Override public void clear() { header.clear(); - body.clear(); } @Override public Iterator<Map.Entry<Field, FieldValue>> iterator() { - return new Iterator<>() { - - private Iterator<Map.Entry<Field, FieldValue>> headerIt = header.iterator(); - private Iterator<Map.Entry<Field, FieldValue>> bodyIt = body.iterator(); - - public boolean hasNext() { - if (headerIt != null) { - if (headerIt.hasNext()) { - return true; - } else { - headerIt = null; - } - } - return bodyIt.hasNext(); - } - - public Map.Entry<Field, FieldValue> next() { - return (headerIt == null ? bodyIt.next() : headerIt.next()); - } - - public void remove() { - if (headerIt == null) { - bodyIt.remove(); - } else { - headerIt.remove(); - } - } - }; + return header.iterator(); } public String toString() { @@ -302,7 +253,7 @@ public class Document extends StructuredFieldValue { if (!(o instanceof Document)) return false; Document other = (Document) o; return (super.equals(o) && docId.equals(other.docId) && - header.equals(other.header) && body.equals(other.body)); + header.equals(other.header)); } @Override @@ -347,7 +298,7 @@ public class Document extends StructuredFieldValue { @Override public int getFieldCount() { - return header.getFieldCount() + body.getFieldCount(); + return header.getFieldCount(); } public void serialize(DocumentWriter writer) { @@ -393,7 +344,6 @@ public class Document extends StructuredFieldValue { return comp; } - comp = body.compareTo(otherValue.body); return comp; } diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java index 23559878fbb..f73fd634e0e 100755 --- a/document/src/main/java/com/yahoo/document/DocumentType.java +++ b/document/src/main/java/com/yahoo/document/DocumentType.java @@ -38,7 +38,6 @@ public class DocumentType extends StructuredDataType { public static final String DOCUMENT = "[document]"; public static final int classId = registerClass(Ids.document + 58, DocumentType.class); private StructDataType headerType; - private StructDataType bodyType; private List<DocumentType> inherits = new ArrayList<>(1); private Map<String, Set<Field>> fieldSets = new HashMap<>(); private final Set<String> importedFieldNames; @@ -52,7 +51,7 @@ public class DocumentType extends StructuredDataType { * @param name The name of the new document type */ public DocumentType(String name) { - this(name, createHeaderStructType(name), createBodyStructType(name)); + this(name, createHeaderStructType(name)); } /** @@ -62,37 +61,46 @@ public class DocumentType extends StructuredDataType { * * @param name The name of the new document type * @param headerType The type of the header struct - * @param bodyType The type of the body struct */ + public DocumentType(String name, StructDataType headerType) { + this(name, headerType, Collections.emptySet()); + } + + /** + * @deprecated //TODO Will be removed on Vespa 8 + */ + @Deprecated public DocumentType(String name, StructDataType headerType, StructDataType bodyType) { - this(name, headerType, bodyType, Collections.emptySet()); + this(name, headerType, Collections.emptySet()); } - public DocumentType(String name, StructDataType headerType, - StructDataType bodyType, Set<String> importedFieldNames) { + public DocumentType(String name, StructDataType headerType, Set<String> importedFieldNames) { super(name); this.headerType = headerType; - this.bodyType = bodyType; this.importedFieldNames = Collections.unmodifiableSet(importedFieldNames); } + /** + * @deprecated //TODO Will be removed on Vespa 8 + */ + @Deprecated + public DocumentType(String name, StructDataType headerType, + StructDataType bodyType, Set<String> importedFieldNames) { + this(name, headerType, importedFieldNames); + } + public DocumentType(String name, Set<String> importedFieldNames) { - this(name, createHeaderStructType(name), createBodyStructType(name), importedFieldNames); + this(name, createHeaderStructType(name), importedFieldNames); } private static StructDataType createHeaderStructType(String name) { return new StructDataType(name + ".header"); } - private static StructDataType createBodyStructType(String name) { - return new StructDataType(name + ".body"); - } - @Override public DocumentType clone() { DocumentType type = (DocumentType) super.clone(); type.headerType = headerType.clone(); - type.bodyType = bodyType.clone(); type.inherits = new ArrayList<>(inherits.size()); type.inherits.addAll(inherits); return type; @@ -136,14 +144,7 @@ public class DocumentType extends StructuredDataType { return contentStruct(); } - @Deprecated // TODO: Remove on Vespa 8 - /** @deprecated use contentStruct instead */ - public StructDataType getBodyType() { - return bodyType; - } - @Override - @SuppressWarnings("deprecation") protected void register(DocumentTypeManager manager, List<DataType> seenTypes) { seenTypes.add(this); for (DocumentType type : getInheritedTypes()) { @@ -153,23 +154,17 @@ public class DocumentType extends StructuredDataType { } // Get parent fields into fields specified in this type StructDataType header = headerType.clone(); - StructDataType body = bodyType.clone(); header.clearFields(); - body.clearFields(); for (Field field : getAllUniqueFields()) { - (field.isHeader() ? header : body).addField(field); + header.addField(field); } headerType.assign(header); - bodyType.assign(body); if (!seenTypes.contains(headerType)) { headerType.register(manager, seenTypes); } - if (!seenTypes.contains(bodyType)) { - bodyType.register(manager, seenTypes); - } manager.registerSingleType(this); } @@ -194,7 +189,6 @@ public class DocumentType extends StructuredDataType { * * @param field the field to add */ - @SuppressWarnings("deprecation") public void addField(Field field) { if (isRegistered()) { throw new IllegalStateException("You cannot add fields to a document type that is already registered."); @@ -227,7 +221,7 @@ public class DocumentType extends StructuredDataType { } /** - * Adds a new body field to this document type and returns the new field object + * Adds a new field to this document type and returns the new field object * * @param name The name of the field to add * @param type The datatype of the field to add @@ -346,9 +340,6 @@ public class DocumentType extends StructuredDataType { */ public Field getField(String name) { Field field = headerType.getField(name); - if (field == null) { - field = bodyType.getField(name); - } if (field == null && !isRegistered()) { for (DocumentType inheritedType : inherits) { field = inheritedType.getField(name); @@ -361,9 +352,6 @@ public class DocumentType extends StructuredDataType { @Override public Field getField(int id) { Field field = headerType.getField(id); - if (field == null) { - field = bodyType.getField(id); - } if (field == null && !isRegistered()) { for (DocumentType inheritedType : inherits) { field = inheritedType.getField(id); @@ -384,7 +372,7 @@ public class DocumentType extends StructuredDataType { } public int getFieldCount() { - return headerType.getFieldCount() + bodyType.getFieldCount(); + return headerType.getFieldCount(); } public Set<String> getImportedFieldNames() { @@ -407,9 +395,6 @@ public class DocumentType extends StructuredDataType { } Field field = headerType.removeField(name); if (field == null) { - field = bodyType.removeField(name); - } - if (field == null) { for (DocumentType inheritedType : inherits) { field = inheritedType.removeField(name); if (field != null) break; @@ -433,7 +418,6 @@ public class DocumentType extends StructuredDataType { } collection.addAll(headerType.getFields()); - collection.addAll(bodyType.getFields()); return ImmutableList.copyOf(collection); } @@ -483,39 +467,14 @@ public class DocumentType extends StructuredDataType { * @return An iterator for iterating the fields in this documenttype. */ public Iterator<Field> fieldIteratorThisTypeOnly() { - return new Iterator<>() { - Iterator<Field> headerIt = headerType.getFields().iterator(); - Iterator<Field> bodyIt = bodyType.getFields().iterator(); - - public boolean hasNext() { - if (headerIt != null) { - if (headerIt.hasNext()) return true; - headerIt = null; - } - return bodyIt.hasNext(); - } - - public Field next() { - return (headerIt != null ? headerIt.next() : bodyIt.next()); - } - - - public void remove() { - if (headerIt != null) { - headerIt.remove(); - } else { - bodyIt.remove(); - } - } - }; + return headerType.getFields().iterator(); } public boolean equals(Object o) { if (!(o instanceof DocumentType)) return false; DocumentType other = (DocumentType) o; // Ignore whether one of them have added inheritance to super Document.0 type - if (super.equals(o) && headerType.equals(other.headerType) && - bodyType.equals(other.bodyType)) { + if (super.equals(o) && headerType.equals(other.headerType)) { if ((inherits.size() > 1 || other.inherits.size() > 1) || (inherits.size() == 1 && other.inherits.size() == 1)) { return inherits.equals(other.inherits); @@ -527,7 +486,7 @@ public class DocumentType extends StructuredDataType { } public int hashCode() { - return super.hashCode() + headerType.hashCode() + bodyType.hashCode() + inherits.hashCode(); + return super.hashCode() + headerType.hashCode() + inherits.hashCode(); } @Override @@ -543,7 +502,6 @@ public class DocumentType extends StructuredDataType { public void visitMembers(ObjectVisitor visitor) { super.visitMembers(visitor); visitor.visit("headertype", headerType); - visitor.visit("bodytype", bodyType); visitor.visit("inherits", inherits); } } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 1d81d9e6e78..c802d2307c0 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -142,14 +142,10 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub @SuppressWarnings("deprecation") private static void registerDocumentType(DocumentTypeManager manager, DocumentmanagerConfig.Datatype.Documenttype doc) { StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct(), ""); - StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct(), ""); - for (Field field : body.getFields()) { - field.setHeader(false); - } var importedFields = doc.importedfield().stream() .map(f -> f.name()) .collect(Collectors.toUnmodifiableSet()); - DocumentType type = new DocumentType(doc.name(), header, body, importedFields); + DocumentType type = new DocumentType(doc.name(), header, importedFields); for (Object j : doc.inherits()) { DocumentmanagerConfig.Datatype.Documenttype.Inherits parent = (DocumentmanagerConfig.Datatype.Documenttype.Inherits) j; diff --git a/document/src/main/java/com/yahoo/document/TestAndSetCondition.java b/document/src/main/java/com/yahoo/document/TestAndSetCondition.java index def1c05a011..6a189fc2969 100644 --- a/document/src/main/java/com/yahoo/document/TestAndSetCondition.java +++ b/document/src/main/java/com/yahoo/document/TestAndSetCondition.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import com.google.common.annotations.Beta; - import java.util.Optional; /** @@ -14,7 +12,6 @@ import java.util.Optional; * * @author Vegard Sjonfjell */ -@Beta public class TestAndSetCondition { public static final TestAndSetCondition NOT_PRESENT_CONDITION = new TestAndSetCondition(); diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java index 7b78f90bc56..51ed93bb6ee 100644 --- a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java +++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java @@ -60,7 +60,7 @@ import static com.yahoo.document.json.JsonSerializationHelper.*; /** * The DocumentUpdateJsonSerializer utility class is used to serialize a DocumentUpdate instance using the JSON format described in - * <a href="http://docs.vespa.ai/documentation/reference/document-json-format.html#update">Document JSON Format: The Update Structure</a> + * <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html#update">Document JSON Format: The Update Structure</a> * * @see #serialize(com.yahoo.document.DocumentUpdate) * @author Vegard Sjonfjell diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java index 769e31818e6..ffc276fc94c 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -14,7 +14,7 @@ import static com.yahoo.document.json.readers.JsonParserHelpers.*; /** * Reads the tensor format defined at - * See <a href="http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor">http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor</a> + * See <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html">https://docs.vespa.ai/documentation/reference/document-json-format.html</a> * * @author geirst * @author bratseth diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java index 27327daab47..cac05fb7879 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java @@ -118,14 +118,12 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu doc.setId(documentId); Struct h = doc.getHeader(); - Struct b = doc.getBody(); h.clear(); - b.clear(); if ((content & 0x2) != 0) { - readHeaderBody(h, b); + readHeaderBody(h); } if ((content & 0x4) != 0) { - readHeaderBody(b, h); + readHeaderBody(h); } if (dataLength != (position() - dataPos)) { @@ -326,7 +324,7 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu buf = bigBuf; } - private void readHeaderBody(Struct primary, Struct alternate) { + private void readHeaderBody(Struct primary) { primary.setVersion(version); if (version < 8) { @@ -371,24 +369,14 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu buf = GrowableByteBuffer.wrap(destination); StructDataType priType = primary.getDataType(); - StructDataType altType = alternate.getDataType(); for (int i=0; i<numberOfFields; ++i) { int posBefore = position(); - Struct s = null; Integer f_id = fieldIdsAndLengths.get(i).first; Field structField = priType.getField(f_id); if (structField != null) { - s = primary; - } else { - structField = altType.getField(f_id); - if (structField != null) { - s = alternate; - } - } - if (s != null) { FieldValue value = structField.getDataType().createFieldValue(); value.deserialize(structField, this); - s.setFieldValue(structField, value); + primary.setFieldValue(structField, value); } //jump to beginning of next field: position(posBefore + fieldIdsAndLengths.get(i).second.intValue()); diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index 630f204c44d..3fca853b4d1 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -96,27 +96,20 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume doc.getId().serialize(this); - Struct head = doc.getHeader(); - Struct body = doc.getBody(); - boolean hasHead = (head.getFieldCount() != 0); - boolean hasBody = (body.getFieldCount() != 0); + boolean hasHead = (doc.getFieldCount() != 0); byte contents = 0x01; // Indicating we have document type which we always have if (hasHead) { contents |= 0x2; // Indicate we have header } - if (hasBody) { - contents |= 0x4; // Indicate we have a body - } + buf.put(contents); doc.getDataType().serialize(this); if (hasHead) { - head.serialize(null, this); - } - if (hasBody) { - body.serialize(null, this); + doc.getHeader().serialize(null, this); } + int finalPos = buf.position(); buf.position(lenPos); buf.putInt(finalPos - lenPos - 4); // Don't include the length itself or the version diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java index 5db98f26141..9dc5b7c2480 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java @@ -106,7 +106,6 @@ public final class XmlDocumentWriter implements DocumentWriter { buffer.addAttribute("lastmodifiedtime", lastModified); } write(null, value.getHeader()); - write(null, value.getBody()); buffer.endTag(); } diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java index 85bc4d032ff..8c6444fb853 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java @@ -55,7 +55,6 @@ public class XmlSerializationHelper { xml.addAttribute("lastmodifiedtime", lastModified); } doc.getHeader().printXml(xml); - doc.getBody().printXml(xml); } public static void printDoubleXml(DoubleFieldValue d, XmlStream xml) { diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/ConditionalFeedOperation.java b/document/src/main/java/com/yahoo/vespaxmlparser/ConditionalFeedOperation.java index e7a06560532..9e69390acd4 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/ConditionalFeedOperation.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/ConditionalFeedOperation.java @@ -4,7 +4,9 @@ package com.yahoo.vespaxmlparser; import com.yahoo.document.TestAndSetCondition; public class ConditionalFeedOperation extends FeedOperation { + private final TestAndSetCondition condition; + protected ConditionalFeedOperation(Type type) { super(type); this.condition = TestAndSetCondition.NOT_PRESENT_CONDITION; @@ -18,4 +20,5 @@ public class ConditionalFeedOperation extends FeedOperation { public TestAndSetCondition getCondition() { return condition; } + } diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/DocumentFeedOperation.java b/document/src/main/java/com/yahoo/vespaxmlparser/DocumentFeedOperation.java index f3ddbc9196c..408741e64f4 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/DocumentFeedOperation.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/DocumentFeedOperation.java @@ -5,7 +5,9 @@ import com.yahoo.document.Document; import com.yahoo.document.TestAndSetCondition; public class DocumentFeedOperation extends ConditionalFeedOperation { + private final Document document; + public DocumentFeedOperation(Document document) { super(Type.DOCUMENT); this.document = document; @@ -20,4 +22,6 @@ public class DocumentFeedOperation extends ConditionalFeedOperation { public Document getDocument() { return document; } + } + diff --git a/document/src/test/document/documentmanager.cfg b/document/src/test/document/documentmanager.cfg index d77c2b17460..e4c581304ce 100644 --- a/document/src/test/document/documentmanager.cfg +++ b/document/src/test/document/documentmanager.cfg @@ -5,10 +5,13 @@ datatype[0].weightedsettype[0] datatype[0].structtype[1] datatype[0].structtype[0].name foobar.header datatype[0].structtype[0].version 9 -datatype[0].structtype[0].field[1] +datatype[0].structtype[0].field[2] datatype[0].structtype[0].field[0].name foobarfield1 datatype[0].structtype[0].field[0].id[0] datatype[0].structtype[0].field[0].datatype 4 +datatype[0].structtype[0].field[1].name foobarfield0 +datatype[0].structtype[0].field[1].id[0] +datatype[0].structtype[0].field[1].datatype 2 datatype[0].documenttype[0] datatype[1].id 278604398 datatype[1].arraytype[0] @@ -16,10 +19,6 @@ datatype[1].weightedsettype[0] datatype[1].structtype[1] datatype[1].structtype[0].name foobar.body datatype[1].structtype[0].version 9 -datatype[1].structtype[0].field[1] -datatype[1].structtype[0].field[0].name foobarfield0 -datatype[1].structtype[0].field[0].id[0] -datatype[1].structtype[0].field[0].datatype 2 datatype[1].documenttype[0] datatype[2].id 378030104 datatype[2].arraytype[0] @@ -48,7 +47,6 @@ datatype[4].weightedsettype[0] datatype[4].structtype[1] datatype[4].structtype[0].name banana.body datatype[4].structtype[0].version 234 -datatype[4].structtype[0].field[0] datatype[4].documenttype[0] datatype[5].id 556449802 datatype[5].arraytype[0] @@ -68,7 +66,13 @@ datatype[6].weightedsettype[0] datatype[6].structtype[1] datatype[6].structtype[0].name customtypes.header datatype[6].structtype[0].version 3 -datatype[6].structtype[0].field[0] +datatype[6].structtype[0].field[2] +datatype[6].structtype[0].field[0].name arrayfloat +datatype[6].structtype[0].field[0].id[0] +datatype[6].structtype[0].field[0].datatype 99 +datatype[6].structtype[0].field[1].name arrayarrayfloat +datatype[6].structtype[0].field[1].id[0] +datatype[6].structtype[0].field[1].datatype 4003 datatype[6].documenttype[0] datatype[7].id 99 datatype[7].arraytype[1] @@ -88,13 +92,6 @@ datatype[9].weightedsettype[0] datatype[9].structtype[1] datatype[9].structtype[0].name customtypes.body datatype[9].structtype[0].version 3 -datatype[9].structtype[0].field[2] -datatype[9].structtype[0].field[0].name arrayfloat -datatype[9].structtype[0].field[0].id[0] -datatype[9].structtype[0].field[0].datatype 99 -datatype[9].structtype[0].field[1].name arrayarrayfloat -datatype[9].structtype[0].field[1].id[0] -datatype[9].structtype[0].field[1].datatype 4003 datatype[9].documenttype[0] datatype[10].id -1500313747 datatype[10].arraytype[0] diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java index fa47c80c6fb..b2be93bfff9 100644 --- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java @@ -116,18 +116,15 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); { doc.getDataType().contentStruct().setCompressionConfig(noncomp); - doc.getDataType().getBodyType().setCompressionConfig(noncomp); FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-uncompressed.dat", false); doc.serialize(fout); fout.close(); } { doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - doc.getDataType().getBodyType().setCompressionConfig(lz4comp); FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-lz4-9.dat", false); doc.serialize(fout); doc.getDataType().contentStruct().setCompressionConfig(noncomp); - doc.getDataType().getBodyType().setCompressionConfig(noncomp); fout.close(); } } diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index dcd4622b3f4..be6544563ed 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -753,12 +753,10 @@ public class DocumentTestCase extends DocumentTestCaseBase { CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - doc.getDataType().getBodyType().setCompressionConfig(lz4comp); buf = new GrowableByteBuffer(size, 2.0f); doc.serialize(buf); doc.getDataType().contentStruct().setCompressionConfig(noncomp); - doc.getDataType().getBodyType().setCompressionConfig(noncomp); fos = new FileOutputStream("src/tests/data/serializejava-compressed.dat"); fos.write(buf.array(), 0, buf.position()); fos.close(); @@ -816,13 +814,11 @@ public class DocumentTestCase extends DocumentTestCaseBase { CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4); doc.getDataType().contentStruct().setCompressionConfig(lz4comp); - doc.getDataType().getBodyType().setCompressionConfig(lz4comp); GrowableByteBuffer data = new GrowableByteBuffer(); doc.serialize(data); int size = doc.getSerializedSize(); doc.getDataType().contentStruct().setCompressionConfig(noncomp); - doc.getDataType().getBodyType().setCompressionConfig(noncomp); assertEquals(size, data.position()); diff --git a/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg b/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg index 3347c3127b5..65ce1b56811 100644 --- a/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg +++ b/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg @@ -4,16 +4,15 @@ datatype[0].id -1407012075 datatype[0].structtype[1] datatype[0].structtype[0].name "outerdoc.body" datatype[0].structtype[0].version 0 -datatype[0].structtype[0].field[1] -datatype[0].structtype[0].field[0].datatype -2035324352 -datatype[0].structtype[0].field[0].name "innerdocuments" datatype[1].id -1686125086 datatype[1].structtype[1] datatype[1].structtype[0].name "docindoc.header" datatype[1].structtype[0].version 0 -datatype[1].structtype[0].field[1] +datatype[1].structtype[0].field[2] datatype[1].structtype[0].field[0].datatype 2 datatype[1].structtype[0].field[0].name "name" +datatype[1].structtype[0].field[1].datatype 2 +datatype[1].structtype[0].field[1].name "content" datatype[2].id -2035324352 datatype[2].arraytype[1] datatype[2].arraytype[0].datatype 1447635645 @@ -21,6 +20,9 @@ datatype[3].id -2040625920 datatype[3].structtype[1] datatype[3].structtype[0].name "outerdoc.header" datatype[3].structtype[0].version 0 +datatype[3].structtype[0].field[1] +datatype[3].structtype[0].field[0].datatype -2035324352 +datatype[3].structtype[0].field[0].name "innerdocuments" datatype[4].id 1447635645 datatype[4].documenttype[1] datatype[4].documenttype[0].bodystruct 2030224503 @@ -37,6 +39,3 @@ datatype[6].id 2030224503 datatype[6].structtype[1] datatype[6].structtype[0].name "docindoc.body" datatype[6].structtype[0].version 0 -datatype[6].structtype[0].field[1] -datatype[6].structtype[0].field[0].datatype 2 -datatype[6].structtype[0].field[0].name "content" diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 2e08814bf43..d1f02ae45e2 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -42,6 +42,7 @@ public class DocumentSelectorTestCase { type.addField("hfloat", DataType.FLOAT); type.addField("hstring", DataType.STRING); type.addField("content", DataType.STRING); + type.addField("truth", DataType.BOOL); StructDataType mystruct = new StructDataType("mystruct"); mystruct.addField(new Field("key", DataType.INT)); @@ -67,6 +68,8 @@ public class DocumentSelectorTestCase { manager.registerDocumentType(new DocumentType("andornot")); manager.registerDocumentType(new DocumentType("idid")); manager.registerDocumentType(new DocumentType("usergroup")); + manager.registerDocumentType(new DocumentType("user")); + manager.registerDocumentType(new DocumentType("group")); } @Test @@ -87,6 +90,7 @@ public class DocumentSelectorTestCase { assertParse("music_.artist = \"*\""); assertParse("music_foo.artist = \"*\""); assertParse("music_foo_.artist = \"*\""); + assertParse("truth"); assertParse("(4 + 3) > 0", "(4+3) > 0"); assertParse("1 + 1 > 0", "1 +1 > 0"); assertParse("1 + -1 > 0", "1 + -1 > 0"); @@ -131,6 +135,8 @@ public class DocumentSelectorTestCase { assertParse(null, "false or false_t or falsetype"); assertParse(null, "true or and_t or andtype"); assertParse(null, "true or or_t or ortype"); + assertParse(null, "user or group"); + assertParse(null, "user.foo or group.bar"); } @Test diff --git a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java index 598369bae39..079e16915e1 100644 --- a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java @@ -68,7 +68,7 @@ public class VespaDocumentSerializerTestCase { CompressionFixture() { docType = new DocumentType("map_of_structs"); - docType.getHeaderType().setCompressionConfig(new CompressionConfig(CompressionType.LZ4)); + docType.contentStruct().setCompressionConfig(new CompressionConfig(CompressionType.LZ4)); nestedType = new StructDataType("nested_type"); nestedType.addField(new Field("str", DataType.STRING)); diff --git a/document/src/tests/data/crossplatform-java-cpp-document.cfg b/document/src/tests/data/crossplatform-java-cpp-document.cfg index 134d31b1831..3ebe56b8671 100644 --- a/document/src/tests/data/crossplatform-java-cpp-document.cfg +++ b/document/src/tests/data/crossplatform-java-cpp-document.cfg @@ -34,7 +34,10 @@ datatype[4].weightedsettype[0] datatype[4].structtype[1] datatype[4].structtype[0].name docindoc.header datatype[4].structtype[0].version 0 -datatype[4].structtype[0].field[0] +datatype[4].structtype[0].field[1] +datatype[4].structtype[0].field[0].name stringindocfield +datatype[4].structtype[0].field[0].id[0] +datatype[4].structtype[0].field[0].datatype 2 datatype[4].documenttype[0] datatype[5].id 2030224503 datatype[5].arraytype[0] @@ -42,10 +45,6 @@ datatype[5].weightedsettype[0] datatype[5].structtype[1] datatype[5].structtype[0].name docindoc.body datatype[5].structtype[0].version 0 -datatype[5].structtype[0].field[1] -datatype[5].structtype[0].field[0].name stringindocfield -datatype[5].structtype[0].field[0].id[0] -datatype[5].structtype[0].field[0].datatype 2 datatype[5].documenttype[0] datatype[6].id 1447635645 datatype[6].arraytype[0] @@ -63,7 +62,7 @@ datatype[7].weightedsettype[0] datatype[7].structtype[1] datatype[7].structtype[0].name serializetest.header datatype[7].structtype[0].version 0 -datatype[7].structtype[0].field[4] +datatype[7].structtype[0].field[15] datatype[7].structtype[0].field[0].name floatfield datatype[7].structtype[0].field[0].id[0] datatype[7].structtype[0].field[0].datatype 1 @@ -76,6 +75,39 @@ datatype[7].structtype[0].field[2].datatype 4 datatype[7].structtype[0].field[3].name urifield datatype[7].structtype[0].field[3].id[0] datatype[7].structtype[0].field[3].datatype 10 +datatype[7].structtype[0].field[4].name intfield +datatype[7].structtype[0].field[4].id[0] +datatype[7].structtype[0].field[4].datatype 0 +datatype[7].structtype[0].field[5].name rawfield +datatype[7].structtype[0].field[5].id[0] +datatype[7].structtype[0].field[5].datatype 3 +datatype[7].structtype[0].field[6].name doublefield +datatype[7].structtype[0].field[6].id[0] +datatype[7].structtype[0].field[6].datatype 5 +datatype[7].structtype[0].field[7].name contentfield +datatype[7].structtype[0].field[7].id[0] +datatype[7].structtype[0].field[7].datatype 2 +datatype[7].structtype[0].field[8].name bytefield +datatype[7].structtype[0].field[8].id[0] +datatype[7].structtype[0].field[8].datatype 16 +datatype[7].structtype[0].field[9].name arrayoffloatfield +datatype[7].structtype[0].field[9].id[0] +datatype[7].structtype[0].field[9].datatype 1001 +datatype[7].structtype[0].field[10].name arrayofarrayoffloatfield +datatype[7].structtype[0].field[10].id[0] +datatype[7].structtype[0].field[10].datatype 2001 +datatype[7].structtype[0].field[11].name docfield +datatype[7].structtype[0].field[11].id[0] +datatype[7].structtype[0].field[11].datatype 8 +datatype[7].structtype[0].field[12].name wsfield +datatype[7].structtype[0].field[12].id[0] +datatype[7].structtype[0].field[12].datatype 437829 +datatype[7].structtype[0].field[13].name mapfield +datatype[7].structtype[0].field[13].id[0] +datatype[7].structtype[0].field[13].datatype 9999 +datatype[7].structtype[0].field[14].name boolfield +datatype[7].structtype[0].field[14].id[0] +datatype[7].structtype[0].field[14].datatype 6 datatype[7].documenttype[0] datatype[8].id 1026122976 datatype[8].arraytype[0] @@ -83,40 +115,7 @@ datatype[8].weightedsettype[0] datatype[8].structtype[1] datatype[8].structtype[0].name serializetest.body datatype[8].structtype[0].version 0 -datatype[8].structtype[0].field[11] -datatype[8].structtype[0].field[0].name intfield -datatype[8].structtype[0].field[0].id[0] -datatype[8].structtype[0].field[0].datatype 0 -datatype[8].structtype[0].field[1].name rawfield -datatype[8].structtype[0].field[1].id[0] -datatype[8].structtype[0].field[1].datatype 3 -datatype[8].structtype[0].field[2].name doublefield -datatype[8].structtype[0].field[2].id[0] -datatype[8].structtype[0].field[2].datatype 5 -datatype[8].structtype[0].field[3].name contentfield -datatype[8].structtype[0].field[3].id[0] -datatype[8].structtype[0].field[3].datatype 2 -datatype[8].structtype[0].field[4].name bytefield -datatype[8].structtype[0].field[4].id[0] -datatype[8].structtype[0].field[4].datatype 16 -datatype[8].structtype[0].field[5].name arrayoffloatfield -datatype[8].structtype[0].field[5].id[0] -datatype[8].structtype[0].field[5].datatype 1001 -datatype[8].structtype[0].field[6].name arrayofarrayoffloatfield -datatype[8].structtype[0].field[6].id[0] -datatype[8].structtype[0].field[6].datatype 2001 -datatype[8].structtype[0].field[7].name docfield -datatype[8].structtype[0].field[7].id[0] -datatype[8].structtype[0].field[7].datatype 8 -datatype[8].structtype[0].field[8].name wsfield -datatype[8].structtype[0].field[8].id[0] -datatype[8].structtype[0].field[8].datatype 437829 -datatype[8].structtype[0].field[9].name mapfield -datatype[8].structtype[0].field[9].id[0] -datatype[8].structtype[0].field[9].datatype 9999 -datatype[8].structtype[0].field[10].name boolfield -datatype[8].structtype[0].field[10].id[0] -datatype[8].structtype[0].field[10].datatype 6 +datatype[8].structtype[0].field[0] datatype[8].documenttype[0] datatype[9].id 1306012852 datatype[9].arraytype[0] diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 9ac402f56ef..b75d094459b 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -76,6 +76,12 @@ namespace { void DocumentSelectParserTest::SetUp() { DocumenttypesConfigBuilderHelper builder(TestDocRepo::getDefaultConfig()); + builder.document(1234567, "with_imported", + Struct("with_imported.header"), + Struct("with_imported.body")) + .imported_field("my_imported_field"); + // Additional document types with names that are (or include) identifiers + // that lex to specific tokens. builder.document(535424777, "notandor", Struct("notandor.header"), Struct("notandor.body")); builder.document(1348665801, "ornotand", @@ -87,10 +93,10 @@ void DocumentSelectParserTest::SetUp() builder.document(-1673092522, "usergroup", Struct("usergroup.header"), Struct("usergroup.body")); - builder.document(1234567, "with_imported", - Struct("with_imported.header"), - Struct("with_imported.body")) - .imported_field("my_imported_field"); + builder.document(875463456, "user", + Struct("user.header"), Struct("user.body")); + builder.document(567463442, "group", + Struct("group.header"), Struct("group.body")); _repo = std::make_unique<DocumentTypeRepo>(builder.config()); _parser = std::make_unique<select::Parser>(*_repo, _bucketIdFactory); @@ -442,6 +448,8 @@ TEST_F(DocumentSelectParserTest, testParseTerminals) verifyParse("andornot"); verifyParse("idid"); verifyParse("usergroup"); + verifyParse("user"); + verifyParse("group"); } TEST_F(DocumentSelectParserTest, testParseBranches) @@ -463,6 +471,7 @@ TEST_F(DocumentSelectParserTest, testParseBranches) verifyParse("not andornot"); verifyParse("idid or not usergroup"); verifyParse("not(andornot or idid)", "not (andornot or idid)"); + verifyParse("not user or not group"); } template <typename ContainsType> @@ -1440,6 +1449,14 @@ TEST_F(DocumentSelectParserTest, test_ambiguous_field_spec_expression_is_handled parse_to_tree("(testdoctype1.foo) AND (testdoctype1.bar)")); } +TEST_F(DocumentSelectParserTest, special_tokens_are_allowed_as_freestanding_identifier_names) { + createDocs(); + EXPECT_EQ("(NOT (DOCTYPE user))", parse_to_tree("not user")); + EXPECT_EQ("(== (ID id.user) (FIELD user user))", parse_to_tree("id.user == user.user")); + EXPECT_EQ("(NOT (DOCTYPE group))", parse_to_tree("not group")); + EXPECT_EQ("(== (ID id.group) (FIELD group group))", parse_to_tree("id.group == group.group")); +} + TEST_F(DocumentSelectParserTest, test_can_build_field_value_from_field_expr_node) { using select::FieldExprNode; diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index 092a29d9293..d53fec43e5d 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -88,7 +88,7 @@ datatype[].documenttype[].inherits[].version int default=0 datatype[].documenttype[].headerstruct int ## Specify a document field id. Must be unique within the document type. -datatype[].documenttype[].bodystruct int +datatype[].documenttype[].bodystruct int default=0 ## Field sets datatype[].documenttype[].fieldsets{}.fields[] string diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index 0f0a9e3e37c..d02e9fe49f2 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -18,7 +18,7 @@ documenttype[].version int default=0 documenttype[].headerstruct int ## Specify a document field id. Must be unique within the document type. -documenttype[].bodystruct int +documenttype[].bodystruct int default=0 ## Specify a document type to inherit (id) documenttype[].inherits[].id int diff --git a/document/src/vespa/document/select/grammar/parser.yy b/document/src/vespa/document/select/grammar/parser.yy index 76b7cb7eeba..9d5b5825330 100644 --- a/document/src/vespa/document/select/grammar/parser.yy +++ b/document/src/vespa/document/select/grammar/parser.yy @@ -219,8 +219,11 @@ doc_type } ; + /* We allow most otherwise reserved tokens to be used as identifiers. */ ident : IDENTIFIER { $$ = $1; } + | USER { $$ = $1; } + | GROUP { $$ = $1; } | SCHEME { $$ = $1; } | TYPE { $$ = $1; } | NAMESPACE { $$ = $1; } diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 80fccbd9161..bb4deed2914 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -293,7 +293,7 @@ "public void <init>(int)", "public void <init>(java.lang.String)", "public void <init>(byte[])", - "public byte[] serialize()", + "public synchronized byte[] serialize()", "public java.lang.String serializeToString()", "public static com.yahoo.documentapi.ProgressToken fromSerializedString(java.lang.String)", "public void addFailedBucket(com.yahoo.document.BucketId, com.yahoo.document.BucketId, java.lang.String)", @@ -957,6 +957,28 @@ ], "fields": [] }, + "com.yahoo.documentapi.local.LocalVisitorSession": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.documentapi.VisitorSession" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.documentapi.local.LocalDocumentAccess, com.yahoo.documentapi.VisitorParameters)", + "public boolean isDone()", + "public com.yahoo.documentapi.ProgressToken getProgress()", + "public com.yahoo.messagebus.Trace getTrace()", + "public boolean waitUntilDone(long)", + "public void ack(com.yahoo.documentapi.AckToken)", + "public void abort()", + "public com.yahoo.documentapi.VisitorResponse getNext()", + "public com.yahoo.documentapi.VisitorResponse getNext(int)", + "public void destroy()" + ], + "fields": [] + }, "com.yahoo.documentapi.messagebus.MessageBusAsyncSession": { "superClass": "java.lang.Object", "interfaces": [ @@ -2082,50 +2104,6 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.LoadBalancer$Node": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.jrt.slobrok.api.Mirror$Entry, com.yahoo.documentapi.messagebus.protocol.LoadBalancer$NodeMetrics)" - ], - "fields": [ - "public com.yahoo.jrt.slobrok.api.Mirror$Entry entry", - "public com.yahoo.documentapi.messagebus.protocol.LoadBalancer$NodeMetrics metrics" - ] - }, - "com.yahoo.documentapi.messagebus.protocol.LoadBalancer$NodeMetrics": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()" - ], - "fields": [ - "public java.util.concurrent.atomic.AtomicLong sent", - "public java.util.concurrent.atomic.AtomicLong busy", - "public double weight" - ] - }, - "com.yahoo.documentapi.messagebus.protocol.LoadBalancer": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)", - "public java.util.List getNodeWeights()", - "public int getIndex(java.lang.String)", - "public com.yahoo.documentapi.messagebus.protocol.LoadBalancer$Node getRecipient(java.util.List)", - "public void received(com.yahoo.documentapi.messagebus.protocol.LoadBalancer$Node, boolean)" - ], - "fields": [] - }, "com.yahoo.documentapi.messagebus.protocol.LoadBalancerPolicy": { "superClass": "com.yahoo.documentapi.messagebus.protocol.SlobrokPolicy", "interfaces": [], diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java index 7e579a3ae6a..540b92461e6 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java @@ -226,7 +226,7 @@ public class ProgressToken { } } - public byte[] serialize() { + public synchronized byte[] serialize() { BufferSerializer out = new BufferSerializer(new GrowableByteBuffer()); out.putInt(null, distributionBits); out.putLong(null, bucketCursor); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java index cc0f6dc7cd5..24fd47ed12c 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java @@ -6,6 +6,7 @@ import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.TestAndSetCondition; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import java.time.Duration; @@ -105,8 +106,11 @@ public interface SyncSession extends Session { * Updates a document. * * @param update the updates to perform - * @return true, if the document was found and updated - * @throws UnsupportedOperationException thrown if this access does not support update + * @return false if the updates could not be applied as the document does not exist and + * {@link DocumentUpdate#setCreateIfNonExistent(boolean) create-if-non-existent} is not set. + * @throws DocumentAccessException on update error, including but not limited to: 1. timeouts, + * 2. the document exists but the {@link DocumentUpdate#setCondition(TestAndSetCondition) condition} + * is not met. */ boolean update(DocumentUpdate update); @@ -115,8 +119,11 @@ public interface SyncSession extends Session { * * @param update the updates to perform. * @param priority the priority with which to perform this operation - * @return true, if the document was found and updated - * @throws UnsupportedOperationException thrown if this access does not support update + * @return false if the updates could not be applied as the document does not exist and + * {@link DocumentUpdate#setCreateIfNonExistent(boolean) create-if-non-existent} is not set. + * @throws DocumentAccessException on update error, including but not limited to: 1. timeouts, + * 2. the document exists but the {@link DocumentUpdate#setCondition(TestAndSetCondition) condition} + * is not met. */ boolean update(DocumentUpdate update, DocumentProtocol.Priority priority); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java index 202929130c7..c69a8fb48de 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java @@ -3,6 +3,7 @@ package com.yahoo.documentapi.local; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; +import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.AsyncParameters; import com.yahoo.documentapi.AsyncSession; import com.yahoo.documentapi.DocumentAccess; @@ -43,8 +44,8 @@ public class LocalDocumentAccess extends DocumentAccess { } @Override - public VisitorSession createVisitorSession(VisitorParameters parameters) { - throw new UnsupportedOperationException("Not supported yet"); + public VisitorSession createVisitorSession(VisitorParameters parameters) throws ParseException { + return new LocalVisitorSession(this, parameters); } @Override diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalVisitorSession.java new file mode 100644 index 00000000000..e107be94008 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalVisitorSession.java @@ -0,0 +1,165 @@ +package com.yahoo.documentapi.local; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentGet; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.Field; +import com.yahoo.document.fieldset.FieldCollection; +import com.yahoo.document.fieldset.FieldSet; +import com.yahoo.document.fieldset.FieldSetRepo; +import com.yahoo.document.select.DocumentSelector; +import com.yahoo.document.select.Result; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.AckToken; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorDataHandler; +import com.yahoo.documentapi.VisitorDataQueue; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorResponse; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.messagebus.Trace; +import com.yahoo.yolean.Exceptions; + +import java.util.Comparator; +import java.util.Map; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Local visitor session that copies and iterates through all items in the local document access. + * Each document must be ack'ed for the session to be done visiting. + * Only document puts are sent by this session, and this is done from a separate thread. + * + * @author jonmv + */ +public class LocalVisitorSession implements VisitorSession { + + private enum State { RUNNING, FAILURE, ABORTED, SUCCESS } + + private final VisitorDataHandler data; + private final VisitorControlHandler control; + private final Map<DocumentId, Document> outstanding; + private final DocumentSelector selector; + private final FieldSet fieldSet; + private final AtomicReference<State> state; + + public LocalVisitorSession(LocalDocumentAccess access, VisitorParameters parameters) throws ParseException { + if (parameters.getResumeToken() != null) + throw new UnsupportedOperationException("Continuation via progress tokens is not supported"); + + if (parameters.getRemoteDataHandler() != null) + throw new UnsupportedOperationException("Remote data handlers are not supported"); + + this.selector = new DocumentSelector(parameters.getDocumentSelection()); + this.fieldSet = new FieldSetRepo().parse(access.getDocumentTypeManager(), parameters.fieldSet()); + + this.data = parameters.getLocalDataHandler() == null ? new VisitorDataQueue() : parameters.getLocalDataHandler(); + this.data.reset(); + this.data.setSession(this); + + this.control = parameters.getControlHandler() == null ? new VisitorControlHandler() : parameters.getControlHandler(); + this.control.reset(); + this.control.setSession(this); + + this.outstanding = new ConcurrentSkipListMap<>(Comparator.comparing(DocumentId::toString)); + this.outstanding.putAll(access.documents); + this.state = new AtomicReference<>(State.RUNNING); + + start(); + } + + void start() { + new Thread(() -> { + try { + // Iterate through all documents and pass on to data handler + outstanding.forEach((id, document) -> { + if (state.get() != State.RUNNING) + return; + + if (selector.accepts(new DocumentPut(document)) != Result.TRUE) + return; + + Document copy = new Document(document.getDataType(), document.getId()); + new FieldSetRepo().copyFields(document, copy, fieldSet); + + data.onMessage(new PutDocumentMessage(new DocumentPut(copy)), + new AckToken(id)); + }); + // Transition to a terminal state when done + state.updateAndGet(current -> { + switch (current) { + case RUNNING: + control.onDone(VisitorControlHandler.CompletionCode.SUCCESS, "Success"); + return State.SUCCESS; + case ABORTED: + control.onDone(VisitorControlHandler.CompletionCode.ABORTED, "Aborted by user"); + return State.ABORTED; + default: + control.onDone(VisitorControlHandler.CompletionCode.FAILURE, "Unexpected state '" + current + "'");; + return State.FAILURE; + } + }); + } + // Transition to failure terminal state on error + catch (Exception e) { + state.set(State.FAILURE); + outstanding.clear(); + control.onDone(VisitorControlHandler.CompletionCode.FAILURE, Exceptions.toMessageString(e)); + } + finally { + data.onDone(); + } + }).start(); + } + + @Override + public boolean isDone() { + return outstanding.isEmpty() // All documents ack'ed + && control.isDone(); // Control handler has been notified + } + + @Override + public ProgressToken getProgress() { + throw new UnsupportedOperationException("Progress tokens are not supported"); + } + + @Override + public Trace getTrace() { + throw new UnsupportedOperationException("Traces are not supported"); + } + + @Override + public boolean waitUntilDone(long timeoutMs) throws InterruptedException { + return control.waitUntilDone(timeoutMs); + } + + @Override + public void ack(AckToken token) { + outstanding.remove((DocumentId) token.ackObject); + } + + @Override + public void abort() { + state.updateAndGet(current -> current == State.RUNNING ? State.ABORTED : current); + outstanding.clear(); + } + + @Override + public VisitorResponse getNext() { + return data.getNext(); + } + + @Override + public VisitorResponse getNext(int timeoutMilliseconds) throws InterruptedException { + return data.getNext(timeoutMilliseconds); + } + + @Override + public void destroy() { + abort(); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index 1ad84cb0b25..69db27686f0 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -643,6 +643,9 @@ public class MessageBusVisitorSession implements VisitorSession { synchronized (progress.getToken()) { try { scheduledSendCreateVisitors = false; + if (done) { + return; // Session already closed; we must not touch anything else. + } while (progress.getIterator().hasNext()) { VisitorIterator.BucketProgress bucket = progress.getIterator().getNext(); Result result = sender.send(createMessage(bucket)); @@ -687,7 +690,6 @@ public class MessageBusVisitorSession implements VisitorSession { private void markSessionCompleted() { // 'done' is only ever written when token mutex is held, so safe to check // outside of completionMonitor lock. - assert(!done) : "Session was marked as completed more than once"; log.log(Level.FINE, "Visitor session '" + sessionName + "' has completed"); if (params.getLocalDataHandler() != null) { params.getLocalDataHandler().onDone(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AdaptiveLoadBalancer.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AdaptiveLoadBalancer.java new file mode 100644 index 00000000000..621064c178e --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AdaptiveLoadBalancer.java @@ -0,0 +1,58 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.jrt.slobrok.api.Mirror; + +import java.util.List; +import java.util.Random; + +/** + * Will pick 2 random candidates and select the one with least pending operations. + * + * @author baldersheim + */ +class AdaptiveLoadBalancer extends LoadBalancer { + private final Random random; + AdaptiveLoadBalancer(String cluster) { + this(cluster, new Random()); + } + AdaptiveLoadBalancer(String cluster, Random random) { + super(cluster); + this.random = random; + } + + @Override + Node getRecipient(List<Mirror.Entry> choices) { + if (choices.isEmpty()) return null; + Mirror.Entry entry; + NodeMetrics metrics; + if (choices.size() == 1) { + entry = choices.get(0); + metrics = getNodeMetrics(entry); + } else { + int candA = random.nextInt(choices.size()); + int candB = random.nextInt(choices.size()); + while (candB == candA) { + candB = random.nextInt(choices.size()); + } + entry = choices.get(candA); + Mirror.Entry entryB = choices.get(candB); + metrics = getNodeMetrics(entry); + NodeMetrics metricsB = getNodeMetrics(entryB); + if (metrics.pending() > metricsB.pending()) { + entry = entryB; + metrics = metricsB; + } + } + metrics.incSend(); + return new Node(entry, metrics); + } + + @Override + void received(Node node, boolean busy) { + node.metrics.incReceived(); + if (busy) { + node.metrics.incBusy(); + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java index 0db239b33bf..dbe6bb2ff45 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java @@ -82,4 +82,5 @@ public abstract class DocumentMessage extends Message { public Utf8String getProtocol() { return DocumentProtocol.NAME; } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 16ab9a017d0..e49cf021fe3 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; -import com.google.common.annotations.Beta; import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.component.VersionSpecification; @@ -147,7 +146,6 @@ public class DocumentProtocol implements Protocol { /** * Test and set condition (selection) failed. */ - @Beta public static final int ERROR_TEST_AND_SET_CONDITION_FAILED = ErrorCode.APP_FATAL_ERROR + 1013; /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java index 054b120cf81..c464bed5b87 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java @@ -1,49 +1,49 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; -import com.google.common.util.concurrent.AtomicDouble; import com.yahoo.jrt.slobrok.api.Mirror; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; -/** - * Load balances over a set of nodes based on statistics gathered from those nodes. - * - * @author thomasg - */ -public class LoadBalancer { - - public static class NodeMetrics { - public AtomicLong sent = new AtomicLong(); - public AtomicLong busy = new AtomicLong(); - public double weight = 1.0; +abstract class LoadBalancer { + static class NodeMetrics { + private AtomicLong sent = new AtomicLong(0); + private AtomicLong received = new AtomicLong(0); + private AtomicLong busy = new AtomicLong(0); + long pending() { return sent.get() - received.get(); } + void incSend() { sent.incrementAndGet(); } + void incReceived() { received.incrementAndGet(); } + void incBusy() { busy.incrementAndGet(); } + long sent() { return sent.get(); } + void reset() { + sent.set(0); + received.set(0); + busy.set(0); + } } + static class Node { + Node(Mirror.Entry e, NodeMetrics m) { entry = e; metrics = m; } - public static class Node { - public Node(Mirror.Entry e, NodeMetrics m) { entry = e; metrics = m; } - - public Mirror.Entry entry; - public NodeMetrics metrics; + Mirror.Entry entry; + NodeMetrics metrics; } + private final Map<String, Integer> cachedIndex = new HashMap<>(); /** Statistics on each node we are load balancing over. Populated lazily. */ - private final List<NodeMetrics> nodeWeights = new CopyOnWriteArrayList<>(); - + private final List<NodeMetrics> nodeWeights = new ArrayList<>(); private final String cluster; - private final AtomicDouble safePosition = new AtomicDouble(0.0); public LoadBalancer(String cluster) { this.cluster = cluster; } - - public List<NodeMetrics> getNodeWeights() { + List<NodeMetrics> getNodeWeights() { return nodeWeights; } - /** Returns the index from a node name string */ - public int getIndex(String nodeName) { + int getIndex(String nodeName) { try { String s = nodeName.substring(cluster.length() + 1); s = s.substring(0, s.indexOf("/")); @@ -54,83 +54,30 @@ public class LoadBalancer { throw new IllegalArgumentException(err, e); } } - - /** - * The load balancing operation: Returns a node choice from the given choices, - * based on previously gathered statistics on the nodes, and a running "position" - * which is increased by 1 on each call to this. - * - * @param choices the node choices, represented as Slobrok entries - * @return the chosen node, or null only if the given choices were zero - */ - public Node getRecipient(List<Mirror.Entry> choices) { - if (choices.isEmpty()) return null; - - double weightSum = 0.0; - Node selectedNode = null; - double position = safePosition.get(); - for (Mirror.Entry entry : choices) { - NodeMetrics nodeMetrics = getNodeMetrics(entry); - - weightSum += nodeMetrics.weight; - - if (weightSum > position) { - selectedNode = new Node(entry, nodeMetrics); - break; - } - } - if (selectedNode == null) { // Position>sum of all weights: Wrap around (but keep the remainder for some reason) - position -= weightSum; - selectedNode = new Node(choices.get(0), getNodeMetrics(choices.get(0))); - } - position += 1.0; - safePosition.set(position); - selectedNode.metrics.sent.incrementAndGet(); - return selectedNode; + int getCachedIndex(String nodeName) { + return cachedIndex.computeIfAbsent(nodeName, key -> getIndex(key)); } - /** * Returns the node metrics at a given index. * If there is no entry at the given index it is created by this call. */ - private NodeMetrics getNodeMetrics(Mirror.Entry entry) { - int index = getIndex(entry.getName()); + protected final synchronized NodeMetrics getNodeMetrics(Mirror.Entry entry) { + int index = getCachedIndex(entry.getName()); // expand node array as needed while (nodeWeights.size() < (index + 1)) nodeWeights.add(null); NodeMetrics nodeMetrics = nodeWeights.get(index); if (nodeMetrics == null) { // initialize statistics for this node - nodeMetrics = new NodeMetrics(); + nodeMetrics = createNodeMetrics(); nodeWeights.set(index, nodeMetrics); } return nodeMetrics; } - /** Scale weights such that ratios are preserved */ - private void increaseWeights() { - for (NodeMetrics n : nodeWeights) { - if (n == null) continue; - double want = n.weight * 1.01010101010101010101; - if (want >= 1.0) { - n.weight = want; - } else { - n.weight = 1.0; - } - } + protected NodeMetrics createNodeMetrics() { + return new NodeMetrics(); } - - public void received(Node node, boolean busy) { - if (busy) { - double wantWeight = node.metrics.weight - 0.01; - if (wantWeight < 1.0) { - increaseWeights(); - node.metrics.weight = 1.0; - } else { - node.metrics.weight = wantWeight; - } - node.metrics.busy.incrementAndGet(); - } - } - + abstract Node getRecipient(List<Mirror.Entry> choices); + abstract void received(Node node, boolean busy); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java index ae4606adaaf..3c670299f3e 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java @@ -26,7 +26,6 @@ import java.util.Map; public class LoadBalancerPolicy extends SlobrokPolicy { private final String session; private final String pattern; - private final LoadBalancer loadBalancer; LoadBalancerPolicy(String param) { @@ -48,7 +47,7 @@ public class LoadBalancerPolicy extends SlobrokPolicy { } pattern = cluster + "/*/" + session; - loadBalancer = new LoadBalancer(cluster); + loadBalancer = new AdaptiveLoadBalancer(cluster); } @Override @@ -95,4 +94,7 @@ public class LoadBalancerPolicy extends SlobrokPolicy { public void destroy() { } + + // For testing + LoadBalancer getLoadBalancer() { return loadBalancer; } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/TestAndSetMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/TestAndSetMessage.java index 5aaf2bde423..280881308f7 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/TestAndSetMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/TestAndSetMessage.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; -import com.google.common.annotations.Beta; import com.yahoo.document.TestAndSetCondition; /** @@ -9,7 +8,6 @@ import com.yahoo.document.TestAndSetCondition; * * @author Vegard Sjonfjell */ -@Beta public abstract class TestAndSetMessage extends DocumentMessage { public abstract void setCondition(TestAndSetCondition condition); public abstract TestAndSetCondition getCondition(); diff --git a/documentapi/src/test/cfg/documentmanager.cfg b/documentapi/src/test/cfg/documentmanager.cfg index eec3a6a06a0..75f205337a1 100644 --- a/documentapi/src/test/cfg/documentmanager.cfg +++ b/documentapi/src/test/cfg/documentmanager.cfg @@ -5,7 +5,10 @@ datatype[0].weightedsettype[0] datatype[0].structtype[1] datatype[0].structtype[0].name music.header datatype[0].structtype[0].version 0 -datatype[0].structtype[0].field[0] +datatype[0].structtype[0].field[1] +datatype[0].structtype[0].field[0].name artist +datatype[0].structtype[0].field[0].id[0] +datatype[0].structtype[0].field[0].datatype 2 datatype[0].documenttype[0] datatype[1].id 993120973 datatype[1].arraytype[0] @@ -13,10 +16,6 @@ datatype[1].weightedsettype[0] datatype[1].structtype[1] datatype[1].structtype[0].name music.body datatype[1].structtype[0].version 0 -datatype[1].structtype[0].field[1] -datatype[1].structtype[0].field[0].name artist -datatype[1].structtype[0].field[0].id[0] -datatype[1].structtype[0].field[0].datatype 2 datatype[1].documenttype[0] datatype[2].id 1412693671 datatype[2].arraytype[0] diff --git a/documentapi/src/test/java/com/yahoo/documentapi/local/LocalDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/local/LocalDocumentApiTestCase.java new file mode 100644 index 00000000000..d1361e50973 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/local/LocalDocumentApiTestCase.java @@ -0,0 +1,242 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.local; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.AsyncSession; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessParams; +import com.yahoo.documentapi.DocumentResponse; +import com.yahoo.documentapi.DumpVisitorDataHandler; +import com.yahoo.documentapi.Response; +import com.yahoo.documentapi.Result; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.test.AbstractDocumentApiTestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.CountDownLatch; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * Runs the superclass tests on this implementation + * + * @author bratseth + */ +public class LocalDocumentApiTestCase extends AbstractDocumentApiTestCase { + + protected LocalDocumentAccess access; + + @Override + protected DocumentAccess access() { + return access; + } + + @Before + public void setUp() { + DocumentAccessParams params = new DocumentAccessParams(); + params.setDocumentManagerConfigId("file:src/test/cfg/documentmanager.cfg"); + access = new LocalDocumentAccess(params); + } + + @After + public void shutdownAccess() { + access.shutdown(); + } + + @Test + public void testNoExceptionFromAsync() { + AsyncSession session = access.createAsyncSession(new AsyncParameters()); + + DocumentType type = access.getDocumentTypeManager().getDocumentType("music"); + DocumentUpdate docUp = new DocumentUpdate(type, new DocumentId("id:ns:music::2")); + + Result result = session.update(docUp); + assertTrue(result.isSuccess()); + Response response = session.getNext(); + assertEquals(result.getRequestId(), response.getRequestId()); + assertFalse(response.isSuccess()); + session.destroy(); + } + + @Test + public void testAsyncFetch() { + AsyncSession session = access.createAsyncSession(new AsyncParameters()); + List<DocumentId> ids = new ArrayList<>(); + ids.add(new DocumentId("id:music:music::1")); + ids.add(new DocumentId("id:music:music::2")); + ids.add(new DocumentId("id:music:music::3")); + for (DocumentId id : ids) + session.put(new Document(access.getDocumentTypeManager().getDocumentType("music"), id)); + int timeout = 100; + + long startTime = System.currentTimeMillis(); + Set<Long> outstandingRequests = new HashSet<>(); + for (DocumentId id : ids) { + Result result = session.get(id); + if ( ! result.isSuccess()) + throw new IllegalStateException("Failed requesting document " + id, result.getError().getCause()); + outstandingRequests.add(result.getRequestId()); + } + + List<Document> documents = new ArrayList<>(); + try { + while ( ! outstandingRequests.isEmpty()) { + int timeSinceStart = (int)(System.currentTimeMillis() - startTime); + Response response = session.getNext(timeout - timeSinceStart); + if (response == null) + throw new RuntimeException("Timed out waiting for documents"); // or return what you have + if ( ! outstandingRequests.contains(response.getRequestId())) continue; // Stale: Ignore + + if (response.isSuccess()) + documents.add(((DocumentResponse)response).getDocument()); + outstandingRequests.remove(response.getRequestId()); + } + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted while waiting for documents", e); + } + + assertEquals(3, documents.size()); + for (Document document : documents) + assertNotNull(document); + } + + @Test + public void testFeedingAndVisiting() throws InterruptedException, ParseException { + DocumentType musicType = access().getDocumentTypeManager().getDocumentType("music"); + Document doc1 = new Document(musicType, "id:ns:music::1"); doc1.setFieldValue("artist", "one"); + Document doc2 = new Document(musicType, "id:ns:music::2"); doc2.setFieldValue("artist", "two"); + Document doc3 = new Document(musicType, "id:ns:music::3"); + + // Select all music documents where the "artist" field is set + VisitorParameters parameters = new VisitorParameters("music.artist"); + parameters.setFieldSet("music:artist"); + VisitorControlHandler control = new VisitorControlHandler(); + parameters.setControlHandler(control); + Set<Document> received = new ConcurrentSkipListSet<>(); + parameters.setLocalDataHandler(new DumpVisitorDataHandler() { + @Override public void onDocument(Document doc, long timeStamp) { + received.add(doc); + } + @Override public void onRemove(DocumentId id) { + throw new IllegalStateException("Not supposed to get here"); + } + }); + + // Visit when there are no documents completes immediately + access.createVisitorSession(parameters).waitUntilDone(0); + assertSame(VisitorControlHandler.CompletionCode.SUCCESS, + control.getResult().getCode()); + assertEquals(Set.of(), + received); + + // Sync-put some documents + SyncSession out = access.createSyncSession(new SyncParameters.Builder().build()); + out.put(new DocumentPut(doc1)); + out.put(new DocumentPut(doc2)); + out.put(new DocumentPut(doc3)); + assertEquals(Map.of(doc1.getId(), doc1, + doc2.getId(), doc2, + doc3.getId(), doc3), + access.documents); + + // Expect a subset of documents to be returned, based on the selection + access.createVisitorSession(parameters).waitUntilDone(0); + assertSame(VisitorControlHandler.CompletionCode.SUCCESS, + control.getResult().getCode()); + assertEquals(Set.of(doc1, doc2), + received); + + // Remove doc2 and set artist for doc3, to see changes are reflected in subsequent visits + out.remove(new DocumentRemove(doc2.getId())); + out.update(new DocumentUpdate(musicType, doc3.getId()).addFieldUpdate(FieldUpdate.createAssign(musicType.getField("artist"), + new StringFieldValue("three")))); + assertEquals(Map.of(doc1.getId(), doc1, + doc3.getId(), doc3), + access.documents); + assertEquals("three", + ((StringFieldValue) doc3.getFieldValue("artist")).getString()); + + // Visit the documents again, retrieving none of the document fields + parameters.setFieldSet("[id]"); + received.clear(); + access.createVisitorSession(parameters).waitUntilDone(0); + assertSame(VisitorControlHandler.CompletionCode.SUCCESS, + control.getResult().getCode()); + assertEquals(Set.of(new Document(musicType, doc1.getId()), new Document(musicType, doc3.getId())), + received); + + // Visit the documents again, throwing an exception in the data handler on doc3 + received.clear(); + parameters.setLocalDataHandler(new DumpVisitorDataHandler() { + @Override public void onDocument(Document doc, long timeStamp) { + if (doc3.getId().equals(doc.getId())) + throw new RuntimeException("SEGFAULT"); + received.add(doc); + } + @Override public void onRemove(DocumentId id) { + throw new IllegalStateException("Not supposed to get here"); + } + }); + access.createVisitorSession(parameters).waitUntilDone(0); + assertSame(VisitorControlHandler.CompletionCode.FAILURE, + control.getResult().getCode()); + assertEquals("SEGFAULT", + control.getResult().getMessage()); + assertEquals(Set.of(new Document(musicType, doc1.getId())), + received); + + // Visit the documents again, aborting after the first document + received.clear(); + CountDownLatch visitLatch = new CountDownLatch(1); + CountDownLatch abortLatch = new CountDownLatch(1); + parameters.setLocalDataHandler(new DumpVisitorDataHandler() { + @Override public void onDocument(Document doc, long timeStamp) { + received.add(doc); + abortLatch.countDown(); + try { + visitLatch.await(); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + @Override public void onRemove(DocumentId id) { throw new IllegalStateException("Not supposed to get here"); } + }); + VisitorSession visit = access.createVisitorSession(parameters); + abortLatch.await(); + control.abort(); + visitLatch.countDown(); + visit.waitUntilDone(0); + assertSame(VisitorControlHandler.CompletionCode.ABORTED, + control.getResult().getCode()); + assertEquals(Set.of(new Document(musicType, doc1.getId())), + received); + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/local/test/LocalDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/local/test/LocalDocumentApiTestCase.java deleted file mode 100644 index 252bf739951..00000000000 --- a/documentapi/src/test/java/com/yahoo/documentapi/local/test/LocalDocumentApiTestCase.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.local.test; - -import com.yahoo.document.*; -import com.yahoo.documentapi.*; -import com.yahoo.documentapi.local.LocalDocumentAccess; -import com.yahoo.documentapi.test.AbstractDocumentApiTestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static org.junit.Assert.*; - -/** - * Runs the superclass tests on this implementation - * - * @author bratseth - */ -public class LocalDocumentApiTestCase extends AbstractDocumentApiTestCase { - - protected DocumentAccess access; - - @Override - protected DocumentAccess access() { - return access; - } - - @Before - public void setUp() { - DocumentAccessParams params = new DocumentAccessParams(); - params.setDocumentManagerConfigId("file:src/test/cfg/documentmanager.cfg"); - access = new LocalDocumentAccess(params); - } - - @After - public void shutdownAccess() { - access.shutdown(); - } - - @Test - public void testNoExceptionFromAsync() { - AsyncSession session = access.createAsyncSession(new AsyncParameters()); - - DocumentType type = access.getDocumentTypeManager().getDocumentType("music"); - DocumentUpdate docUp = new DocumentUpdate(type, new DocumentId("id:ns:music::2")); - - Result result = session.update(docUp); - assertTrue(result.isSuccess()); - Response response = session.getNext(); - assertEquals(result.getRequestId(), response.getRequestId()); - assertFalse(response.isSuccess()); - session.destroy(); - } - - @Test - public void testAsyncFetch() { - AsyncSession session = access.createAsyncSession(new AsyncParameters()); - List<DocumentId> ids = new ArrayList<>(); - ids.add(new DocumentId("id:music:music::1")); - ids.add(new DocumentId("id:music:music::2")); - ids.add(new DocumentId("id:music:music::3")); - for (DocumentId id : ids) - session.put(new Document(access.getDocumentTypeManager().getDocumentType("music"), id)); - int timeout = 100; - - long startTime = System.currentTimeMillis(); - Set<Long> outstandingRequests = new HashSet<>(); - for (DocumentId id : ids) { - Result result = session.get(id); - if ( ! result.isSuccess()) - throw new IllegalStateException("Failed requesting document " + id, result.getError().getCause()); - outstandingRequests.add(result.getRequestId()); - } - - List<Document> documents = new ArrayList<>(); - try { - while ( ! outstandingRequests.isEmpty()) { - int timeSinceStart = (int)(System.currentTimeMillis() - startTime); - Response response = session.getNext(timeout - timeSinceStart); - if (response == null) - throw new RuntimeException("Timed out waiting for documents"); // or return what you have - if ( ! outstandingRequests.contains(response.getRequestId())) continue; // Stale: Ignore - - if (response.isSuccess()) - documents.add(((DocumentResponse)response).getDocument()); - outstandingRequests.remove(response.getRequestId()); - } - } - catch (InterruptedException e) { - throw new RuntimeException("Interrupted while waiting for documents", e); - } - - assertEquals(3, documents.size()); - for (Document document : documents) - assertNotNull(document); - } - -} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerTestCase.java new file mode 100644 index 00000000000..582bd53d8e7 --- /dev/null +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerTestCase.java @@ -0,0 +1,128 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.jrt.slobrok.api.Mirror; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author Simon Thoresen Hult + */ +public class LoadBalancerTestCase { + + @Test + public void requireThatParseExceptionIsReadable() { + assertIllegalArgument("foo", "bar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'bar'."); + assertIllegalArgument("foo", "foobar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foobar'."); + assertIllegalArgument("foo", "foo", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo'."); + assertIllegalArgument("foo", "foo/", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/'."); + assertIllegalArgument("foo", "foo/0", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/0'."); + assertIllegalArgument("foo", "foo/0.", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/0.'."); + assertIllegalArgument("foo", "foo/0.bar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/0.bar'."); + assertIllegalArgument("foo", "foo/bar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/bar'."); + assertIllegalArgument("foo", "foo/bar.", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/bar.'."); + assertIllegalArgument("foo", "foo/bar.0", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/bar.0'."); + } + + private static void assertIllegalArgument(String clusterName, String recipient, String expectedMessage) { + LoadBalancer policy = new AdaptiveLoadBalancer(clusterName); + try { + fail("Expected exception, got index " + policy.getIndex(recipient) + "."); + } catch (IllegalArgumentException e) { + assertEquals(expectedMessage, e.getMessage()); + } + } + + @Test + public void testLoadBalancerCreation() { + LoadBalancerPolicy lbp = new LoadBalancerPolicy("cluster=docproc/cluster.mobile.indexing;session=chain.mobile.indexing"); + assertTrue(lbp.getLoadBalancer() instanceof AdaptiveLoadBalancer); + lbp = new LoadBalancerPolicy("cluster=docproc/cluster.mobile.indexing;session=chain.mobile.indexing;type=legacy"); + assertTrue(lbp.getLoadBalancer() instanceof AdaptiveLoadBalancer); + lbp = new LoadBalancerPolicy("cluster=docproc/cluster.mobile.indexing;session=chain.mobile.indexing;type=adaptive"); + assertTrue(lbp.getLoadBalancer() instanceof AdaptiveLoadBalancer); + } + + @Test + public void testAdaptiveLoadBalancer() { + LoadBalancer lb = new AdaptiveLoadBalancer("foo", new Random(1)); + + List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1"), + new Mirror.Entry("foo/1/default", "tcp/bar:2"), + new Mirror.Entry("foo/2/default", "tcp/bar:3")); + List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); + + for (int i = 0; i < 9999; i++) { + LoadBalancer.Node node = lb.getRecipient(entries); + assertNotNull(node); + } + + long sentSum = 0; + for (var metrics : weights) { + assertTrue(10 > Math.abs(metrics.sent() - 3333)); + sentSum += metrics.sent(); + } + assertEquals(9999, sentSum); + + for (var metrics : weights) { + metrics.reset(); + } + + // Simulate 1/1, 1/2, 1/4 processing capacity + for (int i = 0; i < 9999; i++) { + LoadBalancer.Node node = lb.getRecipient(entries); + assertNotNull(node); + if (node.entry.getName().contains("1")) { + lb.received(node, false); + } else if (node.entry.getName().contains("2")) { + if ((i % 2) == 0) { + lb.received(node, false); + } + } else { + if ((i % 4) == 0) { + lb.received(node, false); + } + } + } + + sentSum = 0; + long sumPending = 0; + for (var metrics : weights) { + System.out.println("m: s=" + metrics.sent() + " p=" + metrics.pending()); + sentSum += metrics.sent(); + sumPending += metrics.pending(); + } + assertEquals(9999, sentSum); + assertEquals(2039, sumPending); + assertEquals(1332, weights.get(0).sent()); + assertEquals(6645, weights.get(1).sent()); + assertEquals(2022, weights.get(2).sent()); + assertEquals(1020, weights.get(0).pending()); + assertEquals(0, weights.get(1).pending()); + assertEquals(1019, weights.get(2).pending()); + } + + private void verifyLoadBalancerOneItemOnly(LoadBalancer lb) { + + List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1") ); + List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); + + assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); + + lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), true); // busy + + assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); + } + @Test + public void testLoadBalancerOneItemOnly() { + verifyLoadBalancerOneItemOnly(new AdaptiveLoadBalancer("foo")); + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java deleted file mode 100644 index 51dd1ac12b8..00000000000 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol.test; - -import com.yahoo.documentapi.messagebus.protocol.LoadBalancer; -import com.yahoo.jrt.slobrok.api.Mirror; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author Simon Thoresen Hult - */ -public class LoadBalancerTestCase { - - @Test - public void requireThatParseExceptionIsReadable() { - assertIllegalArgument("foo", "bar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'bar'."); - assertIllegalArgument("foo", "foobar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foobar'."); - assertIllegalArgument("foo", "foo", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo'."); - assertIllegalArgument("foo", "foo/", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/'."); - assertIllegalArgument("foo", "foo/0", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/0'."); - assertIllegalArgument("foo", "foo/0.", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/0.'."); - assertIllegalArgument("foo", "foo/0.bar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/0.bar'."); - assertIllegalArgument("foo", "foo/bar", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/bar'."); - assertIllegalArgument("foo", "foo/bar.", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/bar.'."); - assertIllegalArgument("foo", "foo/bar.0", "Expected recipient on the form 'foo/x/[y.]number/z', got 'foo/bar.0'."); - } - - private static void assertIllegalArgument(String clusterName, String recipient, String expectedMessage) { - LoadBalancer policy = new LoadBalancer(clusterName); - try { - fail("Expected exception, got index " + policy.getIndex(recipient) + "."); - } catch (IllegalArgumentException e) { - assertEquals(expectedMessage, e.getMessage()); - } - } - - @Test - public void testLoadBalancer() { - LoadBalancer lb = new LoadBalancer("foo"); - - List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1"), - new Mirror.Entry("foo/1/default", "tcp/bar:2"), - new Mirror.Entry("foo/2/default", "tcp/bar:3")); - List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); - - { - for (int i = 0; i < 99; i++) { - LoadBalancer.Node node = lb.getRecipient(entries); - assertEquals("foo/" + (i % 3) + "/default" , node.entry.getName()); - } - - assertEquals(33, weights.get(0).sent.intValue()); - assertEquals(33, weights.get(1).sent.intValue()); - assertEquals(33, weights.get(2).sent.intValue()); - - weights.get(0).sent.set(0); - weights.get(1).sent.set(0); - weights.get(2).sent.set(0); - } - - { - // Simulate that one node is overloaded. It returns busy twice as often as the others. - for (int i = 0; i < 100; i++) { - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), true); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), false); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), false); - - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/2/default", "tcp/bar:3"), weights.get(2)), true); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/2/default", "tcp/bar:3"), weights.get(2)), false); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/2/default", "tcp/bar:3"), weights.get(2)), false); - - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/1/default", "tcp/bar:2"), weights.get(1)), true); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/1/default", "tcp/bar:2"), weights.get(1)), true); - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/1/default", "tcp/bar:2"), weights.get(1)), false); - } - - assertEquals(421, (int)(100 * weights.get(0).weight / weights.get(1).weight)); - assertEquals(100, (int)(100 * weights.get(1).weight)); - assertEquals(421, (int)(100 * weights.get(2).weight / weights.get(1).weight)); - } - - - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/1/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/2/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - } - - @Test - public void testLoadBalancerOneItemOnly() { - LoadBalancer lb = new LoadBalancer("foo"); - - List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1") ); - List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); - - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - - lb.received(new LoadBalancer.Node(new Mirror.Entry("foo/0/default", "tcp/bar:1"), weights.get(0)), true); // busy - - assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); - - } -} diff --git a/documentapi/test/cfg/testdoc.cfg b/documentapi/test/cfg/testdoc.cfg index 89bea273b6e..f218ddcd3b4 100644 --- a/documentapi/test/cfg/testdoc.cfg +++ b/documentapi/test/cfg/testdoc.cfg @@ -17,7 +17,7 @@ datatype[1].weightedsettype[0] datatype[1].structtype[1] datatype[1].structtype[0].name testdoc.header datatype[1].structtype[0].version 0 -datatype[1].structtype[0].field[4] +datatype[1].structtype[0].field[10] datatype[1].structtype[0].field[0].name floatfield datatype[1].structtype[0].field[0].id[0] datatype[1].structtype[0].field[0].datatype 1 @@ -30,6 +30,24 @@ datatype[1].structtype[0].field[2].datatype 4 datatype[1].structtype[0].field[3].name urifield datatype[1].structtype[0].field[3].id[0] datatype[1].structtype[0].field[3].datatype 10 +datatype[1].structtype[0].field[4].name intfield +datatype[1].structtype[0].field[4].id[0] +datatype[1].structtype[0].field[4].datatype 0 +datatype[1].structtype[0].field[5].name rawfield +datatype[1].structtype[0].field[5].id[0] +datatype[1].structtype[0].field[5].datatype 3 +datatype[1].structtype[0].field[6].name doublefield +datatype[1].structtype[0].field[6].id[0] +datatype[1].structtype[0].field[6].datatype 5 +datatype[1].structtype[0].field[7].name contentfield +datatype[1].structtype[0].field[7].id[0] +datatype[1].structtype[0].field[7].datatype 2 +datatype[1].structtype[0].field[8].name bytefield +datatype[1].structtype[0].field[8].id[0] +datatype[1].structtype[0].field[8].datatype 16 +datatype[1].structtype[0].field[9].name foo +datatype[1].structtype[0].field[9].id[0] +datatype[1].structtype[0].field[9].datatype 666999 datatype[1].documenttype[0] datatype[2].id 1878320748 datatype[2].arraytype[0] @@ -37,25 +55,6 @@ datatype[2].weightedsettype[0] datatype[2].structtype[1] datatype[2].structtype[0].name testdoc.body datatype[2].structtype[0].version 0 -datatype[2].structtype[0].field[6] -datatype[2].structtype[0].field[0].name intfield -datatype[2].structtype[0].field[0].id[0] -datatype[2].structtype[0].field[0].datatype 0 -datatype[2].structtype[0].field[1].name rawfield -datatype[2].structtype[0].field[1].id[0] -datatype[2].structtype[0].field[1].datatype 3 -datatype[2].structtype[0].field[2].name doublefield -datatype[2].structtype[0].field[2].id[0] -datatype[2].structtype[0].field[2].datatype 5 -datatype[2].structtype[0].field[3].name contentfield -datatype[2].structtype[0].field[3].id[0] -datatype[2].structtype[0].field[3].datatype 2 -datatype[2].structtype[0].field[4].name bytefield -datatype[2].structtype[0].field[4].id[0] -datatype[2].structtype[0].field[4].datatype 16 -datatype[2].structtype[0].field[5].name foo -datatype[2].structtype[0].field[5].id[0] -datatype[2].structtype[0].field[5].datatype 666999 datatype[2].documenttype[0] datatype[3].id -1175657560 datatype[3].arraytype[0] @@ -73,7 +72,10 @@ datatype[4].weightedsettype[0] datatype[4].structtype[1] datatype[4].structtype[0].name other.header datatype[4].structtype[0].version 0 -datatype[4].structtype[0].field[0] +datatype[4].structtype[0].field[1] +datatype[4].structtype[0].field[0].name intfield +datatype[4].structtype[0].field[0].id[0] +datatype[4].structtype[0].field[0].datatype 0 datatype[4].documenttype[0] datatype[5].id -72846462 datatype[5].arraytype[0] @@ -81,10 +83,6 @@ datatype[5].weightedsettype[0] datatype[5].structtype[1] datatype[5].structtype[0].name other.body datatype[5].structtype[0].version 0 -datatype[5].structtype[0].field[1] -datatype[5].structtype[0].field[0].name intfield -datatype[5].structtype[0].field[0].id[0] -datatype[5].structtype[0].field[0].datatype 0 datatype[5].documenttype[0] datatype[6].id -1146158894 datatype[6].arraytype[0] diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 29bee2e9e3e..91b786e20f5 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -870,25 +870,21 @@ public class DocumentGenPluginTest { final Book book = getBook(); assertEquals(book.getMystruct().getD1(), (Double)56.777); assertEquals(book.getMystruct().getCompressionType(), CompressionType.NONE); - assertEquals(book.getBody().getFieldCount(), 0); - assertEquals(book.getHeader().getFieldCount(), 13); + assertEquals(book.getFieldCount(), 13); assertEquals(book.getMystruct().getFieldCount(), 4); assertEquals(book.getContent().get(0), 3); assertEquals(book.getContent().get(1), 4); assertEquals(book.getContent().get(2), 5); final Document des = roundtripSerialize(book, typeManagerForBookType()); - assertEquals(des.getBody().getFieldCount(), 0); - assertEquals(des.getHeader().getFieldCount(), 13); + assertEquals(des.getFieldCount(), 13); assertEquals(des.getDataType().getName(), "book"); assertEquals(((Raw) des.getFieldValue("content")).getByteBuffer().get(0), 3); assertEquals(((Raw) des.getFieldValue("content")).getByteBuffer().get(1), 4); assertEquals(((Raw) des.getFieldValue("content")).getByteBuffer().get(2), 5); assertEquals(des.getFieldValue("author").toString(), "Herman Melville"); assertEquals(des.getFieldValue("title").toString(), "Moby Dick - Or The Whale"); - assertEquals(des.getHeader().getFieldValue("title").toString(), "Moby Dick - Or The Whale"); - assertNull(des.getBody().getFieldValue("title")); - assertEquals(des.getHeader().getFieldValue("author").toString(), "Herman Melville"); - assertNull(des.getBody().getFieldValue("author")); + assertEquals(des.getFieldValue("title").toString(), "Moby Dick - Or The Whale"); + assertEquals(des.getFieldValue("author").toString(), "Herman Melville"); Struct mystruct = (Struct)des.getFieldValue("mystruct"); FieldValue d1 = mystruct.getFieldValue("d1"); diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 29d3c192139..3e81521550a 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -17,7 +17,9 @@ vespa_define_module( src/tests/eval/function src/tests/eval/function_speed src/tests/eval/gbdt + src/tests/eval/inline_operation src/tests/eval/interpreted_function + src/tests/eval/multiply_add src/tests/eval/node_tools src/tests/eval/node_types src/tests/eval/param_usage @@ -34,17 +36,22 @@ vespa_define_module( src/tests/tensor/dense_fast_rename_optimizer src/tests/tensor/dense_generic_join src/tests/tensor/dense_inplace_join_function - src/tests/tensor/dense_inplace_map_function src/tests/tensor/dense_matmul_function src/tests/tensor/dense_multi_matmul_function + src/tests/tensor/dense_number_join_function + src/tests/tensor/dense_pow_as_map_optimizer src/tests/tensor/dense_remove_dimension_optimizer src/tests/tensor/dense_replace_type_function + src/tests/tensor/dense_simple_expand_function + src/tests/tensor/dense_simple_join_function + src/tests/tensor/dense_simple_map_function src/tests/tensor/dense_single_reduce_function src/tests/tensor/dense_tensor_create_function src/tests/tensor/dense_tensor_peek_function src/tests/tensor/dense_xw_product_function src/tests/tensor/direct_dense_tensor_builder src/tests/tensor/direct_sparse_tensor_builder + src/tests/tensor/index_lookup_table src/tests/tensor/tensor_add_operation src/tests/tensor/tensor_address src/tests/tensor/tensor_conformance diff --git a/eval/src/tests/ann/nns-l2.h b/eval/src/tests/ann/nns-l2.h index 82a95741200..de24df50b6c 100644 --- a/eval/src/tests/ann/nns-l2.h +++ b/eval/src/tests/ann/nns-l2.h @@ -36,7 +36,7 @@ template <typename FltType = float> struct L2DistCalc { const vespalib::hwaccelrated::IAccelrated & _hw; - L2DistCalc() : _hw(vespalib::hwaccelrated::IAccelrated::getAccelrator()) {} + L2DistCalc() : _hw(vespalib::hwaccelrated::IAccelrated::getAccelerator()) {} using Arr = vespalib::ArrayRef<FltType>; using ConstArr = vespalib::ConstArrayRef<FltType>; diff --git a/eval/src/tests/eval/compile_cache/compile_cache_test.cpp b/eval/src/tests/eval/compile_cache/compile_cache_test.cpp index 5176a4b3e94..a0dad889d9a 100644 --- a/eval/src/tests/eval/compile_cache/compile_cache_test.cpp +++ b/eval/src/tests/eval/compile_cache/compile_cache_test.cpp @@ -5,12 +5,16 @@ #include <vespa/eval/eval/test/eval_spec.h> #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/util/stringfmt.h> #include <thread> #include <set> using namespace vespalib; using namespace vespalib::eval; +using vespalib::make_string_short::fmt; + struct MyExecutor : public Executor { std::vector<Executor::Task::UP> tasks; Executor::Task::UP execute(Executor::Task::UP task) override { @@ -157,7 +161,7 @@ TEST("require that cache usage works") { } TEST("require that async cache usage works") { - ThreadStackExecutor executor(8, 256*1024); + auto executor = std::make_shared<ThreadStackExecutor>(8, 256*1024); auto binding = CompileCache::bind(executor); CompileCache::Token::UP token_a = CompileCache::compile(*Function::parse("x+y"), PassParams::SEPARATE); EXPECT_EQUAL(5.0, token_a->get().get_function<2>()(2.0, 3.0)); @@ -166,7 +170,6 @@ TEST("require that async cache usage works") { CompileCache::Token::UP token_c = CompileCache::compile(*Function::parse("x+y"), PassParams::SEPARATE); EXPECT_EQUAL(5.0, token_c->get().get_function<2>()(2.0, 3.0)); EXPECT_EQUAL(CompileCache::num_cached(), 2u); - executor.sync(); // wait for compile threads to drop all compile cache tokens token_a.reset(); TEST_DO(verify_cache(2, 2)); token_b.reset(); @@ -176,24 +179,24 @@ TEST("require that async cache usage works") { } TEST("require that compile tasks are run in the most recently bound executor") { - MyExecutor exe1; - MyExecutor exe2; + auto exe1 = std::make_shared<MyExecutor>(); + auto exe2 = std::make_shared<MyExecutor>(); auto token0 = CompileCache::compile(*Function::parse("a+b"), PassParams::SEPARATE); EXPECT_EQUAL(CompileCache::num_bound(), 0u); - EXPECT_EQUAL(exe1.tasks.size(), 0u); - EXPECT_EQUAL(exe2.tasks.size(), 0u); + EXPECT_EQUAL(exe1->tasks.size(), 0u); + EXPECT_EQUAL(exe2->tasks.size(), 0u); { auto bind1 = CompileCache::bind(exe1); auto token1 = CompileCache::compile(*Function::parse("a-b"), PassParams::SEPARATE); EXPECT_EQUAL(CompileCache::num_bound(), 1u); - EXPECT_EQUAL(exe1.tasks.size(), 1u); - EXPECT_EQUAL(exe2.tasks.size(), 0u); + EXPECT_EQUAL(exe1->tasks.size(), 1u); + EXPECT_EQUAL(exe2->tasks.size(), 0u); { auto bind2 = CompileCache::bind(exe2); auto token2 = CompileCache::compile(*Function::parse("a*b"), PassParams::SEPARATE); EXPECT_EQUAL(CompileCache::num_bound(), 2u); - EXPECT_EQUAL(exe1.tasks.size(), 1u); - EXPECT_EQUAL(exe2.tasks.size(), 1u); + EXPECT_EQUAL(exe1->tasks.size(), 1u); + EXPECT_EQUAL(exe2->tasks.size(), 1u); } EXPECT_EQUAL(CompileCache::num_bound(), 1u); } @@ -201,9 +204,9 @@ TEST("require that compile tasks are run in the most recently bound executor") { } TEST("require that executors may be unbound in any order") { - MyExecutor exe1; - MyExecutor exe2; - MyExecutor exe3; + auto exe1 = std::make_shared<MyExecutor>(); + auto exe2 = std::make_shared<MyExecutor>(); + auto exe3 = std::make_shared<MyExecutor>(); auto bind1 = CompileCache::bind(exe1); auto bind2 = CompileCache::bind(exe2); auto bind3 = CompileCache::bind(exe3); @@ -213,13 +216,13 @@ TEST("require that executors may be unbound in any order") { bind3.reset(); EXPECT_EQUAL(CompileCache::num_bound(), 1u); auto token = CompileCache::compile(*Function::parse("a+b"), PassParams::SEPARATE); - EXPECT_EQUAL(exe1.tasks.size(), 1u); - EXPECT_EQUAL(exe2.tasks.size(), 0u); - EXPECT_EQUAL(exe3.tasks.size(), 0u); + EXPECT_EQUAL(exe1->tasks.size(), 1u); + EXPECT_EQUAL(exe2->tasks.size(), 0u); + EXPECT_EQUAL(exe3->tasks.size(), 0u); } TEST("require that the same executor can be bound multiple times") { - MyExecutor exe1; + auto exe1 = std::make_shared<MyExecutor>(); auto bind1 = CompileCache::bind(exe1); auto bind2 = CompileCache::bind(exe1); auto bind3 = CompileCache::bind(exe1); @@ -230,7 +233,7 @@ TEST("require that the same executor can be bound multiple times") { EXPECT_EQUAL(CompileCache::num_bound(), 1u); auto token = CompileCache::compile(*Function::parse("a+b"), PassParams::SEPARATE); EXPECT_EQUAL(CompileCache::num_bound(), 1u); - EXPECT_EQUAL(exe1.tasks.size(), 1u); + EXPECT_EQUAL(exe1->tasks.size(), 1u); } struct CompileCheck : test::EvalSpec::EvalTest { @@ -275,6 +278,7 @@ TEST_F("compile sequentially, then run all conformance tests", test::EvalSpec()) auto t0 = steady_clock::now(); f1.each_case(test); auto t1 = steady_clock::now(); + CompileCache::wait_pending(); auto t2 = steady_clock::now(); test.verify(); auto t3 = steady_clock::now(); @@ -285,9 +289,9 @@ TEST_F("compile sequentially, then run all conformance tests", test::EvalSpec()) TEST_F("compile concurrently (8 threads), then run all conformance tests", test::EvalSpec()) { f1.add_all_cases(); - ThreadStackExecutor executor(8, 256*1024); + auto executor = std::make_shared<ThreadStackExecutor>(8, 256*1024); auto binding = CompileCache::bind(executor); - while (executor.num_idle_workers() < 8) { + while (executor->num_idle_workers() < 8) { std::this_thread::sleep_for(1ms); } for (size_t i = 0; i < 2; ++i) { @@ -295,7 +299,7 @@ TEST_F("compile concurrently (8 threads), then run all conformance tests", test: auto t0 = steady_clock::now(); f1.each_case(test); auto t1 = steady_clock::now(); - executor.sync(); + CompileCache::wait_pending(); auto t2 = steady_clock::now(); test.verify(); auto t3 = steady_clock::now(); @@ -304,6 +308,43 @@ TEST_F("compile concurrently (8 threads), then run all conformance tests", test: } } +struct MyCompileTask : public Executor::Task { + size_t seed; + size_t loop; + MyCompileTask(size_t seed_in, size_t loop_in) : seed(seed_in), loop(loop_in) {} + void run() override { + for (size_t i = 0; i < loop; ++i) { + // use custom constant to make a unique function that needs compilation + auto token = CompileCache::compile(*Function::parse(fmt("%zu", seed + i)), PassParams::SEPARATE); + } + } +}; + +TEST_MT_FF("require that deadlock is avoided with blocking executor", 8, std::shared_ptr<Executor>(nullptr), TimeBomb(300)) { + size_t loop = 16; + if (thread_id == 0) { + auto t0 = steady_clock::now(); + f1 = std::make_shared<BlockingThreadStackExecutor>(2, 256*1024, 3); + auto binding = CompileCache::bind(f1); + TEST_BARRIER(); // #1 + for (size_t i = 0; i < num_threads; ++i) { + f1->execute(std::make_unique<MyCompileTask>(i * loop, loop)); + } + TEST_BARRIER(); // #2 + auto t1 = steady_clock::now(); + fprintf(stderr, "deadlock test took %" PRIu64 " ms\n", count_ms(t1 - t0)); + + } else { + TEST_BARRIER(); // #1 + size_t seed = (10000 + (thread_id * loop)); + for (size_t i = 0; i < loop; ++i) { + // use custom constant to make a unique function that needs compilation + auto token = CompileCache::compile(*Function::parse(fmt("%zu", seed + i)), PassParams::SEPARATE); + } + TEST_BARRIER(); // #2 + } +} + //----------------------------------------------------------------------------- TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/eval/inline_operation/CMakeLists.txt b/eval/src/tests/eval/inline_operation/CMakeLists.txt new file mode 100644 index 00000000000..04cdbca3abf --- /dev/null +++ b/eval/src/tests/eval/inline_operation/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_inline_operation_test_app TEST + SOURCES + inline_operation_test.cpp + DEPENDS + vespaeval + gtest +) +vespa_add_test(NAME eval_inline_operation_test_app COMMAND eval_inline_operation_test_app) diff --git a/eval/src/tests/eval/inline_operation/inline_operation_test.cpp b/eval/src/tests/eval/inline_operation/inline_operation_test.cpp new file mode 100644 index 00000000000..fe9396398da --- /dev/null +++ b/eval/src/tests/eval/inline_operation/inline_operation_test.cpp @@ -0,0 +1,356 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/inline_operation.h> +#include <vespa/eval/eval/function.h> +#include <vespa/vespalib/util/typify.h> +#include <vespa/vespalib/gtest/gtest.h> + +using vespalib::typify_invoke; +using namespace vespalib::eval; +using namespace vespalib::eval::operation; + +const int my_value = 42; +struct AsValue { template <typename T> static int invoke() { return my_value; } }; +struct AsRef { template <typename T> static const int &invoke() { return my_value; } }; + +template <typename T> void test_op1(op1_t ref, double a, double expect) { + bool need_ref = std::is_same_v<T,CallOp1>; + T op = need_ref ? T(ref) : T(nullptr); + EXPECT_DOUBLE_EQ(ref(a), expect); + EXPECT_DOUBLE_EQ(op(a), expect); +}; + +template <typename T> void test_op2(op2_t ref, double a, double b, double expect) { + bool need_ref = std::is_same_v<T,CallOp2>; + T op = need_ref ? T(ref) : T(nullptr); + EXPECT_DOUBLE_EQ(ref(a, b), expect); + EXPECT_DOUBLE_EQ(op(a, b), expect); +}; + +op1_t as_op1(const vespalib::string &str) { + auto fun = Function::parse({"a"}, str); + auto res = lookup_op1(*fun); + EXPECT_TRUE(res.has_value()); + return res.value(); +} + +op2_t as_op2(const vespalib::string &str) { + auto fun = Function::parse({"a", "b"}, str); + auto res = lookup_op2(*fun); + EXPECT_TRUE(res.has_value()); + return res.value(); +} + +TEST(InlineOperationTest, op1_lambdas_are_recognized) { + EXPECT_EQ(as_op1("-a"), &Neg::f); + EXPECT_EQ(as_op1("!a"), &Not::f); + EXPECT_EQ(as_op1("cos(a)"), &Cos::f); + EXPECT_EQ(as_op1("sin(a)"), &Sin::f); + EXPECT_EQ(as_op1("tan(a)"), &Tan::f); + EXPECT_EQ(as_op1("cosh(a)"), &Cosh::f); + EXPECT_EQ(as_op1("sinh(a)"), &Sinh::f); + EXPECT_EQ(as_op1("tanh(a)"), &Tanh::f); + EXPECT_EQ(as_op1("acos(a)"), &Acos::f); + EXPECT_EQ(as_op1("asin(a)"), &Asin::f); + EXPECT_EQ(as_op1("atan(a)"), &Atan::f); + EXPECT_EQ(as_op1("exp(a)"), &Exp::f); + EXPECT_EQ(as_op1("log10(a)"), &Log10::f); + EXPECT_EQ(as_op1("log(a)"), &Log::f); + EXPECT_EQ(as_op1("sqrt(a)"), &Sqrt::f); + EXPECT_EQ(as_op1("ceil(a)"), &Ceil::f); + EXPECT_EQ(as_op1("fabs(a)"), &Fabs::f); + EXPECT_EQ(as_op1("floor(a)"), &Floor::f); + EXPECT_EQ(as_op1("isNan(a)"), &IsNan::f); + EXPECT_EQ(as_op1("relu(a)"), &Relu::f); + EXPECT_EQ(as_op1("sigmoid(a)"), &Sigmoid::f); + EXPECT_EQ(as_op1("elu(a)"), &Elu::f); + //------------------------------------------- + EXPECT_EQ(as_op1("1/a"), &Inv::f); + EXPECT_EQ(as_op1("1.0/a"), &Inv::f); + EXPECT_EQ(as_op1("a*a"), &Square::f); + EXPECT_EQ(as_op1("a^2"), &Square::f); + EXPECT_EQ(as_op1("a^2.0"), &Square::f); + EXPECT_EQ(as_op1("pow(a,2)"), &Square::f); + EXPECT_EQ(as_op1("pow(a,2.0)"), &Square::f); + EXPECT_EQ(as_op1("a*a*a"), &Cube::f); + EXPECT_EQ(as_op1("(a*a)*a"), &Cube::f); + EXPECT_EQ(as_op1("a*(a*a)"), &Cube::f); + EXPECT_EQ(as_op1("a^3"), &Cube::f); + EXPECT_EQ(as_op1("a^3.0"), &Cube::f); + EXPECT_EQ(as_op1("pow(a,3)"), &Cube::f); + EXPECT_EQ(as_op1("pow(a,3.0)"), &Cube::f); +} + +TEST(InlineOperationTest, op1_lambdas_are_recognized_with_different_parameter_names) { + EXPECT_EQ(lookup_op1(*Function::parse({"x"}, "-x")).value(), &Neg::f); + EXPECT_EQ(lookup_op1(*Function::parse({"x"}, "!x")).value(), &Not::f); +} + +TEST(InlineOperationTest, non_op1_lambdas_are_not_recognized) { + EXPECT_FALSE(lookup_op1(*Function::parse({"a"}, "a*a+3")).has_value()); + EXPECT_FALSE(lookup_op1(*Function::parse({"a", "b"}, "a+b")).has_value()); +} + +TEST(InlineOperationTest, op2_lambdas_are_recognized) { + EXPECT_EQ(as_op2("a+b"), &Add::f); + EXPECT_EQ(as_op2("a-b"), &Sub::f); + EXPECT_EQ(as_op2("a*b"), &Mul::f); + EXPECT_EQ(as_op2("a/b"), &Div::f); + EXPECT_EQ(as_op2("a%b"), &Mod::f); + EXPECT_EQ(as_op2("a^b"), &Pow::f); + EXPECT_EQ(as_op2("a==b"), &Equal::f); + EXPECT_EQ(as_op2("a!=b"), &NotEqual::f); + EXPECT_EQ(as_op2("a~=b"), &Approx::f); + EXPECT_EQ(as_op2("a<b"), &Less::f); + EXPECT_EQ(as_op2("a<=b"), &LessEqual::f); + EXPECT_EQ(as_op2("a>b"), &Greater::f); + EXPECT_EQ(as_op2("a>=b"), &GreaterEqual::f); + EXPECT_EQ(as_op2("a&&b"), &And::f); + EXPECT_EQ(as_op2("a||b"), &Or::f); + EXPECT_EQ(as_op2("atan2(a,b)"), &Atan2::f); + EXPECT_EQ(as_op2("ldexp(a,b)"), &Ldexp::f); + EXPECT_EQ(as_op2("pow(a,b)"), &Pow::f); + EXPECT_EQ(as_op2("fmod(a,b)"), &Mod::f); + EXPECT_EQ(as_op2("min(a,b)"), &Min::f); + EXPECT_EQ(as_op2("max(a,b)"), &Max::f); +} + +TEST(InlineOperationTest, op2_lambdas_are_recognized_with_different_parameter_names) { + EXPECT_EQ(lookup_op2(*Function::parse({"x", "y"}, "x+y")).value(), &Add::f); + EXPECT_EQ(lookup_op2(*Function::parse({"x", "y"}, "x-y")).value(), &Sub::f); +} + +TEST(InlineOperationTest, non_op2_lambdas_are_not_recognized) { + EXPECT_FALSE(lookup_op2(*Function::parse({"a"}, "-a")).has_value()); + EXPECT_FALSE(lookup_op2(*Function::parse({"a", "b"}, "b+a")).has_value()); +} + +TEST(InlineOperationTest, generic_op1_wrapper_works) { + CallOp1 op(Neg::f); + EXPECT_EQ(op(3), -3); + EXPECT_EQ(op(-5), 5); +} + +TEST(InlineOperationTest, generic_op2_wrapper_works) { + CallOp2 op(Add::f); + EXPECT_EQ(op(2,3), 5); + EXPECT_EQ(op(3,7), 10); +} + +TEST(InlineOperationTest, op1_typifier_forwards_return_value_correctly) { + auto a = typify_invoke<1,TypifyOp1,AsValue>(Neg::f); + auto b = typify_invoke<1,TypifyOp1,AsRef>(Neg::f); + EXPECT_EQ(a, my_value); + EXPECT_EQ(b, my_value); + bool same_memory = (&(typify_invoke<1,TypifyOp1,AsRef>(Neg::f)) == &my_value); + EXPECT_EQ(same_memory, true); +} + +TEST(InlineOperationTest, op2_typifier_forwards_return_value_correctly) { + auto a = typify_invoke<1,TypifyOp2,AsValue>(Add::f); + auto b = typify_invoke<1,TypifyOp2,AsRef>(Add::f); + EXPECT_EQ(a, my_value); + EXPECT_EQ(b, my_value); + bool same_memory = (&(typify_invoke<1,TypifyOp2,AsRef>(Add::f)) == &my_value); + EXPECT_EQ(same_memory, true); +} + +TEST(InlineOperationTest, inline_op1_example_works) { + op1_t ignored = nullptr; + InlineOp1<Inv> op(ignored); + EXPECT_EQ(op(2.0), 0.5); + EXPECT_EQ(op(4.0f), 0.25f); + EXPECT_EQ(op(8.0), 0.125); +} + +TEST(InlineOperationTest, inline_op2_example_works) { + op2_t ignored = nullptr; + InlineOp2<Add> op(ignored); + EXPECT_EQ(op(2.0, 3.0), 5.0); + EXPECT_EQ(op(3.0, 7.0), 10.0); +} + +TEST(InlineOperationTest, parameter_swap_wrapper_works) { + CallOp2 op(Sub::f); + SwapArgs2<CallOp2> swap_op(Sub::f); + EXPECT_EQ(op(2,3), -1); + EXPECT_EQ(swap_op(2,3), 1); + EXPECT_EQ(op(3,7), -4); + EXPECT_EQ(swap_op(3,7), 4); +} + +//----------------------------------------------------------------------------- + +TEST(InlineOperationTest, op1_cube_is_inlined) { + TypifyOp1::resolve(Cube::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Cube>>; + op1_t ref = Cube::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 2.0, 8.0); + test_op1<T>(ref, 3.0, 27.0); + test_op1<T>(ref, 7.0, 343.0); + }); +} + +TEST(InlineOperationTest, op1_exp_is_inlined) { + TypifyOp1::resolve(Exp::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Exp>>; + op1_t ref = Exp::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 2.0, std::exp(2.0)); + test_op1<T>(ref, 3.0, std::exp(3.0)); + test_op1<T>(ref, 7.0, std::exp(7.0)); + }); +} + +TEST(InlineOperationTest, op1_inv_is_inlined) { + TypifyOp1::resolve(Inv::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Inv>>; + op1_t ref = Inv::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 2.0, 1.0/2.0); + test_op1<T>(ref, 4.0, 1.0/4.0); + test_op1<T>(ref, 8.0, 1.0/8.0); + }); +} + +TEST(InlineOperationTest, op1_sqrt_is_inlined) { + TypifyOp1::resolve(Sqrt::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Sqrt>>; + op1_t ref = Sqrt::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 2.0, sqrt(2.0)); + test_op1<T>(ref, 4.0, sqrt(4.0)); + test_op1<T>(ref, 64.0, sqrt(64.0)); + }); +} + +TEST(InlineOperationTest, op1_square_is_inlined) { + TypifyOp1::resolve(Square::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Square>>; + op1_t ref = Square::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 2.0, 4.0); + test_op1<T>(ref, 3.0, 9.0); + test_op1<T>(ref, 7.0, 49.0); + }); +} + +TEST(InlineOperationTest, op1_tanh_is_inlined) { + TypifyOp1::resolve(Tanh::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp1<Tanh>>; + op1_t ref = Tanh::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 0.1, std::tanh(0.1)); + test_op1<T>(ref, 0.3, std::tanh(0.3)); + test_op1<T>(ref, 0.7, std::tanh(0.7)); + }); +} + +TEST(InlineOperationTest, op1_neg_is_not_inlined) { + TypifyOp1::resolve(Neg::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,CallOp1>; + op1_t ref = Neg::f; + EXPECT_TRUE(type_ok); + test_op1<T>(ref, 3.0, -3.0); + test_op1<T>(ref, 5.0, -5.0); + test_op1<T>(ref, -2.0, 2.0); + }); +} + +//----------------------------------------------------------------------------- + +TEST(InlineOperationTest, op2_add_is_inlined) { + TypifyOp2::resolve(Add::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp2<Add>>; + op2_t ref = Add::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 2.0, 2.0, 4.0); + test_op2<T>(ref, 3.0, 8.0, 11.0); + test_op2<T>(ref, 7.0, 1.0, 8.0); + }); +} + +TEST(InlineOperationTest, op2_div_is_inlined) { + TypifyOp2::resolve(Div::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp2<Div>>; + op2_t ref = Div::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 2.0, 2.0, 1.0); + test_op2<T>(ref, 3.0, 8.0, 3.0 / 8.0); + test_op2<T>(ref, 7.0, 5.0, 7.0 / 5.0); + }); +} + +TEST(InlineOperationTest, op2_mul_is_inlined) { + TypifyOp2::resolve(Mul::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp2<Mul>>; + op2_t ref = Mul::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 2.0, 2.0, 4.0); + test_op2<T>(ref, 3.0, 8.0, 24.0); + test_op2<T>(ref, 7.0, 5.0, 35.0); + }); +} + +TEST(InlineOperationTest, op2_pow_is_inlined) { + TypifyOp2::resolve(Pow::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp2<Pow>>; + op2_t ref = Pow::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 2.0, 2.0, std::pow(2.0, 2.0)); + test_op2<T>(ref, 3.0, 8.0, std::pow(3.0, 8.0)); + test_op2<T>(ref, 7.0, 5.0, std::pow(7.0, 5.0)); + }); +} + +TEST(InlineOperationTest, op2_sub_is_inlined) { + TypifyOp2::resolve(Sub::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,InlineOp2<Sub>>; + op2_t ref = Sub::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 3.0, 2.0, 1.0); + test_op2<T>(ref, 3.0, 8.0, -5.0); + test_op2<T>(ref, 7.0, 5.0, 2.0); + }); +} + +TEST(InlineOperationTest, op2_mod_is_not_inlined) { + TypifyOp2::resolve(Mod::f, [](auto t) + { + using T = typename decltype(t)::type; + bool type_ok = std::is_same_v<T,CallOp2>; + op2_t ref = Mod::f; + EXPECT_TRUE(type_ok); + test_op2<T>(ref, 3.0, 2.0, std::fmod(3.0, 2.0)); + test_op2<T>(ref, 3.0, 8.0, std::fmod(3.0, 8.0)); + test_op2<T>(ref, 7.0, 5.0, std::fmod(7.0, 5.0)); + }); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/eval/multiply_add/CMakeLists.txt b/eval/src/tests/eval/multiply_add/CMakeLists.txt new file mode 100644 index 00000000000..c50aa4f50a2 --- /dev/null +++ b/eval/src/tests/eval/multiply_add/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_multiply_add_test_app TEST + SOURCES + multiply_add_test.cpp + DEPENDS + vespaeval + gtest +) +vespa_add_test(NAME eval_multiply_add_test_app COMMAND eval_multiply_add_test_app) diff --git a/eval/src/tests/eval/multiply_add/multiply_add_test.cpp b/eval/src/tests/eval/multiply_add/multiply_add_test.cpp new file mode 100644 index 00000000000..35cab0a6030 --- /dev/null +++ b/eval/src/tests/eval/multiply_add/multiply_add_test.cpp @@ -0,0 +1,44 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/function.h> +#include <vespa/eval/eval/llvm/compiled_function.h> +#include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval; + +using Engine = vespalib::tensor::DefaultTensorEngine; + +double gcc_fun(double a, double b) { + return (a * 3) + b; +} + +TEST(MultiplyAddTest, multiply_add_gives_same_result) { + auto fun = Function::parse("a*3+b"); + CompiledFunction cfun(*fun, PassParams::ARRAY); + NodeTypes node_types = NodeTypes(*fun, {ValueType::double_type(), ValueType::double_type()}); + InterpretedFunction ifun(Engine::ref(), *fun, node_types); + auto llvm_fun = cfun.get_function(); + //------------------------------------------------------------------------- + double a = -1.0/3.0; + double b = 1.0; + std::vector<double> ab({a, b}); + SimpleParams params(ab); + InterpretedFunction::Context ictx(ifun); + //------------------------------------------------------------------------- + const Value &result_value = ifun.eval(ictx, params); + double ifun_res = result_value.as_double(); + double llvm_res = llvm_fun(&ab[0]); + double gcc_res = gcc_fun(a, b); + fprintf(stderr, "ifun_res: %a\n", ifun_res); + fprintf(stderr, "llvm_res: %a\n", llvm_res); + fprintf(stderr, "gcc_res: %a\n", gcc_res); + EXPECT_EQ(ifun_res, llvm_res); + EXPECT_DOUBLE_EQ(llvm_res + 1.0, gcc_res + 1.0); + if (llvm_res != gcc_res) { + fprintf(stderr, "WARNING: diverging results caused by fused multiply add\n"); + } +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/eval/tensor_function/tensor_function_test.cpp b/eval/src/tests/eval/tensor_function/tensor_function_test.cpp index 1eb9912abd2..b205205f52e 100644 --- a/eval/src/tests/eval/tensor_function/tensor_function_test.cpp +++ b/eval/src/tests/eval/tensor_function/tensor_function_test.cpp @@ -38,7 +38,7 @@ struct EvalCtx { ictx = std::make_unique<InterpretedFunction::Context>(*ifun); return ifun->eval(*ictx, SimpleObjectParams(params)); } - const TensorFunction &compile(const tensor_function::Node &expr) { + const TensorFunction &compile(const TensorFunction &expr) { return engine.optimize(expr, stash); } Value::UP make_double(double value) { @@ -391,15 +391,15 @@ TEST("require that if_node works") { TEST("require that if_node result is mutable only when both children produce mutable results") { Stash stash; - const Node &cond = inject(DoubleValue::double_type(), 0, stash); - const Node &a = inject(ValueType::from_spec("tensor(x[2])"), 0, stash); - const Node &b = inject(ValueType::from_spec("tensor(x[3])"), 0, stash); - const Node &c = inject(ValueType::from_spec("tensor(x[5])"), 0, stash); - const Node &tmp = concat(a, b, "x", stash); // will be mutable - const Node &if_con_con = if_node(cond, c, c, stash); - const Node &if_mut_con = if_node(cond, tmp, c, stash); - const Node &if_con_mut = if_node(cond, c, tmp, stash); - const Node &if_mut_mut = if_node(cond, tmp, tmp, stash); + const TensorFunction &cond = inject(DoubleValue::double_type(), 0, stash); + const TensorFunction &a = inject(ValueType::from_spec("tensor(x[2])"), 0, stash); + const TensorFunction &b = inject(ValueType::from_spec("tensor(x[3])"), 0, stash); + const TensorFunction &c = inject(ValueType::from_spec("tensor(x[5])"), 0, stash); + const TensorFunction &tmp = concat(a, b, "x", stash); // will be mutable + const TensorFunction &if_con_con = if_node(cond, c, c, stash); + const TensorFunction &if_mut_con = if_node(cond, tmp, c, stash); + const TensorFunction &if_con_mut = if_node(cond, c, tmp, stash); + const TensorFunction &if_mut_mut = if_node(cond, tmp, tmp, stash); EXPECT_EQUAL(if_con_con.result_type(), c.result_type()); EXPECT_EQUAL(if_con_mut.result_type(), c.result_type()); EXPECT_EQUAL(if_mut_con.result_type(), c.result_type()); @@ -412,13 +412,13 @@ TEST("require that if_node result is mutable only when both children produce mut TEST("require that if_node gets expected result type") { Stash stash; - const Node &a = inject(DoubleValue::double_type(), 0, stash); - const Node &b = inject(ValueType::from_spec("tensor(x[2])"), 0, stash); - const Node &c = inject(ValueType::from_spec("tensor(x[3])"), 0, stash); - const Node &d = inject(ValueType::from_spec("error"), 0, stash); - const Node &if_same = if_node(a, b, b, stash); - const Node &if_different = if_node(a, b, c, stash); - const Node &if_with_error = if_node(a, b, d, stash); + const TensorFunction &a = inject(DoubleValue::double_type(), 0, stash); + const TensorFunction &b = inject(ValueType::from_spec("tensor(x[2])"), 0, stash); + const TensorFunction &c = inject(ValueType::from_spec("tensor(x[3])"), 0, stash); + const TensorFunction &d = inject(ValueType::from_spec("error"), 0, stash); + const TensorFunction &if_same = if_node(a, b, b, stash); + const TensorFunction &if_different = if_node(a, b, c, stash); + const TensorFunction &if_with_error = if_node(a, b, d, stash); EXPECT_EQUAL(if_same.result_type(), ValueType::from_spec("tensor(x[2])")); EXPECT_EQUAL(if_different.result_type(), ValueType::from_spec("error")); EXPECT_EQUAL(if_with_error.result_type(), ValueType::from_spec("error")); @@ -426,10 +426,10 @@ TEST("require that if_node gets expected result type") { TEST("require that push_children works") { Stash stash; - std::vector<Node::Child::CREF> refs; - const Node &a = inject(DoubleValue::double_type(), 0, stash); - const Node &b = inject(DoubleValue::double_type(), 1, stash); - const Node &c = const_value(stash.create<DoubleValue>(1.0), stash); + std::vector<TensorFunction::Child::CREF> refs; + const TensorFunction &a = inject(DoubleValue::double_type(), 0, stash); + const TensorFunction &b = inject(DoubleValue::double_type(), 1, stash); + const TensorFunction &c = const_value(stash.create<DoubleValue>(1.0), stash); a.push_children(refs); b.push_children(refs); c.push_children(refs); diff --git a/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp b/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp index 3c35f90c521..c8091fd7c6e 100644 --- a/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp +++ b/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp @@ -8,6 +8,7 @@ #include <vespa/eval/tensor/dense/dense_replace_type_function.h> #include <vespa/eval/tensor/dense/dense_cell_range_function.h> #include <vespa/eval/tensor/dense/dense_lambda_peek_function.h> +#include <vespa/eval/tensor/dense/dense_lambda_function.h> #include <vespa/eval/tensor/dense/dense_fast_rename_optimizer.h> #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/eval/test/tensor_model.hpp> @@ -23,11 +24,27 @@ using namespace vespalib::eval::test; using namespace vespalib::tensor; using namespace vespalib::eval::tensor_function; +using EvalMode = DenseLambdaFunction::EvalMode; + +namespace vespalib::tensor { + +std::ostream &operator<<(std::ostream &os, EvalMode eval_mode) +{ + switch(eval_mode) { + case EvalMode::COMPILED: return os << "COMPILED"; + case EvalMode::INTERPRETED: return os << "INTERPRETED"; + } + abort(); +} + +} + const TensorEngine &prod_engine = DefaultTensorEngine::ref(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() .add("a", spec(1)) + .add("b", spec(2)) .add("x3", spec({x(3)}, N())) .add("x3f", spec(float_cells({x(3)}), N())) .add("x3m", spec({x({"0", "1", "2"})}, N())) @@ -55,8 +72,14 @@ void verify_impl(const vespalib::string &expr, const vespalib::string &expect) { verify_impl<T>(expr, expect, [](const T*){}); } -void verify_generic(const vespalib::string &expr, const vespalib::string &expect) { - verify_impl<Lambda>(expr, expect); +void verify_generic(const vespalib::string &expr, const vespalib::string &expect, + EvalMode expect_eval_mode) +{ + verify_impl<DenseLambdaFunction>(expr, expect, + [&](const DenseLambdaFunction *info) + { + EXPECT_EQUAL(info->eval_mode(), expect_eval_mode); + }); } void verify_reshape(const vespalib::string &expr, const vespalib::string &expect) { @@ -67,8 +90,8 @@ void verify_range(const vespalib::string &expr, const vespalib::string &expect) verify_impl<DenseCellRangeFunction>(expr, expect); } -void verify_compiled(const vespalib::string &expr, const vespalib::string &expect, - const vespalib::string &expect_idx_fun) +void verify_idx_fun(const vespalib::string &expr, const vespalib::string &expect, + const vespalib::string &expect_idx_fun) { verify_impl<DenseLambdaPeekFunction>(expr, expect, [&](const DenseLambdaPeekFunction *info) @@ -88,22 +111,32 @@ TEST("require that simple constant tensor lambda works") { } TEST("require that simple dynamic tensor lambda works") { - TEST_DO(verify_generic("tensor(x[3])(x+a)", "tensor(x[3]):[1,2,3]")); + TEST_DO(verify_generic("tensor(x[3])(x+a)", "tensor(x[3]):[1,2,3]", EvalMode::COMPILED)); +} + +TEST("require that compiled multi-dimensional multi-param dynamic tensor lambda works") { + TEST_DO(verify_generic("tensor(x[3],y[2])((b-a)+x+y)", "tensor(x[3],y[2]):[[1,2],[2,3],[3,4]]", EvalMode::COMPILED)); + TEST_DO(verify_generic("tensor<float>(x[3],y[2])((b-a)+x+y)", "tensor<float>(x[3],y[2]):[[1,2],[2,3],[3,4]]", EvalMode::COMPILED)); +} + +TEST("require that interpreted multi-dimensional multi-param dynamic tensor lambda works") { + TEST_DO(verify_generic("tensor(x[3],y[2])((x3{x:(a)}-a)+x+y)", "tensor(x[3],y[2]):[[1,2],[2,3],[3,4]]", EvalMode::INTERPRETED)); + TEST_DO(verify_generic("tensor<float>(x[3],y[2])((x3{x:(a)}-a)+x+y)", "tensor<float>(x[3],y[2]):[[1,2],[2,3],[3,4]]", EvalMode::INTERPRETED)); } TEST("require that tensor lambda can be used for tensor slicing") { - TEST_DO(verify_generic("tensor(x[2])(x3{x:(x+a)})", "tensor(x[2]):[2,3]")); - TEST_DO(verify_generic("tensor(x[2])(a+x3{x:(x)})", "tensor(x[2]):[2,3]")); + TEST_DO(verify_generic("tensor(x[2])(x3{x:(x+a)})", "tensor(x[2]):[2,3]", EvalMode::INTERPRETED)); + TEST_DO(verify_generic("tensor(x[2])(a+x3{x:(x)})", "tensor(x[2]):[2,3]", EvalMode::INTERPRETED)); } TEST("require that tensor lambda can be used for cell type casting") { - TEST_DO(verify_compiled("tensor(x[3])(x3f{x:(x)})", "tensor(x[3]):[1,2,3]", "f(x)(x)")); - TEST_DO(verify_compiled("tensor<float>(x[3])(x3{x:(x)})", "tensor<float>(x[3]):[1,2,3]", "f(x)(x)")); + TEST_DO(verify_idx_fun("tensor(x[3])(x3f{x:(x)})", "tensor(x[3]):[1,2,3]", "f(x)(x)")); + TEST_DO(verify_idx_fun("tensor<float>(x[3])(x3{x:(x)})", "tensor<float>(x[3]):[1,2,3]", "f(x)(x)")); } TEST("require that tensor lambda can be used to convert from sparse to dense tensors") { - TEST_DO(verify_generic("tensor(x[3])(x3m{x:(x)})", "tensor(x[3]):[1,2,3]")); - TEST_DO(verify_generic("tensor(x[2])(x3m{x:(x)})", "tensor(x[2]):[1,2]")); + TEST_DO(verify_generic("tensor(x[3])(x3m{x:(x)})", "tensor(x[3]):[1,2,3]", EvalMode::INTERPRETED)); + TEST_DO(verify_generic("tensor(x[2])(x3m{x:(x)})", "tensor(x[2]):[1,2]", EvalMode::INTERPRETED)); } TEST("require that constant nested tensor lambda using tensor peek works") { @@ -111,7 +144,7 @@ TEST("require that constant nested tensor lambda using tensor peek works") { } TEST("require that dynamic nested tensor lambda using tensor peek works") { - TEST_DO(verify_generic("tensor(x[2])(tensor(y[2])((x+y)+a){y:(x)})", "tensor(x[2]):[1,3]")); + TEST_DO(verify_generic("tensor(x[2])(tensor(y[2])((x+y)+a){y:(x)})", "tensor(x[2]):[1,3]", EvalMode::INTERPRETED)); } TEST("require that tensor reshape is optimized") { @@ -121,10 +154,10 @@ TEST("require that tensor reshape is optimized") { } TEST("require that tensor reshape with non-matching cell type requires cell copy") { - TEST_DO(verify_compiled("tensor(x[15])(x3y5f{x:(x/5),y:(x%5)})", "x15", "f(x)((floor((x/5))*5)+(x%5))")); - TEST_DO(verify_compiled("tensor<float>(x[15])(x3y5{x:(x/5),y:(x%5)})", "x15f", "f(x)((floor((x/5))*5)+(x%5))")); - TEST_DO(verify_compiled("tensor(x[3],y[5])(x15f{x:(x*5+y)})", "x3y5", "f(x,y)((x*5)+y)")); - TEST_DO(verify_compiled("tensor<float>(x[3],y[5])(x15{x:(x*5+y)})", "x3y5f", "f(x,y)((x*5)+y)")); + TEST_DO(verify_idx_fun("tensor(x[15])(x3y5f{x:(x/5),y:(x%5)})", "x15", "f(x)((floor((x/5))*5)+(x%5))")); + TEST_DO(verify_idx_fun("tensor<float>(x[15])(x3y5{x:(x/5),y:(x%5)})", "x15f", "f(x)((floor((x/5))*5)+(x%5))")); + TEST_DO(verify_idx_fun("tensor(x[3],y[5])(x15f{x:(x*5+y)})", "x3y5", "f(x,y)((x*5)+y)")); + TEST_DO(verify_idx_fun("tensor<float>(x[3],y[5])(x15{x:(x*5+y)})", "x3y5f", "f(x,y)((x*5)+y)")); } TEST("require that tensor cell subrange view is optimized") { @@ -135,22 +168,22 @@ TEST("require that tensor cell subrange view is optimized") { } TEST("require that tensor cell subrange with non-matching cell type requires cell copy") { - TEST_DO(verify_compiled("tensor(x[3])(x15f{x:(x+5)})", "tensor(x[3]):[6,7,8]", "f(x)(x+5)")); - TEST_DO(verify_compiled("tensor<float>(x[3])(x15{x:(x+5)})", "tensor<float>(x[3]):[6,7,8]", "f(x)(x+5)")); + TEST_DO(verify_idx_fun("tensor(x[3])(x15f{x:(x+5)})", "tensor(x[3]):[6,7,8]", "f(x)(x+5)")); + TEST_DO(verify_idx_fun("tensor<float>(x[3])(x15{x:(x+5)})", "tensor<float>(x[3]):[6,7,8]", "f(x)(x+5)")); } TEST("require that non-continuous cell extraction is optimized") { - TEST_DO(verify_compiled("tensor(x[3])(x3y5{x:(x),y:2})", "x3y5{y:2}", "f(x)((floor(x)*5)+2)")); - TEST_DO(verify_compiled("tensor(x[3])(x3y5f{x:(x),y:2})", "x3y5{y:2}", "f(x)((floor(x)*5)+2)")); - TEST_DO(verify_compiled("tensor<float>(x[3])(x3y5{x:(x),y:2})", "x3y5f{y:2}", "f(x)((floor(x)*5)+2)")); - TEST_DO(verify_compiled("tensor<float>(x[3])(x3y5f{x:(x),y:2})", "x3y5f{y:2}", "f(x)((floor(x)*5)+2)")); + TEST_DO(verify_idx_fun("tensor(x[3])(x3y5{x:(x),y:2})", "x3y5{y:2}", "f(x)((floor(x)*5)+2)")); + TEST_DO(verify_idx_fun("tensor(x[3])(x3y5f{x:(x),y:2})", "x3y5{y:2}", "f(x)((floor(x)*5)+2)")); + TEST_DO(verify_idx_fun("tensor<float>(x[3])(x3y5{x:(x),y:2})", "x3y5f{y:2}", "f(x)((floor(x)*5)+2)")); + TEST_DO(verify_idx_fun("tensor<float>(x[3])(x3y5f{x:(x),y:2})", "x3y5f{y:2}", "f(x)((floor(x)*5)+2)")); } TEST("require that out-of-bounds cell extraction is not optimized") { - TEST_DO(verify_generic("tensor(x[3])(x3y5{x:1,y:(x+3)})", "tensor(x[3]):[9,10,0]")); - TEST_DO(verify_generic("tensor(x[3])(x3y5{x:1,y:(x-1)})", "tensor(x[3]):[0,6,7]")); - TEST_DO(verify_generic("tensor(x[3])(x3y5{x:(x+1),y:(x)})", "tensor(x[3]):[6,12,0]")); - TEST_DO(verify_generic("tensor(x[3])(x3y5{x:(x-1),y:(x)})", "tensor(x[3]):[0,2,8]")); + TEST_DO(verify_generic("tensor(x[3])(x3y5{x:1,y:(x+3)})", "tensor(x[3]):[9,10,0]", EvalMode::INTERPRETED)); + TEST_DO(verify_generic("tensor(x[3])(x3y5{x:1,y:(x-1)})", "tensor(x[3]):[0,6,7]", EvalMode::INTERPRETED)); + TEST_DO(verify_generic("tensor(x[3])(x3y5{x:(x+1),y:(x)})", "tensor(x[3]):[6,12,0]", EvalMode::INTERPRETED)); + TEST_DO(verify_generic("tensor(x[3])(x3y5{x:(x-1),y:(x)})", "tensor(x[3]):[0,2,8]", EvalMode::INTERPRETED)); } TEST("require that non-double result from inner tensor lambda function fails type resolving") { diff --git a/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp b/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp index 80321ac3d22..0a34e1123de 100644 --- a/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp +++ b/eval/src/tests/tensor/dense_inplace_join_function/dense_inplace_join_function_test.cpp @@ -5,7 +5,6 @@ #include <vespa/eval/eval/simple_tensor.h> #include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/tensor/dense/dense_inplace_join_function.h> #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/eval/eval/test/eval_fixture.h> @@ -53,7 +52,7 @@ EvalFixture::ParamRepo make_params() { } EvalFixture::ParamRepo param_repo = make_params(); -void verify_optimized(const vespalib::string &expr, size_t cnt, size_t param_idx) { +void verify_optimized(const vespalib::string &expr, size_t param_idx) { EvalFixture fixture(prod_engine, expr, param_repo, true, true); EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); for (size_t i = 0; i < fixture.num_params(); ++i) { @@ -64,23 +63,18 @@ void verify_optimized(const vespalib::string &expr, size_t cnt, size_t param_idx EXPECT_NOT_EQUAL(fixture.get_param(i), fixture.result()); } } - auto info = fixture.find_all<DenseInplaceJoinFunction>(); - ASSERT_EQUAL(info.size(), cnt); - for (size_t i = 0; i < cnt; ++i) { - EXPECT_TRUE(info[i]->result_is_mutable()); - } } -void verify_p0_optimized(const vespalib::string &expr, size_t cnt) { - verify_optimized(expr, cnt, 0); +void verify_p0_optimized(const vespalib::string &expr) { + verify_optimized(expr, 0); } -void verify_p1_optimized(const vespalib::string &expr, size_t cnt) { - verify_optimized(expr, cnt, 1); +void verify_p1_optimized(const vespalib::string &expr) { + verify_optimized(expr, 1); } -void verify_p2_optimized(const vespalib::string &expr, size_t cnt) { - verify_optimized(expr, cnt, 2); +void verify_p2_optimized(const vespalib::string &expr) { + verify_optimized(expr, 2); } void verify_not_optimized(const vespalib::string &expr) { @@ -89,37 +83,35 @@ void verify_not_optimized(const vespalib::string &expr) { for (size_t i = 0; i < fixture.num_params(); ++i) { EXPECT_NOT_EQUAL(fixture.get_param(i), fixture.result()); } - auto info = fixture.find_all<DenseInplaceJoinFunction>(); - EXPECT_TRUE(info.empty()); } TEST("require that mutable dense concrete tensors are optimized") { - TEST_DO(verify_p0_optimized("mut_x5_A-mut_x5_B", 1)); - TEST_DO(verify_p0_optimized("mut_x5_A-con_x5_B", 1)); - TEST_DO(verify_p1_optimized("con_x5_A-mut_x5_B", 1)); - TEST_DO(verify_p0_optimized("mut_x5y3_A-mut_x5y3_B", 1)); - TEST_DO(verify_p0_optimized("mut_x5y3_A-con_x5y3_B", 1)); - TEST_DO(verify_p1_optimized("con_x5y3_A-mut_x5y3_B", 1)); + TEST_DO(verify_p1_optimized("mut_x5_A-mut_x5_B")); + TEST_DO(verify_p0_optimized("mut_x5_A-con_x5_B")); + TEST_DO(verify_p1_optimized("con_x5_A-mut_x5_B")); + TEST_DO(verify_p1_optimized("mut_x5y3_A-mut_x5y3_B")); + TEST_DO(verify_p0_optimized("mut_x5y3_A-con_x5y3_B")); + TEST_DO(verify_p1_optimized("con_x5y3_A-mut_x5y3_B")); } TEST("require that self-join operations can be optimized") { - TEST_DO(verify_p0_optimized("mut_x5_A+mut_x5_A", 1)); + TEST_DO(verify_p0_optimized("mut_x5_A+mut_x5_A")); } -TEST("require that join(tensor,scalar) operations are not optimized") { - TEST_DO(verify_not_optimized("mut_x5_A-mut_dbl_B")); - TEST_DO(verify_not_optimized("mut_dbl_A-mut_x5_B")); +TEST("require that join(tensor,scalar) operations are optimized") { + TEST_DO(verify_p0_optimized("mut_x5_A-mut_dbl_B")); + TEST_DO(verify_p1_optimized("mut_dbl_A-mut_x5_B")); } -TEST("require that join with different tensor shapes are not optimized") { - TEST_DO(verify_not_optimized("mut_x5_A*mut_x5y3_B")); +TEST("require that join with different tensor shapes are optimized") { + TEST_DO(verify_p1_optimized("mut_x5_A*mut_x5y3_B")); } TEST("require that inplace join operations can be chained") { - TEST_DO(verify_p0_optimized("mut_x5_A-(mut_x5_B-mut_x5_C)", 2)); - TEST_DO(verify_p0_optimized("(mut_x5_A-con_x5_B)-con_x5_C", 2)); - TEST_DO(verify_p1_optimized("con_x5_A-(mut_x5_B-con_x5_C)", 2)); - TEST_DO(verify_p2_optimized("con_x5_A-(con_x5_B-mut_x5_C)", 2)); + TEST_DO(verify_p2_optimized("mut_x5_A+(mut_x5_B+mut_x5_C)")); + TEST_DO(verify_p0_optimized("(mut_x5_A+con_x5_B)+con_x5_C")); + TEST_DO(verify_p1_optimized("con_x5_A+(mut_x5_B+con_x5_C)")); + TEST_DO(verify_p2_optimized("con_x5_A+(con_x5_B+mut_x5_C)")); } TEST("require that non-mutable tensors are not optimized") { @@ -136,21 +128,13 @@ TEST("require that mapped tensors are not optimized") { TEST_DO(verify_not_optimized("mut_x_sparse+mut_x_sparse")); } -TEST("require that inplace join can be debug dumped") { - EvalFixture fixture(prod_engine, "con_x5_A-mut_x5_B", param_repo, true, true); - auto info = fixture.find_all<DenseInplaceJoinFunction>(); - ASSERT_EQUAL(info.size(), 1u); - EXPECT_TRUE(info[0]->result_is_mutable()); - fprintf(stderr, "%s\n", info[0]->as_string().c_str()); -} - TEST("require that optimization works with float cells") { - TEST_DO(verify_p0_optimized("mut_x5f_D-mut_x5f_E", 1)); + TEST_DO(verify_p1_optimized("mut_x5f_D-mut_x5f_E")); } TEST("require that overwritten value must have same cell type as result") { - TEST_DO(verify_p0_optimized("mut_x5_A-mut_x5f_D", 1)); - TEST_DO(verify_p1_optimized("mut_x5f_D-mut_x5_A", 1)); + TEST_DO(verify_p0_optimized("mut_x5_A-mut_x5f_D")); + TEST_DO(verify_p1_optimized("mut_x5f_D-mut_x5_A")); TEST_DO(verify_not_optimized("con_x5_A-mut_x5f_D")); TEST_DO(verify_not_optimized("mut_x5f_D-con_x5_A")); } diff --git a/eval/src/tests/tensor/dense_inplace_map_function/CMakeLists.txt b/eval/src/tests/tensor/dense_inplace_map_function/CMakeLists.txt deleted file mode 100644 index 3bd1dbaf271..00000000000 --- a/eval/src/tests/tensor/dense_inplace_map_function/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_dense_inplace_map_function_test_app TEST - SOURCES - dense_inplace_map_function_test.cpp - DEPENDS - vespaeval -) -vespa_add_test(NAME eval_dense_inplace_map_function_test_app COMMAND eval_dense_inplace_map_function_test_app) diff --git a/eval/src/tests/tensor/dense_inplace_map_function/dense_inplace_map_function_test.cpp b/eval/src/tests/tensor/dense_inplace_map_function/dense_inplace_map_function_test.cpp deleted file mode 100644 index f85742b4e0f..00000000000 --- a/eval/src/tests/tensor/dense_inplace_map_function/dense_inplace_map_function_test.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/eval/eval/tensor_function.h> -#include <vespa/eval/eval/simple_tensor.h> -#include <vespa/eval/eval/simple_tensor_engine.h> -#include <vespa/eval/tensor/default_tensor_engine.h> -#include <vespa/eval/tensor/dense/dense_inplace_map_function.h> -#include <vespa/eval/tensor/dense/dense_tensor.h> -#include <vespa/eval/eval/test/tensor_model.hpp> -#include <vespa/eval/eval/test/eval_fixture.h> - -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/stash.h> - -using namespace vespalib; -using namespace vespalib::eval; -using namespace vespalib::eval::test; -using namespace vespalib::tensor; -using namespace vespalib::eval::tensor_function; - -const TensorEngine &prod_engine = DefaultTensorEngine::ref(); - -EvalFixture::ParamRepo make_params() { - return EvalFixture::ParamRepo() - .add("x5", spec({x(5)}, N())) - .add_mutable("_d", spec(5.0)) - .add_mutable("_x5", spec({x(5)}, N())) - .add_mutable("_x5f", spec(float_cells({x(5)}), N())) - .add_mutable("_x5y3", spec({x(5),y(3)}, N())) - .add_mutable("_x_m", spec({x({"a", "b", "c"})}, N())); -} -EvalFixture::ParamRepo param_repo = make_params(); - -void verify_optimized(const vespalib::string &expr, size_t cnt) { - EvalFixture fixture(prod_engine, expr, param_repo, true, true); - EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); - EXPECT_EQUAL(fixture.get_param(0), fixture.result()); - auto info = fixture.find_all<DenseInplaceMapFunction>(); - ASSERT_EQUAL(info.size(), cnt); - for (size_t i = 0; i < cnt; ++i) { - EXPECT_TRUE(info[i]->result_is_mutable()); - } -} - -void verify_not_optimized(const vespalib::string &expr) { - EvalFixture fixture(prod_engine, expr, param_repo, true, true); - EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); - EXPECT_NOT_EQUAL(fixture.get_param(0), fixture.result()); - auto info = fixture.find_all<DenseInplaceMapFunction>(); - EXPECT_TRUE(info.empty()); -} - -TEST("require that mutable dense concrete tensors are optimized") { - TEST_DO(verify_optimized("map(_x5,f(x)(x+10))", 1)); - TEST_DO(verify_optimized("map(_x5y3,f(x)(x+10))", 1)); -} - -TEST("require that inplace map operations can be chained") { - TEST_DO(verify_optimized("map(map(_x5,f(x)(x+10)),f(x)(x-5))", 2)); -} - -TEST("require that non-mutable tensors are not optimized") { - TEST_DO(verify_not_optimized("map(x5,f(x)(x+10))")); -} - -TEST("require that scalar values are not optimized") { - TEST_DO(verify_not_optimized("map(_d,f(x)(x+10))")); -} - -TEST("require that mapped tensors are not optimized") { - TEST_DO(verify_not_optimized("map(_x_m,f(x)(x+10))")); -} - -TEST("require that optimization works for float cells") { - TEST_DO(verify_optimized("map(_x5f,f(x)(x+10))", 1)); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp b/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp index a571837b8e9..92fdbfade46 100644 --- a/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp +++ b/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp @@ -67,7 +67,6 @@ TEST("require that matmul can be optimized") { TEST("require that matmul with lambda can be optimized") { TEST_DO(verify_optimized("reduce(join(a2d3,b5d3,f(x,y)(x*y)),sum,d)", 2, 3, 5, true, true)); - TEST_DO(verify_optimized("reduce(join(a2d3,b5d3,f(x,y)(y*x)),sum,d)", 2, 3, 5, true, true)); } TEST("require that expressions similar to matmul are not optimized") { @@ -75,6 +74,7 @@ TEST("require that expressions similar to matmul are not optimized") { TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,sum,b)")); TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,prod,d)")); TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,sum)")); + TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(y*x)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(x+y)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(x*x)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(y*y)),sum,d)")); diff --git a/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp b/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp index c0823248538..f9c563c9bf8 100644 --- a/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp +++ b/eval/src/tests/tensor/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp @@ -78,7 +78,6 @@ TEST("require that single multi matmul can be optimized") { TEST("require that multi matmul with lambda can be optimized") { TEST_DO(verify_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(x*y)),sum,d)", 2, 3, 5, 6, true, true)); - TEST_DO(verify_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(y*x)),sum,d)", 2, 3, 5, 6, true, true)); } TEST("require that expressions similar to multi matmul are not optimized") { @@ -86,6 +85,7 @@ TEST("require that expressions similar to multi matmul are not optimized") { TEST_DO(verify_not_optimized("reduce(A2B1C3a2d3*A2B1C3b5d3,sum,b)")); TEST_DO(verify_not_optimized("reduce(A2B1C3a2d3*A2B1C3b5d3,prod,d)")); TEST_DO(verify_not_optimized("reduce(A2B1C3a2d3*A2B1C3b5d3,sum)")); + TEST_DO(verify_not_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(y*x)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(x+y)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(x*x)),sum,d)")); TEST_DO(verify_not_optimized("reduce(join(A2B1C3a2d3,A2B1C3b5d3,f(x,y)(y*y)),sum,d)")); diff --git a/eval/src/tests/tensor/dense_number_join_function/CMakeLists.txt b/eval/src/tests/tensor/dense_number_join_function/CMakeLists.txt new file mode 100644 index 00000000000..73c544cab38 --- /dev/null +++ b/eval/src/tests/tensor/dense_number_join_function/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_dense_number_join_function_test_app TEST + SOURCES + dense_number_join_function_test.cpp + DEPENDS + vespaeval +) +vespa_add_test(NAME eval_dense_number_join_function_test_app COMMAND eval_dense_number_join_function_test_app) diff --git a/eval/src/tests/tensor/dense_number_join_function/dense_number_join_function_test.cpp b/eval/src/tests/tensor/dense_number_join_function/dense_number_join_function_test.cpp new file mode 100644 index 00000000000..0255c33b295 --- /dev/null +++ b/eval/src/tests/tensor/dense_number_join_function/dense_number_join_function_test.cpp @@ -0,0 +1,119 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/eval/simple_tensor.h> +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/dense/dense_number_join_function.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/tensor_model.hpp> + +#include <vespa/vespalib/util/stringfmt.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; +using namespace vespalib::tensor; +using namespace vespalib::eval::tensor_function; + +using vespalib::make_string_short::fmt; + +using Primary = DenseNumberJoinFunction::Primary; + +namespace vespalib::tensor { + +std::ostream &operator<<(std::ostream &os, Primary primary) +{ + switch(primary) { + case Primary::LHS: return os << "LHS"; + case Primary::RHS: return os << "RHS"; + } + abort(); +} + +} + +const TensorEngine &prod_engine = DefaultTensorEngine::ref(); + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add("a", spec(1.5)) + .add("number", spec(2.5)) + .add("sparse", spec({x({"a"})}, N())) + .add("dense", spec({y(5)}, N())) + .add("mixed", spec({x({"a"}),y(5)}, N())) + .add_matrix("x", 3, "y", 5); +} +EvalFixture::ParamRepo param_repo = make_params(); + +void verify_optimized(const vespalib::string &expr, Primary primary, bool inplace) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true, true); + EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQUAL(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseNumberJoinFunction>(); + ASSERT_EQUAL(info.size(), 1u); + EXPECT_TRUE(info[0]->result_is_mutable()); + EXPECT_EQUAL(info[0]->primary(), primary); + EXPECT_EQUAL(info[0]->inplace(), inplace); + int p_inplace = inplace ? ((primary == Primary::LHS) ? 0 : 1) : -1; + EXPECT_TRUE((p_inplace == -1) || (fixture.num_params() > size_t(p_inplace))); + for (size_t i = 0; i < fixture.num_params(); ++i) { + if (i == size_t(p_inplace)) { + EXPECT_EQUAL(fixture.get_param(i), fixture.result()); + } else { + EXPECT_NOT_EQUAL(fixture.get_param(i), fixture.result()); + } + } +} + +void verify_not_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true); + EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQUAL(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseNumberJoinFunction>(); + EXPECT_TRUE(info.empty()); +} + +TEST("require that dense number join can be optimized") { + TEST_DO(verify_optimized("x3y5+a", Primary::LHS, false)); + TEST_DO(verify_optimized("a+x3y5", Primary::RHS, false)); + TEST_DO(verify_optimized("x3y5f*a", Primary::LHS, false)); + TEST_DO(verify_optimized("a*x3y5f", Primary::RHS, false)); +} + +TEST("require that dense number join can be inplace") { + TEST_DO(verify_optimized("@x3y5*a", Primary::LHS, true)); + TEST_DO(verify_optimized("a*@x3y5", Primary::RHS, true)); + TEST_DO(verify_optimized("@x3y5f+a", Primary::LHS, true)); + TEST_DO(verify_optimized("a+@x3y5f", Primary::RHS, true)); +} + +TEST("require that asymmetric operations work") { + TEST_DO(verify_optimized("x3y5/a", Primary::LHS, false)); + TEST_DO(verify_optimized("a/x3y5", Primary::RHS, false)); + TEST_DO(verify_optimized("x3y5f-a", Primary::LHS, false)); + TEST_DO(verify_optimized("a-x3y5f", Primary::RHS, false)); +} + +TEST("require that inappropriate cases are not optimized") { + int optimized = 0; + for (vespalib::string lhs: {"number", "dense", "sparse", "mixed"}) { + for (vespalib::string rhs: {"number", "dense", "sparse", "mixed"}) { + if (((lhs == "number") && (rhs == "dense")) || + ((lhs == "dense") && (rhs == "number"))) + { + ++optimized; + } else { + auto expr = fmt("%s+%s", lhs.c_str(), rhs.c_str()); + TEST_STATE(expr.c_str()); + verify_not_optimized(expr); + } + } + } + EXPECT_EQUAL(optimized, 2); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/dense_pow_as_map_optimizer/CMakeLists.txt b/eval/src/tests/tensor/dense_pow_as_map_optimizer/CMakeLists.txt new file mode 100644 index 00000000000..cf5103f8b4f --- /dev/null +++ b/eval/src/tests/tensor/dense_pow_as_map_optimizer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_dense_pow_as_map_optimizer_test_app TEST + SOURCES + dense_pow_as_map_optimizer_test.cpp + DEPENDS + vespaeval + gtest +) +vespa_add_test(NAME eval_dense_pow_as_map_optimizer_test_app COMMAND eval_dense_pow_as_map_optimizer_test_app) diff --git a/eval/src/tests/tensor/dense_pow_as_map_optimizer/dense_pow_as_map_optimizer_test.cpp b/eval/src/tests/tensor/dense_pow_as_map_optimizer/dense_pow_as_map_optimizer_test.cpp new file mode 100644 index 00000000000..38d9ac8aeef --- /dev/null +++ b/eval/src/tests/tensor/dense_pow_as_map_optimizer/dense_pow_as_map_optimizer_test.cpp @@ -0,0 +1,92 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/dense/dense_simple_map_function.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval::operation; +using namespace vespalib::eval::tensor_function; +using namespace vespalib::eval::test; +using namespace vespalib::eval; +using namespace vespalib::tensor; +//using namespace vespalib; + +const TensorEngine &prod_engine = DefaultTensorEngine::ref(); + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add("a", spec(1.5)) + .add("b", spec(2.5)) + .add("sparse", spec({x({"a"})}, N())) + .add("mixed", spec({x({"a"}),y(5)}, N())) + .add_matrix("x", 5, "y", 3); +} +EvalFixture::ParamRepo param_repo = make_params(); + +void verify_optimized(const vespalib::string &expr, op1_t op1, bool inplace = false) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true, true); + EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseSimpleMapFunction>(); + ASSERT_EQ(info.size(), 1u); + EXPECT_TRUE(info[0]->result_is_mutable()); + EXPECT_EQ(info[0]->function(), op1); + EXPECT_EQ(info[0]->inplace(), inplace); + ASSERT_EQ(fixture.num_params(), 1); + if (inplace) { + EXPECT_EQ(fixture.get_param(0), fixture.result()); + } else { + EXPECT_TRUE(!(fixture.get_param(0) == fixture.result())); + } +} + +void verify_not_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true); + EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<Map>(); + EXPECT_TRUE(info.empty()); +} + +TEST(PowAsMapTest, squared_dense_tensor_is_optimized) { + verify_optimized("x5y3^2.0", Square::f); + verify_optimized("pow(x5y3,2.0)", Square::f); + verify_optimized("join(x5y3,2.0,f(x,y)(x^y))", Square::f); + verify_optimized("join(x5y3,2.0,f(x,y)(pow(x,y)))", Square::f); + verify_optimized("join(x5y3f,2.0,f(x,y)(pow(x,y)))", Square::f); + verify_optimized("join(@x5y3,2.0,f(x,y)(pow(x,y)))", Square::f, true); + verify_optimized("join(@x5y3f,2.0,f(x,y)(pow(x,y)))", Square::f, true); +} + +TEST(PowAsMapTest, cubed_dense_tensor_is_optimized) { + verify_optimized("x5y3^3.0", Cube::f); + verify_optimized("pow(x5y3,3.0)", Cube::f); + verify_optimized("join(x5y3,3.0,f(x,y)(x^y))", Cube::f); + verify_optimized("join(x5y3,3.0,f(x,y)(pow(x,y)))", Cube::f); + verify_optimized("join(x5y3f,3.0,f(x,y)(pow(x,y)))", Cube::f); + verify_optimized("join(@x5y3,3.0,f(x,y)(pow(x,y)))", Cube::f, true); + verify_optimized("join(@x5y3f,3.0,f(x,y)(pow(x,y)))", Cube::f, true); +} + +TEST(PowAsMapTest, hypercubed_dense_tensor_is_not_optimized) { + verify_not_optimized("join(x5y3,4.0,f(x,y)(pow(x,y)))"); +} + +TEST(PowAsMapTest, scalar_join_is_not_optimized) { + verify_not_optimized("join(a,2.0,f(x,y)(pow(x,y)))"); +} + +TEST(PowAsMapTest, sparse_join_is_not_optimized) { + verify_not_optimized("join(sparse,2.0,f(x,y)(pow(x,y)))"); +} + +TEST(PowAsMapTest, mixed_join_is_not_optimized) { + verify_not_optimized("join(mixed,2.0,f(x,y)(pow(x,y)))"); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/dense_simple_expand_function/CMakeLists.txt b/eval/src/tests/tensor/dense_simple_expand_function/CMakeLists.txt new file mode 100644 index 00000000000..4e7409a6139 --- /dev/null +++ b/eval/src/tests/tensor/dense_simple_expand_function/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_dense_simple_expand_function_test_app TEST + SOURCES + dense_simple_expand_function_test.cpp + DEPENDS + vespaeval + gtest +) +vespa_add_test(NAME eval_dense_simple_expand_function_test_app COMMAND eval_dense_simple_expand_function_test_app) diff --git a/eval/src/tests/tensor/dense_simple_expand_function/dense_simple_expand_function_test.cpp b/eval/src/tests/tensor/dense_simple_expand_function/dense_simple_expand_function_test.cpp new file mode 100644 index 00000000000..4b870bc0153 --- /dev/null +++ b/eval/src/tests/tensor/dense_simple_expand_function/dense_simple_expand_function_test.cpp @@ -0,0 +1,130 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/eval/simple_tensor.h> +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/dense/dense_simple_expand_function.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; +using namespace vespalib::eval::tensor_function; +using namespace vespalib::tensor; + +using Inner = DenseSimpleExpandFunction::Inner; + +const TensorEngine &prod_engine = DefaultTensorEngine::ref(); + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add("a", spec(1.5)) + .add("sparse", spec({x({"a"})}, N())) + .add("mixed", spec({y({"a"}),z(5)}, N())) + .add_vector("a", 5) + .add_vector("b", 3) + .add_cube("A", 1, "a", 5, "c", 1) + .add_cube("B", 1, "b", 3, "c", 1) + .add_matrix("a", 5, "c", 3) + .add_matrix("x", 3, "y", 2) + .add_cube("a", 1, "b", 1, "c", 1) + .add_cube("x", 1, "y", 1, "z", 1); +} + +EvalFixture::ParamRepo param_repo = make_params(); + +void verify_optimized(const vespalib::string &expr, Inner inner) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true, true); + EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseSimpleExpandFunction>(); + ASSERT_EQ(info.size(), 1u); + EXPECT_TRUE(info[0]->result_is_mutable()); + EXPECT_EQ(info[0]->inner(), inner); + ASSERT_EQ(fixture.num_params(), 2); + EXPECT_TRUE(!(fixture.get_param(0) == fixture.result())); + EXPECT_TRUE(!(fixture.get_param(1) == fixture.result())); +} + +void verify_not_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true); + EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseSimpleExpandFunction>(); + EXPECT_TRUE(info.empty()); +} + +TEST(ExpandTest, simple_expand_is_optimized) { + verify_optimized("join(a5,b3,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(b3,a5,f(x,y)(x*y))", Inner::LHS); +} + +TEST(ExpandTest, multiple_dimensions_are_supported) { + verify_optimized("join(a5,x3y2,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(x3y2,a5,f(x,y)(x*y))", Inner::LHS); + verify_optimized("join(a5c3,x3y2,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(x3y2,a5c3,f(x,y)(x*y))", Inner::LHS); +} + +TEST(ExpandTest, trivial_dimensions_are_ignored) { + verify_optimized("join(A1a5c1,B1b3c1,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(B1b3c1,A1a5c1,f(x,y)(x*y))", Inner::LHS); +} + +TEST(ExpandTest, simple_expand_handles_asymmetric_operations_correctly) { + verify_optimized("join(a5,b3,f(x,y)(x-y))", Inner::RHS); + verify_optimized("join(b3,a5,f(x,y)(x-y))", Inner::LHS); + verify_optimized("join(a5,b3,f(x,y)(x/y))", Inner::RHS); + verify_optimized("join(b3,a5,f(x,y)(x/y))", Inner::LHS); +} + +TEST(ExpandTest, simple_expand_can_have_various_cell_types) { + verify_optimized("join(a5,b3f,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(a5f,b3,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(a5f,b3f,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(b3,a5f,f(x,y)(x*y))", Inner::LHS); + verify_optimized("join(b3f,a5,f(x,y)(x*y))", Inner::LHS); + verify_optimized("join(b3f,a5f,f(x,y)(x*y))", Inner::LHS); +} + +TEST(ExpandTest, simple_expand_is_never_inplace) { + verify_optimized("join(@a5,@b3,f(x,y)(x*y))", Inner::RHS); + verify_optimized("join(@b3,@a5,f(x,y)(x*y))", Inner::LHS); +} + +TEST(ExpandTest, interleaved_dimensions_are_not_optimized) { + verify_not_optimized("join(a5c3,b3,f(x,y)(x*y))"); + verify_not_optimized("join(b3,a5c3,f(x,y)(x*y))"); +} + +TEST(ExpandTest, matching_dimensions_are_not_expanding) { + verify_not_optimized("join(a5c3,a5,f(x,y)(x*y))"); + verify_not_optimized("join(a5,a5c3,f(x,y)(x*y))"); +} + +TEST(ExpandTest, scalar_is_not_expanding) { + verify_not_optimized("join(a5,a,f(x,y)(x*y))"); +} + +TEST(ExpandTest, unit_tensor_is_not_expanding) { + verify_not_optimized("join(a5,x1y1z1,f(x,y)(x+y))"); + verify_not_optimized("join(x1y1z1,a5,f(x,y)(x+y))"); + verify_not_optimized("join(a1b1c1,x1y1z1,f(x,y)(x+y))"); +} + +TEST(ExpandTest, sparse_expand_is_not_optimized) { + verify_not_optimized("join(a5,sparse,f(x,y)(x*y))"); + verify_not_optimized("join(sparse,a5,f(x,y)(x*y))"); +} + +TEST(ExpandTest, mixed_expand_is_not_optimized) { + verify_not_optimized("join(a5,mixed,f(x,y)(x*y))"); + verify_not_optimized("join(mixed,a5,f(x,y)(x*y))"); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/dense_simple_join_function/CMakeLists.txt b/eval/src/tests/tensor/dense_simple_join_function/CMakeLists.txt new file mode 100644 index 00000000000..8a2df392145 --- /dev/null +++ b/eval/src/tests/tensor/dense_simple_join_function/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_dense_simple_join_function_test_app TEST + SOURCES + dense_simple_join_function_test.cpp + DEPENDS + vespaeval +) +vespa_add_test(NAME eval_dense_simple_join_function_test_app COMMAND eval_dense_simple_join_function_test_app) diff --git a/eval/src/tests/tensor/dense_simple_join_function/dense_simple_join_function_test.cpp b/eval/src/tests/tensor/dense_simple_join_function/dense_simple_join_function_test.cpp new file mode 100644 index 00000000000..b0927fd7e90 --- /dev/null +++ b/eval/src/tests/tensor/dense_simple_join_function/dense_simple_join_function_test.cpp @@ -0,0 +1,228 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/eval/simple_tensor.h> +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/dense/dense_simple_join_function.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/tensor_model.hpp> + +#include <vespa/vespalib/util/stringfmt.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; +using namespace vespalib::tensor; +using namespace vespalib::eval::tensor_function; + +using vespalib::make_string_short::fmt; + +using Primary = DenseSimpleJoinFunction::Primary; +using Overlap = DenseSimpleJoinFunction::Overlap; + +namespace vespalib::tensor { + +std::ostream &operator<<(std::ostream &os, Primary primary) +{ + switch(primary) { + case Primary::LHS: return os << "LHS"; + case Primary::RHS: return os << "RHS"; + } + abort(); +} + +std::ostream &operator<<(std::ostream &os, Overlap overlap) +{ + switch(overlap) { + case Overlap::FULL: return os << "FULL"; + case Overlap::INNER: return os << "INNER"; + case Overlap::OUTER: return os << "OUTER"; + } + abort(); +} + +} + +const TensorEngine &prod_engine = DefaultTensorEngine::ref(); + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add("a", spec(1.5)) + .add("b", spec(2.5)) + .add("sparse", spec({x({"a"})}, N())) + .add("mixed", spec({x({"a"}),y(5)}, N())) + .add_cube("a", 1, "b", 1, "c", 1) + .add_cube("x", 1, "y", 1, "z", 1) + .add_cube("x", 3, "y", 5, "z", 3) + .add_vector("x", 5) + .add_dense({{"c", 5}, {"d", 1}}) + .add_dense({{"b", 1}, {"c", 5}}) + .add_matrix("x", 3, "y", 5, [](size_t idx){ return double((idx * 2) + 3); }) + .add_matrix("x", 3, "y", 5, [](size_t idx){ return double((idx * 3) + 2); }) + .add_vector("y", 5, [](size_t idx){ return double((idx * 2) + 3); }) + .add_vector("y", 5, [](size_t idx){ return double((idx * 3) + 2); }) + .add_matrix("y", 5, "z", 3, [](size_t idx){ return double((idx * 2) + 3); }) + .add_matrix("y", 5, "z", 3, [](size_t idx){ return double((idx * 3) + 2); }); +} +EvalFixture::ParamRepo param_repo = make_params(); + +void verify_optimized(const vespalib::string &expr, Primary primary, Overlap overlap, bool pri_mut, size_t factor, int p_inplace = -1) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true, true); + EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQUAL(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseSimpleJoinFunction>(); + ASSERT_EQUAL(info.size(), 1u); + EXPECT_TRUE(info[0]->result_is_mutable()); + EXPECT_EQUAL(info[0]->primary(), primary); + EXPECT_EQUAL(info[0]->overlap(), overlap); + EXPECT_EQUAL(info[0]->primary_is_mutable(), pri_mut); + EXPECT_EQUAL(info[0]->factor(), factor); + EXPECT_TRUE((p_inplace == -1) || (fixture.num_params() > size_t(p_inplace))); + for (size_t i = 0; i < fixture.num_params(); ++i) { + if (i == size_t(p_inplace)) { + EXPECT_EQUAL(fixture.get_param(i), fixture.result()); + } else { + EXPECT_NOT_EQUAL(fixture.get_param(i), fixture.result()); + } + } +} + +void verify_not_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true); + EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQUAL(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseSimpleJoinFunction>(); + EXPECT_TRUE(info.empty()); +} + +TEST("require that basic join is optimized") { + TEST_DO(verify_optimized("y5+y5$2", Primary::RHS, Overlap::FULL, false, 1)); +} + +TEST("require that unit join is optimized") { + TEST_DO(verify_optimized("a1b1c1+x1y1z1", Primary::RHS, Overlap::FULL, false, 1)); +} + +TEST("require that trivial dimensions do not affect overlap calculation") { + TEST_DO(verify_optimized("c5d1+b1c5", Primary::RHS, Overlap::FULL, false, 1)); +} + +TEST("require that outer nesting is preferred to inner nesting") { + TEST_DO(verify_optimized("a1b1c1+y5", Primary::RHS, Overlap::OUTER, false, 5)); +} + +TEST("require that non-subset join is not optimized") { + TEST_DO(verify_not_optimized("x5+y5")); +} + +TEST("require that subset join with complex overlap is not optimized") { + TEST_DO(verify_not_optimized("x3y5z3+y5")); +} + +struct LhsRhs { + vespalib::string lhs; + vespalib::string rhs; + size_t lhs_size; + size_t rhs_size; + Overlap overlap; + size_t factor; + LhsRhs(const vespalib::string &lhs_in, const vespalib::string &rhs_in, + size_t lhs_size_in, size_t rhs_size_in, Overlap overlap_in) noexcept + : lhs(lhs_in), rhs(rhs_in), lhs_size(lhs_size_in), rhs_size(rhs_size_in), overlap(overlap_in), factor(1) + { + if (lhs_size > rhs_size) { + ASSERT_EQUAL(lhs_size % rhs_size, 0u); + factor = (lhs_size / rhs_size); + } else { + ASSERT_EQUAL(rhs_size % lhs_size, 0u); + factor = (rhs_size / lhs_size); + } + } +}; + +vespalib::string adjust_param(const vespalib::string &str, bool float_cells, bool mut_cells, bool is_rhs) { + vespalib::string result = str; + if (mut_cells) { + result = "@" + result; + } + if (float_cells) { + result += "f"; + } + if (is_rhs) { + result += "$2"; + } + return result; +} + +TEST("require that various parameter combinations work") { + for (bool left_float: {false, true}) { + for (bool right_float: {false, true}) { + bool float_result = (left_float && right_float); + for (bool left_mut: {false, true}) { + for (bool right_mut: {false, true}) { + for (const char *op_pattern: {"%s+%s", "%s-%s", "%s*%s"}) { + for (const LhsRhs ¶ms: + { LhsRhs("y5", "y5", 5, 5, Overlap::FULL), + LhsRhs("y5", "x3y5", 5, 15, Overlap::INNER), + LhsRhs("y5", "y5z3", 5, 15, Overlap::OUTER), + LhsRhs("x3y5", "y5", 15, 5, Overlap::INNER), + LhsRhs("y5z3", "y5", 15, 5, Overlap::OUTER)}) + { + vespalib::string left = adjust_param(params.lhs, left_float, left_mut, false); + vespalib::string right = adjust_param(params.rhs, right_float, right_mut, true); + vespalib::string expr = fmt(op_pattern, left.c_str(), right.c_str()); + TEST_STATE(expr.c_str()); + Primary primary = Primary::RHS; + if (params.overlap == Overlap::FULL) { + bool w_lhs = ((left_float == float_result) && left_mut); + bool w_rhs = ((right_float == float_result) && right_mut); + if (w_lhs && !w_rhs) { + primary = Primary::LHS; + } + } else if (params.lhs_size > params.rhs_size) { + primary = Primary::LHS; + } + bool pri_mut = (primary == Primary::LHS) ? left_mut : right_mut; + bool pri_float = (primary == Primary::LHS) ? left_float : right_float; + int p_inplace = -1; + if (pri_mut && (pri_float == float_result)) { + p_inplace = (primary == Primary::LHS) ? 0 : 1; + } + verify_optimized(expr, primary, params.overlap, pri_mut, params.factor, p_inplace); + } + } + } + } + } + } +} + +TEST("require that scalar values are not optimized") { + TEST_DO(verify_not_optimized("a+b")); + TEST_DO(verify_not_optimized("a+y5")); + TEST_DO(verify_not_optimized("y5+b")); + TEST_DO(verify_not_optimized("a+sparse")); + TEST_DO(verify_not_optimized("sparse+a")); + TEST_DO(verify_not_optimized("a+mixed")); + TEST_DO(verify_not_optimized("mixed+a")); +} + +TEST("require that mapped tensors are not optimized") { + TEST_DO(verify_not_optimized("sparse+sparse")); + TEST_DO(verify_not_optimized("sparse+y5")); + TEST_DO(verify_not_optimized("y5+sparse")); + TEST_DO(verify_not_optimized("sparse+mixed")); + TEST_DO(verify_not_optimized("mixed+sparse")); +} + +TEST("require mixed tensors are not optimized") { + TEST_DO(verify_not_optimized("mixed+mixed")); + TEST_DO(verify_not_optimized("mixed+y5")); + TEST_DO(verify_not_optimized("y5+mixed")); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/dense_simple_map_function/CMakeLists.txt b/eval/src/tests/tensor/dense_simple_map_function/CMakeLists.txt new file mode 100644 index 00000000000..766986bd628 --- /dev/null +++ b/eval/src/tests/tensor/dense_simple_map_function/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_dense_simple_map_function_test_app TEST + SOURCES + dense_simple_map_function_test.cpp + DEPENDS + vespaeval + gtest +) +vespa_add_test(NAME eval_dense_simple_map_function_test_app COMMAND eval_dense_simple_map_function_test_app) diff --git a/eval/src/tests/tensor/dense_simple_map_function/dense_simple_map_function_test.cpp b/eval/src/tests/tensor/dense_simple_map_function/dense_simple_map_function_test.cpp new file mode 100644 index 00000000000..99ec719fa2f --- /dev/null +++ b/eval/src/tests/tensor/dense_simple_map_function/dense_simple_map_function_test.cpp @@ -0,0 +1,78 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/eval/simple_tensor.h> +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/dense/dense_simple_map_function.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; +using namespace vespalib::eval::tensor_function; +using namespace vespalib::tensor; + +const TensorEngine &prod_engine = DefaultTensorEngine::ref(); + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add("a", spec(1.5)) + .add("b", spec(2.5)) + .add("sparse", spec({x({"a"})}, N())) + .add("mixed", spec({x({"a"}),y(5)}, N())) + .add_matrix("x", 5, "y", 3); +} +EvalFixture::ParamRepo param_repo = make_params(); + +void verify_optimized(const vespalib::string &expr, bool inplace) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true, true); + EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseSimpleMapFunction>(); + ASSERT_EQ(info.size(), 1u); + EXPECT_TRUE(info[0]->result_is_mutable()); + EXPECT_EQ(info[0]->inplace(), inplace); + ASSERT_EQ(fixture.num_params(), 1); + if (inplace) { + EXPECT_EQ(fixture.get_param(0), fixture.result()); + } else { + EXPECT_TRUE(!(fixture.get_param(0) == fixture.result())); + } +} + +void verify_not_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_engine, expr, param_repo, false); + EvalFixture fixture(prod_engine, expr, param_repo, true); + EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fixture.result(), slow_fixture.result()); + auto info = fixture.find_all<DenseSimpleMapFunction>(); + EXPECT_TRUE(info.empty()); +} + +TEST(MapTest, dense_map_is_optimized) { + verify_optimized("map(x5y3,f(x)(x+10))", false); + verify_optimized("map(x5y3f,f(x)(x+10))", false); +} + +TEST(MapTest, simple_dense_map_can_be_inplace) { + verify_optimized("map(@x5y3,f(x)(x+10))", true); + verify_optimized("map(@x5y3f,f(x)(x+10))", true); +} + +TEST(MapTest, scalar_map_is_not_optimized) { + verify_not_optimized("map(a,f(x)(x+10))"); +} + +TEST(MapTest, sparse_map_is_not_optimized) { + verify_not_optimized("map(sparse,f(x)(x+10))"); +} + +TEST(MapTest, mixed_map_is_not_optimized) { + verify_not_optimized("map(mixed,f(x)(x+10))"); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp index 0b924451907..3ecc3f66cda 100644 --- a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp @@ -130,13 +130,13 @@ TEST("require that xw product gives same results as reference join/reduce") { TEST("require that various variants of xw product can be optimized") { TEST_DO(verify_optimized("reduce(join(y3,x2y3,f(x,y)(x*y)),sum,y)", 3, 2, true)); - TEST_DO(verify_optimized("reduce(join(y3,x2y3,f(x,y)(y*x)),sum,y)", 3, 2, true)); } TEST("require that expressions similar to xw product are not optimized") { TEST_DO(verify_not_optimized("reduce(y3*x2y3,sum,x)")); TEST_DO(verify_not_optimized("reduce(y3*x2y3,prod,y)")); TEST_DO(verify_not_optimized("reduce(y3*x2y3,sum)")); + TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(y*x)),sum,y)")); TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(x+y)),sum,y)")); TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(x*x)),sum,y)")); TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(y*y)),sum,y)")); diff --git a/eval/src/tests/tensor/index_lookup_table/CMakeLists.txt b/eval/src/tests/tensor/index_lookup_table/CMakeLists.txt new file mode 100644 index 00000000000..f2dc5d31045 --- /dev/null +++ b/eval/src/tests/tensor/index_lookup_table/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_index_lookup_table_test_app TEST + SOURCES + index_lookup_table_test.cpp + DEPENDS + vespaeval + gtest +) +vespa_add_test(NAME eval_index_lookup_table_test_app COMMAND eval_index_lookup_table_test_app) diff --git a/eval/src/tests/tensor/index_lookup_table/index_lookup_table_test.cpp b/eval/src/tests/tensor/index_lookup_table/index_lookup_table_test.cpp new file mode 100644 index 00000000000..3f806e8252c --- /dev/null +++ b/eval/src/tests/tensor/index_lookup_table/index_lookup_table_test.cpp @@ -0,0 +1,118 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/tensor/dense/index_lookup_table.h> +#include <vespa/eval/eval/function.h> +#include <vespa/eval/eval/value_type.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval; +using namespace vespalib::tensor; + +std::vector<uint32_t> make_table(std::vector<uint32_t> list) { return list; } + +TEST(IndexLookupTableTest, single_dimension_lookup_table_is_correct) +{ + auto idx_fun = Function::parse({"x"}, "5-x"); + auto type = ValueType::from_spec("tensor(x[6])"); + auto table = IndexLookupTable::create(*idx_fun, type); + + EXPECT_EQ(IndexLookupTable::num_cached(), 1); + EXPECT_EQ(IndexLookupTable::count_refs(), 1); + EXPECT_EQ(table->get(), make_table({5,4,3,2,1,0})); +} + +TEST(IndexLookupTableTest, dual_dimension_lookup_table_is_correct) +{ + auto idx_fun = Function::parse({"x","y"}, "5-(x*2+y)"); + auto type = ValueType::from_spec("tensor(x[3],y[2])"); + auto table = IndexLookupTable::create(*idx_fun, type); + + EXPECT_EQ(IndexLookupTable::num_cached(), 1); + EXPECT_EQ(IndexLookupTable::count_refs(), 1); + EXPECT_EQ(table->get(), make_table({5,4,3,2,1,0})); +} + +TEST(IndexLookupTableTest, multi_dimension_lookup_table_is_correct) +{ + auto idx_fun = Function::parse({"a","b","c","d"}, "11-(a*6+b*2+c*2+d)"); + auto type = ValueType::from_spec("tensor(a[2],b[3],c[1],d[2])"); + auto table = IndexLookupTable::create(*idx_fun, type); + + EXPECT_EQ(IndexLookupTable::num_cached(), 1); + EXPECT_EQ(IndexLookupTable::count_refs(), 1); + EXPECT_EQ(table->get(), make_table({11,10,9,8,7,6,5,4,3,2,1,0})); +} + +TEST(IndexLookupTableTest, lookup_tables_can_be_shared) +{ + auto idx_fun1 = Function::parse({"x"}, "5-x"); + auto type1 = ValueType::from_spec("tensor(x[6])"); + auto table1 = IndexLookupTable::create(*idx_fun1, type1); + + auto idx_fun2 = Function::parse({"x"}, "5-x"); + auto type2 = ValueType::from_spec("tensor(x[6])"); + auto table2 = IndexLookupTable::create(*idx_fun2, type2); + + EXPECT_EQ(IndexLookupTable::num_cached(), 1); + EXPECT_EQ(IndexLookupTable::count_refs(), 2); + EXPECT_EQ(&table1->get(), &table2->get()); + EXPECT_EQ(table1->get(), make_table({5,4,3,2,1,0})); +} + +TEST(IndexLookupTableTest, lookup_tables_with_different_index_functions_are_not_shared) +{ + auto idx_fun1 = Function::parse({"x"}, "5-x"); + auto type1 = ValueType::from_spec("tensor(x[6])"); + auto table1 = IndexLookupTable::create(*idx_fun1, type1); + + auto idx_fun2 = Function::parse({"x"}, "x"); + auto type2 = ValueType::from_spec("tensor(x[6])"); + auto table2 = IndexLookupTable::create(*idx_fun2, type2); + + EXPECT_EQ(IndexLookupTable::num_cached(), 2); + EXPECT_EQ(IndexLookupTable::count_refs(), 2); + EXPECT_NE(&table1->get(), &table2->get()); + EXPECT_EQ(table1->get(), make_table({5,4,3,2,1,0})); + EXPECT_EQ(table2->get(), make_table({0,1,2,3,4,5})); +} + +TEST(IndexLookupTableTest, lookup_tables_with_different_value_types_are_not_shared) +{ + auto idx_fun1 = Function::parse({"x"}, "x"); + auto type1 = ValueType::from_spec("tensor(x[6])"); + auto table1 = IndexLookupTable::create(*idx_fun1, type1); + + auto idx_fun2 = Function::parse({"x"}, "x"); + auto type2 = ValueType::from_spec("tensor(x[5])"); + auto table2 = IndexLookupTable::create(*idx_fun2, type2); + + EXPECT_EQ(IndexLookupTable::num_cached(), 2); + EXPECT_EQ(IndexLookupTable::count_refs(), 2); + EXPECT_NE(&table1->get(), &table2->get()); + EXPECT_EQ(table1->get(), make_table({0,1,2,3,4,5})); + EXPECT_EQ(table2->get(), make_table({0,1,2,3,4})); +} + +TEST(IndexLookupTableTest, identical_lookup_tables_might_not_be_shared) +{ + auto idx_fun1 = Function::parse({"x"}, "5-x"); + auto type1 = ValueType::from_spec("tensor(x[6])"); + auto table1 = IndexLookupTable::create(*idx_fun1, type1); + + auto idx_fun2 = Function::parse({"x","y"}, "5-(x*2+y)"); + auto type2 = ValueType::from_spec("tensor(x[3],y[2])"); + auto table2 = IndexLookupTable::create(*idx_fun2, type2); + + EXPECT_EQ(IndexLookupTable::num_cached(), 2); + EXPECT_EQ(IndexLookupTable::count_refs(), 2); + EXPECT_NE(&table1->get(), &table2->get()); + EXPECT_EQ(table1->get(), make_table({5,4,3,2,1,0})); + EXPECT_EQ(table2->get(), make_table({5,4,3,2,1,0})); +} + +TEST(IndexLookupTableTest, unused_lookup_tables_are_discarded) { + EXPECT_EQ(IndexLookupTable::num_cached(), 0); + EXPECT_EQ(IndexLookupTable::count_refs(), 0); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/vespa/eval/eval/aggr.cpp b/eval/src/vespa/eval/eval/aggr.cpp index d10bbc4abb8..8efb0ec9fe7 100644 --- a/eval/src/vespa/eval/eval/aggr.cpp +++ b/eval/src/vespa/eval/eval/aggr.cpp @@ -71,15 +71,11 @@ Aggregator::~Aggregator() Aggregator & Aggregator::create(Aggr aggr, Stash &stash) { - switch (aggr) { - case Aggr::AVG: return stash.create<Wrapper<aggr::Avg<double>>>(); - case Aggr::COUNT: return stash.create<Wrapper<aggr::Count<double>>>(); - case Aggr::PROD: return stash.create<Wrapper<aggr::Prod<double>>>(); - case Aggr::SUM: return stash.create<Wrapper<aggr::Sum<double>>>(); - case Aggr::MAX: return stash.create<Wrapper<aggr::Max<double>>>(); - case Aggr::MIN: return stash.create<Wrapper<aggr::Min<double>>>(); - } - LOG_ABORT("should not be reached"); + return TypifyAggr::resolve(aggr, [&stash](auto t)->Aggregator& + { + using T = typename decltype(t)::template templ<double>; + return stash.create<Wrapper<T>>(); + }); } std::vector<Aggr> diff --git a/eval/src/vespa/eval/eval/aggr.h b/eval/src/vespa/eval/eval/aggr.h index 8dea54d8abc..169f0b1d2af 100644 --- a/eval/src/vespa/eval/eval/aggr.h +++ b/eval/src/vespa/eval/eval/aggr.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/vespalib/util/typify.h> #include <vespa/vespalib/stllike/string.h> #include <vector> #include <map> @@ -118,5 +119,21 @@ public: }; } // namespave vespalib::eval::aggr + +struct TypifyAggr { + template <template<typename> typename TT> using Result = TypifyResultSimpleTemplate<TT>; + template <typename F> static decltype(auto) resolve(Aggr aggr, F &&f) { + switch (aggr) { + case Aggr::AVG: return f(Result<aggr::Avg>()); + case Aggr::COUNT: return f(Result<aggr::Count>()); + case Aggr::PROD: return f(Result<aggr::Prod>()); + case Aggr::SUM: return f(Result<aggr::Sum>()); + case Aggr::MAX: return f(Result<aggr::Max>()); + case Aggr::MIN: return f(Result<aggr::Min>()); + } + abort(); + } +}; + } // namespace vespalib::eval } // namespace vespalib diff --git a/eval/src/vespa/eval/eval/inline_operation.h b/eval/src/vespa/eval/eval/inline_operation.h new file mode 100644 index 00000000000..21516c4d94e --- /dev/null +++ b/eval/src/vespa/eval/eval/inline_operation.h @@ -0,0 +1,148 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "operation.h" +#include <vespa/vespalib/util/typify.h> +#include <cmath> + +namespace vespalib::eval::operation { + +//----------------------------------------------------------------------------- + +struct CallOp1 { + op1_t my_op1; + CallOp1(op1_t op1) : my_op1(op1) {} + double operator()(double a) const { return my_op1(a); } +}; + +template <typename T> struct InlineOp1; +template <> struct InlineOp1<Cube> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return (a * a * a); } +}; +template <> struct InlineOp1<Exp> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return exp(a); } +}; +template <> struct InlineOp1<Inv> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return (A{1}/a); } +}; +template <> struct InlineOp1<Sqrt> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return std::sqrt(a); } +}; +template <> struct InlineOp1<Square> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return (a * a); } +}; +template <> struct InlineOp1<Tanh> { + InlineOp1(op1_t) {} + template <typename A> constexpr auto operator()(A a) const { return std::tanh(a); } +}; + +struct TypifyOp1 { + template <typename T> using Result = TypifyResultType<T>; + template <typename F> static decltype(auto) resolve(op1_t value, F &&f) { + if (value == Cube::f) { + return f(Result<InlineOp1<Cube>>()); + } else if (value == Exp::f) { + return f(Result<InlineOp1<Exp>>()); + } else if (value == Inv::f) { + return f(Result<InlineOp1<Inv>>()); + } else if (value == Sqrt::f) { + return f(Result<InlineOp1<Sqrt>>()); + } else if (value == Square::f) { + return f(Result<InlineOp1<Square>>()); + } else if (value == Tanh::f) { + return f(Result<InlineOp1<Tanh>>()); + } else { + return f(Result<CallOp1>()); + } + } +}; + +//----------------------------------------------------------------------------- + +struct CallOp2 { + op2_t my_op2; + CallOp2(op2_t op2) : my_op2(op2) {} + op2_t get() const { return my_op2; } + double operator()(double a, double b) const { return my_op2(a, b); } +}; + +template <typename Op2> +struct SwapArgs2 { + Op2 op2; + SwapArgs2(op2_t op2_in) : op2(op2_in) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return op2(b, a); } +}; + +template <typename T> struct InlineOp2; +template <> struct InlineOp2<Add> { + InlineOp2(op2_t) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return (a+b); } +}; +template <> struct InlineOp2<Div> { + InlineOp2(op2_t) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return (a/b); } +}; +template <> struct InlineOp2<Mul> { + InlineOp2(op2_t) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return (a*b); } +}; +template <> struct InlineOp2<Pow> { + InlineOp2(op2_t) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return std::pow(a,b); } +}; +template <> struct InlineOp2<Sub> { + InlineOp2(op2_t) {} + template <typename A, typename B> constexpr auto operator()(A a, B b) const { return (a-b); } +}; + +struct TypifyOp2 { + template <typename T> using Result = TypifyResultType<T>; + template <typename F> static decltype(auto) resolve(op2_t value, F &&f) { + if (value == Add::f) { + return f(Result<InlineOp2<Add>>()); + } else if (value == Div::f) { + return f(Result<InlineOp2<Div>>()); + } else if (value == Mul::f) { + return f(Result<InlineOp2<Mul>>()); + } else if (value == Pow::f) { + return f(Result<InlineOp2<Pow>>()); + } else if (value == Sub::f) { + return f(Result<InlineOp2<Sub>>()); + } else { + return f(Result<CallOp2>()); + } + } +}; + +//----------------------------------------------------------------------------- + +template <typename A, typename OP1> +void apply_op1_vec(A *dst, const A *src, size_t n, OP1 &&f) { + for (size_t i = 0; i < n; ++i) { + dst[i] = f(src[i]); + } +} + +template <typename D, typename A, typename B, typename OP2> +void apply_op2_vec_num(D *dst, const A *a, B b, size_t n, OP2 &&f) { + for (size_t i = 0; i < n; ++i) { + dst[i] = f(a[i], b); + } +} + +template <typename D, typename A, typename B, typename OP2> +void apply_op2_vec_vec(D *dst, const A *a, const B *b, size_t n, OP2 &&f) { + for (size_t i = 0; i < n; ++i) { + dst[i] = f(a[i], b[i]); + } +} + +//----------------------------------------------------------------------------- + +} diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp index 9a262f6dca5..e2674a6e4d6 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp @@ -10,14 +10,14 @@ namespace eval { std::mutex CompileCache::_lock{}; CompileCache::Map CompileCache::_cached{}; uint64_t CompileCache::_executor_tag{0}; -std::vector<std::pair<uint64_t,Executor*>> CompileCache::_executor_stack{}; +std::vector<std::pair<uint64_t,std::shared_ptr<Executor>>> CompileCache::_executor_stack{}; const CompiledFunction & CompileCache::Value::wait_for_result() { - std::unique_lock<std::mutex> guard(_lock); - cond.wait(guard, [this](){ return bool(compiled_function); }); - return *compiled_function; + std::unique_lock<std::mutex> guard(result->lock); + result->cond.wait(guard, [this](){ return bool(result->compiled_function); }); + return *(result->compiled_function); } void @@ -30,10 +30,10 @@ CompileCache::release(Map::iterator entry) } uint64_t -CompileCache::attach_executor(Executor &executor) +CompileCache::attach_executor(std::shared_ptr<Executor> executor) { std::lock_guard<std::mutex> guard(_lock); - _executor_stack.emplace_back(++_executor_tag, &executor); + _executor_stack.emplace_back(++_executor_tag, std::move(executor)); return _executor_tag; } @@ -52,6 +52,7 @@ CompileCache::compile(const Function &function, PassParams pass_params) { Token::UP token; Executor::Task::UP task; + std::shared_ptr<Executor> executor; vespalib::string key = gen_key(function, pass_params); { std::lock_guard<std::mutex> guard(_lock); @@ -63,20 +64,42 @@ CompileCache::compile(const Function &function, PassParams pass_params) auto res = _cached.emplace(std::move(key), Value::ctor_tag()); assert(res.second); token = std::make_unique<Token>(res.first, Token::ctor_tag()); - ++(res.first->second.num_refs); - task = std::make_unique<CompileTask>(function, pass_params, - std::make_unique<Token>(res.first, Token::ctor_tag())); + task = std::make_unique<CompileTask>(function, pass_params, res.first->second.result); if (!_executor_stack.empty()) { - task = _executor_stack.back().second->execute(std::move(task)); + executor = _executor_stack.back().second; } } } + if (executor) { + task = executor->execute(std::move(task)); + } if (task) { std::thread([&task](){ task.get()->run(); }).join(); } return token; } +void +CompileCache::wait_pending() +{ + std::vector<Token::UP> pending; + { + std::lock_guard<std::mutex> guard(_lock); + for (auto entry = _cached.begin(); entry != _cached.end(); ++entry) { + if (entry->second.result->cf.load(std::memory_order_acquire) == nullptr) { + ++(entry->second.num_refs); + pending.push_back(std::make_unique<Token>(entry, Token::ctor_tag())); + } + } + } + { + for (const auto &token: pending) { + const CompiledFunction &fun = token->get(); + (void) fun; + } + } +} + size_t CompileCache::num_cached() { @@ -108,7 +131,7 @@ CompileCache::count_pending() std::lock_guard<std::mutex> guard(_lock); size_t pending = 0; for (const auto &entry: _cached) { - if (entry.second.compiled_function.get() == nullptr) { + if (entry.second.result->cf.load(std::memory_order_acquire) == nullptr) { ++pending; } } @@ -118,12 +141,11 @@ CompileCache::count_pending() void CompileCache::CompileTask::run() { - auto &entry = token->_entry->second; - auto result = std::make_unique<CompiledFunction>(*function, pass_params); - std::lock_guard<std::mutex> guard(_lock); - entry.compiled_function = std::move(result); - entry.cf.store(entry.compiled_function.get(), std::memory_order_release); - entry.cond.notify_all(); + auto compiled = std::make_unique<CompiledFunction>(*function, pass_params); + std::lock_guard<std::mutex> guard(result->lock); + result->compiled_function = std::move(compiled); + result->cf.store(result->compiled_function.get(), std::memory_order_release); + result->cond.notify_all(); } } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.h b/eval/src/vespa/eval/eval/llvm/compile_cache.h index aaadec772a5..61d0cc83d94 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.h +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.h @@ -23,16 +23,22 @@ class CompileCache { private: using Key = vespalib::string; - struct Value { - size_t num_refs; + struct Result { + using SP = std::shared_ptr<Result>; std::atomic<const CompiledFunction *> cf; + std::mutex lock; std::condition_variable cond; CompiledFunction::UP compiled_function; + Result() : cf(nullptr), lock(), cond(), compiled_function(nullptr) {} + }; + struct Value { + size_t num_refs; + Result::SP result; struct ctor_tag {}; - Value(ctor_tag) : num_refs(1), cf(nullptr), cond(), compiled_function() {} + Value(ctor_tag) : num_refs(1), result(std::make_shared<Result>()) {} const CompiledFunction &wait_for_result(); const CompiledFunction &get() { - const CompiledFunction *ptr = cf.load(std::memory_order_acquire); + const CompiledFunction *ptr = result->cf.load(std::memory_order_acquire); if (ptr == nullptr) { return wait_for_result(); } @@ -43,10 +49,10 @@ private: static std::mutex _lock; static Map _cached; static uint64_t _executor_tag; - static std::vector<std::pair<uint64_t,Executor*>> _executor_stack; + static std::vector<std::pair<uint64_t,std::shared_ptr<Executor>>> _executor_stack; static void release(Map::iterator entry); - static uint64_t attach_executor(Executor &executor); + static uint64_t attach_executor(std::shared_ptr<Executor> executor); static void detach_executor(uint64_t tag); public: @@ -54,7 +60,6 @@ public: { private: friend class CompileCache; - friend class CompileTask; struct ctor_tag {}; CompileCache::Map::iterator _entry; public: @@ -79,13 +84,15 @@ public: ExecutorBinding &operator=(ExecutorBinding &&) = delete; ExecutorBinding &operator=(const ExecutorBinding &) = delete; using UP = std::unique_ptr<ExecutorBinding>; - explicit ExecutorBinding(Executor &executor, ctor_tag) : _tag(attach_executor(executor)) {} + explicit ExecutorBinding(std::shared_ptr<Executor> executor, ctor_tag) + : _tag(attach_executor(std::move(executor))) {} ~ExecutorBinding() { detach_executor(_tag); } }; static Token::UP compile(const Function &function, PassParams pass_params); - static ExecutorBinding::UP bind(Executor &executor) { - return std::make_unique<ExecutorBinding>(executor, ExecutorBinding::ctor_tag()); + static void wait_pending(); + static ExecutorBinding::UP bind(std::shared_ptr<Executor> executor) { + return std::make_unique<ExecutorBinding>(std::move(executor), ExecutorBinding::ctor_tag()); } static size_t num_cached(); static size_t num_bound(); @@ -96,9 +103,9 @@ private: struct CompileTask : public Executor::Task { std::shared_ptr<Function const> function; PassParams pass_params; - Token::UP token; - CompileTask(const Function &function_in, PassParams pass_params_in, Token::UP token_in) - : function(function_in.shared_from_this()), pass_params(pass_params_in), token(std::move(token_in)) {} + Result::SP result; + CompileTask(const Function &function_in, PassParams pass_params_in, Result::SP result_in) + : function(function_in.shared_from_this()), pass_params(pass_params_in), result(std::move(result_in)) {} void run() override; }; }; diff --git a/eval/src/vespa/eval/eval/make_tensor_function.cpp b/eval/src/vespa/eval/eval/make_tensor_function.cpp index 3a73a3b8784..e80633b5c41 100644 --- a/eval/src/vespa/eval/eval/make_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/make_tensor_function.cpp @@ -15,25 +15,6 @@ namespace vespalib::eval { namespace { using namespace nodes; -using map_fun_t = double (*)(double); -using join_fun_t = double (*)(double, double); - -//----------------------------------------------------------------------------- - -// TODO(havardpe): generic function pointer resolving for all single -// operation lambdas. - -template <typename OP2> -bool is_op2(const Function &lambda) { - if (lambda.num_params() == 2) { - if (auto op2 = as<OP2>(lambda.root())) { - auto sym1 = as<Symbol>(op2->lhs()); - auto sym2 = as<Symbol>(op2->rhs()); - return (sym1 && sym2 && (sym1->id() != sym2->id())); - } - } - return false; -} //----------------------------------------------------------------------------- @@ -41,7 +22,7 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { Stash &stash; const TensorEngine &tensor_engine; const NodeTypes &types; - std::vector<tensor_function::Node::CREF> stack; + std::vector<TensorFunction::CREF> stack; TensorFunctionBuilder(Stash &stash_in, const TensorEngine &tensor_engine_in, const NodeTypes &types_in) : stash(stash_in), tensor_engine(tensor_engine_in), types(types_in), stack() {} @@ -63,13 +44,13 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { stack.back() = tensor_function::reduce(a, aggr, dimensions, stash); } - void make_map(const Node &, map_fun_t function) { + void make_map(const Node &, operation::op1_t function) { assert(stack.size() >= 1); const auto &a = stack.back().get(); stack.back() = tensor_function::map(a, function, stash); } - void make_join(const Node &, join_fun_t function) { + void make_join(const Node &, operation::op2_t function) { assert(stack.size() >= 2); const auto &b = stack.back().get(); stack.pop_back(); @@ -77,7 +58,7 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { stack.back() = tensor_function::join(a, b, function, stash); } - void make_merge(const Node &, join_fun_t function) { + void make_merge(const Node &, operation::op2_t function) { assert(stack.size() >= 2); const auto &b = stack.back().get(); stack.pop_back(); @@ -113,7 +94,7 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void make_create(const TensorCreate &node) { assert(stack.size() >= node.num_children()); - std::map<TensorSpec::Address, tensor_function::Node::CREF> spec; + std::map<TensorSpec::Address, TensorFunction::CREF> spec; for (size_t idx = node.num_children(); idx-- > 0; ) { spec.emplace(node.get_child_address(idx), stack.back()); stack.pop_back(); @@ -134,8 +115,8 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void make_peek(const TensorPeek &node) { assert(stack.size() >= node.num_children()); - const tensor_function::Node ¶m = stack[stack.size()-node.num_children()]; - std::map<vespalib::string, std::variant<TensorSpec::Label, tensor_function::Node::CREF>> spec; + const TensorFunction ¶m = stack[stack.size()-node.num_children()]; + std::map<vespalib::string, std::variant<TensorSpec::Label, TensorFunction::CREF>> spec; for (auto pos = node.dim_list().rbegin(); pos != node.dim_list().rend(); ++pos) { if (pos->second.is_expr()) { spec.emplace(pos->first, stack.back()); @@ -203,14 +184,16 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { abort(); } void visit(const TensorMap &node) override { - const auto &token = stash.create<CompileCache::Token::UP>(CompileCache::compile(node.lambda(), PassParams::SEPARATE)); - make_map(node, token.get()->get().get_function<1>()); + if (auto op1 = operation::lookup_op1(node.lambda())) { + make_map(node, op1.value()); + } else { + const auto &token = stash.create<CompileCache::Token::UP>(CompileCache::compile(node.lambda(), PassParams::SEPARATE)); + make_map(node, token.get()->get().get_function<1>()); + } } void visit(const TensorJoin &node) override { - if (is_op2<Mul>(node.lambda())) { - make_join(node, operation::Mul::f); - } else if (is_op2<Add>(node.lambda())) { - make_join(node, operation::Add::f); + if (auto op2 = operation::lookup_op2(node.lambda())) { + make_join(node, op2.value()); } else { const auto &token = stash.create<CompileCache::Token::UP>(CompileCache::compile(node.lambda(), PassParams::SEPARATE)); make_join(node, token.get()->get().get_function<2>()); diff --git a/eval/src/vespa/eval/eval/operation.cpp b/eval/src/vespa/eval/eval/operation.cpp index fa0a99de461..fa8be4d20bc 100644 --- a/eval/src/vespa/eval/eval/operation.cpp +++ b/eval/src/vespa/eval/eval/operation.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "operation.h" +#include "function.h" +#include "key_gen.h" #include <vespa/vespalib/util/approx.h> #include <algorithm> @@ -47,5 +49,111 @@ double IsNan::f(double a) { return std::isnan(a) ? 1.0 : 0.0; } double Relu::f(double a) { return std::max(a, 0.0); } double Sigmoid::f(double a) { return 1.0 / (1.0 + std::exp(-1.0 * a)); } double Elu::f(double a) { return (a < 0) ? std::exp(a) - 1 : a; } +//----------------------------------------------------------------------------- +double Inv::f(double a) { return (1.0 / a); } +double Square::f(double a) { return (a * a); } +double Cube::f(double a) { return (a * a * a); } + +namespace { + +template <typename T> +void add_op(std::map<vespalib::string,T> &map, const Function &fun, T op) { + assert(!fun.has_error()); + auto key = gen_key(fun, PassParams::SEPARATE); + auto res = map.emplace(key, op); + assert(res.second); +} + +template <typename T> +std::optional<T> lookup_op(const std::map<vespalib::string,T> &map, const Function &fun) { + auto key = gen_key(fun, PassParams::SEPARATE); + auto pos = map.find(key); + if (pos != map.end()) { + return pos->second; + } + return std::nullopt; +} + +void add_op1(std::map<vespalib::string,op1_t> &map, const vespalib::string &expr, op1_t op) { + add_op(map, *Function::parse({"a"}, expr), op); +} + +void add_op2(std::map<vespalib::string,op2_t> &map, const vespalib::string &expr, op2_t op) { + add_op(map, *Function::parse({"a", "b"}, expr), op); +} + +std::map<vespalib::string,op1_t> make_op1_map() { + std::map<vespalib::string,op1_t> map; + add_op1(map, "-a", Neg::f); + add_op1(map, "!a", Not::f); + add_op1(map, "cos(a)", Cos::f); + add_op1(map, "sin(a)", Sin::f); + add_op1(map, "tan(a)", Tan::f); + add_op1(map, "cosh(a)", Cosh::f); + add_op1(map, "sinh(a)", Sinh::f); + add_op1(map, "tanh(a)", Tanh::f); + add_op1(map, "acos(a)", Acos::f); + add_op1(map, "asin(a)", Asin::f); + add_op1(map, "atan(a)", Atan::f); + add_op1(map, "exp(a)", Exp::f); + add_op1(map, "log10(a)", Log10::f); + add_op1(map, "log(a)", Log::f); + add_op1(map, "sqrt(a)", Sqrt::f); + add_op1(map, "ceil(a)", Ceil::f); + add_op1(map, "fabs(a)", Fabs::f); + add_op1(map, "floor(a)", Floor::f); + add_op1(map, "isNan(a)", IsNan::f); + add_op1(map, "relu(a)", Relu::f); + add_op1(map, "sigmoid(a)", Sigmoid::f); + add_op1(map, "elu(a)", Elu::f); + //------------------------------------- + add_op1(map, "1/a", Inv::f); + add_op1(map, "a*a", Square::f); + add_op1(map, "a^2", Square::f); + add_op1(map, "pow(a,2)", Square::f); + add_op1(map, "(a*a)*a", Cube::f); + add_op1(map, "a*(a*a)", Cube::f); + add_op1(map, "a^3", Cube::f); + add_op1(map, "pow(a,3)", Cube::f); + return map; +} + +std::map<vespalib::string,op2_t> make_op2_map() { + std::map<vespalib::string,op2_t> map; + add_op2(map, "a+b", Add::f); + add_op2(map, "a-b", Sub::f); + add_op2(map, "a*b", Mul::f); + add_op2(map, "a/b", Div::f); + add_op2(map, "a%b", Mod::f); + add_op2(map, "a^b", Pow::f); + add_op2(map, "a==b", Equal::f); + add_op2(map, "a!=b", NotEqual::f); + add_op2(map, "a~=b", Approx::f); + add_op2(map, "a<b", Less::f); + add_op2(map, "a<=b", LessEqual::f); + add_op2(map, "a>b", Greater::f); + add_op2(map, "a>=b", GreaterEqual::f); + add_op2(map, "a&&b", And::f); + add_op2(map, "a||b", Or::f); + add_op2(map, "atan2(a,b)", Atan2::f); + add_op2(map, "ldexp(a,b)", Ldexp::f); + add_op2(map, "pow(a,b)", Pow::f); + add_op2(map, "fmod(a,b)", Mod::f); + add_op2(map, "min(a,b)", Min::f); + add_op2(map, "max(a,b)", Max::f); + return map; +} + +} // namespace <unnamed> + +std::optional<op1_t> lookup_op1(const Function &fun) { + static const std::map<vespalib::string,op1_t> map = make_op1_map(); + return lookup_op(map, fun); +} + +std::optional<op2_t> lookup_op2(const Function &fun) { + static const std::map<vespalib::string,op2_t> map = make_op2_map(); + return lookup_op(map, fun); +} } diff --git a/eval/src/vespa/eval/eval/operation.h b/eval/src/vespa/eval/eval/operation.h index fa99f51a308..02d3322f867 100644 --- a/eval/src/vespa/eval/eval/operation.h +++ b/eval/src/vespa/eval/eval/operation.h @@ -1,6 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <optional> + +namespace vespalib::eval { class Function; } namespace vespalib::eval::operation { @@ -45,5 +48,15 @@ struct IsNan { static double f(double a); }; struct Relu { static double f(double a); }; struct Sigmoid { static double f(double a); }; struct Elu { static double f(double a); }; +//----------------------------------------------------------------------------- +struct Inv { static double f(double a); }; +struct Square { static double f(double a); }; +struct Cube { static double f(double a); }; + +using op1_t = double (*)(double); +using op2_t = double (*)(double, double); + +std::optional<op1_t> lookup_op1(const Function &fun); +std::optional<op2_t> lookup_op2(const Function &fun); } diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp index 2656e240a5b..85079b7a8e3 100644 --- a/eval/src/vespa/eval/eval/tensor_function.cpp +++ b/eval/src/vespa/eval/eval/tensor_function.cpp @@ -134,9 +134,16 @@ void op_tensor_create(State &state, uint64_t param) { state.pop_n_push(i, result); } +struct LambdaParams { + const Lambda &parent; + InterpretedFunction fun; + LambdaParams(const Lambda &parent_in, InterpretedFunction fun_in) + : parent(parent_in), fun(std::move(fun_in)) {} +}; + void op_tensor_lambda(State &state, uint64_t param) { - const Lambda::Self &self = unwrap_param<Lambda::Self>(param); - TensorSpec spec = self.parent.create_spec(*state.params, self.fun); + const LambdaParams ¶ms = unwrap_param<LambdaParams>(param); + TensorSpec spec = params.parent.create_spec(*state.params, params.fun); const Value &result = *state.stash.create<Value::UP>(state.engine.from_spec(spec)); state.stack.emplace_back(result); } @@ -439,13 +446,8 @@ InterpretedFunction::Instruction Lambda::compile_self(const TensorEngine &engine, Stash &stash) const { InterpretedFunction fun(engine, _lambda->root(), _lambda_types); - Self &self = stash.create<Self>(*this, std::move(fun)); - return Instruction(op_tensor_lambda, wrap_param<Self>(self)); -} - -void -Lambda::push_children(std::vector<Child::CREF> &) const -{ + LambdaParams ¶ms = stash.create<LambdaParams>(*this, std::move(fun)); + return Instruction(op_tensor_lambda, wrap_param<LambdaParams>(params)); } void @@ -543,48 +545,48 @@ If::visit_children(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- -const Node &const_value(const Value &value, Stash &stash) { +const TensorFunction &const_value(const Value &value, Stash &stash) { return stash.create<ConstValue>(value); } -const Node &inject(const ValueType &type, size_t param_idx, Stash &stash) { +const TensorFunction &inject(const ValueType &type, size_t param_idx, Stash &stash) { return stash.create<Inject>(type, param_idx); } -const Node &reduce(const Node &child, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) { +const TensorFunction &reduce(const TensorFunction &child, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash) { ValueType result_type = child.result_type().reduce(dimensions); return stash.create<Reduce>(result_type, child, aggr, dimensions); } -const Node &map(const Node &child, map_fun_t function, Stash &stash) { +const TensorFunction &map(const TensorFunction &child, map_fun_t function, Stash &stash) { ValueType result_type = child.result_type(); return stash.create<Map>(result_type, child, function); } -const Node &join(const Node &lhs, const Node &rhs, join_fun_t function, Stash &stash) { +const TensorFunction &join(const TensorFunction &lhs, const TensorFunction &rhs, join_fun_t function, Stash &stash) { ValueType result_type = ValueType::join(lhs.result_type(), rhs.result_type()); return stash.create<Join>(result_type, lhs, rhs, function); } -const Node &merge(const Node &lhs, const Node &rhs, join_fun_t function, Stash &stash) { +const TensorFunction &merge(const TensorFunction &lhs, const TensorFunction &rhs, join_fun_t function, Stash &stash) { ValueType result_type = ValueType::merge(lhs.result_type(), rhs.result_type()); return stash.create<Merge>(result_type, lhs, rhs, function); } -const Node &concat(const Node &lhs, const Node &rhs, const vespalib::string &dimension, Stash &stash) { +const TensorFunction &concat(const TensorFunction &lhs, const TensorFunction &rhs, const vespalib::string &dimension, Stash &stash) { ValueType result_type = ValueType::concat(lhs.result_type(), rhs.result_type(), dimension); return stash.create<Concat>(result_type, lhs, rhs, dimension); } -const Node &create(const ValueType &type, const std::map<TensorSpec::Address,Node::CREF> &spec, Stash &stash) { +const TensorFunction &create(const ValueType &type, const std::map<TensorSpec::Address,TensorFunction::CREF> &spec, Stash &stash) { return stash.create<Create>(type, spec); } -const Node &lambda(const ValueType &type, const std::vector<size_t> &bindings, const Function &function, NodeTypes node_types, Stash &stash) { +const TensorFunction &lambda(const ValueType &type, const std::vector<size_t> &bindings, const Function &function, NodeTypes node_types, Stash &stash) { return stash.create<Lambda>(type, bindings, function, std::move(node_types)); } -const Node &peek(const Node ¶m, const std::map<vespalib::string, std::variant<TensorSpec::Label, Node::CREF>> &spec, Stash &stash) { +const TensorFunction &peek(const TensorFunction ¶m, const std::map<vespalib::string, std::variant<TensorSpec::Label, TensorFunction::CREF>> &spec, Stash &stash) { std::vector<vespalib::string> dimensions; for (const auto &dim_spec: spec) { dimensions.push_back(dim_spec.first); @@ -593,12 +595,12 @@ const Node &peek(const Node ¶m, const std::map<vespalib::string, std::varian return stash.create<Peek>(result_type, param, spec); } -const Node &rename(const Node &child, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash) { +const TensorFunction &rename(const TensorFunction &child, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash) { ValueType result_type = child.result_type().rename(from, to); return stash.create<Rename>(result_type, child, from, to); } -const Node &if_node(const Node &cond, const Node &true_child, const Node &false_child, Stash &stash) { +const TensorFunction &if_node(const TensorFunction &cond, const TensorFunction &true_child, const TensorFunction &false_child, Stash &stash) { ValueType result_type = ValueType::either(true_child.result_type(), false_child.result_type()); return stash.create<If>(result_type, cond, true_child, false_child); } diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index e1961079017..20631108775 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -52,6 +52,7 @@ class Tensor; **/ struct TensorFunction { + using CREF = std::reference_wrapper<const TensorFunction>; TensorFunction(const TensorFunction &) = delete; TensorFunction &operator=(const TensorFunction &) = delete; TensorFunction(TensorFunction &&) = delete; @@ -132,7 +133,6 @@ class Node : public TensorFunction private: ValueType _result_type; public: - using CREF = std::reference_wrapper<const Node>; Node(const ValueType &result_type_in) : _result_type(result_type_in) {} const ValueType &result_type() const final override { return _result_type; } }; @@ -183,7 +183,7 @@ class ConstValue : public Leaf private: const Value &_value; public: - ConstValue(const Value &value_in) : Leaf(value_in.type()), _value(value_in) {} + ConstValue(const Value &value_in) : Super(value_in.type()), _value(value_in) {} const Value &value() const { return _value; } bool result_is_mutable() const override { return false; } InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override; @@ -199,7 +199,7 @@ private: size_t _param_idx; public: Inject(const ValueType &result_type_in, size_t param_idx_in) - : Leaf(result_type_in), _param_idx(param_idx_in) {} + : Super(result_type_in), _param_idx(param_idx_in) {} size_t param_idx() const { return _param_idx; } bool result_is_mutable() const override { return false; } InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override; @@ -219,7 +219,7 @@ public: const TensorFunction &child_in, Aggr aggr_in, const std::vector<vespalib::string> &dimensions_in) - : Op1(result_type_in, child_in), _aggr(aggr_in), _dimensions(dimensions_in) {} + : Super(result_type_in, child_in), _aggr(aggr_in), _dimensions(dimensions_in) {} Aggr aggr() const { return _aggr; } const std::vector<vespalib::string> &dimensions() const { return _dimensions; } bool result_is_mutable() const override { return true; } @@ -238,7 +238,7 @@ public: Map(const ValueType &result_type_in, const TensorFunction &child_in, map_fun_t function_in) - : Op1(result_type_in, child_in), _function(function_in) {} + : Super(result_type_in, child_in), _function(function_in) {} map_fun_t function() const { return _function; } bool result_is_mutable() const override { return true; } InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const override; @@ -257,7 +257,7 @@ public: const TensorFunction &lhs_in, const TensorFunction &rhs_in, join_fun_t function_in) - : Op2(result_type_in, lhs_in, rhs_in), _function(function_in) {} + : Super(result_type_in, lhs_in, rhs_in), _function(function_in) {} join_fun_t function() const { return _function; } bool result_is_mutable() const override { return true; } InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const override; @@ -276,7 +276,7 @@ public: const TensorFunction &lhs_in, const TensorFunction &rhs_in, join_fun_t function_in) - : Op2(result_type_in, lhs_in, rhs_in), _function(function_in) {} + : Super(result_type_in, lhs_in, rhs_in), _function(function_in) {} join_fun_t function() const { return _function; } bool result_is_mutable() const override { return true; } InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const override; @@ -295,7 +295,7 @@ public: const TensorFunction &lhs_in, const TensorFunction &rhs_in, const vespalib::string &dimension_in) - : Op2(result_type_in, lhs_in, rhs_in), _dimension(dimension_in) {} + : Super(result_type_in, lhs_in, rhs_in), _dimension(dimension_in) {} const vespalib::string &dimension() const { return _dimension; } bool result_is_mutable() const override { return true; } InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override; @@ -310,8 +310,8 @@ class Create : public Node private: std::map<TensorSpec::Address, Child> _spec; public: - Create(const ValueType &result_type_in, const std::map<TensorSpec::Address, Node::CREF> &spec_in) - : Node(result_type_in), _spec() + Create(const ValueType &result_type_in, const std::map<TensorSpec::Address, TensorFunction::CREF> &spec_in) + : Super(result_type_in), _spec() { for (const auto &cell: spec_in) { _spec.emplace(cell.first, Child(cell.second)); @@ -326,23 +326,16 @@ public: //----------------------------------------------------------------------------- -class Lambda : public Node +class Lambda : public Leaf { - using Super = Node; -public: - struct Self { - const Lambda &parent; - InterpretedFunction fun; - Self(const Lambda &parent_in, InterpretedFunction fun_in) - : parent(parent_in), fun(std::move(fun_in)) {} - }; + using Super = Leaf; private: std::vector<size_t> _bindings; std::shared_ptr<Function const> _lambda; NodeTypes _lambda_types; public: Lambda(const ValueType &result_type_in, const std::vector<size_t> &bindings_in, const Function &lambda_in, NodeTypes lambda_types_in) - : Node(result_type_in), _bindings(bindings_in), _lambda(lambda_in.shared_from_this()), _lambda_types(std::move(lambda_types_in)) {} + : Super(result_type_in), _bindings(bindings_in), _lambda(lambda_in.shared_from_this()), _lambda_types(std::move(lambda_types_in)) {} const std::vector<size_t> &bindings() const { return _bindings; } const Function &lambda() const { return *_lambda; } const NodeTypes &types() const { return _lambda_types; } @@ -352,7 +345,6 @@ public: } bool result_is_mutable() const override { return true; } InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override; - void push_children(std::vector<Child::CREF> &children) const final override; void visit_self(vespalib::ObjectVisitor &visitor) const override; }; @@ -367,9 +359,9 @@ private: Child _param; std::map<vespalib::string, MyLabel> _spec; public: - Peek(const ValueType &result_type_in, const Node ¶m, - const std::map<vespalib::string, std::variant<TensorSpec::Label, Node::CREF>> &spec) - : Node(result_type_in), _param(param), _spec() + Peek(const ValueType &result_type_in, const TensorFunction ¶m, + const std::map<vespalib::string, std::variant<TensorSpec::Label, TensorFunction::CREF>> &spec) + : Super(result_type_in), _param(param), _spec() { for (const auto &dim: spec) { std::visit(vespalib::overload @@ -377,7 +369,7 @@ public: [&](const TensorSpec::Label &label) { _spec.emplace(dim.first, label); }, - [&](const Node::CREF &ref) { + [&](const TensorFunction::CREF &ref) { _spec.emplace(dim.first, ref.get()); } }, dim.second); @@ -404,7 +396,7 @@ public: const TensorFunction &child_in, const std::vector<vespalib::string> &from_in, const std::vector<vespalib::string> &to_in) - : Op1(result_type_in, child_in), _from(from_in), _to(to_in) {} + : Super(result_type_in, child_in), _from(from_in), _to(to_in) {} const std::vector<vespalib::string> &from() const { return _from; } const std::vector<vespalib::string> &to() const { return _to; } bool result_is_mutable() const override { return true; } @@ -440,18 +432,18 @@ public: //----------------------------------------------------------------------------- -const Node &const_value(const Value &value, Stash &stash); -const Node &inject(const ValueType &type, size_t param_idx, Stash &stash); -const Node &reduce(const Node &child, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash); -const Node &map(const Node &child, map_fun_t function, Stash &stash); -const Node &join(const Node &lhs, const Node &rhs, join_fun_t function, Stash &stash); -const Node &merge(const Node &lhs, const Node &rhs, join_fun_t function, Stash &stash); -const Node &concat(const Node &lhs, const Node &rhs, const vespalib::string &dimension, Stash &stash); -const Node &create(const ValueType &type, const std::map<TensorSpec::Address, Node::CREF> &spec, Stash &stash); -const Node &lambda(const ValueType &type, const std::vector<size_t> &bindings, const Function &function, NodeTypes node_types, Stash &stash); -const Node &peek(const Node ¶m, const std::map<vespalib::string, std::variant<TensorSpec::Label, Node::CREF>> &spec, Stash &stash); -const Node &rename(const Node &child, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash); -const Node &if_node(const Node &cond, const Node &true_child, const Node &false_child, Stash &stash); +const TensorFunction &const_value(const Value &value, Stash &stash); +const TensorFunction &inject(const ValueType &type, size_t param_idx, Stash &stash); +const TensorFunction &reduce(const TensorFunction &child, Aggr aggr, const std::vector<vespalib::string> &dimensions, Stash &stash); +const TensorFunction &map(const TensorFunction &child, map_fun_t function, Stash &stash); +const TensorFunction &join(const TensorFunction &lhs, const TensorFunction &rhs, join_fun_t function, Stash &stash); +const TensorFunction &merge(const TensorFunction &lhs, const TensorFunction &rhs, join_fun_t function, Stash &stash); +const TensorFunction &concat(const TensorFunction &lhs, const TensorFunction &rhs, const vespalib::string &dimension, Stash &stash); +const TensorFunction &create(const ValueType &type, const std::map<TensorSpec::Address, TensorFunction::CREF> &spec, Stash &stash); +const TensorFunction &lambda(const ValueType &type, const std::vector<size_t> &bindings, const Function &function, NodeTypes node_types, Stash &stash); +const TensorFunction &peek(const TensorFunction ¶m, const std::map<vespalib::string, std::variant<TensorSpec::Label, TensorFunction::CREF>> &spec, Stash &stash); +const TensorFunction &rename(const TensorFunction &child, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash); +const TensorFunction &if_node(const TensorFunction &cond, const TensorFunction &true_child, const TensorFunction &false_child, Stash &stash); } // namespace vespalib::eval::tensor_function } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp index dd27ba1f147..0b27ba1f2df 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp +++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp @@ -127,6 +127,13 @@ TensorSpec make_dense(const vespalib::string &type, } // namespace vespalib::eval::test ParamRepo & +ParamRepo::add(const vespalib::string &name, TensorSpec value_in, bool is_mutable_in) { + ASSERT_TRUE(map.find(name) == map.end()); + map.insert_or_assign(name, Param(std::move(value_in), is_mutable_in)); + return *this; +} + +ParamRepo & EvalFixture::ParamRepo::add_vector(const char *d1, size_t s1, gen_fun_t gen) { return add_dense({{d1, s1}}, gen); @@ -159,8 +166,15 @@ EvalFixture::ParamRepo::add_dense(const std::vector<std::pair<vespalib::string, type += fmt("%s[%zu]", dim.first.c_str(), dim.second); prev = dim.first; } - add(name, make_dense(fmt("tensor(%s)", type.c_str()), dims, gen)); - add(name + "f", make_dense(fmt("tensor<float>(%s)", type.c_str()), dims, gen)); + int cpy = 1; + vespalib::string suffix = ""; + while (map.find(name + suffix) != map.end()) { + suffix = fmt("$%d", ++cpy); + } + add(name + suffix, make_dense(fmt("tensor(%s)", type.c_str()), dims, gen)); + add(name + "f" + suffix, make_dense(fmt("tensor<float>(%s)", type.c_str()), dims, gen)); + add_mutable("@" + name + suffix, make_dense(fmt("tensor(%s)", type.c_str()), dims, gen)); + add_mutable("@" + name + "f" + suffix, make_dense(fmt("tensor<float>(%s)", type.c_str()), dims, gen)); return *this; } diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h index 1010b5e58a8..31fcee49782 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.h +++ b/eval/src/vespa/eval/eval/test/eval_fixture.h @@ -30,10 +30,7 @@ public: using gen_fun_t = std::function<double(size_t)>; static double gen_N(size_t seq) { return (seq + 1); } ParamRepo() : map() {} - ParamRepo &add(const vespalib::string &name, TensorSpec value_in, bool is_mutable_in) { - map.insert_or_assign(name, Param(std::move(value_in), is_mutable_in)); - return *this; - } + ParamRepo &add(const vespalib::string &name, TensorSpec value_in, bool is_mutable_in); ParamRepo &add(const vespalib::string &name, const TensorSpec &value) { return add(name, value, false); } diff --git a/eval/src/vespa/eval/eval/test/eval_spec.cpp b/eval/src/vespa/eval/eval/test/eval_spec.cpp index 709234a1a2c..dbc20dcf606 100644 --- a/eval/src/vespa/eval/eval/test/eval_spec.cpp +++ b/eval/src/vespa/eval/eval/test/eval_spec.cpp @@ -162,11 +162,11 @@ EvalSpec::add_function_call_cases() { void EvalSpec::add_tensor_operation_cases() { add_rule({"a", -1.0, 1.0}, "map(a,f(x)(sin(x)))", [](double x){ return std::sin(x); }); - add_rule({"a", -1.0, 1.0}, "map(a,f(x)(x+x*3))", [](double x){ return (x + (x * 3)); }); + add_rule({"a", -1.0, 1.0}, "map(a,f(x)(x*x*3))", [](double x){ return ((x * x) * 3); }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "join(a,b,f(x,y)(x+y))", [](double x, double y){ return (x + y); }); - add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "join(a,b,f(x,y)(x+y*3))", [](double x, double y){ return (x + (y * 3)); }); + add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "join(a,b,f(x,y)(x*y*3))", [](double x, double y){ return ((x * y) * 3); }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "merge(a,b,f(x,y)(x+y))", [](double x, double y){ return (x + y); }); - add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "merge(a,b,f(x,y)(x+y*3))", [](double x, double y){ return (x + (y * 3)); }); + add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "merge(a,b,f(x,y)(x*y*3))", [](double x, double y){ return ((x * y) * 3); }); add_rule({"a", -1.0, 1.0}, "reduce(a,avg)", [](double a){ return a; }); add_rule({"a", -1.0, 1.0}, "reduce(a,count)", [](double){ return 1.0; }); add_rule({"a", -1.0, 1.0}, "reduce(a,prod)", [](double a){ return a; }); diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index 3e91240048b..a8ae9c44bb0 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/vespalib/util/typify.h> #include <vespa/vespalib/stllike/string.h> #include <vector> @@ -104,4 +105,15 @@ template <typename CT> inline ValueType::CellType get_cell_type(); template <> inline ValueType::CellType get_cell_type<double>() { return ValueType::CellType::DOUBLE; } template <> inline ValueType::CellType get_cell_type<float>() { return ValueType::CellType::FLOAT; } +struct TypifyCellType { + template <typename T> using Result = TypifyResultType<T>; + template <typename F> static decltype(auto) resolve(ValueType::CellType value, F &&f) { + switch(value) { + case ValueType::CellType::DOUBLE: return f(Result<double>()); + case ValueType::CellType::FLOAT: return f(Result<float>()); + } + abort(); + } +}; + } // namespace diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index a817c1454d8..ca14e40e4d0 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -17,8 +17,12 @@ #include "dense/dense_single_reduce_function.h" #include "dense/dense_remove_dimension_optimizer.h" #include "dense/dense_lambda_peek_optimizer.h" -#include "dense/dense_inplace_join_function.h" -#include "dense/dense_inplace_map_function.h" +#include "dense/dense_lambda_function.h" +#include "dense/dense_simple_expand_function.h" +#include "dense/dense_simple_join_function.h" +#include "dense/dense_number_join_function.h" +#include "dense/dense_pow_as_map_optimizer.h" +#include "dense/dense_simple_map_function.h" #include "dense/vector_from_doubles_function.h" #include "dense/dense_tensor_create_function.h" #include "dense/dense_tensor_peek_function.h" @@ -174,7 +178,7 @@ DefaultTensorEngine::to_spec(const Value &value) const struct CallDenseTensorBuilder { template <typename CT> static Value::UP - call(const ValueType &type, const TensorSpec &spec) + invoke(const ValueType &type, const TensorSpec &spec) { TypedDenseTensorBuilder<CT> builder(type); for (const auto &cell: spec.cells()) { @@ -189,6 +193,8 @@ struct CallDenseTensorBuilder { } }; +using MyTypify = eval::TypifyCellType; + Value::UP DefaultTensorEngine::from_spec(const TensorSpec &spec) const { @@ -199,7 +205,7 @@ DefaultTensorEngine::from_spec(const TensorSpec &spec) const double value = spec.cells().empty() ? 0.0 : spec.cells().begin()->second.value; return std::make_unique<DoubleValue>(value); } else if (type.is_dense()) { - return dispatch_0<CallDenseTensorBuilder>(type.cell_type(), type, spec); + return typify_invoke<1,MyTypify,CallDenseTensorBuilder>(type.cell_type(), type, spec); } else if (type.is_sparse()) { DirectSparseTensorBuilder builder(type); SparseTensorAddressBuilder address_builder; @@ -283,15 +289,19 @@ DefaultTensorEngine::optimize(const TensorFunction &expr, Stash &stash) const } while (!nodes.empty()) { const Child &child = nodes.back().get(); + child.set(DenseSimpleExpandFunction::optimize(child.get(), stash)); child.set(DenseAddDimensionOptimizer::optimize(child.get(), stash)); child.set(DenseRemoveDimensionOptimizer::optimize(child.get(), stash)); child.set(VectorFromDoublesFunction::optimize(child.get(), stash)); child.set(DenseTensorCreateFunction::optimize(child.get(), stash)); child.set(DenseTensorPeekFunction::optimize(child.get(), stash)); child.set(DenseLambdaPeekOptimizer::optimize(child.get(), stash)); + child.set(DenseLambdaFunction::optimize(child.get(), stash)); child.set(DenseFastRenameOptimizer::optimize(child.get(), stash)); - child.set(DenseInplaceMapFunction::optimize(child.get(), stash)); - child.set(DenseInplaceJoinFunction::optimize(child.get(), stash)); + child.set(DensePowAsMapOptimizer::optimize(child.get(), stash)); + child.set(DenseSimpleMapFunction::optimize(child.get(), stash)); + child.set(DenseSimpleJoinFunction::optimize(child.get(), stash)); + child.set(DenseNumberJoinFunction::optimize(child.get(), stash)); child.set(DenseSingleReduceFunction::optimize(child.get(), stash)); nodes.pop_back(); } @@ -445,7 +455,7 @@ const Value &concat_vectors(const Value &a, const Value &b, const vespalib::stri struct CallConcatVectors { template <typename OCT> - static const Value &call(const Value &a, const Value &b, const vespalib::string &dimension, size_t vector_size, Stash &stash) { + static const Value &invoke(const Value &a, const Value &b, const vespalib::string &dimension, size_t vector_size, Stash &stash) { return concat_vectors<OCT>(a, b, dimension, vector_size, stash); } }; @@ -457,7 +467,7 @@ DefaultTensorEngine::concat(const Value &a, const Value &b, const vespalib::stri size_t b_size = vector_size(b.type(), dimension); if ((a_size > 0) && (b_size > 0)) { CellType result_cell_type = ValueType::unify_cell_types(a.type(), b.type()); - return dispatch_0<CallConcatVectors>(result_cell_type, a, b, dimension, (a_size + b_size), stash); + return typify_invoke<1,MyTypify,CallConcatVectors>(result_cell_type, a, b, dimension, (a_size + b_size), stash); } return to_default(simple_engine().concat(to_simple(a, stash), to_simple(b, stash), dimension, stash), stash); } diff --git a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt index 0131ff28398..c4b8138148c 100644 --- a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt @@ -6,14 +6,18 @@ vespa_add_library(eval_tensor_dense OBJECT dense_dimension_combiner.cpp dense_dot_product_function.cpp dense_fast_rename_optimizer.cpp - dense_inplace_join_function.cpp - dense_inplace_map_function.cpp + dense_lambda_function.cpp dense_lambda_peek_function.cpp dense_lambda_peek_optimizer.cpp dense_matmul_function.cpp dense_multi_matmul_function.cpp + dense_number_join_function.cpp + dense_pow_as_map_optimizer.cpp dense_remove_dimension_optimizer.cpp dense_replace_type_function.cpp + dense_simple_expand_function.cpp + dense_simple_join_function.cpp + dense_simple_map_function.cpp dense_single_reduce_function.cpp dense_tensor.cpp dense_tensor_address_mapper.cpp @@ -24,6 +28,7 @@ vespa_add_library(eval_tensor_dense OBJECT dense_tensor_reduce.cpp dense_tensor_view.cpp dense_xw_product_function.cpp + index_lookup_table.cpp mutable_dense_tensor_view.cpp typed_cells.cpp typed_dense_tensor_builder.cpp diff --git a/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp index 9b93f5e7d72..84da53c8488 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_cell_range_function.cpp @@ -25,7 +25,7 @@ void my_cell_range_op(eval::InterpretedFunction::State &state, uint64_t param) { struct MyCellRangeOp { template <typename CT> - static auto get_fun() { return my_cell_range_op<CT>; } + static auto invoke() { return my_cell_range_op<CT>; } }; } // namespace vespalib::tensor::<unnamed> @@ -46,7 +46,9 @@ DenseCellRangeFunction::compile_self(const TensorEngine &, Stash &) const { static_assert(sizeof(uint64_t) == sizeof(this)); assert(result_type().cell_type() == child().result_type().cell_type()); - auto op = select_1<MyCellRangeOp>(result_type().cell_type()); + + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<1,MyTypify,MyCellRangeOp>(result_type().cell_type()); return eval::InterpretedFunction::Instruction(op, (uint64_t)this); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp index c9ff57e4a65..9e30451cd67 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp @@ -48,7 +48,7 @@ void my_cblas_float_dot_product_op(eval::InterpretedFunction::State &state, uint struct MyDotProductOp { template <typename LCT, typename RCT> - static auto get_fun() { return my_dot_product_op<LCT,RCT>; } + static auto invoke() { return my_dot_product_op<LCT,RCT>; } }; eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct) { @@ -60,7 +60,8 @@ eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct) { return my_cblas_float_dot_product_op; } } - return select_2<MyDotProductOp>(lct, rct); + using MyTypify = eval::TypifyCellType; + return typify_invoke<2,MyTypify,MyDotProductOp>(lct, rct); } } // namespace vespalib::tensor::<unnamed> diff --git a/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.cpp deleted file mode 100644 index 2107c7661f2..00000000000 --- a/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.cpp +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "dense_inplace_join_function.h" -#include "dense_tensor_view.h" -#include <vespa/vespalib/objects/objectvisitor.h> -#include <vespa/eval/eval/value.h> - -namespace vespalib::tensor { - -using eval::Value; -using eval::ValueType; -using eval::TensorFunction; -using eval::TensorEngine; -using eval::as; -using namespace eval::tensor_function; - -namespace { - -template <typename LCT, typename RCT> -void my_inplace_join_left_op(eval::InterpretedFunction::State &state, uint64_t param) { - join_fun_t function = (join_fun_t)param; - auto lhs_cells = unconstify(DenseTensorView::typify_cells<LCT>(state.peek(1))); - auto rhs_cells = DenseTensorView::typify_cells<RCT>(state.peek(0)); - for (size_t i = 0; i < lhs_cells.size(); ++i) { - lhs_cells[i] = function(lhs_cells[i], rhs_cells[i]); - } - state.stack.pop_back(); -} - -template <typename LCT, typename RCT> -void my_inplace_join_right_op(eval::InterpretedFunction::State &state, uint64_t param) { - join_fun_t function = (join_fun_t)param; - auto lhs_cells = DenseTensorView::typify_cells<LCT>(state.peek(1)); - auto rhs_cells = unconstify(DenseTensorView::typify_cells<RCT>(state.peek(0))); - for (size_t i = 0; i < rhs_cells.size(); ++i) { - rhs_cells[i] = function(lhs_cells[i], rhs_cells[i]); - } - const Value &result = state.stack.back(); - state.pop_pop_push(result); -} - -struct MyInplaceJoinLeftOp { - template <typename LCT, typename RCT> - static auto get_fun() { return my_inplace_join_left_op<LCT,RCT>; } -}; - -struct MyInplaceJoinRightOp { - template <typename LCT, typename RCT> - static auto get_fun() { return my_inplace_join_right_op<LCT,RCT>; } -}; - -eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool write_left) { - if (write_left) { - return select_2<MyInplaceJoinLeftOp>(lct, rct); - } else { - return select_2<MyInplaceJoinRightOp>(lct, rct); - } -} - -} // namespace vespalib::tensor::<unnamed> - - -DenseInplaceJoinFunction::DenseInplaceJoinFunction(const ValueType &result_type, - const TensorFunction &lhs, - const TensorFunction &rhs, - join_fun_t function_in, - bool write_left_in) - : eval::tensor_function::Join(result_type, lhs, rhs, function_in), - _write_left(write_left_in) -{ -} - -DenseInplaceJoinFunction::~DenseInplaceJoinFunction() -{ -} - -eval::InterpretedFunction::Instruction -DenseInplaceJoinFunction::compile_self(const TensorEngine &, Stash &) const -{ - auto op = my_select(lhs().result_type().cell_type(), - rhs().result_type().cell_type(), _write_left); - return eval::InterpretedFunction::Instruction(op, (uint64_t)function()); -} - -void -DenseInplaceJoinFunction::visit_self(vespalib::ObjectVisitor &visitor) const -{ - Super::visit_self(visitor); - visitor.visitBool("write_left", _write_left); -} - -const TensorFunction & -DenseInplaceJoinFunction::optimize(const eval::TensorFunction &expr, Stash &stash) -{ - if (auto join = as<Join>(expr)) { - const TensorFunction &lhs = join->lhs(); - const TensorFunction &rhs = join->rhs(); - if (lhs.result_type().is_dense() && - (lhs.result_type().dimensions() == rhs.result_type().dimensions())) - { - if (lhs.result_is_mutable() && (lhs.result_type() == expr.result_type())) { - return stash.create<DenseInplaceJoinFunction>(join->result_type(), lhs, rhs, - join->function(), /* write left: */ true); - } - if (rhs.result_is_mutable() && (rhs.result_type() == expr.result_type())) { - return stash.create<DenseInplaceJoinFunction>(join->result_type(), lhs, rhs, - join->function(), /* write left: */ false); - } - } - } - return expr; -} - -} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.h b/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.h deleted file mode 100644 index acd1a2d716b..00000000000 --- a/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/eval/eval/tensor_function.h> - -namespace vespalib::tensor { - -/** - * Tensor function for inplace join operation on mutable dense tensors. - **/ -class DenseInplaceJoinFunction : public eval::tensor_function::Join -{ - using Super = eval::tensor_function::Join; -public: - using join_fun_t = ::vespalib::eval::tensor_function::join_fun_t; -private: - bool _write_left; -public: - DenseInplaceJoinFunction(const eval::ValueType &result_type, - const TensorFunction &lhs, - const TensorFunction &rhs, - join_fun_t function_in, - bool write_left_in); - ~DenseInplaceJoinFunction(); - bool write_left() const { return _write_left; } - bool result_is_mutable() const override { return true; } - eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override; - void visit_self(vespalib::ObjectVisitor &visitor) const override; - static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash); -}; - -} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.cpp deleted file mode 100644 index 62434073f8e..00000000000 --- a/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.cpp +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "dense_inplace_map_function.h" -#include "dense_tensor_view.h" -#include <vespa/eval/eval/value.h> - -namespace vespalib::tensor { - -using eval::Value; -using eval::ValueType; -using eval::TensorFunction; -using eval::TensorEngine; -using eval::as; -using namespace eval::tensor_function; - -namespace { - -template <typename CT> -void my_inplace_map_op(eval::InterpretedFunction::State &state, uint64_t param) { - map_fun_t function = (map_fun_t)param; - ArrayRef<CT> cells = unconstify(DenseTensorView::typify_cells<CT>(state.peek(0))); - for (CT &cell: cells) { - cell = function(cell); - } -} - -struct MyInplaceMapOp { - template <typename CT> - static auto get_fun() { return my_inplace_map_op<CT>; } -}; - -} // namespace vespalib::tensor::<unnamed> - -DenseInplaceMapFunction::DenseInplaceMapFunction(const eval::ValueType &result_type, - const eval::TensorFunction &child, - map_fun_t function_in) - : eval::tensor_function::Map(result_type, child, function_in) -{ -} - -DenseInplaceMapFunction::~DenseInplaceMapFunction() -{ -} - -eval::InterpretedFunction::Instruction -DenseInplaceMapFunction::compile_self(const TensorEngine &, Stash &) const -{ - auto op = select_1<MyInplaceMapOp>(result_type().cell_type()); - return eval::InterpretedFunction::Instruction(op, (uint64_t)function()); -} - -const TensorFunction & -DenseInplaceMapFunction::optimize(const eval::TensorFunction &expr, Stash &stash) -{ - if (auto map = as<Map>(expr)) { - if (map->child().result_is_mutable() && map->result_type().is_dense()) { - assert(map->result_type().cell_type() == map->child().result_type().cell_type()); - return stash.create<DenseInplaceMapFunction>(map->result_type(), map->child(), map->function()); - } - } - return expr; -} - -} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.h b/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.h deleted file mode 100644 index 52122f4d8dc..00000000000 --- a/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/eval/eval/tensor_function.h> - -namespace vespalib::tensor { - -/** - * Tensor function for inplace map operation on mutable dense tensors. - **/ -class DenseInplaceMapFunction : public eval::tensor_function::Map -{ -public: - using map_fun_t = ::vespalib::eval::tensor_function::map_fun_t; - DenseInplaceMapFunction(const eval::ValueType &result_type, - const eval::TensorFunction &child, - map_fun_t function_in); - ~DenseInplaceMapFunction(); - bool result_is_mutable() const override { return true; } - eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override; - static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash); -}; - -} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp new file mode 100644 index 00000000000..e373ca09e11 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_lambda_function.cpp @@ -0,0 +1,190 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_lambda_function.h" +#include "dense_tensor_view.h" +#include <vespa/vespalib/objects/objectvisitor.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/eval/llvm/compiled_function.h> +#include <vespa/eval/eval/llvm/compile_cache.h> +#include <assert.h> + +namespace vespalib::tensor { + +using eval::CompileCache; +using eval::CompiledFunction; +using eval::InterpretedFunction; +using eval::LazyParams; +using eval::PassParams; +using eval::TensorEngine; +using eval::TensorFunction; +using eval::Value; +using eval::DoubleValue; +using eval::ValueType; +using eval::as; +using vespalib::Stash; + +using Instruction = InterpretedFunction::Instruction; +using State = InterpretedFunction::State; + +using namespace eval::tensor_function; + +const TensorEngine &prod_engine = DefaultTensorEngine::ref(); + +namespace { + +//----------------------------------------------------------------------------- + +bool step_labels(double *labels, const ValueType &type) { + for (size_t idx = type.dimensions().size(); idx-- > 0; ) { + if ((labels[idx] += 1.0) < type.dimensions()[idx].size) { + return true; + } else { + labels[idx] = 0.0; + } + } + return false; +} + +struct ParamProxy : public LazyParams { + const std::vector<double> &labels; + const LazyParams ¶ms; + const std::vector<size_t> &bindings; + ParamProxy(const std::vector<double> &labels_in, const LazyParams ¶ms_in, const std::vector<size_t> &bindings_in) + : labels(labels_in), params(params_in), bindings(bindings_in) {} + const Value &resolve(size_t idx, Stash &stash) const override { + if (idx < labels.size()) { + return stash.create<DoubleValue>(labels[idx]); + } + return params.resolve(bindings[idx - labels.size()], stash); + } +}; + +//----------------------------------------------------------------------------- + +struct CompiledParams { + const ValueType &result_type; + const std::vector<size_t> &bindings; + size_t num_cells; + CompileCache::Token::UP token; + CompiledParams(const Lambda &lambda) + : result_type(lambda.result_type()), + bindings(lambda.bindings()), + num_cells(result_type.dense_subspace_size()), + token(CompileCache::compile(lambda.lambda(), PassParams::ARRAY)) + { + assert(lambda.lambda().num_params() == (result_type.dimensions().size() + bindings.size())); + } +}; + +template <typename CT> +void my_compiled_lambda_op(eval::InterpretedFunction::State &state, uint64_t param) { + const CompiledParams ¶ms = *(const CompiledParams*)param; + std::vector<double> args(params.result_type.dimensions().size() + params.bindings.size(), 0.0); + double *bind_next = &args[params.result_type.dimensions().size()]; + for (size_t binding: params.bindings) { + *bind_next++ = state.params->resolve(binding, state.stash).as_double(); + } + auto fun = params.token->get().get_function(); + ArrayRef<CT> dst_cells = state.stash.create_array<CT>(params.num_cells); + CT *dst = &dst_cells[0]; + do { + *dst++ = fun(&args[0]); + } while (step_labels(&args[0], params.result_type)); + state.stack.push_back(state.stash.create<DenseTensorView>(params.result_type, TypedCells(dst_cells))); +} + +struct MyCompiledLambdaOp { + template <typename CT> + static auto invoke() { return my_compiled_lambda_op<CT>; } +}; + +//----------------------------------------------------------------------------- + +struct InterpretedParams { + const ValueType &result_type; + const std::vector<size_t> &bindings; + size_t num_cells; + InterpretedFunction fun; + InterpretedParams(const Lambda &lambda) + : result_type(lambda.result_type()), + bindings(lambda.bindings()), + num_cells(result_type.dense_subspace_size()), + fun(prod_engine, lambda.lambda().root(), lambda.types()) + { + assert(lambda.lambda().num_params() == (result_type.dimensions().size() + bindings.size())); + } +}; + +template <typename CT> +void my_interpreted_lambda_op(eval::InterpretedFunction::State &state, uint64_t param) { + const InterpretedParams ¶ms = *(const InterpretedParams*)param; + std::vector<double> labels(params.result_type.dimensions().size(), 0.0); + ParamProxy param_proxy(labels, *state.params, params.bindings); + InterpretedFunction::Context ctx(params.fun); + ArrayRef<CT> dst_cells = state.stash.create_array<CT>(params.num_cells); + CT *dst = &dst_cells[0]; + do { + *dst++ = params.fun.eval(ctx, param_proxy).as_double(); + } while (step_labels(&labels[0], params.result_type)); + state.stack.push_back(state.stash.create<DenseTensorView>(params.result_type, TypedCells(dst_cells))); +} + +struct MyInterpretedLambdaOp { + template <typename CT> + static auto invoke() { return my_interpreted_lambda_op<CT>; } +}; + +//----------------------------------------------------------------------------- + +} + +DenseLambdaFunction::DenseLambdaFunction(const Lambda &lambda_in) + : Super(lambda_in.result_type()), + _lambda(lambda_in) +{ +} + +DenseLambdaFunction::~DenseLambdaFunction() = default; + +DenseLambdaFunction::EvalMode +DenseLambdaFunction::eval_mode() const +{ + if (!CompiledFunction::detect_issues(_lambda.lambda()) && + _lambda.types().all_types_are_double()) + { + return EvalMode::COMPILED; + } else { + return EvalMode::INTERPRETED; + } +} + +Instruction +DenseLambdaFunction::compile_self(const TensorEngine &engine, Stash &stash) const +{ + assert(&engine == &prod_engine); + auto mode = eval_mode(); + using MyTypify = eval::TypifyCellType; + if (mode == EvalMode::COMPILED) { + CompiledParams ¶ms = stash.create<CompiledParams>(_lambda); + auto op = typify_invoke<1,MyTypify,MyCompiledLambdaOp>(result_type().cell_type()); + static_assert(sizeof(¶ms) == sizeof(uint64_t)); + return Instruction(op, (uint64_t)(¶ms)); + } else { + assert(mode == EvalMode::INTERPRETED); + InterpretedParams ¶ms = stash.create<InterpretedParams>(_lambda); + auto op = typify_invoke<1,MyTypify,MyInterpretedLambdaOp>(result_type().cell_type()); + static_assert(sizeof(¶ms) == sizeof(uint64_t)); + return Instruction(op, (uint64_t)(¶ms)); + } +} + +const eval::TensorFunction & +DenseLambdaFunction::optimize(const TensorFunction &expr, Stash &stash) +{ + if (auto lambda = as<Lambda>(expr)) { + return stash.create<DenseLambdaFunction>(*lambda); + } + return expr; +} + +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_lambda_function.h b/eval/src/vespa/eval/tensor/dense/dense_lambda_function.h new file mode 100644 index 00000000000..a1b6e5a1551 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_lambda_function.h @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::tensor { + +/** + * Tensor function for generic tensor lambda producing dense tensor + * views directly. This is the catch-all fall-back used by the default + * (production) tensor engine to avoid having a TensorSpec as an + * intermediate result. + **/ +class DenseLambdaFunction : public eval::tensor_function::Leaf +{ + using Super = eval::tensor_function::Leaf; +private: + const eval::tensor_function::Lambda &_lambda; +public: + enum class EvalMode : uint8_t { COMPILED, INTERPRETED }; + DenseLambdaFunction(const eval::tensor_function::Lambda &lambda_in); + ~DenseLambdaFunction() override; + bool result_is_mutable() const override { return true; } + EvalMode eval_mode() const; + eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override; + static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash); +}; + +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp index 0c7debde4ef..70bdc8ae7d6 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_function.cpp @@ -1,13 +1,12 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dense_lambda_peek_function.h" +#include "index_lookup_table.h" #include "dense_tensor_view.h" #include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/llvm/compile_cache.h> namespace vespalib::tensor { -using eval::CompileCache; using eval::Function; using eval::InterpretedFunction; using eval::PassParams; @@ -22,41 +21,31 @@ namespace { struct Self { const ValueType &result_type; - CompileCache::Token::UP compile_token; + IndexLookupTable::Token::UP table_token; Self(const ValueType &result_type_in, const Function &function) : result_type(result_type_in), - compile_token(CompileCache::compile(function, PassParams::ARRAY)) {} -}; - -bool step_params(std::vector<double> ¶ms, const ValueType &type) { - const auto &dims = type.dimensions(); - for (size_t idx = params.size(); idx-- > 0; ) { - if (size_t(params[idx] += 1.0) < dims[idx].size) { - return true; - } else { - params[idx] = 0.0; - } + table_token(IndexLookupTable::create(function, result_type_in)) + { + assert(table_token->get().size() == result_type.dense_subspace_size()); } - return false; -} +}; template <typename DST_CT, typename SRC_CT> void my_lambda_peek_op(InterpretedFunction::State &state, uint64_t param) { const auto *self = (const Self *)(param); + const std::vector<uint32_t> &lookup_table = self->table_token->get(); auto src_cells = DenseTensorView::typify_cells<SRC_CT>(state.peek(0)); - ArrayRef<DST_CT> dst_cells = state.stash.create_array<DST_CT>(self->result_type.dense_subspace_size()); + ArrayRef<DST_CT> dst_cells = state.stash.create_array<DST_CT>(lookup_table.size()); DST_CT *dst = &dst_cells[0]; - std::vector<double> params(self->result_type.dimensions().size(), 0.0); - auto idx_fun = self->compile_token->get().get_function(); - do { - *dst++ = src_cells[size_t(idx_fun(¶ms[0]))]; - } while(step_params(params, self->result_type)); + for (uint32_t idx: lookup_table) { + *dst++ = src_cells[idx]; + } state.pop_push(state.stash.create<DenseTensorView>(self->result_type, TypedCells(dst_cells))); } struct MyLambdaPeekOp { template <typename DST_CT, typename SRC_CT> - static auto get_fun() { return my_lambda_peek_op<DST_CT, SRC_CT>; } + static auto invoke() { return my_lambda_peek_op<DST_CT, SRC_CT>; } }; } // namespace vespalib::tensor::<unnamed> @@ -75,7 +64,8 @@ InterpretedFunction::Instruction DenseLambdaPeekFunction::compile_self(const TensorEngine &, Stash &stash) const { const Self &self = stash.create<Self>(result_type(), *_idx_fun); - auto op = select_2<MyLambdaPeekOp>(result_type().cell_type(), child().result_type().cell_type()); + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<2,MyTypify,MyLambdaPeekOp>(result_type().cell_type(), child().result_type().cell_type()); static_assert(sizeof(uint64_t) == sizeof(&self)); assert(child().result_type().is_dense()); return InterpretedFunction::Instruction(op, (uint64_t)&self); diff --git a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp index 695e0fddd08..9c18cf285d4 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp @@ -80,47 +80,6 @@ void my_cblas_float_matmul_op(eval::InterpretedFunction::State &state, uint64_t state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); } -template <bool lhs_common_inner, bool rhs_common_inner> -struct MyMatMulOp { - template <typename LCT, typename RCT> - static auto get_fun() { return my_matmul_op<LCT,RCT,lhs_common_inner,rhs_common_inner>; } -}; - -template <bool lhs_common_inner, bool rhs_common_inner> -eval::InterpretedFunction::op_function my_select3(CellType lct, CellType rct) -{ - if (lct == rct) { - if (lct == ValueType::CellType::DOUBLE) { - return my_cblas_double_matmul_op<lhs_common_inner,rhs_common_inner>; - } - if (lct == ValueType::CellType::FLOAT) { - return my_cblas_float_matmul_op<lhs_common_inner,rhs_common_inner>; - } - } - return select_2<MyMatMulOp<lhs_common_inner,rhs_common_inner>>(lct, rct); -} - -template <bool lhs_common_inner> -eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct, - bool rhs_common_inner) -{ - if (rhs_common_inner) { - return my_select3<lhs_common_inner,true>(lct, rct); - } else { - return my_select3<lhs_common_inner,false>(lct, rct); - } -} - -eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, - bool lhs_common_inner, bool rhs_common_inner) -{ - if (lhs_common_inner) { - return my_select2<true>(lct, rct, rhs_common_inner); - } else { - return my_select2<false>(lct, rct, rhs_common_inner); - } -} - bool is_matrix(const ValueType &type) { return (type.is_dense() && (type.dimensions().size() == 2)); } @@ -160,6 +119,18 @@ const TensorFunction &create_matmul(const TensorFunction &a, const TensorFunctio } } +struct MyGetFun { + template<typename R1, typename R2, typename R3, typename R4> static auto invoke() { + if (std::is_same_v<R1,double> && std::is_same_v<R2,double>) { + return my_cblas_double_matmul_op<R3::value, R4::value>; + } else if (std::is_same_v<R1,float> && std::is_same_v<R2,float>) { + return my_cblas_float_matmul_op<R3::value, R4::value>; + } else { + return my_matmul_op<R1, R2, R3::value, R4::value>; + } + } +}; + } // namespace vespalib::tensor::<unnamed> DenseMatMulFunction::Self::Self(const eval::ValueType &result_type_in, @@ -197,9 +168,11 @@ DenseMatMulFunction::~DenseMatMulFunction() = default; eval::InterpretedFunction::Instruction DenseMatMulFunction::compile_self(const TensorEngine &, Stash &stash) const { + using MyTypify = TypifyValue<eval::TypifyCellType,TypifyBool>; Self &self = stash.create<Self>(result_type(), _lhs_size, _common_size, _rhs_size); - auto op = my_select(lhs().result_type().cell_type(), rhs().result_type().cell_type(), - _lhs_common_inner, _rhs_common_inner); + auto op = typify_invoke<4,MyTypify,MyGetFun>( + lhs().result_type().cell_type(), rhs().result_type().cell_type(), + _lhs_common_inner, _rhs_common_inner); return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self)); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp new file mode 100644 index 00000000000..925627c5684 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp @@ -0,0 +1,123 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_number_join_function.h" +#include "dense_tensor_view.h" +#include <vespa/vespalib/util/typify.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/inline_operation.h> + +namespace vespalib::tensor { + +using vespalib::ArrayRef; + +using eval::Value; +using eval::ValueType; +using eval::TensorFunction; +using eval::TensorEngine; +using eval::TypifyCellType; +using eval::as; + +using namespace eval::operation; +using namespace eval::tensor_function; + +using Primary = DenseNumberJoinFunction::Primary; + +using op_function = eval::InterpretedFunction::op_function; +using Instruction = eval::InterpretedFunction::Instruction; +using State = eval::InterpretedFunction::State; + +namespace { + +template <typename CT, bool inplace> +ArrayRef<CT> make_dst_cells(ConstArrayRef<CT> src_cells, Stash &stash) { + if (inplace) { + return unconstify(src_cells); + } else { + return stash.create_array<CT>(src_cells.size()); + } +} + +template <typename CT, typename Fun, bool inplace, bool swap> +void my_number_join_op(State &state, uint64_t param) { + using OP = typename std::conditional<swap,SwapArgs2<Fun>,Fun>::type; + OP my_op((join_fun_t)param); + const Value &tensor = state.peek(swap ? 0 : 1); + CT number = state.peek(swap ? 1 : 0).as_double(); + auto src_cells = DenseTensorView::typify_cells<CT>(tensor); + auto dst_cells = make_dst_cells<CT, inplace>(src_cells, state.stash); + apply_op2_vec_num(dst_cells.begin(), src_cells.begin(), number, dst_cells.size(), my_op); + if (inplace) { + state.pop_pop_push(tensor); + } else { + state.pop_pop_push(state.stash.create<DenseTensorView>(tensor.type(), TypedCells(dst_cells))); + } +} + +//----------------------------------------------------------------------------- + +struct MyGetFun { + template <typename R1, typename R2, typename R3, typename R4> static auto invoke() { + return my_number_join_op<R1, R2, R3::value, R4::value>; + } +}; + +using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool>; + +bool is_dense(const TensorFunction &tf) { return tf.result_type().is_dense(); } +bool is_double(const TensorFunction &tf) { return tf.result_type().is_double(); } +ValueType::CellType cell_type(const TensorFunction &tf) { return tf.result_type().cell_type(); } + +} // namespace vespalib::tensor::<unnamed> + +//----------------------------------------------------------------------------- + +DenseNumberJoinFunction::DenseNumberJoinFunction(const ValueType &result_type, + const TensorFunction &lhs, + const TensorFunction &rhs, + join_fun_t function_in, + Primary primary_in) + : Join(result_type, lhs, rhs, function_in), + _primary(primary_in) +{ +} + +DenseNumberJoinFunction::~DenseNumberJoinFunction() = default; + +bool +DenseNumberJoinFunction::inplace() const +{ + if (_primary == Primary::LHS) { + return lhs().result_is_mutable(); + } else { + return rhs().result_is_mutable(); + } +} + +Instruction +DenseNumberJoinFunction::compile_self(const TensorEngine &, Stash &) const +{ + auto op = typify_invoke<4,MyTypify,MyGetFun>(result_type().cell_type(), function(), + inplace(), (_primary == Primary::RHS)); + static_assert(sizeof(uint64_t) == sizeof(function())); + return Instruction(op, (uint64_t)(function())); +} + +const TensorFunction & +DenseNumberJoinFunction::optimize(const TensorFunction &expr, Stash &stash) +{ + if (auto join = as<Join>(expr)) { + const TensorFunction &lhs = join->lhs(); + const TensorFunction &rhs = join->rhs(); + if (is_dense(lhs) && is_double(rhs)) { + assert(cell_type(expr) == cell_type(lhs)); + return stash.create<DenseNumberJoinFunction>(join->result_type(), lhs, rhs, join->function(), Primary::LHS); + } else if (is_double(lhs) && is_dense(rhs)) { + assert(cell_type(expr) == cell_type(rhs)); + return stash.create<DenseNumberJoinFunction>(join->result_type(), lhs, rhs, join->function(), Primary::RHS); + } + } + return expr; +} + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.h b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.h new file mode 100644 index 00000000000..1a9e92b860f --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.h @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::tensor { + +/** + * Tensor function for join operations between dense tensors and + * numbers. + **/ +class DenseNumberJoinFunction : public eval::tensor_function::Join +{ +public: + enum class Primary : uint8_t { LHS, RHS }; + using join_fun_t = ::vespalib::eval::tensor_function::join_fun_t; +private: + Primary _primary; +public: + DenseNumberJoinFunction(const eval::ValueType &result_type, + const TensorFunction &lhs, + const TensorFunction &rhs, + join_fun_t function_in, + Primary primary_in); + ~DenseNumberJoinFunction() override; + Primary primary() const { return _primary; } + bool inplace() const; + eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override; + static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash); +}; + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.cpp b/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.cpp new file mode 100644 index 00000000000..f78c23c80ac --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.cpp @@ -0,0 +1,38 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_pow_as_map_optimizer.h" +#include "dense_simple_map_function.h" +#include <vespa/eval/eval/operation.h> + +namespace vespalib::tensor { + +using eval::TensorFunction; +using eval::as; + +using namespace eval::tensor_function; +using namespace eval::operation; + +const TensorFunction & +DensePowAsMapOptimizer::optimize(const TensorFunction &expr, Stash &stash) +{ + if (auto join = as<Join>(expr)) { + const TensorFunction &lhs = join->lhs(); + const TensorFunction &rhs = join->rhs(); + if ((join->function() == Pow::f) && + lhs.result_type().is_dense() && + rhs.result_type().is_double()) + { + if (auto const_value = as<ConstValue>(rhs)) { + if (const_value->value().as_double() == 2.0) { + return map(lhs, Square::f, stash); + } + if (const_value->value().as_double() == 3.0) { + return map(lhs, Cube::f, stash); + } + } + } + } + return expr; +} + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.h b/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.h new file mode 100644 index 00000000000..4849a10c070 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_pow_as_map_optimizer.h @@ -0,0 +1,18 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::tensor { + +/** + * Tensor function optimizer for converting join expressions on the + * form 'join(tensor,<small integer constant>,f(x,y)(pow(x,y))' to + * expressions on the form 'map(tensor,f(x)(x*x...))'. + **/ +struct DensePowAsMapOptimizer { + static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash); +}; + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp new file mode 100644 index 00000000000..d45e0d936a9 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.cpp @@ -0,0 +1,141 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_simple_expand_function.h" +#include "dense_tensor_view.h" +#include <vespa/vespalib/objects/objectvisitor.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/inline_operation.h> +#include <vespa/vespalib/util/typify.h> +#include <optional> +#include <algorithm> + +namespace vespalib::tensor { + +using vespalib::ArrayRef; + +using eval::Value; +using eval::ValueType; +using eval::TensorFunction; +using eval::TensorEngine; +using eval::TypifyCellType; +using eval::as; + +using namespace eval::operation; +using namespace eval::tensor_function; + +using Inner = DenseSimpleExpandFunction::Inner; + +using op_function = eval::InterpretedFunction::op_function; +using Instruction = eval::InterpretedFunction::Instruction; +using State = eval::InterpretedFunction::State; + +namespace { + +struct ExpandParams { + const ValueType &result_type; + size_t result_size; + join_fun_t function; + ExpandParams(const ValueType &result_type_in, size_t result_size_in, join_fun_t function_in) + : result_type(result_type_in), result_size(result_size_in), function(function_in) {} +}; + +template <typename LCT, typename RCT, typename Fun, bool rhs_inner> +void my_simple_expand_op(State &state, uint64_t param) { + using ICT = typename std::conditional<rhs_inner,RCT,LCT>::type; + using OCT = typename std::conditional<rhs_inner,LCT,RCT>::type; + using DCT = typename eval::UnifyCellTypes<ICT,OCT>::type; + using OP = typename std::conditional<rhs_inner,SwapArgs2<Fun>,Fun>::type; + const ExpandParams ¶ms = *(ExpandParams*)param; + OP my_op(params.function); + auto inner_cells = DenseTensorView::typify_cells<ICT>(state.peek(rhs_inner ? 0 : 1)); + auto outer_cells = DenseTensorView::typify_cells<OCT>(state.peek(rhs_inner ? 1 : 0)); + auto dst_cells = state.stash.create_array<DCT>(params.result_size); + DCT *dst = dst_cells.begin(); + for (OCT outer_cell: outer_cells) { + apply_op2_vec_num(dst, inner_cells.begin(), outer_cell, inner_cells.size(), my_op); + dst += inner_cells.size(); + } + state.pop_pop_push(state.stash.create<DenseTensorView>(params.result_type, TypedCells(dst_cells))); +} + +//----------------------------------------------------------------------------- + +struct MyGetFun { + template <typename R1, typename R2, typename R3, typename R4> static auto invoke() { + return my_simple_expand_op<R1, R2, R3, R4::value>; + } +}; + +using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool>; + +//----------------------------------------------------------------------------- + +std::vector<ValueType::Dimension> strip_trivial(const std::vector<ValueType::Dimension> &dim_list) { + std::vector<ValueType::Dimension> result; + std::copy_if(dim_list.begin(), dim_list.end(), std::back_inserter(result), + [](const auto &dim){ return (dim.size != 1); }); + return result; +} + +std::optional<Inner> detect_simple_expand(const TensorFunction &lhs, const TensorFunction &rhs) { + std::vector<ValueType::Dimension> a = strip_trivial(lhs.result_type().dimensions()); + std::vector<ValueType::Dimension> b = strip_trivial(rhs.result_type().dimensions()); + if (a.empty() || b.empty()) { + return std::nullopt; + } else if (a.back().name < b.front().name) { + return Inner::RHS; + } else if (b.back().name < a.front().name) { + return Inner::LHS; + } else { + return std::nullopt; + } +} + +} // namespace vespalib::tensor::<unnamed> + +//----------------------------------------------------------------------------- + +DenseSimpleExpandFunction::DenseSimpleExpandFunction(const ValueType &result_type, + const TensorFunction &lhs, + const TensorFunction &rhs, + join_fun_t function_in, + Inner inner_in) + : Join(result_type, lhs, rhs, function_in), + _inner(inner_in) +{ +} + +DenseSimpleExpandFunction::~DenseSimpleExpandFunction() = default; + +Instruction +DenseSimpleExpandFunction::compile_self(const TensorEngine &, Stash &stash) const +{ + size_t result_size = result_type().dense_subspace_size(); + const ExpandParams ¶ms = stash.create<ExpandParams>(result_type(), result_size, function()); + auto op = typify_invoke<4,MyTypify,MyGetFun>(lhs().result_type().cell_type(), + rhs().result_type().cell_type(), + function(), (_inner == Inner::RHS)); + static_assert(sizeof(uint64_t) == sizeof(¶ms)); + return Instruction(op, (uint64_t)(¶ms)); +} + +const TensorFunction & +DenseSimpleExpandFunction::optimize(const TensorFunction &expr, Stash &stash) +{ + if (auto join = as<Join>(expr)) { + const TensorFunction &lhs = join->lhs(); + const TensorFunction &rhs = join->rhs(); + if (lhs.result_type().is_dense() && rhs.result_type().is_dense()) { + if (std::optional<Inner> inner = detect_simple_expand(lhs, rhs)) { + assert(expr.result_type().dense_subspace_size() == + (lhs.result_type().dense_subspace_size() * + rhs.result_type().dense_subspace_size())); + return stash.create<DenseSimpleExpandFunction>(join->result_type(), lhs, rhs, join->function(), inner.value()); + } + } + } + return expr; +} + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.h b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.h new file mode 100644 index 00000000000..b4b303901a7 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_expand_function.h @@ -0,0 +1,38 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::tensor { + +/** + * Tensor function for simple expanding join operations on dense + * tensors. An expanding operation is a join between tensors resulting + * in a larger tensor where the input tensors have no matching + * dimensions (trivial dimensions are ignored). A simple expanding + * operation is an expanding operation where all the dimensions of one + * input is nested inside all the dimensions from the other input + * within the result (trivial dimensions are again ignored). + **/ +class DenseSimpleExpandFunction : public eval::tensor_function::Join +{ + using Super = eval::tensor_function::Join; +public: + enum class Inner : uint8_t { LHS, RHS }; + using join_fun_t = ::vespalib::eval::tensor_function::join_fun_t; +private: + Inner _inner; +public: + DenseSimpleExpandFunction(const eval::ValueType &result_type, + const TensorFunction &lhs, + const TensorFunction &rhs, + join_fun_t function_in, + Inner inner_in); + ~DenseSimpleExpandFunction() override; + Inner inner() const { return _inner; } + eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override; + static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash); +}; + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp new file mode 100644 index 00000000000..5f8fbcac9bb --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp @@ -0,0 +1,231 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_simple_join_function.h" +#include "dense_tensor_view.h" +#include <vespa/vespalib/objects/objectvisitor.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/inline_operation.h> +#include <vespa/vespalib/util/typify.h> +#include <optional> +#include <algorithm> + +namespace vespalib::tensor { + +using vespalib::ArrayRef; + +using eval::Value; +using eval::ValueType; +using eval::TensorFunction; +using eval::TensorEngine; +using eval::TypifyCellType; +using eval::as; + +using namespace eval::operation; +using namespace eval::tensor_function; + +using Primary = DenseSimpleJoinFunction::Primary; +using Overlap = DenseSimpleJoinFunction::Overlap; + +using op_function = eval::InterpretedFunction::op_function; +using Instruction = eval::InterpretedFunction::Instruction; +using State = eval::InterpretedFunction::State; + +namespace { + +struct TypifyOverlap { + template <Overlap VALUE> using Result = TypifyResultValue<Overlap, VALUE>; + template <typename F> static decltype(auto) resolve(Overlap value, F &&f) { + switch (value) { + case Overlap::INNER: return f(Result<Overlap::INNER>()); + case Overlap::OUTER: return f(Result<Overlap::OUTER>()); + case Overlap::FULL: return f(Result<Overlap::FULL>()); + } + abort(); + } +}; + +struct JoinParams { + const ValueType &result_type; + size_t factor; + join_fun_t function; + JoinParams(const ValueType &result_type_in, size_t factor_in, join_fun_t function_in) + : result_type(result_type_in), factor(factor_in), function(function_in) {} +}; + +template <typename OCT, bool pri_mut, typename PCT> +ArrayRef<OCT> make_dst_cells(ConstArrayRef<PCT> pri_cells, Stash &stash) { + if constexpr (pri_mut && std::is_same<PCT,OCT>::value) { + return unconstify(pri_cells); + } else { + return stash.create_array<OCT>(pri_cells.size()); + } +} + +template <typename LCT, typename RCT, typename Fun, bool swap, Overlap overlap, bool pri_mut> +void my_simple_join_op(State &state, uint64_t param) { + using PCT = typename std::conditional<swap,RCT,LCT>::type; + using SCT = typename std::conditional<swap,LCT,RCT>::type; + using OCT = typename eval::UnifyCellTypes<PCT,SCT>::type; + using OP = typename std::conditional<swap,SwapArgs2<Fun>,Fun>::type; + const JoinParams ¶ms = *(JoinParams*)param; + OP my_op(params.function); + auto pri_cells = DenseTensorView::typify_cells<PCT>(state.peek(swap ? 0 : 1)); + auto sec_cells = DenseTensorView::typify_cells<SCT>(state.peek(swap ? 1 : 0)); + auto dst_cells = make_dst_cells<OCT, pri_mut>(pri_cells, state.stash); + if (overlap == Overlap::FULL) { + apply_op2_vec_vec(dst_cells.begin(), pri_cells.begin(), sec_cells.begin(), dst_cells.size(), my_op); + } else if (overlap == Overlap::OUTER) { + size_t offset = 0; + size_t factor = params.factor; + for (SCT cell: sec_cells) { + apply_op2_vec_num(dst_cells.begin() + offset, pri_cells.begin() + offset, cell, factor, my_op); + offset += factor; + } + } else { + assert(overlap == Overlap::INNER); + size_t offset = 0; + size_t factor = params.factor; + for (size_t i = 0; i < factor; ++i) { + apply_op2_vec_vec(dst_cells.begin() + offset, pri_cells.begin() + offset, sec_cells.begin(), sec_cells.size(), my_op); + offset += sec_cells.size(); + } + } + state.pop_pop_push(state.stash.create<DenseTensorView>(params.result_type, TypedCells(dst_cells))); +} + +//----------------------------------------------------------------------------- + +struct MyGetFun { + template <typename R1, typename R2, typename R3, typename R4, typename R5, typename R6> static auto invoke() { + return my_simple_join_op<R1, R2, R3, R4::value, R5::value, R6::value>; + } +}; + +using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool,TypifyOverlap>; + +//----------------------------------------------------------------------------- + +bool can_use_as_output(const TensorFunction &fun, ValueType::CellType result_cell_type) { + return (fun.result_is_mutable() && (fun.result_type().cell_type() == result_cell_type)); +} + +Primary select_primary(const TensorFunction &lhs, const TensorFunction &rhs, ValueType::CellType result_cell_type) { + size_t lhs_size = lhs.result_type().dense_subspace_size(); + size_t rhs_size = rhs.result_type().dense_subspace_size(); + if (lhs_size > rhs_size) { + return Primary::LHS; + } else if (rhs_size > lhs_size) { + return Primary::RHS; + } else { + bool can_write_lhs = can_use_as_output(lhs, result_cell_type); + bool can_write_rhs = can_use_as_output(rhs, result_cell_type); + if (can_write_lhs && !can_write_rhs) { + return Primary::LHS; + } else { + // prefer using rhs as output due to write recency + return Primary::RHS; + } + } +} + +std::vector<ValueType::Dimension> strip_trivial(const std::vector<ValueType::Dimension> &dim_list) { + std::vector<ValueType::Dimension> result; + std::copy_if(dim_list.begin(), dim_list.end(), std::back_inserter(result), + [](const auto &dim){ return (dim.size != 1); }); + return result; +} + +std::optional<Overlap> detect_overlap(const TensorFunction &primary, const TensorFunction &secondary) { + std::vector<ValueType::Dimension> a = strip_trivial(primary.result_type().dimensions()); + std::vector<ValueType::Dimension> b = strip_trivial(secondary.result_type().dimensions()); + if (b.size() > a.size()) { + return std::nullopt; + } else if (b == a) { + return Overlap::FULL; + } else if (std::equal(b.begin(), b.end(), a.begin())) { + // prefer OUTER to INNER (for empty b) due to loop nesting + return Overlap::OUTER; + } else if (std::equal(b.rbegin(), b.rend(), a.rbegin())) { + return Overlap::INNER; + } else { + return std::nullopt; + } +} + +std::optional<Overlap> detect_overlap(const TensorFunction &lhs, const TensorFunction &rhs, Primary primary) { + return (primary == Primary::LHS) ? detect_overlap(lhs, rhs) : detect_overlap(rhs, lhs); +} + +} // namespace vespalib::tensor::<unnamed> + +//----------------------------------------------------------------------------- + +DenseSimpleJoinFunction::DenseSimpleJoinFunction(const ValueType &result_type, + const TensorFunction &lhs, + const TensorFunction &rhs, + join_fun_t function_in, + Primary primary_in, + Overlap overlap_in) + : Join(result_type, lhs, rhs, function_in), + _primary(primary_in), + _overlap(overlap_in) +{ +} + +DenseSimpleJoinFunction::~DenseSimpleJoinFunction() = default; + +bool +DenseSimpleJoinFunction::primary_is_mutable() const +{ + if (_primary == Primary::LHS) { + return lhs().result_is_mutable(); + } else { + return rhs().result_is_mutable(); + } +} + +size_t +DenseSimpleJoinFunction::factor() const +{ + const TensorFunction &p = (_primary == Primary::LHS) ? lhs() : rhs(); + const TensorFunction &s = (_primary == Primary::LHS) ? rhs() : lhs(); + size_t a = p.result_type().dense_subspace_size(); + size_t b = s.result_type().dense_subspace_size(); + assert((a % b) == 0); + return (a / b); +} + +Instruction +DenseSimpleJoinFunction::compile_self(const TensorEngine &, Stash &stash) const +{ + const JoinParams ¶ms = stash.create<JoinParams>(result_type(), factor(), function()); + auto op = typify_invoke<6,MyTypify,MyGetFun>(lhs().result_type().cell_type(), + rhs().result_type().cell_type(), + function(), (_primary == Primary::RHS), + _overlap, primary_is_mutable()); + static_assert(sizeof(uint64_t) == sizeof(¶ms)); + return Instruction(op, (uint64_t)(¶ms)); +} + +const TensorFunction & +DenseSimpleJoinFunction::optimize(const TensorFunction &expr, Stash &stash) +{ + if (auto join = as<Join>(expr)) { + const TensorFunction &lhs = join->lhs(); + const TensorFunction &rhs = join->rhs(); + if (lhs.result_type().is_dense() && rhs.result_type().is_dense()) { + Primary primary = select_primary(lhs, rhs, join->result_type().cell_type()); + std::optional<Overlap> overlap = detect_overlap(lhs, rhs, primary); + if (overlap.has_value()) { + const TensorFunction &ptf = (primary == Primary::LHS) ? lhs : rhs; + assert(ptf.result_type().dense_subspace_size() == join->result_type().dense_subspace_size()); + return stash.create<DenseSimpleJoinFunction>(join->result_type(), lhs, rhs, join->function(), + primary, overlap.value()); + } + } + } + return expr; +} + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.h b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.h new file mode 100644 index 00000000000..b4b52fcb8ab --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.h @@ -0,0 +1,38 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::tensor { + +/** + * Tensor function for simple join operations on dense tensors. + **/ +class DenseSimpleJoinFunction : public eval::tensor_function::Join +{ + using Super = eval::tensor_function::Join; +public: + enum class Primary : uint8_t { LHS, RHS }; + enum class Overlap : uint8_t { INNER, OUTER, FULL }; + using join_fun_t = ::vespalib::eval::tensor_function::join_fun_t; +private: + Primary _primary; + Overlap _overlap; +public: + DenseSimpleJoinFunction(const eval::ValueType &result_type, + const TensorFunction &lhs, + const TensorFunction &rhs, + join_fun_t function_in, + Primary primary_in, + Overlap overlap_in); + ~DenseSimpleJoinFunction() override; + Primary primary() const { return _primary; } + Overlap overlap() const { return _overlap; } + bool primary_is_mutable() const; + size_t factor() const; + eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override; + static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash); +}; + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp new file mode 100644 index 00000000000..b5f46fca70c --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.cpp @@ -0,0 +1,93 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dense_simple_map_function.h" +#include "dense_tensor_view.h" +#include <vespa/vespalib/util/typify.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/inline_operation.h> + +namespace vespalib::tensor { + +using vespalib::ArrayRef; + +using eval::Value; +using eval::ValueType; +using eval::TensorFunction; +using eval::TensorEngine; +using eval::TypifyCellType; +using eval::as; + +using namespace eval::operation; +using namespace eval::tensor_function; + +using op_function = eval::InterpretedFunction::op_function; +using Instruction = eval::InterpretedFunction::Instruction; +using State = eval::InterpretedFunction::State; + +namespace { + +template <typename CT, bool inplace> +ArrayRef<CT> make_dst_cells(ConstArrayRef<CT> src_cells, Stash &stash) { + if (inplace) { + return unconstify(src_cells); + } else { + return stash.create_array<CT>(src_cells.size()); + } +} + +template <typename CT, typename Fun, bool inplace> +void my_simple_map_op(State &state, uint64_t param) { + Fun my_fun((map_fun_t)param); + auto const &child = state.peek(0); + auto src_cells = DenseTensorView::typify_cells<CT>(child); + auto dst_cells = make_dst_cells<CT, inplace>(src_cells, state.stash); + apply_op1_vec(dst_cells.begin(), src_cells.begin(), dst_cells.size(), my_fun); + if (!inplace) { + state.pop_push(state.stash.create<DenseTensorView>(child.type(), TypedCells(dst_cells))); + } +} + +//----------------------------------------------------------------------------- + +struct MyGetFun { + template <typename R1, typename R2, typename R3> static auto invoke() { + return my_simple_map_op<R1, R2, R3::value>; + } +}; + +using MyTypify = TypifyValue<TypifyCellType,TypifyOp1,TypifyBool>; + +} // namespace vespalib::tensor::<unnamed> + +//----------------------------------------------------------------------------- + +DenseSimpleMapFunction::DenseSimpleMapFunction(const ValueType &result_type, + const TensorFunction &child, + map_fun_t function_in) + : Map(result_type, child, function_in) +{ +} + +DenseSimpleMapFunction::~DenseSimpleMapFunction() = default; + +Instruction +DenseSimpleMapFunction::compile_self(const TensorEngine &, Stash &) const +{ + auto op = typify_invoke<3,MyTypify,MyGetFun>(result_type().cell_type(), function(), inplace()); + static_assert(sizeof(uint64_t) == sizeof(function())); + return Instruction(op, (uint64_t)(function())); +} + +const TensorFunction & +DenseSimpleMapFunction::optimize(const TensorFunction &expr, Stash &stash) +{ + if (auto map = as<Map>(expr)) { + if (map->child().result_type().is_dense()) { + return stash.create<DenseSimpleMapFunction>(map->result_type(), map->child(), map->function()); + } + } + return expr; +} + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.h b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.h new file mode 100644 index 00000000000..daf35a44adc --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_map_function.h @@ -0,0 +1,25 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::tensor { + +/** + * Tensor function for simple map operations on dense tensors. + **/ +class DenseSimpleMapFunction : public eval::tensor_function::Map +{ +public: + using map_fun_t = ::vespalib::eval::tensor_function::map_fun_t; + DenseSimpleMapFunction(const eval::ValueType &result_type, + const TensorFunction &child, + map_fun_t function_in); + ~DenseSimpleMapFunction() override; + bool inplace() const { return child().result_is_mutable(); } + eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override; + static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash); +}; + +} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp index 663993b6c26..571bcb79c9f 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp @@ -2,6 +2,7 @@ #include "dense_single_reduce_function.h" #include "dense_tensor_view.h" +#include <vespa/vespalib/util/typify.h> #include <vespa/eval/eval/value.h> namespace vespalib::tensor { @@ -12,6 +13,8 @@ using eval::TensorEngine; using eval::TensorFunction; using eval::Value; using eval::ValueType; +using eval::TypifyCellType; +using eval::TypifyAggr; using eval::as; using namespace eval::tensor_function; @@ -66,28 +69,13 @@ void my_single_reduce_op(InterpretedFunction::State &state, uint64_t param) { state.pop_push(state.stash.create<DenseTensorView>(params.result_type, TypedCells(dst_cells))); } -template <typename CT> -InterpretedFunction::op_function my_select_2(Aggr aggr) { - switch (aggr) { - case Aggr::AVG: return my_single_reduce_op<CT, Avg<CT>>; - case Aggr::COUNT: return my_single_reduce_op<CT, Count<CT>>; - case Aggr::PROD: return my_single_reduce_op<CT, Prod<CT>>; - case Aggr::SUM: return my_single_reduce_op<CT, Sum<CT>>; - case Aggr::MAX: return my_single_reduce_op<CT, Max<CT>>; - case Aggr::MIN: return my_single_reduce_op<CT, Min<CT>>; +struct MyGetFun { + template <typename R1, typename R2> static auto invoke() { + return my_single_reduce_op<R1, typename R2::template templ<R1>>; } - abort(); -} +}; -InterpretedFunction::op_function my_select(CellType cell_type, Aggr aggr) { - if (cell_type == ValueType::CellType::DOUBLE) { - return my_select_2<double>(aggr); - } - if (cell_type == ValueType::CellType::FLOAT) { - return my_select_2<float>(aggr); - } - abort(); -} +using MyTypify = TypifyValue<TypifyCellType,TypifyAggr>; bool check_input_type(const ValueType &type) { return (type.is_dense() && ((type.cell_type() == CellType::FLOAT) || (type.cell_type() == CellType::DOUBLE))); @@ -109,7 +97,7 @@ DenseSingleReduceFunction::~DenseSingleReduceFunction() = default; InterpretedFunction::Instruction DenseSingleReduceFunction::compile_self(const TensorEngine &, Stash &stash) const { - auto op = my_select(result_type().cell_type(), _aggr); + auto op = typify_invoke<2,MyTypify,MyGetFun>(result_type().cell_type(), _aggr); auto ¶ms = stash.create<Params>(result_type(), child().result_type(), _dim_idx); static_assert(sizeof(uint64_t) == sizeof(¶ms)); return InterpretedFunction::Instruction(op, (uint64_t)¶ms); diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp index 3533ab20175..7e887d4df34 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp @@ -34,7 +34,7 @@ void my_tensor_create_op(eval::InterpretedFunction::State &state, uint64_t param struct MyTensorCreateOp { template <typename CT> - static auto get_fun() { return my_tensor_create_op<CT>; } + static auto invoke() { return my_tensor_create_op<CT>; } }; size_t get_index(const TensorSpec::Address &addr, const ValueType &type) { @@ -72,7 +72,9 @@ eval::InterpretedFunction::Instruction DenseTensorCreateFunction::compile_self(const TensorEngine &, Stash &) const { static_assert(sizeof(uint64_t) == sizeof(&_self)); - auto op = select_1<MyTensorCreateOp>(result_type().cell_type()); + + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<1,MyTypify,MyTensorCreateOp>(result_type().cell_type()); return eval::InterpretedFunction::Instruction(op, (uint64_t)&_self); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp index 5cb1cbfd88f..16c0b01b169 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp @@ -44,7 +44,7 @@ void my_tensor_peek_op(eval::InterpretedFunction::State &state, uint64_t param) struct MyTensorPeekOp { template <typename CT> - static auto get_fun() { return my_tensor_peek_op<CT>; } + static auto invoke() { return my_tensor_peek_op<CT>; } }; } // namespace vespalib::tensor::<unnamed> @@ -71,7 +71,8 @@ eval::InterpretedFunction::Instruction DenseTensorPeekFunction::compile_self(const TensorEngine &, Stash &) const { static_assert(sizeof(uint64_t) == sizeof(&_spec)); - auto op = select_1<MyTensorPeekOp>(_children[0].get().result_type().cell_type()); + using MyTypify = eval::TypifyCellType; + auto op = typify_invoke<1,MyTypify,MyTensorPeekOp>(_children[0].get().result_type().cell_type()); return eval::InterpretedFunction::Instruction(op, (uint64_t)&_spec); } diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp index a0d63a1ce1e..968308d69c9 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp @@ -76,33 +76,6 @@ void my_cblas_float_xw_product_op(eval::InterpretedFunction::State &state, uint6 state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells))); } -template <bool common_inner> -struct MyXWProductOp { - template <typename LCT, typename RCT> - static auto get_fun() { return my_xw_product_op<LCT,RCT,common_inner>; } -}; - -template <bool common_inner> -eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct) { - if (lct == rct) { - if (lct == ValueType::CellType::DOUBLE) { - return my_cblas_double_xw_product_op<common_inner>; - } - if (lct == ValueType::CellType::FLOAT) { - return my_cblas_float_xw_product_op<common_inner>; - } - } - return select_2<MyXWProductOp<common_inner>>(lct, rct); -} - -eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool common_inner) { - if (common_inner) { - return my_select2<true>(lct, rct); - } else { - return my_select2<false>(lct, rct); - } -} - bool isDenseTensor(const ValueType &type, size_t d) { return (type.is_dense() && (type.dimensions().size() == d)); } @@ -132,6 +105,18 @@ const TensorFunction &createDenseXWProduct(const ValueType &res, const TensorFun common_inner); } +struct MyXWProductOp { + template<typename R1, typename R2, typename R3> static auto invoke() { + if (std::is_same_v<R1,double> && std::is_same_v<R2,double>) { + return my_cblas_double_xw_product_op<R3::value>; + } else if (std::is_same_v<R1,float> && std::is_same_v<R2,float>) { + return my_cblas_float_xw_product_op<R3::value>; + } else { + return my_xw_product_op<R1, R2, R3::value>; + } + } +}; + } // namespace vespalib::tensor::<unnamed> DenseXWProductFunction::Self::Self(const eval::ValueType &result_type_in, @@ -160,8 +145,10 @@ eval::InterpretedFunction::Instruction DenseXWProductFunction::compile_self(const TensorEngine &, Stash &stash) const { Self &self = stash.create<Self>(result_type(), _vector_size, _result_size); - auto op = my_select(lhs().result_type().cell_type(), - rhs().result_type().cell_type(), _common_inner); + using MyTypify = TypifyValue<eval::TypifyCellType,vespalib::TypifyBool>; + auto op = typify_invoke<3,MyTypify,MyXWProductOp>(lhs().result_type().cell_type(), + rhs().result_type().cell_type(), + _common_inner); return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self)); } diff --git a/eval/src/vespa/eval/tensor/dense/index_lookup_table.cpp b/eval/src/vespa/eval/tensor/dense/index_lookup_table.cpp new file mode 100644 index 00000000000..99a05762a34 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/index_lookup_table.cpp @@ -0,0 +1,98 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "index_lookup_table.h" +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/key_gen.h> +#include <vespa/eval/eval/llvm/compiled_function.h> + +namespace vespalib::tensor { + +using eval::CompiledFunction; +using eval::Function; +using eval::PassParams; +using eval::ValueType; + +namespace { + +bool step_params(std::vector<double> ¶ms, const ValueType &type) { + const auto &dims = type.dimensions(); + for (size_t idx = params.size(); idx-- > 0; ) { + if (size_t(params[idx] += 1.0) < dims[idx].size) { + return true; + } else { + params[idx] = 0.0; + } + } + return false; +} + +std::vector<uint32_t> make_index_table(const Function &idx_fun, const ValueType &type) { + std::vector<uint32_t> result; + result.reserve(type.dense_subspace_size()); + std::vector<double> params(type.dimensions().size(), 0.0); + CompiledFunction cfun(idx_fun, PassParams::ARRAY); + auto fun = cfun.get_function(); + do { + result.push_back(uint32_t(fun(¶ms[0]))); + } while(step_params(params, type)); + assert(result.size() == type.dense_subspace_size()); + return result; +} + +} + +std::mutex IndexLookupTable::_lock{}; +IndexLookupTable::Map IndexLookupTable::_cached{}; + +size_t +IndexLookupTable::num_cached() +{ + std::lock_guard<std::mutex> guard(_lock); + return _cached.size(); +} + +size_t +IndexLookupTable::count_refs() +{ + std::lock_guard<std::mutex> guard(_lock); + size_t refs = 0; + for (const auto &entry: _cached) { + refs += entry.second.num_refs; + } + return refs; +} + +IndexLookupTable::Token::UP +IndexLookupTable::create(const Function &idx_fun, const ValueType &type) +{ + assert(type.is_dense()); + assert(idx_fun.num_params() == type.dimensions().size()); + assert(!CompiledFunction::detect_issues(idx_fun)); + auto key = type.to_spec() + gen_key(idx_fun, PassParams::ARRAY); + { + std::lock_guard<std::mutex> guard(_lock); + auto pos = _cached.find(key); + if (pos != _cached.end()) { + ++(pos->second.num_refs); + return std::make_unique<Token>(pos, Token::ctor_tag()); + } + } + // avoid holding the lock while making the index table + auto table = make_index_table(idx_fun, type); + { + std::lock_guard<std::mutex> guard(_lock); + auto pos = _cached.find(key); + if (pos != _cached.end()) { + ++(pos->second.num_refs); + // in case of a race; return the same table for all callers + return std::make_unique<Token>(pos, Token::ctor_tag()); + } else { + auto res = _cached.emplace(std::move(key), Value::ctor_tag()); + assert(res.second); + res.first->second.data = std::move(table); + return std::make_unique<Token>(res.first, Token::ctor_tag()); + } + } +} + +} diff --git a/eval/src/vespa/eval/tensor/dense/index_lookup_table.h b/eval/src/vespa/eval/tensor/dense/index_lookup_table.h new file mode 100644 index 00000000000..ee5776ef5ca --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/index_lookup_table.h @@ -0,0 +1,69 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <mutex> +#include <vector> +#include <map> + +namespace vespalib::eval { +class ValueType; +class Function; +} + +namespace vespalib::tensor { + +/** + * Pre-computed index tables used by DenseLambdaPeekFunction. The + * underlying index tables are shared between operations. + **/ +class IndexLookupTable +{ +private: + using Key = vespalib::string; + struct Value { + size_t num_refs; + std::vector<uint32_t> data; + struct ctor_tag {}; + Value(ctor_tag) : num_refs(1), data() {} + }; + using Map = std::map<Key,Value>; + static std::mutex _lock; + static Map _cached; + + static void release(Map::iterator entry) { + std::lock_guard<std::mutex> guard(_lock); + if (--(entry->second.num_refs) == 0) { + _cached.erase(entry); + } + } + +public: + /** + * A token represents shared ownership of a cached index lookup + * table. + **/ + class Token + { + private: + friend class IndexLookupTable; + struct ctor_tag {}; + IndexLookupTable::Map::iterator _entry; + public: + Token(Token &&) = delete; + Token(const Token &) = delete; + Token &operator=(Token &&) = delete; + Token &operator=(const Token &) = delete; + using UP = std::unique_ptr<Token>; + Token(IndexLookupTable::Map::iterator entry, ctor_tag) : _entry(entry) {} + const std::vector<uint32_t> &get() const { return _entry->second.data; } + ~Token() { IndexLookupTable::release(_entry); } + }; + IndexLookupTable() = delete; + static size_t num_cached(); + static size_t count_refs(); + static Token::UP create(const eval::Function &idx_fun, const eval::ValueType &type); +}; + +} diff --git a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp index 7a4b5917f00..57f727f7968 100644 --- a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp @@ -19,7 +19,7 @@ namespace { struct CallVectorFromDoubles { template <typename CT> static TypedCells - call(eval::InterpretedFunction::State &state, size_t numCells) { + invoke(eval::InterpretedFunction::State &state, size_t numCells) { ArrayRef<CT> outputCells = state.stash.create_array<CT>(numCells); for (size_t i = numCells; i-- > 0; ) { outputCells[i] = (CT) state.peek(0).as_double(); @@ -33,7 +33,8 @@ void my_vector_from_doubles_op(eval::InterpretedFunction::State &state, uint64_t const auto *self = (const VectorFromDoublesFunction::Self *)(param); CellType ct = self->resultType.cell_type(); size_t numCells = self->resultSize; - TypedCells cells = dispatch_0<CallVectorFromDoubles>(ct, state, numCells); + using MyTypify = eval::TypifyCellType; + TypedCells cells = typify_invoke<1,MyTypify,CallVectorFromDoubles>(ct, state, numCells); const Value &result = state.stash.create<DenseTensorView>(self->resultType, cells); state.stack.emplace_back(result); } diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp index 493e4af3caf..4d6cfb1c9af 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp @@ -92,7 +92,7 @@ DenseBinaryFormat::serialize(nbostream &stream, const DenseTensorView &tensor) struct CallDecodeCells { template <typename CT> static std::unique_ptr<DenseTensorView> - call(nbostream &stream, size_t numCells, ValueType &&newType) { + invoke(nbostream &stream, size_t numCells, ValueType &&newType) { std::vector<CT> newCells; newCells.reserve(numCells); decodeCells<CT>(stream, numCells, newCells); @@ -106,7 +106,8 @@ DenseBinaryFormat::deserialize(nbostream &stream, CellType cell_type) std::vector<Dimension> dimensions; size_t numCells = decodeDimensions(stream, dimensions); ValueType newType = ValueType::tensor_type(std::move(dimensions), cell_type); - return dispatch_0<CallDecodeCells>(cell_type, stream, numCells, std::move(newType)); + using MyTypify = eval::TypifyCellType; + return typify_invoke<1,MyTypify,CallDecodeCells>(cell_type, stream, numCells, std::move(newType)); } template <typename T> diff --git a/fastlib/src/vespa/fastlib/testsuite/suite.h b/fastlib/src/vespa/fastlib/testsuite/suite.h index 94e756387bf..ee8775518a2 100644 --- a/fastlib/src/vespa/fastlib/testsuite/suite.h +++ b/fastlib/src/vespa/fastlib/testsuite/suite.h @@ -68,6 +68,8 @@ #include <cassert> +namespace fast::testsuite { + class TestSuiteError; class Suite @@ -258,3 +260,7 @@ void Suite::Reset() m_tests[i]->Reset(); } } + +} + +using fast::testsuite::Suite; diff --git a/fastlib/src/vespa/fastlib/testsuite/test.cpp b/fastlib/src/vespa/fastlib/testsuite/test.cpp index d7541d163e7..a29e5053c98 100644 --- a/fastlib/src/vespa/fastlib/testsuite/test.cpp +++ b/fastlib/src/vespa/fastlib/testsuite/test.cpp @@ -2,6 +2,8 @@ #include "test.h" +namespace fast::testsuite { + Test::Test(std::ostream* osptr, const char*name) : m_osptr(osptr), name_(name), @@ -135,3 +137,5 @@ long Test::Report(int padSpaces) const } return m_nFail; } + +} diff --git a/fastlib/src/vespa/fastlib/testsuite/test.h b/fastlib/src/vespa/fastlib/testsuite/test.h index ea17e64dba2..96f38c1577b 100644 --- a/fastlib/src/vespa/fastlib/testsuite/test.h +++ b/fastlib/src/vespa/fastlib/testsuite/test.h @@ -73,6 +73,8 @@ do_equality_test((lhs), (rhs), #lhs, __FILE__, __LINE__) #define _fail(str) do_fail((str), __FILE__, __LINE__) +namespace fast::testsuite { + class Test { public: @@ -143,3 +145,7 @@ bool Test::do_equality_test(const t1& lhs, const t2& rhs, const char* lbl, } return false; } + +} + +using fast::testsuite::Test; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index a68d2860874..67fcff2224b 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -24,7 +24,7 @@ import java.util.logging.Logger; * * @author hmusum */ -public class FileDownloader { +public class FileDownloader implements AutoCloseable { private final static Logger log = Logger.getLogger(FileDownloader.class.getName()); public static File defaultDownloadDirectory = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution")); @@ -34,7 +34,11 @@ public class FileDownloader { private final FileReferenceDownloader fileReferenceDownloader; public FileDownloader(ConnectionPool connectionPool) { - this(connectionPool, defaultDownloadDirectory , defaultDownloadDirectory , Duration.ofMinutes(15), Duration.ofSeconds(10)); + this(connectionPool, defaultDownloadDirectory ); + } + + public FileDownloader(ConnectionPool connectionPool, File downloadDirectory) { + this(connectionPool, downloadDirectory , downloadDirectory , Duration.ofMinutes(15), Duration.ofSeconds(10)); } FileDownloader(ConnectionPool connectionPool, File downloadDirectory, File tmpDirectory, Duration timeout, Duration sleepBetweenRetries) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index 5897a5ab58b..89c4f16e27b 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -51,6 +51,9 @@ public class FetchVector { */ VESPA_VERSION, + /** Email address of user - provided by auth0 in console. */ + CONSOLE_USER_EMAIL, + /** * Zone from ZoneId::value of the form environment.region. * diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 20e3216166f..8ec6174e8ae 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -3,13 +3,14 @@ package com.yahoo.vespa.flags; import com.yahoo.component.Vtag; import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.flags.custom.PreprovisionCapacity; +import com.yahoo.vespa.flags.custom.HostCapacity; import java.util.List; import java.util.Optional; import java.util.TreeMap; import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.CONSOLE_USER_EMAIL; import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; @@ -39,7 +40,8 @@ import static com.yahoo.vespa.flags.FetchVector.Dimension.ZONE_ID; public class Flags { private static volatile TreeMap<FlagId, FlagDefinition> flags = new TreeMap<>(); - public static final UnboundIntFlag DROP_CACHES = defineIntFlag("drop-caches", 3, + public static final UnboundIntFlag DROP_CACHES = defineIntFlag( + "drop-caches", 3, "The int value to write into /proc/sys/vm/drop_caches for each tick. " + "1 is page cache, 2 is dentries inodes, 3 is both page cache and dentries inodes, etc.", "Takes effect on next tick.", @@ -67,13 +69,6 @@ public class Flags { "Takes effect on next host admin tick.", HOSTNAME); - public static final UnboundBooleanFlag USE_NEW_VESPA_RPMS = defineFeatureFlag( - "use-new-vespa-rpms", false, - "Whether to use the new vespa-rpms YUM repo when upgrading/downgrading. The vespa-version " + - "when fetching the flag value is the wanted version of the host.", - "Takes effect when upgrading or downgrading host admin to a different version.", - HOSTNAME, NODE_TYPE, VESPA_VERSION); - public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag( "disabled-host-admin-tasks", List.of(), String.class, "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask) that should be skipped", @@ -108,12 +103,6 @@ public class Flags { "Takes effect on restart of Docker container", NODE_TYPE, APPLICATION_ID, HOSTNAME); - public static final UnboundBooleanFlag USE_ADAPTIVE_DISPATCH = defineFeatureFlag( - "use-adaptive-dispatch", false, - "Should adaptive dispatch be used over round robin", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundIntFlag REBOOT_INTERVAL_IN_DAYS = defineIntFlag( "reboot-interval-in-days", 30, "No reboots are scheduled 0x-1x reboot intervals after the previous reboot, while reboot is " + @@ -127,11 +116,12 @@ public class Flags { "Takes effect on the next run of RetiredExpirer.", HOSTNAME); - public static final UnboundListFlag<PreprovisionCapacity> PREPROVISION_CAPACITY = defineListFlag( - "preprovision-capacity", List.of(), PreprovisionCapacity.class, - "List of node resources and their count that should be present in zone to receive new deployments. When a " + - "preprovisioned is taken, new will be provisioned within next iteration of maintainer.", - "Takes effect on next iteration of HostProvisionMaintainer."); + public static final UnboundListFlag<HostCapacity> TARGET_CAPACITY = defineListFlag( + "preprovision-capacity", List.of(), HostCapacity.class, + "List of node resources and their count that should be provisioned." + + "In a dynamically provisioned zone this specifies the unallocated (i.e. pre-provisioned) capacity. " + + "Otherwise it specifies the total (unallocated or not) capacity.", + "Takes effect on next iteration of DynamicProvisioningMaintainer."); public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, @@ -139,11 +129,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundDoubleFlag DEFAULT_SOFT_START_SECONDS = defineDoubleFlag( - "default-soft-start-seconds", 0.0, - "Default number of seconds that a soft start shall use", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag DEFAULT_THREADPOOL_SIZE_FACTOR = defineDoubleFlag( "default-threadpool-size-factor", 0.0, "Default multiplication factor when computing maxthreads for main container threadpool based on available cores", @@ -154,15 +139,10 @@ public class Flags { "Default multiplication factor when computing queuesize for burst handling", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundDoubleFlag DEFAULT_TOP_K_PROBABILITY = defineDoubleFlag( - "default-top-k-probability", 1.0, - "Default probability that you will get the globally top K documents when merging many partitions.", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundIntFlag DEFAULT_NUM_RESPONSE_THREADS = defineIntFlag( - "default-num-response-threads", 0, - "Default number of threads used for processing put/update/remove responses.", + public static final UnboundStringFlag JVM_GC_OPTIONS = defineStringFlag( + "jvm-gc-options", "", + "Sets deafult jvm gc options", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -190,18 +170,12 @@ public class Flags { "Takes effect on next host-admin tick.", HOSTNAME); - public static final UnboundStringFlag ZOOKEEPER_SERVER_MAJOR_MINOR_VERSION = defineStringFlag( - "zookeeper-server-version", "3.5", - "The version of ZooKeeper server to use (major.minor, not full version)", + public static final UnboundStringFlag ZOOKEEPER_SERVER_VERSION = defineStringFlag( + "zookeeper-server-version", "3.5.6", + "ZooKeeper server version, a jar file zookeeper-server-<ZOOKEEPER_SERVER_VERSION>-jar-with-dependencies.jar must exist", "Takes effect on restart of Docker container", NODE_TYPE, APPLICATION_ID, HOSTNAME); - public static final UnboundStringFlag TLS_FOR_ZOOKEEPER_QUORUM_COMMUNICATION = defineStringFlag( - "tls-for-zookeeper-quorum-communication", "TLS_WITH_PORT_UNIFICATION", - "How to setup TLS for ZooKeeper quorum communication. Valid values are OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY", - "Takes effect on restart of config server", - NODE_TYPE, HOSTNAME); - public static final UnboundStringFlag TLS_FOR_ZOOKEEPER_CLIENT_SERVER_COMMUNICATION = defineStringFlag( "tls-for-zookeeper-client-server-communication", "OFF", "How to setup TLS for ZooKeeper client/server communication. Valid values are OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY", @@ -265,17 +239,49 @@ public class Flags { "Takes effect on next application redeploy", APPLICATION_ID); - public static final UnboundLongFlag CONFIGSERVER_SESSIONS_EXPIRY_INTERVAL_IN_DAYS = defineLongFlag( - "configserver-sessions-expiry-interval-in-days", 28, - "Expiry time for unused sessions in config server", - "Takes effect on next run of config server maintainer SessionsMaintainer"); - public static final UnboundBooleanFlag USE_CLOUD_INIT_FORMAT = defineFeatureFlag( "use-cloud-init", false, "Use the cloud-init format when provisioning hosts", "Takes effect immediately", ZONE_ID); + public static final UnboundBooleanFlag CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE = defineFeatureFlag( + "configserver-distribute-application-package", false, + "Whether the application package should be distributed to other config servers during a deployment", + "Takes effect immediately"); + + public static final UnboundBooleanFlag PROVISION_APPLICATION_ROLES = defineFeatureFlag( + "provision-application-roles", false, + "Whether application roles should be provisioned", + "Takes effect on next deployment (controller)", + ZONE_ID); + + public static final UnboundIntFlag JDISC_HEALTH_CHECK_PROXY_CLIENT_TIMEOUT = defineIntFlag( + "jdisc-health-check-proxy-client-timeout", 1000, + "Temporary flag to rollout reduced timeout for JDisc's health check proxy client. Timeout in milliseconds", + "Takes effect on next internal redeployment", + APPLICATION_ID); + + public static final UnboundBooleanFlag APPLICATION_IAM_ROLE = defineFeatureFlag( + "application-iam-roles", false, + "Allow separate iam roles when provisioning/assigning hosts", + "Takes effect immediately on new hosts, on next redeploy for applications", + APPLICATION_ID); + + public static final UnboundBooleanFlag ENABLE_PUBLIC_SIGNUP_FLOW = defineFeatureFlag( + "enable-public-signup-flow", false, + "Show the public signup flow for a user in the console", + "takes effect on browser reload of api/user/v1/user", + CONSOLE_USER_EMAIL + ); + + public static final UnboundBooleanFlag CONFIGSERVER_PROVISION_LB = defineFeatureFlag( + "configserver-provision-lb", false, + "Provision load balancer for config server cluster", + "Takes effect when zone-config-servers application is redeployed", + ZONE_ID + ); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/custom/PreprovisionCapacity.java b/flags/src/main/java/com/yahoo/vespa/flags/custom/HostCapacity.java index 01eab8dfb9c..947520ca2d7 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/custom/PreprovisionCapacity.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/custom/HostCapacity.java @@ -10,7 +10,7 @@ import java.util.Objects; * @author freva */ @JsonIgnoreProperties(ignoreUnknown = true) -public class PreprovisionCapacity { +public class HostCapacity { @JsonProperty("vcpu") private final double vcpu; @@ -23,10 +23,10 @@ public class PreprovisionCapacity { @JsonProperty("count") private final int count; - public PreprovisionCapacity(@JsonProperty("vcpu") double vcpu, - @JsonProperty("memoryGb") double memoryGb, - @JsonProperty("diskGb") double diskGb, - @JsonProperty("count") int count) { + public HostCapacity(@JsonProperty("vcpu") double vcpu, + @JsonProperty("memoryGb") double memoryGb, + @JsonProperty("diskGb") double diskGb, + @JsonProperty("count") int count) { this.vcpu = requirePositive("vcpu", vcpu); this.memoryGb = requirePositive("memoryGb", memoryGb); this.diskGb = requirePositive("diskGb", diskGb); @@ -59,7 +59,7 @@ public class PreprovisionCapacity { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - PreprovisionCapacity that = (PreprovisionCapacity) o; + HostCapacity that = (HostCapacity) o; return Double.compare(that.vcpu, vcpu) == 0 && Double.compare(that.memoryGb, memoryGb) == 0 && Double.compare(that.diskGb, diskGb) == 0 && @@ -70,4 +70,4 @@ public class PreprovisionCapacity { public int hashCode() { return Objects.hash(vcpu, memoryGb, diskGb, count); } -}
\ No newline at end of file +} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java index c7081ca72ab..4b989b8f819 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java @@ -19,6 +19,7 @@ public class DimensionHelper { serializedDimensions.put(FetchVector.Dimension.NODE_TYPE, "node-type"); serializedDimensions.put(FetchVector.Dimension.CLUSTER_TYPE, "cluster-type"); serializedDimensions.put(FetchVector.Dimension.VESPA_VERSION, "vespa-version"); + serializedDimensions.put(FetchVector.Dimension.CONSOLE_USER_EMAIL, "console-user-email"); if (serializedDimensions.size() != FetchVector.Dimension.values().length) { throw new IllegalStateException(FetchVectorHelper.class.getName() + " is not in sync with " + diff --git a/fsa/src/alltest/alltest.sh b/fsa/src/alltest/alltest.sh index e04dae481ea..684cf2e513d 100755 --- a/fsa/src/alltest/alltest.sh +++ b/fsa/src/alltest/alltest.sh @@ -6,8 +6,6 @@ if [ -z "$SOURCE_DIRECTORY" ]; then SOURCE_DIRECTORY="." fi -export LD_PRELOAD=../vespa/fsa/libfsa.so:../vespa/fsamanagers/libfsamanagers.so - # first create the FSA ./fsa_fsa_create_test_app diff --git a/hosted-tenant-base/OWNERS b/hosted-tenant-base/OWNERS new file mode 100644 index 00000000000..ff9741f2060 --- /dev/null +++ b/hosted-tenant-base/OWNERS @@ -0,0 +1,2 @@ +mortent +bjorncs
\ No newline at end of file diff --git a/hosted-tenant-base/README b/hosted-tenant-base/README new file mode 100644 index 00000000000..fed0672dcec --- /dev/null +++ b/hosted-tenant-base/README @@ -0,0 +1 @@ +Base pom for Vespa cloud application poms diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml new file mode 100644 index 00000000000..93ef008b8f7 --- /dev/null +++ b/hosted-tenant-base/pom.xml @@ -0,0 +1,388 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <!-- <version>7-SNAPSHOT</version>--> + <!-- <url>https://github.com/vespa-engine</url>--> + <!-- <packaging>pom</packaging>--> + + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-tenant-base</artifactId> + <version>7-SNAPSHOT</version> + <name>Base pom for all tenant base poms</name> + <description>Parent POM for all Vespa base poms.</description> + <url>https://github.com/vespa-engine</url> + <packaging>pom</packaging> + + <licenses> + <license> + <name>The Apache License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + </license> + </licenses> + <developers> + <developer> + <name>Vespa</name> + <url>https://github.com/vespa-engine</url> + </developer> + </developers> + <scm> + <connection>scm:git:git@github.com:vespa-engine/vespa.git</connection> + <developerConnection>scm:git:git@github.com:vespa-engine/vespa.git</developerConnection> + <url>git@github.com:vespa-engine/vespa.git</url> + </scm> + + <properties> + <vespaversion>${project.version}</vespaversion> + <test-framework.version>${project.version}</test-framework.version> + <target_jdk_version>11</target_jdk_version> + <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version> + <maven-surefire-plugin.version>2.22.0</maven-surefire-plugin.version> + <junit.version>5.4.2</junit.version> + <endpoint>https://api.vespa-external.aws.oath.cloud:4443</endpoint> + <test.categories>!integration</test.categories> + </properties> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dependency-versions</artifactId> + <version>${vespaversion}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <version>${junit.version}</version> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <version>${junit.version}</version> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container</artifactId> + <version>${vespaversion}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${vespaversion}</version> + <scope>runtime</scope> + <exclusions> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-cd-api</artifactId> + <version>${test-framework.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + + <profiles> + <profile> + <!-- Build *-fat-test.jar file that includes all non-test classes and resources + that are part of the class path during test and and test.jar that includes + all test classes and resources, and put it inside a zip: + 1. application classes and resources + 2. test classes and resources + 3. classes and resources in all dependencies of both (1) and (2) + 4. copy the fat-test-jar and test-jar to application-test/artifacts directory + 5. zip application-test --> + <id>fat-test-application</id> + <build> + <plugins> + <plugin> + <!-- dependencies, see (3) above --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <!-- JAR-like dependencies --> + <id>unpack-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <configuration> + <includeTypes>jar,test-jar</includeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + <!-- WARNING(2018-06-27): bcpkix-jdk15on-1.58.jar and + bcprov-jdk15on-1.58.jar are pulled in via + container-dev and both contains the same set of + bouncycastle signature files in META-INF: + BC1024KE.DSA, BC1024KE.SF, BC2048KE.DSA, and + BC2048KE.SF. By merging any of these two with any + other JAR file like we're doing here, the signatures + are wrong. Worse, what we're doing is WRONG but not + yet fatal. + + The symptom of this happening is that the tester fails + to load the SystemTest class(!?), and subsequently + tries to run all test-like files in the fat test JAR. + + The solution is to exclude such files. This happens + automatically with maven-assembly-plugin. --> + <excludes>META-INF/*.SF,META-INF/*.DSA</excludes> + </configuration> + </execution> + <execution> + <!-- non-JAR-like dependencies --> + <id>non-jar-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <excludeTypes>jar,test-jar</excludeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>copy-resources</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>target/fat-test-classes</outputDirectory> + <resources> + <!-- application classes and resources, see 1. above --> + <resource> + <directory>target/classes</directory> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>fat-test-jar</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <classesDirectory>target/fat-test-classes</classesDirectory> + <classifier>fat-test</classifier> + </configuration> + </execution> + <execution> + <id>test-jar</id> + <phase>package</phase> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>attach-artifact</id> + <phase>package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <tasks> + <!-- copy fat test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-fat-test.jar" + todir="target/application-test/artifacts/" /> + + <!-- copy slim test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-tests.jar" + todir="target/application-test/artifacts/" /> + + <!-- zip application-test, see 5. above --> + <zip destfile="target/application-test.zip" + basedir="target/application-test/" /> + </tasks> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + + <profile> <!-- Alias vespaversion with a more descriptive vespa.compile.version --> + <id>set-vespa-compile-version</id> + <activation> + <property> + <name>vespa.compile.version</name> + </property> + </activation> + <properties> + <vespaversion>${vespa.compile.version}</vespaversion> + </properties> + </profile> + + <profile> <!-- Alias vespaVersion with a more descriptive vespa.runtime.version --> + <id>set-vespa-runtime-version</id> + <activation> + <property> + <name>vespa.runtime.version</name> + </property> + </activation> + <properties> + <vespaVersion>${vespa.runtime.version}</vespaVersion> + </properties> + </profile> + </profiles> + + <build> + <finalName>${project.artifactId}</finalName> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <groups>${test.categories}</groups> + <redirectTestOutputToFile>false</redirectTestOutputToFile> + <trimStackTrace>false</trimStackTrace> + <systemPropertyVariables> + <application>${application}</application> + <tenant>${tenant}</tenant> + <instance>${instance}</instance> + <environment>${environment}</environment> + <region>${region}</region> + <endpoint>${endpoint}</endpoint> + <apiKeyFile>${apiKeyFile}</apiKeyFile> + <apiCertificateFile>${apiCertificateFile}</apiCertificateFile> + <dataPlaneKeyFile>${dataPlaneKeyFile}</dataPlaneKeyFile> + <dataPlaneCertificateFile>${dataPlaneCertificateFile}</dataPlaneCertificateFile> + </systemPropertyVariables> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + </configuration> + </plugin> + </plugins> + </pluginManagement> + + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>3.0.0-M2</version> + <executions> + <execution> + <id>enforce-java</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireJavaVersion> + <version>[11, )</version> + </requireJavaVersion> + <requireMavenVersion> + <version>[3.5, )</version> + </requireMavenVersion> + </rules> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>${maven-compiler-plugin.version}</version> + <configuration> + <source>${target_jdk_version}</source> + <target>${target_jdk_version}</target> + <showWarnings>true</showWarnings> + <showDeprecation>true</showDeprecation> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-maven-plugin</artifactId> + <version>${vespaversion}</version> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-application-maven-plugin</artifactId> + <version>${vespaversion}</version> + <executions> + <execution> + <goals> + <goal>packageApplication</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${vespaversion}</version> + <extensions>true</extensions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/hosted-zone-api/CMakeLists.txt b/hosted-zone-api/CMakeLists.txt new file mode 100644 index 00000000000..cc6b2953759 --- /dev/null +++ b/hosted-zone-api/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(hosted-zone-api) diff --git a/hosted-zone-api/OWNERS b/hosted-zone-api/OWNERS new file mode 100644 index 00000000000..569bf1cc3a1 --- /dev/null +++ b/hosted-zone-api/OWNERS @@ -0,0 +1 @@ +bjorncs diff --git a/hosted-zone-api/README.md b/hosted-zone-api/README.md new file mode 100644 index 00000000000..8f44632dde1 --- /dev/null +++ b/hosted-zone-api/README.md @@ -0,0 +1,4 @@ +<!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +# hosted-zone-api + +Contains hosted Zone API for user facing Vespa Java APIs diff --git a/hosted-zone-api/abi-spec.json b/hosted-zone-api/abi-spec.json new file mode 100644 index 00000000000..e5d1db476c2 --- /dev/null +++ b/hosted-zone-api/abi-spec.json @@ -0,0 +1,51 @@ +{ + "ai.vespa.cloud.Environment": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static ai.vespa.cloud.Environment[] values()", + "public static ai.vespa.cloud.Environment valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum ai.vespa.cloud.Environment dev", + "public static final enum ai.vespa.cloud.Environment perf", + "public static final enum ai.vespa.cloud.Environment test", + "public static final enum ai.vespa.cloud.Environment staging", + "public static final enum ai.vespa.cloud.Environment prod" + ] + }, + "ai.vespa.cloud.SystemInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(ai.vespa.cloud.Zone)", + "public ai.vespa.cloud.Zone zone()" + ], + "fields": [] + }, + "ai.vespa.cloud.Zone": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(ai.vespa.cloud.Environment, java.lang.String)", + "public ai.vespa.cloud.Environment environment()", + "public java.lang.String region()", + "public java.lang.String toString()", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public static ai.vespa.cloud.Zone from(java.lang.String)" + ], + "fields": [] + } +}
\ No newline at end of file diff --git a/hosted-zone-api/pom.xml b/hosted-zone-api/pom.xml new file mode 100644 index 00000000000..05a291c52bf --- /dev/null +++ b/hosted-zone-api/pom.xml @@ -0,0 +1,55 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>hosted-zone-api</artifactId> + <packaging>container-plugin</packaging> + <version>7-SNAPSHOT</version> + + <dependencies> + <!-- provided --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <!-- required for bundle-plugin to generate import-package statements for Java's standard library --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <!-- test --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>abi-check-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/container-core/src/main/java/ai/vespa/cloud/Environment.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/Environment.java index 8f1d9fc962a..8f1d9fc962a 100644 --- a/container-core/src/main/java/ai/vespa/cloud/Environment.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/Environment.java diff --git a/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java index 0524ae072cd..0ac93861275 100644 --- a/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java @@ -1,9 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.cloud; -import com.google.inject.Inject; -import com.yahoo.cloud.config.ConfigserverConfig; - /** * Provides information about the system in which this container is running. * This is available and can be injected when running in a cloud environment. @@ -14,13 +11,6 @@ public class SystemInfo { private final Zone zone; - /** Do not use */ - @Inject - public SystemInfo(ConfigserverConfig config) { - this.zone = new Zone(Environment.valueOf(config.environment()), config.region()); - } - - /** Create an instance for testing */ public SystemInfo(Zone zone) { this.zone = zone; } diff --git a/container-core/src/main/java/ai/vespa/cloud/Zone.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/Zone.java index 48293aa7908..48293aa7908 100644 --- a/container-core/src/main/java/ai/vespa/cloud/Zone.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/Zone.java diff --git a/container-core/src/main/java/ai/vespa/cloud/package-info.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/package-info.java index 259a2bda258..259a2bda258 100644 --- a/container-core/src/main/java/ai/vespa/cloud/package-info.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/package-info.java diff --git a/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java b/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java index 6bc8b395e00..6bc8b395e00 100644 --- a/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java +++ b/hosted-zone-api/src/test/java/ai/vespa/cloud/SystemInfoTest.java diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java b/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java index 06a7359f307..25f5b9eb627 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java +++ b/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java @@ -8,7 +8,7 @@ import java.time.Duration; * * @author bjorncs */ -interface Sleeper { +public interface Sleeper { void sleep(Duration duration); class Default implements Sleeper { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java index 6056a9b0ca5..14b5af53b5a 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java @@ -23,6 +23,7 @@ public final class ExactExpression extends Expression { public ExactExpression() { super(DataType.STRING); } + @Override protected void doExecute(ExecutionContext ctx) { StringFieldValue input = (StringFieldValue)ctx.getValue(); @@ -70,4 +71,5 @@ public final class ExactExpression extends Expression { public int hashCode() { return getClass().hashCode(); } + } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java index 14a6c8f9b04..8dcf360a6ee 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java @@ -9,6 +9,7 @@ import com.yahoo.document.datatypes.StringFieldValue; * @author Simon Thoresen Hult */ public final class TrimExpression extends Expression { + public TrimExpression() { super(DataType.STRING); } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java index 466e74202c1..25194a5502a 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java @@ -116,20 +116,15 @@ public class Request extends AbstractResource { setUri(uri); } - /** - * <p>Returns the {@link Container} for which this Request was created.</p> - * - * @return The container instance. - */ + /** Returns the {@link Container} for which this Request was created */ public Container container() { return parent != null ? parent.container() : container; } /** - * <p>Returns the Uniform Resource Identifier used by the {@link Container} to resolve the appropriate {@link - * RequestHandler} for this Request.</p> + * Returns the Uniform Resource Identifier used by the {@link Container} to resolve the appropriate {@link + * RequestHandler} for this Request * - * @return The resource identifier. * @see #setUri(URI) */ public URI getUri() { @@ -137,12 +132,12 @@ public class Request extends AbstractResource { } /** - * <p>Sets the Uniform Resource Identifier used by the {@link Container} to resolve the appropriate {@link + * Sets the Uniform Resource Identifier used by the {@link Container} to resolve the appropriate {@link * RequestHandler} for this Request. Because access to the URI is not guarded by any lock, any changes made after - * calling {@link #connect(ResponseHandler)} might never become visible to other threads.</p> + * calling {@link #connect(ResponseHandler)} might never become visible to other threads. * - * @param uri The URI to set. - * @return This, to allow chaining. + * @param uri the URI to set + * @return this, to allow chaining * @see #getUri() */ public Request setUri(URI uri) { @@ -151,22 +146,22 @@ public class Request extends AbstractResource { } /** - * <p>Returns whether or not this Request was created by a {@link ServerProvider}. The value of this is used by - * {@link Container#resolveHandler(Request)} to decide whether to match against server- or client-bindings.</p> + * Returns whether or not this Request was created by a {@link ServerProvider}. The value of this is used by + * {@link Container#resolveHandler(Request)} to decide whether to match against server- or client-bindings. * - * @return True, if this is a server request. + * @return true, if this is a server request */ public boolean isServerRequest() { return serverRequest; } /** - * <p>Sets whether or not this Request was created by a {@link ServerProvider}. The constructor that accepts a + * Sets whether or not this Request was created by a {@link ServerProvider}. The constructor that accepts a * {@link CurrentContainer} sets this to <em>true</em>, whereas the constructor that accepts a parent Request sets - * this to <em>false</em>.</p> + * this to <em>false</em>. * - * @param serverRequest Whether or not this is a server request. - * @return This, to allow chaining. + * @param serverRequest whether or not this is a server request + * @return this, to allow chaining * @see #isServerRequest() */ public Request setServerRequest(boolean serverRequest) { @@ -175,13 +170,13 @@ public class Request extends AbstractResource { } /** - * <p>Returns the last resolved {@link BindingMatch}, or null if none has been resolved yet. This is set + * Returns the last resolved {@link BindingMatch}, or null if none has been resolved yet. This is set * automatically when calling the {@link Container#resolveHandler(Request)} method. The BindingMatch object holds * information about the match of this Request's {@link #getUri() URI} to the {@link UriPattern} of the resolved * {@link RequestHandler}. It allows you to reflect on the parts of the URI that were matched by wildcards in the - * UriPattern.</p> + * UriPattern. * - * @return The last resolved BindingMatch, or null. + * @return the last resolved BindingMatch, or null * @see #setBindingMatch(BindingMatch) * @see Container#resolveHandler(Request) */ diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index c5a0a676a70..68b68dca068 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -73,6 +73,7 @@ "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy)", "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder enable(boolean)", "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder port(int)", + "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder clientTimeout(double)", "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy build()" ], "fields": [] @@ -87,7 +88,8 @@ "methods": [ "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder)", "public boolean enable()", - "public int port()" + "public int port()", + "public double clientTimeout()" ], "fields": [] }, diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java index d93d738bb91..6fdd5e8534f 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java @@ -20,7 +20,6 @@ import org.eclipse.jetty.server.handler.HandlerWrapper; import org.eclipse.jetty.util.ssl.SslContextFactory; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLException; import javax.servlet.ServletException; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; @@ -60,7 +59,8 @@ class HealthCheckProxyHandler extends HandlerWrapper { for (JDiscServerConnector connector : connectors) { ConnectorConfig.HealthCheckProxy proxyConfig = connector.connectorConfig().healthCheckProxy(); if (proxyConfig.enable()) { - mapping.put(connector.listenPort(), createProxyTarget(proxyConfig.port(), connectors)); + Duration targetTimeout = Duration.ofMillis((int) (proxyConfig.clientTimeout() * 1000)); + mapping.put(connector.listenPort(), createProxyTarget(proxyConfig.port(), targetTimeout, connectors)); log.info(String.format("Port %1$d is configured as a health check proxy for port %2$d. " + "HTTP requests to '%3$s' on %1$d are proxied as HTTPS to %2$d.", connector.listenPort(), proxyConfig.port(), HEALTH_CHECK_PATH)); @@ -69,7 +69,7 @@ class HealthCheckProxyHandler extends HandlerWrapper { return mapping; } - private static ProxyTarget createProxyTarget(int targetPort, List<JDiscServerConnector> connectors) { + private static ProxyTarget createProxyTarget(int targetPort, Duration targetTimeout, List<JDiscServerConnector> connectors) { JDiscServerConnector targetConnector = connectors.stream() .filter(connector -> connector.listenPort() == targetPort) .findAny() @@ -80,12 +80,13 @@ class HealthCheckProxyHandler extends HandlerWrapper { .map(detectorConnFactory -> detectorConnFactory.getBean(SslConnectionFactory.class))) .map(connFactory -> (SslContextFactory.Server) connFactory.getSslContextFactory()) .orElseThrow(() -> new IllegalArgumentException("Health check proxy can only target https port")); - return new ProxyTarget(targetPort, sslContextFactory); + return new ProxyTarget(targetPort, targetTimeout, sslContextFactory); } @Override public void handle(String target, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException, ServletException { - ProxyTarget proxyTarget = portToProxyTargetMapping.get(getConnectorLocalPort(servletRequest)); + int localPort = getConnectorLocalPort(servletRequest); + ProxyTarget proxyTarget = portToProxyTargetMapping.get(localPort); if (proxyTarget != null) { if (servletRequest.getRequestURI().equals(HEALTH_CHECK_PATH)) { try (CloseableHttpResponse proxyResponse = proxyTarget.requestStatusHtml()) { @@ -101,10 +102,14 @@ class HealthCheckProxyHandler extends HandlerWrapper { entity.getContent().transferTo(output); } } - } catch (Exception e) { - String message = "Unable to proxy health check request: " + e.getMessage(); - log.log(Level.WARNING, e, () -> message); + } catch (Exception e) { // Typically timeouts which are reported as SSLHandshakeException + String message = String.format("Health check request from port %d to %d failed: %s", localPort, proxyTarget.port, e.getMessage()); + log.log(Level.WARNING, message); + log.log(Level.FINE, e.toString(), e); servletResponse.sendError(Response.Status.INTERNAL_SERVER_ERROR, message); + if (Duration.ofSeconds(1).compareTo(proxyTarget.timeout) >= 0) { // TODO bjorncs: remove call to close() if client is correctly pruning bad connections (VESPA-17628) + proxyTarget.close(); + } } } else { servletResponse.sendError(NOT_FOUND); @@ -124,23 +129,19 @@ class HealthCheckProxyHandler extends HandlerWrapper { private static class ProxyTarget implements AutoCloseable { final int port; + final Duration timeout; final SslContextFactory.Server sslContextFactory; volatile CloseableHttpClient client; - ProxyTarget(int port, SslContextFactory.Server sslContextFactory) { + ProxyTarget(int port, Duration timeout, SslContextFactory.Server sslContextFactory) { this.port = port; + this.timeout = timeout; this.sslContextFactory = sslContextFactory; } CloseableHttpResponse requestStatusHtml() throws IOException { - try { - HttpGet request = new HttpGet("https://localhost:" + port + HEALTH_CHECK_PATH); - return client().execute(request); - } catch (SSLException e) { - log.log(Level.SEVERE, "SSL connection failed. Closing existing client, a new client will be created on next request", e); - close(); - throw e; - } + return client() + .execute(new HttpGet("https://localhost:" + port + HEALTH_CHECK_PATH)); } // Client construction must be delayed to ensure that the SslContextFactory is started before calling getSslContext(). @@ -148,18 +149,19 @@ class HealthCheckProxyHandler extends HandlerWrapper { if (client == null) { synchronized (this) { if (client == null) { + int timeoutMillis = (int) timeout.toMillis(); client = HttpClientBuilder.create() .disableAutomaticRetries() .setMaxConnPerRoute(4) .setSSLContext(getSslContext(sslContextFactory)) - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) // Certificate may not match "localhost" .setUserTokenHandler(context -> null) // https://stackoverflow.com/a/42112034/1615280 .setUserAgent("health-check-proxy-client") .setDefaultRequestConfig( RequestConfig.custom() - .setConnectTimeout((int) Duration.ofSeconds(4).toMillis()) - .setConnectionRequestTimeout((int) Duration.ofSeconds(4).toMillis()) - .setSocketTimeout((int) Duration.ofSeconds(8).toMillis()) + .setConnectTimeout(timeoutMillis) + .setConnectionRequestTimeout(timeoutMillis) + .setSocketTimeout(timeoutMillis) .build()) .build(); } diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def index fa7ed6657d9..4c86c8b9bb6 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def @@ -101,6 +101,9 @@ healthCheckProxy.enable bool default=false # Which port to proxy healthCheckProxy.port int default=8080 +# Low-level timeout for proxy client (socket connect, socket read, connection pool). Aggregate timeout will be longer. +healthCheckProxy.clientTimeout double default=1.0 + # Enable PROXY protocol V1/V2 support (only for https connectors). proxyProtocol.enabled bool default=false diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index 37f717437ee..a67f6919727 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -758,12 +758,23 @@ public class HttpServerTest { } private ContentResponse sendJettyClientRequest(TestDriver testDriver, HttpClient client, Object tag) - throws InterruptedException, ExecutionException, TimeoutException { - ContentResponse response = client.newRequest(URI.create("https://localhost:" + testDriver.server().getListenPort() + "/")) - .tag(tag) - .send(); - assertEquals(200, response.getStatus()); - return response; + throws InterruptedException, TimeoutException { + int maxAttempts = 3; + for (int attempt = 0; attempt < maxAttempts; attempt++) { + try { + ContentResponse response = client.newRequest(URI.create("https://localhost:" + testDriver.server().getListenPort() + "/")) + .tag(tag) + .send(); + assertEquals(200, response.getStatus()); + return response; + } catch (ExecutionException e) { + // Retry when the server closes the connection before the TLS handshake is completed. This have been observed in CI. + // We have been unable to reproduce this locally. The cause is therefor currently unknown. + log.log(Level.WARNING, String.format("Attempt %d failed: %s", attempt, e.getMessage()), e); + Thread.sleep(10); + } + } + throw new AssertionError("Failed to send request, see log for details"); } // Using Jetty's http client as Apache httpclient does not support the proxy-protocol v1/v2. diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java index d3f6fcf2ee3..93599fa7dbe 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -15,6 +15,7 @@ import java.util.logging.Logger; import java.util.logging.Level; public class OpenNlpTokenizer implements Tokenizer { + private final static int SPACE_CODE = 32; private static final Logger log = Logger.getLogger(OpenNlpTokenizer.class.getName()); private final Normalizer normalizer; diff --git a/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java b/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java index ce0291c85e5..59ae664e79e 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java +++ b/linguistics/src/main/java/com/yahoo/language/process/CharacterClasses.java @@ -17,7 +17,7 @@ public class CharacterClasses { if (Character.isDigit(c) && ! isLatin(c)) return true; // Not considering these digits, so treat them as letters // if (c == '_') return true; - // Ticket 3864695, some CJK punctuation YST defined as word characters + // Some CJK punctuation defined as word characters if (c == '\u3008' || c == '\u3009' || c == '\u300a' || c == '\u300b' || c == '\u300c' || c == '\u300d' || c == '\u300e' || c == '\u300f' || c == '\u3010' || c == '\u3011') { @@ -52,4 +52,5 @@ public class CharacterClasses { public boolean isLetterOrDigit(int c) { return isLetter(c) || isDigit(c); } + } diff --git a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java index 94fd0e08493..aa7ae59edf9 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java +++ b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java @@ -39,12 +39,8 @@ public class GramSplitter { * @throws IllegalArgumentException if n is less than 1 */ public GramSplitterIterator split(String input, int n) { - if (input == null) { - throw new NullPointerException("input cannot be null"); - } - if (n < 1) { - throw new IllegalArgumentException("n (gram size) cannot be smaller than 1, was " + n); - } + if (input == null) throw new NullPointerException("input cannot be null"); + if (n < 1) throw new IllegalArgumentException("n (gram size) cannot be smaller than 1, was " + n); return new GramSplitterIterator(input, n, characterClasses); } @@ -52,29 +48,19 @@ public class GramSplitter { private final CharacterClasses characterClasses; - /** - * Text to split - */ + /** Text to split */ private final String input; - /** - * Gram size - */ + /** Gram size */ private final int n; - /** - * Current index - */ + /** Current index */ private int i = 0; - /** - * Whether the last thing that happened was being on a separator (including the start of the string) - */ + /** Whether the last thing that happened was being on a separator (including the start of the string) */ private boolean isFirstAfterSeparator = true; - /** - * The next gram or null if not determined yet - */ + /** The next gram or null if not determined yet */ private Gram nextGram = null; public GramSplitterIterator(String input, int n, CharacterClasses characterClasses) { @@ -85,9 +71,7 @@ public class GramSplitter { @Override public boolean hasNext() { - if (nextGram != null) { - return true; - } + if (nextGram != null) return true; nextGram = findNext(); return nextGram != null; } @@ -95,12 +79,10 @@ public class GramSplitter { @Override public Gram next() { Gram currentGram = nextGram; - if (currentGram == null) { + if (currentGram == null) currentGram = findNext(); - } - if (currentGram == null) { + if (currentGram == null) throw new NoSuchElementException("No next gram at position " + i); - } nextGram = null; return currentGram; } @@ -111,24 +93,21 @@ public class GramSplitter { i++; isFirstAfterSeparator = true; } - if (i >= input.length()) { - return null; - } + if (i >= input.length()) return null; String gram = input.substring(i, Math.min(i + n, input.length())); int nonWordChar = indexOfNonWordChar(gram); - if (nonWordChar == 0) { - throw new RuntimeException("Programming error"); - } - if (nonWordChar > 0) { + if (nonWordChar == 0) throw new RuntimeException("Programming error"); + + if (nonWordChar > 0) gram = gram.substring(0, nonWordChar); - } if (gram.length() == n) { // normal case: got a full length gram i++; isFirstAfterSeparator = false; return new Gram(i - 1, gram.length()); - } else { // gram is too short due either to a non-word separator or end of string + } + else { // gram is too short due either to a non-word separator or end of string if (isFirstAfterSeparator) { // make a gram anyway i++; isFirstAfterSeparator = false; @@ -143,9 +122,8 @@ public class GramSplitter { private int indexOfNonWordChar(String s) { for (int i = 0; i < s.length(); i++) { - if (!characterClasses.isLetterOrDigit(s.codePointAt(i))) { + if ( ! characterClasses.isLetterOrDigit(s.codePointAt(i))) return i; - } } return -1; } @@ -162,9 +140,8 @@ public class GramSplitter { */ public List<String> toExtractedList() { List<String> gramList = new ArrayList<>(); - while (hasNext()) { + while (hasNext()) gramList.add(next().extractFrom(input)); - } return Collections.unmodifiableList(gramList); } } @@ -189,31 +166,19 @@ public class GramSplitter { return length; } - /** - * Returns this gram as a string from the input string - */ + /** Returns this gram as a string from the input string */ public String extractFrom(String input) { return input.substring(start, start + length); } @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof Gram)) { - return false; - } + if (this == o) return true; + if ( ! (o instanceof Gram)) return false; Gram gram = (Gram)o; - - if (length != gram.length) { - return false; - } - if (start != gram.start) { - return false; - } - + if (length != gram.length) return false; + if (start != gram.start) return false; return true; } @@ -223,5 +188,7 @@ public class GramSplitter { result = 31 * result + length; return result; } + } + } diff --git a/linguistics/src/main/java/com/yahoo/language/process/Normalizer.java b/linguistics/src/main/java/com/yahoo/language/process/Normalizer.java index 0e34f88f4ca..044d249f077 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Normalizer.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Normalizer.java @@ -9,11 +9,11 @@ package com.yahoo.language.process; public interface Normalizer { /** - * <p>NFKC normalizes a String.</p> + * NFKC normalizes a String. * - * @param input String to normalize. - * @return The normalized String. - * @throws ProcessingException If underlying library throws an Exception. + * @param input the string to normalize + * @return the normalized string + * @throws ProcessingException if underlying library throws an Exception */ String normalize(String input); diff --git a/linguistics/src/main/java/com/yahoo/language/process/ProcessingException.java b/linguistics/src/main/java/com/yahoo/language/process/ProcessingException.java index 941afa07347..752992f5a26 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/ProcessingException.java +++ b/linguistics/src/main/java/com/yahoo/language/process/ProcessingException.java @@ -2,7 +2,7 @@ package com.yahoo.language.process; /** - * <p>Exception class indicating that a fatal error occured during linguistic processing.</p> + * Exception class indicating that a fatal error occured during linguistic processing. * * @author Simon Thoresen Hult */ diff --git a/linguistics/src/main/java/com/yahoo/language/process/Transformer.java b/linguistics/src/main/java/com/yahoo/language/process/Transformer.java index 46f3c060d4e..4927edc98c9 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Transformer.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Transformer.java @@ -13,8 +13,8 @@ public interface Transformer { /** * Remove accents from input text. * - * @param input text to transform. - * @param language language of input text. + * @param input text to transform + * @param language language of input text * @return text with accents removed, or input-text if the feature is unavailable * @throws ProcessingException thrown if there is an exception stemming this input */ diff --git a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp index 0588ce5d2e7..c5c19aa2c3f 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp @@ -27,7 +27,7 @@ runSplunk(const vespalib::string &prefix, std::vector<const char *> args) dbg.append(arg); dbg.append("'"); } - LOG(debug, "starting splunk forwarder with command: %s", dbg.c_str()); + LOG(info, "trigger splunk with command: %s", dbg.c_str()); args.push_back(nullptr); fflush(stdout); pid_t child = fork(); @@ -58,13 +58,13 @@ runSplunk(const vespalib::string &prefix, std::vector<const char *> args) return; } if (WIFEXITED(waitStatus)) { - LOG(warning, "failed starting splunk forwarder (exit status %d)", + LOG(warning, "failed triggering splunk (exit status %d)", WEXITSTATUS(waitStatus)); } else if (WIFSIGNALED(waitStatus)) { - LOG(warning, "failed starting splunk forwarder (exit on signal %d)", + LOG(warning, "failed triggering splunk (exit on signal %d)", WTERMSIG(waitStatus)); } else { - LOG(warning, "failed starting splunk forwarder (abnormal exit status %d)", + LOG(warning, "failed triggering splunk (abnormal exit status %d)", waitStatus); } } @@ -76,14 +76,15 @@ void ChildHandler::startChild(const vespalib::string &prefix) { if (! _childRunning) { + // it is possible that splunk was already running anyway, so + // make sure we restart it to get new config activated: + runSplunk(prefix, {"stop"}); + sleep(1); runSplunk(prefix, {"start", "--answer-yes", "--no-prompt", "--accept-license"}); _childRunning = true; - // it is possible that splunk was already running, and - // then the above won't do anything, so we need to - // *also* do the restart below, after a small delay. - sleep(1); + } else { + runSplunk(prefix, {"restart"}); } - runSplunk(prefix, {"restart"}); } void diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Message.java b/messagebus/src/main/java/com/yahoo/messagebus/Message.java index 43f5c8d2dfd..6848ffc55d8 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/Message.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Message.java @@ -47,21 +47,10 @@ public abstract class Message extends Routable { } } - /** - * <p>Return the route of this routable.</p> - * - * @return The route. - */ - public Route getRoute() { - return route; - } + /** Returns the route of this routable */ + public Route getRoute() { return route; } - /** - * <p>Set a new route for this routable.</p> - * - * @param route The new route. - * @return This, to allow chaining. - */ + /** Sets a new route for this routable */ public Message setRoute(Route route) { this.route = new Route(route); return this; @@ -157,11 +146,9 @@ public abstract class Message extends Routable { } /** - * <p>Returns the identifier used to order messages. Any two messages that have the same sequence id are ensured to + * Returns the identifier used to order messages. Any two messages that have the same sequence id are ensured to * arrive at the recipient in the order they were sent by the client. This value is only respected if the {@link - * #hasSequenceId()} method returns true.</p> - * - * @return The sequence identifier. + * #hasSequenceId()} method returns true. */ public long getSequenceId() { return 0; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java index f0894b39e28..c05c88ec68a 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/prometheus/PrometheusUtil.java @@ -28,8 +28,8 @@ public class PrometheusUtil { List<MetricFamilySamples> metricFamilySamples = new ArrayList<>(packetsByService.size()); + Map<String, List<Sample>> samples = new HashMap<>(); packetsByService.forEach(((serviceId, packets) -> { - Map<String, List<Sample>> samples = new HashMap<>(); var serviceName = Collector.sanitizeMetricName(serviceId.id); for (var packet : packets) { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java index 5c7d8b3ce83..f60a3491a81 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java @@ -3,6 +3,8 @@ package ai.vespa.metricsproxy.telegraf; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; + +import java.io.File; import java.util.logging.Level; import com.yahoo.system.execution.ProcessExecutor; import com.yahoo.system.execution.ProcessResult; @@ -40,7 +42,7 @@ public class Telegraf extends AbstractComponent { public Telegraf(TelegrafRegistry telegrafRegistry, TelegrafConfig telegrafConfig) { this.telegrafRegistry = telegrafRegistry; telegrafRegistry.addInstance(this); - writeConfig(telegrafConfig, uncheck(() -> new FileWriter(TELEGRAF_CONFIG_PATH)), TELEGRAF_LOG_FILE_PATH); + writeConfig(telegrafConfig, getConfigWriter(), TELEGRAF_LOG_FILE_PATH); restartTelegraf(); } @@ -93,6 +95,12 @@ public class Telegraf extends AbstractComponent { } + private static Writer getConfigWriter() { + File configFile = new File(TELEGRAF_CONFIG_PATH); + configFile.getParentFile().mkdirs(); + return uncheck(() -> new FileWriter(configFile)); + } + @Override public void deconstruct() { telegrafRegistry.removeInstance(this); diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index fd398f6d7d7..7ac912416de 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -74,7 +74,6 @@ MetricManager::MetricManager(std::unique_ptr<Timer> timer) false)), _timer(std::move(timer)), _lastProcessedTime(0), - _forceEventLogging(false), _snapshotUnsetMetrics(false), _consumerConfigChanged(false), _metricManagerMetrics("metricmanager", {}, "Metrics for the metric manager upkeep tasks"), @@ -719,7 +718,6 @@ MetricManager::forceEventLogging() LOG(debug, "Forcing event logging to happen."); // Ensure background thread is not in a current cycle during change. vespalib::MonitorGuard sync(_waiter); - _forceEventLogging = true; sync.signal(); } @@ -788,7 +786,7 @@ MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) // Set next work time to the time we want to take next snapshot. time_t nextWorkTime = _snapshots[0]->getToTime() + _snapshots[0]->getPeriod(); time_t nextUpdateHookTime; - if (nextWorkTime <= currentTime || _forceEventLogging) { + if (nextWorkTime <= currentTime) { // If taking a new snapshot or logging, force calls to all // update hooks. LOG(debug, "%s. Calling update hooks.", nextWorkTime <= currentTime diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index b728510b6e4..6d1f8f4dcf7 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -111,7 +111,6 @@ private: MetricSnapshot::SP _totalMetrics; std::unique_ptr<Timer> _timer; std::atomic<time_t> _lastProcessedTime; - bool _forceEventLogging; // Should be added to config, but wont now due to problems with // upgrading bool _snapshotUnsetMetrics; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java index af134fac6cf..6e637c72d0f 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java @@ -59,7 +59,7 @@ public abstract class IntermediateOperation { IntermediateOperation(String modelName, String name, List<IntermediateOperation> inputs) { this.name = name; - this.modelName = modelName; + this.modelName = ensureValidAsDimensionName(modelName); this.inputs = new ArrayList<>(inputs); this.inputs.forEach(i -> i.outputs.add(this)); } @@ -351,6 +351,11 @@ public abstract class IntermediateOperation { public abstract String operationName(); + /** Required due to tensor dimension name restrictions */ + private static String ensureValidAsDimensionName(String modelName) { + return modelName.replaceAll("[^\\w\\d\\$@_]", "_"); + } + @Override public String toString() { return operationName() + "(" + diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reshape.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reshape.java index d8e806ae7e4..78eb5a0e7bb 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reshape.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reshape.java @@ -166,7 +166,7 @@ public class Reshape extends IntermediateOperation { ExpressionNode previousSize = new ConstantNode(new DoubleValue(previousInnerSize)); ExpressionNode mod = new ArithmeticNode(unrolled, ArithmeticOperator.MODULO, previousSize); ExpressionNode div = new ArithmeticNode(new EmbracedNode(mod), ArithmeticOperator.DIVIDE, size); - inputDimensionExpression = new EmbracedNode(new FunctionNode(Function.floor, div)); + inputDimensionExpression = new EmbracedNode(div); } dimensionValues.add(new com.yahoo.tensor.functions.Slice.DimensionValue<>(Optional.of(inputDimensionName), wrapScalar(inputDimensionExpression))); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index 76616a3a8f5..f7f231d5e0c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -27,7 +27,6 @@ public class NodeSpec { private final NodeState state; private final NodeType type; private final String flavor; - private final Optional<Double> cpuCores; private final Optional<DockerImage> wantedDockerImage; private final Optional<DockerImage> currentDockerImage; @@ -68,7 +67,6 @@ public class NodeSpec { NodeState state, NodeType type, String flavor, - Optional<Double> cpuCores, Optional<Version> wantedVespaVersion, Optional<Version> currentVespaVersion, Optional<Version> wantedOsVersion, @@ -103,7 +101,6 @@ public class NodeSpec { this.state = Objects.requireNonNull(state); this.type = Objects.requireNonNull(type); this.flavor = Objects.requireNonNull(flavor); - this.cpuCores = Objects.requireNonNull(cpuCores); this.modelName = Objects.requireNonNull(modelName); this.wantedVespaVersion = Objects.requireNonNull(wantedVespaVersion); this.currentVespaVersion = Objects.requireNonNull(currentVespaVersion); @@ -141,10 +138,6 @@ public class NodeSpec { return flavor; } - public Optional<Double> cpuCores() { - return cpuCores; - } - public Optional<DockerImage> wantedDockerImage() { return wantedDockerImage; } @@ -264,7 +257,6 @@ public class NodeSpec { Objects.equals(state, that.state) && Objects.equals(type, that.type) && Objects.equals(flavor, that.flavor) && - Objects.equals(cpuCores, that.cpuCores) && Objects.equals(wantedVespaVersion, that.wantedVespaVersion) && Objects.equals(currentVespaVersion, that.currentVespaVersion) && Objects.equals(wantedOsVersion, that.wantedOsVersion) && @@ -294,7 +286,6 @@ public class NodeSpec { state, type, flavor, - cpuCores, wantedVespaVersion, currentVespaVersion, wantedOsVersion, @@ -324,7 +315,6 @@ public class NodeSpec { + " state=" + state + " type=" + type + " flavor=" + flavor - + " cpuCores=" + cpuCores + " wantedVespaVersion=" + wantedVespaVersion + " currentVespaVersion=" + currentVespaVersion + " wantedOsVersion=" + wantedOsVersion @@ -351,7 +341,6 @@ public class NodeSpec { private NodeState state; private NodeType type; private String flavor; - private Optional<Double> cpuCores = Optional.empty(); private Optional<DockerImage> wantedDockerImage = Optional.empty(); private Optional<DockerImage> currentDockerImage = Optional.empty(); private Optional<Version> wantedVespaVersion = Optional.empty(); @@ -387,7 +376,6 @@ public class NodeSpec { wantedRebootGeneration(node.wantedRebootGeneration); currentRebootGeneration(node.currentRebootGeneration); reports(new NodeReports(node.reports)); - node.cpuCores.ifPresent(this::cpuCores); node.wantedDockerImage.ifPresent(this::wantedDockerImage); node.currentDockerImage.ifPresent(this::currentDockerImage); node.wantedVespaVersion.ifPresent(this::wantedVespaVersion); @@ -434,11 +422,6 @@ public class NodeSpec { return this; } - public Builder cpuCores(double cpuCores) { - this.cpuCores = Optional.of(cpuCores); - return this; - } - public Builder wantedVespaVersion(Version wantedVespaVersion) { this.wantedVespaVersion = Optional.of(wantedVespaVersion); return this; @@ -658,7 +641,7 @@ public class NodeSpec { } public NodeSpec build() { - return new NodeSpec(hostname, wantedDockerImage, currentDockerImage, state, type, flavor, cpuCores, + return new NodeSpec(hostname, wantedDockerImage, currentDockerImage, state, type, flavor, wantedVespaVersion, currentVespaVersion, wantedOsVersion, currentOsVersion, allowedToBeDown, owner, membership, wantedRestartGeneration, currentRestartGeneration, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index ea737773289..2e47410f1b5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -157,7 +157,6 @@ public class RealNodeRepository implements NodeRepository { nodeState, nodeType, node.flavor, - Optional.ofNullable(node.cpuCores), Optional.ofNullable(node.wantedVespaVersion).map(Version::fromString), Optional.ofNullable(node.vespaVersion).map(Version::fromString), Optional.ofNullable(node.wantedOsVersion).map(Version::fromString), diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java index 4c1f74290e9..932c1384e7e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java @@ -28,8 +28,6 @@ public class NodeRepositoryNode { public String openStackId; @JsonProperty("flavor") public String flavor; - @JsonProperty("cpuCores") - public Double cpuCores; @JsonProperty("resources") public NodeResources resources; @JsonProperty("membership") @@ -92,7 +90,6 @@ public class NodeRepositoryNode { ", openStackId='" + openStackId + '\'' + ", modelName='" + modelName + '\'' + ", flavor='" + flavor + '\'' + - ", cpuCores=" + cpuCores + ", resources=" + resources + ", membership=" + membership + ", owner=" + owner + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index cdf5687d61a..9a1c01044f9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; +import com.yahoo.config.provision.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; @@ -11,6 +12,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; @@ -29,15 +31,14 @@ public class CoreCollector { private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'"); private final DockerOperations docker; - private final Path gdb; - public CoreCollector(DockerOperations docker, Path pathToGdbInContainer) { + public CoreCollector(DockerOperations docker) { this.docker = docker; - this.gdb = pathToGdbInContainer; } Path readBinPathFallback(NodeAgentContext context, Path coredumpPath) { - String command = gdb + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'"; + String command = GDBPath(context).toString() + + " -n -batch -core " + coredumpPath + " | grep \'^Core was generated by\'"; String[] wrappedCommand = {"/bin/sh", "-c", command}; ProcessResult result = docker.executeCommandInContainerAsRoot(context, wrappedCommand); @@ -49,6 +50,17 @@ public class CoreCollector { return Paths.get(matcher.group("path").split(" ")[0]); } + Path GDBPath(NodeAgentContext context) { + Optional<DockerImage> image = context.node().currentDockerImage(); + + if (image.isPresent() && image.get().repository().endsWith("vespa/ci")) { + return context.pathInNodeUnderVespaHome("bin64/gdb"); + } + else { + return Paths.get("/opt/rh/devtoolset-9/root/bin/gdb"); + } + } + Path readBinPath(NodeAgentContext context, Path coredumpPath) { String[] command = {"file", coredumpPath.toString()}; try { @@ -76,7 +88,7 @@ public class CoreCollector { List<String> readBacktrace(NodeAgentContext context, Path coredumpPath, Path binPath, boolean allThreads) { String threads = allThreads ? "thread apply all bt" : "bt"; - String[] command = {gdb.toString(), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()}; + String[] command = {GDBPath(context).toString(), "-n", "-ex", threads, "-batch", binPath.toString(), coredumpPath.toString()}; ProcessResult result = docker.executeCommandInContainerAsRoot(context, command); if (result.getExitStatus() != 0) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index 453f4f4b4c1..9d44f125efc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -27,7 +27,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Supplier; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java index 237b7d0daf7..ad67e13e044 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java @@ -22,6 +22,7 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg private boolean wantFrozen = false; private boolean isFrozen = true; private boolean pendingInterrupt = false; + private boolean isWaitingForNextContext = false; public NodeAgentContextManager(Clock clock, NodeAgentContext context) { this.clock = clock; @@ -61,6 +62,9 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg @Override public NodeAgentContext nextContext() throws InterruptedException { synchronized (monitor) { + nextContext = null; // Reset any previous context and wait for the next one + isWaitingForNextContext = true; + monitor.notify(); Duration untilNextContext = Duration.ZERO; while (setAndGetIsFrozen(wantFrozen) || nextContext == null || @@ -75,8 +79,8 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg } catch (InterruptedException ignored) { } } + isWaitingForNextContext = false; currentContext = nextContext; - nextContext = null; return currentContext; } } @@ -105,4 +109,15 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg return this.isFrozen; } } + + /** FOR TESTING ONLY */ + void waitUntilWaitingForNextContext() { + synchronized (monitor) { + while (!isWaitingForNextContext) { + try { + monitor.wait(); + } catch (InterruptedException ignored) { } + } + } + } }
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java index 40a11f61d3f..c76c872902d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java @@ -13,10 +13,11 @@ public class DiskSize { private static final char[] UNITS = "kMGTPE".toCharArray(); public enum Unit { - kB(1000), kiB(1 << 10), + kB(1_000), kiB(1 << 10), MB(1_000_000), MiB(1 << 20), GB(1_000_000_000), GiB(1 << 30), - PB(1_000_000_000_000L), PiB(1L << 40); + TB(1_000_000_000_000L), TiB(1L << 40), + PB(1_000_000_000_000_000L), PiB(1L << 50); private final long size; Unit(long size) { this.size = size; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index bcec01fe91d..f8d14eea996 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -21,10 +21,10 @@ import static org.mockito.Mockito.when; * @author freva */ public class CoreCollectorTest { - private final String GDB_PATH = "/my/path/to/gdb"; + private final String GDB_PATH = "/opt/rh/devtoolset-9/root/bin/gdb"; private final String JDK_PATH = "/path/to/jdk/java"; private final DockerOperations docker = mock(DockerOperations.class); - private final CoreCollector coreCollector = new CoreCollector(docker, Paths.get(GDB_PATH)); + private final CoreCollector coreCollector = new CoreCollector(docker); private final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld").build(); private final Path TEST_CORE_PATH = Paths.get("/tmp/core.1234"); @@ -85,7 +85,7 @@ public class CoreCollectorTest { fail("Expected not to be able to get bin path"); } catch (RuntimeException e) { assertEquals("Failed to extract binary path from GDB, result: ProcessResult { exitStatus=1 output= errors=Error 123 }, command: " + - "[/bin/sh, -c, /my/path/to/gdb -n -batch -core /tmp/core.1234 | grep '^Core was generated by']", e.getMessage()); + "[/bin/sh, -c, /opt/rh/devtoolset-9/root/bin/gdb -n -batch -core /tmp/core.1234 | grep '^Core was generated by']", e.getMessage()); } } @@ -102,7 +102,7 @@ public class CoreCollectorTest { fail("Expected not to be able to read backtrace"); } catch (RuntimeException e) { assertEquals("Failed to read backtrace ProcessResult { exitStatus=1 output= errors=Failure }, Command: " + - "[/my/path/to/gdb, -n, -ex, bt, -batch, /usr/bin/program, /tmp/core.1234]", e.getMessage()); + "[/opt/rh/devtoolset-9/root/bin/gdb, -n, -ex, bt, -batch, /usr/bin/program, /tmp/core.1234]", e.getMessage()); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java index 3771774c9a5..a082addd775 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java @@ -7,6 +7,8 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -25,34 +27,42 @@ public class NodeAgentContextManagerTest { private final NodeAgentContextManager manager = new NodeAgentContextManager(clock, initialContext); @Test(timeout = TIMEOUT) - public void returns_immediately_if_next_context_is_ready() throws InterruptedException { + public void context_is_ignored_unless_scheduled_while_waiting() { NodeAgentContext context1 = generateContext(); manager.scheduleTickWith(context1, clock.instant()); - assertSame(initialContext, manager.currentContext()); - assertSame(context1, manager.nextContext()); - assertSame(context1, manager.currentContext()); + + AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); + manager.waitUntilWaitingForNextContext(); + assertFalse(async.isCompleted()); + + NodeAgentContext context2 = generateContext(); + manager.scheduleTickWith(context2, clock.instant()); + + assertSame(context2, async.awaitResult().response.get()); + assertSame(context2, manager.currentContext()); } @Test(timeout = TIMEOUT) - public void returns_no_earlier_than_at_given_time() throws InterruptedException { + public void returns_no_earlier_than_at_given_time() { + AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); + manager.waitUntilWaitingForNextContext(); + NodeAgentContext context1 = generateContext(); Instant returnAt = clock.instant().plusMillis(500); manager.scheduleTickWith(context1, returnAt); - assertSame(initialContext, manager.currentContext()); - assertSame(context1, manager.nextContext()); + assertSame(context1, async.awaitResult().response.get()); assertSame(context1, manager.currentContext()); // Is accurate to a millisecond assertFalse(clock.instant().plusMillis(1).isBefore(returnAt)); } @Test(timeout = TIMEOUT) - public void blocks_in_nextContext_until_one_is_scheduled() throws InterruptedException { + public void blocks_in_nextContext_until_one_is_scheduled() { AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - assertFalse(async.response.isPresent()); - Thread.sleep(10); - assertFalse(async.response.isPresent()); + manager.waitUntilWaitingForNextContext(); + assertFalse(async.isCompleted()); NodeAgentContext context1 = generateContext(); manager.scheduleTickWith(context1, clock.instant()); @@ -63,11 +73,10 @@ public class NodeAgentContextManagerTest { } @Test(timeout = TIMEOUT) - public void blocks_in_nextContext_until_interrupt() throws InterruptedException { + public void blocks_in_nextContext_until_interrupt() { AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); - assertFalse(async.response.isPresent()); - Thread.sleep(10); - assertFalse(async.response.isPresent()); + manager.waitUntilWaitingForNextContext(); + assertFalse(async.isCompleted()); manager.interrupt(); @@ -77,13 +86,15 @@ public class NodeAgentContextManagerTest { } @Test(timeout = TIMEOUT) - public void setFrozen_does_not_block_with_no_timeout() throws InterruptedException { + public void setFrozen_does_not_block_with_no_timeout() { assertFalse(manager.setFrozen(false, Duration.ZERO)); // Generate new context and get it from the supplier, this completes the unfreeze NodeAgentContext context1 = generateContext(); + AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); + manager.waitUntilWaitingForNextContext(); manager.scheduleTickWith(context1, clock.instant()); - assertSame(context1, manager.nextContext()); + assertSame(context1, async.awaitResult().response.get()); assertTrue(manager.setFrozen(false, Duration.ZERO)); } @@ -100,17 +111,34 @@ public class NodeAgentContextManagerTest { @Test(timeout = TIMEOUT) public void setFrozen_is_successful_if_converged_in_time() throws InterruptedException { - AsyncExecutor<Boolean> async = new AsyncExecutor<>(() -> manager.setFrozen(false, Duration.ofMillis(500))); - - assertFalse(async.response.isPresent()); + AsyncExecutor<NodeAgentContext> asyncConsumer1 = new AsyncExecutor<>(() -> { + NodeAgentContext context = manager.nextContext(); + Thread.sleep(200); // Simulate running NodeAgent::converge + return context; + }); + manager.waitUntilWaitingForNextContext(); NodeAgentContext context1 = generateContext(); manager.scheduleTickWith(context1, clock.instant()); - assertSame(context1, manager.nextContext()); + Thread.sleep(10); - async.awaitResult(); - assertEquals(Optional.of(true), async.response); - assertFalse(async.exception.isPresent()); + // Scheduler wants to freeze + AsyncExecutor<Boolean> asyncScheduler = new AsyncExecutor<>(() -> manager.setFrozen(true, Duration.ofMillis(500))); + Thread.sleep(20); + assertFalse(asyncConsumer1.isCompleted()); // Still running NodeAgent::converge + assertSame(context1, asyncConsumer1.awaitResult().response.get()); + assertFalse(asyncScheduler.isCompleted()); // Still waiting for consumer to converge to frozen + + AsyncExecutor<NodeAgentContext> asyncConsumer2 = new AsyncExecutor<>(manager::nextContext); + manager.waitUntilWaitingForNextContext(); + assertFalse(asyncConsumer2.isCompleted()); // Waiting for next context + asyncScheduler.awaitResult(); // We should be able to converge to frozen now + + // Interrupt manager to end asyncConsumer2 + manager.interrupt(); + asyncConsumer2.awaitResult(); + + assertEquals(Optional.of(true), asyncScheduler.response); } private static NodeAgentContext generateContext() { @@ -118,39 +146,30 @@ public class NodeAgentContextManagerTest { } private static class AsyncExecutor<T> { - private final Object monitor = new Object(); - private final Thread thread; + private final CountDownLatch latch = new CountDownLatch(1); private volatile Optional<T> response = Optional.empty(); private volatile Optional<Exception> exception = Optional.empty(); - private boolean completed = false; - private AsyncExecutor(ThrowingSupplier<T> supplier) { - this.thread = new Thread(() -> { + private AsyncExecutor(Callable<T> supplier) { + new Thread(() -> { try { - response = Optional.of(supplier.get()); + response = Optional.of(supplier.call()); } catch (Exception e) { exception = Optional.of(e); } - synchronized (monitor) { - completed = true; - monitor.notifyAll(); - } - }); - this.thread.start(); + latch.countDown(); + }).start(); } - private void awaitResult() { - synchronized (monitor) { - while (!completed) { - try { - monitor.wait(); - } catch (InterruptedException ignored) { } - } - } + private AsyncExecutor<T> awaitResult() { + try { + latch.await(); + } catch (InterruptedException ignored) { } + return this; } - } - private interface ThrowingSupplier<T> { - T get() throws Exception; + private boolean isCompleted() { + return latch.getCount() == 0; + } } }
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java index b90906f8974..662bcd33a17 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java @@ -24,11 +24,10 @@ public class LockedNodeList extends NodeList { private final Mutex lock; public LockedNodeList(List<Node> nodes, Mutex lock) { - super(nodes); + super(nodes, false); this.lock = Objects.requireNonNull(lock, "lock must be non-null"); } - @Override public LockedNodeList filter(Predicate<Node> predicate) { return asList().stream() .filter(predicate) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index d42efd7ba29..b6237886dc7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -128,6 +128,8 @@ public final class Node { return parentHostname.isPresent() && parentHostname.get().equals(hostname); } + public NodeResources resources() { return flavor.resources(); } + /** Returns the flavor of this node */ public Flavor flavor() { return flavor; } @@ -168,17 +170,25 @@ public final class Node { public Optional<TenantName> reservedTo() { return reservedTo; } /** - * Returns a copy of this node with wantToRetire set to the given value and updated history. - * If given wantToRetire is equal to the current, the method is no-op. + * Returns a copy of this node with wantToRetire and wantToDeprovision set to the given values and updated history. + * + * If both given wantToRetire and wantToDeprovision are equal to the current values, the method is no-op. */ - public Node withWantToRetire(boolean wantToRetire, Agent agent, Instant at) { - if (wantToRetire == status.wantToRetire()) return this; - Node node = this.with(status.withWantToRetire(wantToRetire)); + public Node withWantToRetire(boolean wantToRetire, boolean wantToDeprovision, Agent agent, Instant at) { + if (!type.isDockerHost() && wantToDeprovision) + throw new IllegalArgumentException("wantToDeprovision can only be set for hosts"); + if (wantToRetire == status.wantToRetire() && + wantToDeprovision == status.wantToDeprovision()) return this; + Node node = this.with(status.withWantToRetire(wantToRetire, wantToDeprovision)); if (wantToRetire) node = node.with(history.with(new History.Event(History.Event.Type.wantToRetire, agent, at))); return node; } + public Node withWantToRetire(boolean wantToRetire, Agent agent, Instant at) { + return withWantToRetire(wantToRetire, status.wantToDeprovision(), agent, at); + } + /** * Returns a copy of this node which is retired. * If the node was already retired it is returned as-is. @@ -398,7 +408,7 @@ public final class Node { public enum State { - /** This host has been requested (from OpenStack) but is not yet ready for use */ + /** This node has been requested, but is not yet ready for use */ provisioned, /** This node is free and ready for use */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index 93e5b160524..cbc5a44ae94 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -1,6 +1,7 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; +import com.yahoo.collections.AbstractFilteringList; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; @@ -8,13 +9,10 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; -import java.util.Iterator; import java.util.List; import java.util.Optional; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -26,109 +24,112 @@ import static java.util.stream.Collectors.collectingAndThen; * @author bratseth * @author mpolden */ -public class NodeList implements Iterable<Node> { +public class NodeList extends AbstractFilteringList<Node, NodeList> { - private final List<Node> nodes; - private final boolean negate; - - NodeList(List<Node> nodes) { - this(nodes, true, false); - } - - private NodeList(List<Node> nodes, boolean copy, boolean negate) { - this.nodes = copy ? List.copyOf(nodes) : Collections.unmodifiableList(nodes); - this.negate = negate; - } - - /** Invert the next filter operation. All other methods that return a {@link NodeList} clears the negation. */ - public NodeList not() { - return new NodeList(nodes, false, true); + protected NodeList(List<Node> nodes, boolean negate) { + super(nodes, negate, NodeList::new); } /** Returns the subset of nodes which are retired */ public NodeList retired() { - return filter(node -> node.allocation().get().membership().retired()); + return matching(node -> node.allocation().isPresent() && node.allocation().get().membership().retired()); + } + + /** Returns the subset of nodes that are being deprovisioned */ + public NodeList deprovisioning() { + return matching(node -> node.status().wantToRetire() && node.status().wantToDeprovision()); } /** Returns the subset of nodes which are removable */ public NodeList removable() { - return filter(node -> node.allocation().get().isRemovable()); + return matching(node -> node.allocation().isPresent() && node.allocation().get().isRemovable()); } /** Returns the subset of nodes having exactly the given resources */ - public NodeList resources(NodeResources resources) { return filter(node -> node.flavor().resources().equals(resources)); } + public NodeList resources(NodeResources resources) { return matching(node -> node.resources().equals(resources)); } + + /** Returns the subset of nodes which satisfy the given resources */ + public NodeList satisfies(NodeResources resources) { return matching(node -> node.resources().satisfies(resources)); } /** Returns the subset of nodes of the given flavor */ public NodeList flavor(String flavor) { - return filter(node -> node.flavor().name().equals(flavor)); + return matching(node -> node.flavor().name().equals(flavor)); + } + + /** Returns the subset of nodes not in the given collection */ + public NodeList except(Collection<Node> nodes) { + return matching(node -> ! nodes.contains(node)); } /** Returns the subset of nodes assigned to the given cluster type */ public NodeList type(ClusterSpec.Type type) { - return filter(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().type().equals(type)); + return matching(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().type().equals(type)); } /** Returns the subset of nodes that run containers */ public NodeList container() { - return filter(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().type().isContainer()); + return matching(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().type().isContainer()); } /** Returns the subset of nodes that are currently changing their Vespa version */ public NodeList changingVersion() { - return filter(node -> node.status().vespaVersion().isPresent() && - node.allocation().isPresent() && - !node.status().vespaVersion().get().equals(node.allocation().get().membership().cluster().vespaVersion())); + return matching(node -> node.status().vespaVersion().isPresent() && + node.allocation().isPresent() && + !node.status().vespaVersion().get().equals(node.allocation().get().membership().cluster().vespaVersion())); } /** Returns the subset of nodes that are currently changing their OS version to given version */ public NodeList changingOsVersionTo(Version version) { - return filter(node -> node.status().osVersion().changingTo(version)); + return matching(node -> node.status().osVersion().changingTo(version)); } /** Returns the subset of nodes that are currently changing their OS version */ public NodeList changingOsVersion() { - return filter(node -> node.status().osVersion().changing()); + return matching(node -> node.status().osVersion().changing()); } /** Returns a copy of this sorted by current OS version (lowest to highest) */ public NodeList byIncreasingOsVersion() { - return nodes.stream() - .sorted(Comparator.comparing(node -> node.status() - .osVersion() - .current() - .orElse(Version.emptyVersion))) - .collect(collectingAndThen(Collectors.toList(), NodeList::wrap)); + return sortedBy(Comparator.comparing(node -> node.status() + .osVersion() + .current() + .orElse(Version.emptyVersion))); } /** Returns the subset of nodes that are currently on the given OS version */ public NodeList onOsVersion(Version version) { - return filter(node -> node.status().osVersion().matches(version)); + return matching(node -> node.status().osVersion().matches(version)); } /** Returns the subset of nodes assigned to the given cluster */ public NodeList cluster(ClusterSpec.Id cluster) { - return filter(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().id().equals(cluster)); + return matching(node -> node.allocation().isPresent() && node.allocation().get().membership().cluster().id().equals(cluster)); } /** Returns the subset of nodes owned by the given application */ public NodeList owner(ApplicationId application) { - return filter(node -> node.allocation().map(a -> a.owner().equals(application)).orElse(false)); + return matching(node -> node.allocation().map(a -> a.owner().equals(application)).orElse(false)); } /** Returns the subset of nodes matching the given node type(s) */ public NodeList nodeType(NodeType first, NodeType... rest) { EnumSet<NodeType> nodeTypes = EnumSet.of(first, rest); - return filter(node -> nodeTypes.contains(node.type())); + return matching(node -> nodeTypes.contains(node.type())); + } + + /** Returns the subset of nodes of the host type */ + public NodeList hosts() { + return matching(node -> node.type() == NodeType.host); } /** Returns the subset of nodes that are parents */ public NodeList parents() { - return filter(n -> n.parentHostname().isEmpty()); + return matching(n -> n.parentHostname().isEmpty()); } /** Returns the child nodes of the given parent node */ public NodeList childrenOf(String hostname) { - return filter(n -> n.parentHostname().map(hostname::equals).orElse(false)); + return matching(n -> n.parentHostname().map(hostname::equals).orElse(false)); } public NodeList childrenOf(Node parent) { @@ -142,7 +143,12 @@ public class NodeList implements Iterable<Node> { /** Returns the subset of nodes that are in any of the given state(s) */ public NodeList state(Collection<Node.State> nodeStates) { - return filter(node -> nodeStates.contains(node.state())); + return matching(node -> nodeStates.contains(node.state())); + } + + /** Returns the subset of nodes which wantToRetire set true */ + public NodeList wantToRetire() { + return matching((node -> node.status().wantToRetire())); } /** Returns the parent nodes of the given child nodes */ @@ -151,49 +157,31 @@ public class NodeList implements Iterable<Node> { .map(this::parentOf) .filter(Optional::isPresent) .flatMap(Optional::stream) - .collect(collectingAndThen(Collectors.toList(), NodeList::wrap)); + .collect(collectingAndThen(Collectors.toList(), NodeList::copyOf)); + } + + public NodeList group(int index) { + return matching(n -> ( n.allocation().isPresent() && + n.allocation().get().membership().cluster().group().equals(Optional.of(ClusterSpec.Group.from(index))))); } /** Returns the parent node of the given child node */ public Optional<Node> parentOf(Node child) { return child.parentHostname() - .flatMap(parentHostname -> nodes.stream() - .filter(node -> node.hostname().equals(parentHostname)) - .findFirst()); + .flatMap(parentHostname -> stream().filter(node -> node.hostname().equals(parentHostname)) + .findFirst()); } - /** Returns the first n nodes in this */ - public NodeList first(int n) { - n = Math.min(n, nodes.size()); - return wrap(nodes.subList(negate ? n : 0, - negate ? nodes.size() : n)); - } - - public int size() { return nodes.size(); } - - /** Returns the immutable list of nodes in this */ - public List<Node> asList() { return nodes; } - /** Returns the nodes of this as a stream */ public Stream<Node> stream() { return asList().stream(); } - public NodeList filter(Predicate<Node> predicate) { - return nodes.stream().filter(negate ? predicate.negate() : predicate) - .collect(collectingAndThen(Collectors.toList(), NodeList::wrap)); + public static NodeList copyOf(List<Node> nodes) { + return new NodeList(nodes, false); } @Override - public Iterator<Node> iterator() { - return nodes.iterator(); - } - - /** Create a new list containing the given nodes, without copying */ - private static NodeList wrap(List<Node> nodes) { - return new NodeList(nodes, false, false); - } - - public static NodeList copyOf(List<Node> nodes) { - return new NodeList(nodes, true, false); + public String toString() { + return asList().toString(); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 436b1e49a54..bec35e7ee4f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -102,6 +102,7 @@ public class NodeRepository extends AbstractComponent { private final DockerImages dockerImages; private final JobControl jobControl; private final Applications applications; + private final boolean canProvisionHosts; /** * Creates a node repository from a zookeeper provider. @@ -119,7 +120,8 @@ public class NodeRepository extends AbstractComponent { Clock.systemUTC(), zone, new DnsNameResolver(), - DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache()); + DockerImage.fromString(config.dockerImage()), config.useCuratorClientCache(), + provisionServiceProvider.getHostProvisioner().isPresent()); } /** @@ -133,7 +135,8 @@ public class NodeRepository extends AbstractComponent { Zone zone, NameResolver nameResolver, DockerImage dockerImage, - boolean useCuratorClientCache) { + boolean useCuratorClientCache, + boolean canProvisionHosts) { this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache); this.zone = zone; this.clock = clock; @@ -146,6 +149,7 @@ public class NodeRepository extends AbstractComponent { this.dockerImages = new DockerImages(db, dockerImage); this.jobControl = new JobControl(db); this.applications = new Applications(db); + this.canProvisionHosts = canProvisionHosts; // read and write all nodes to make sure they are stored in the latest version of the serialized format for (State state : State.values()) @@ -430,7 +434,7 @@ public class NodeRepository extends AbstractComponent { .map(node -> { if (node.state() != State.provisioned && node.state() != State.dirty) illegal("Can not set " + node + " ready. It is not provisioned or dirty."); - return node.with(node.status().withWantToRetire(false).withWantToDeprovision(false)); + return node.withWantToRetire(false, false, Agent.system, clock.instant()); }) .collect(Collectors.toList()); @@ -646,7 +650,7 @@ public class NodeRepository extends AbstractComponent { try (Mutex lock = lockUnallocated()) { requireRemovable(node, false, force); - if (node.type() == NodeType.host) { + if (node.type().isDockerHost()) { List<Node> children = list().childrenOf(node).asList(); children.forEach(child -> requireRemovable(child, true, force)); db.removeNodes(children); @@ -661,8 +665,9 @@ public class NodeRepository extends AbstractComponent { return removed; } else { - db.removeNodes(List.of(node)); - return List.of(node); + List<Node> removed = List.of(node); + db.removeNodes(removed); + return removed; } } } @@ -677,8 +682,8 @@ public class NodeRepository extends AbstractComponent { /** * Throws if the given node cannot be removed. Removal is allowed if: * - Tenant node: node is unallocated - * - Non-Docker-container node: iff in state provisioned|failed|parked - * - Docker-container-node: + * - Host node: iff in state provisioned|failed|parked + * - Child node: * If only removing the container node: node in state ready * If also removing the parent node: child is in state provisioned|failed|parked|dirty|ready */ @@ -690,7 +695,7 @@ public class NodeRepository extends AbstractComponent { if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER && !removingAsChild) { if (node.state() != State.ready) - illegal(node + " can not be removed as it is not in the state [ready]"); + illegal(node + " can not be removed as it is not in the state " + State.ready); } else if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER) { // removing a child node Set<State> legalStates = EnumSet.of(State.provisioned, State.failed, State.parked, State.dirty, State.ready); @@ -791,16 +796,19 @@ public class NodeRepository extends AbstractComponent { } public boolean canAllocateTenantNodeTo(Node host) { - if (!host.type().canRun(NodeType.tenant)) return false; - - // Do not allocate to hosts we want to retire or are currently retiring - if (host.status().wantToRetire() || host.allocation().map(alloc -> alloc.membership().retired()).orElse(false)) - return false; + if ( ! host.type().canRun(NodeType.tenant)) return false; + if (host.status().wantToRetire()) return false; + if (host.allocation().map(alloc -> alloc.membership().retired()).orElse(false)) return false; - if (!zone.getCloud().dynamicProvisioning()) return host.state() == State.active; - else return EnumSet.of(State.active, State.ready, State.provisioned).contains(host.state()); + if ( canProvisionHosts()) + return EnumSet.of(State.active, State.ready, State.provisioned).contains(host.state()); + else + return host.state() == State.active; } + /** Returns whether this repository can provision hosts on demand */ + public boolean canProvisionHosts() { return canProvisionHosts; } + /** Returns the time keeper of this system */ public Clock clock() { return clock; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java index 3a3d3e4ec2e..e9e91910281 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -53,12 +53,12 @@ public class Application { * Returns an application with the given cluster having the min and max resource limits of the given cluster. * If the cluster has a target which is not inside the new limits, the target is removed. */ - public Application withClusterLimits(ClusterSpec.Id id, ClusterResources min, ClusterResources max) { + public Application withCluster(ClusterSpec.Id id, boolean exclusive, ClusterResources min, ClusterResources max) { Cluster cluster = clusters.get(id); if (cluster == null) - cluster = new Cluster(id, min, max, Optional.empty(), Optional.empty()); + cluster = new Cluster(id, exclusive, min, max, Optional.empty(), Optional.empty()); else - cluster = cluster.withLimits(min, max); + cluster = cluster.withConfiguration(exclusive, min, max); return with(cluster); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 847ec1290f6..3aae47a9088 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -19,16 +19,19 @@ import java.util.Optional; public class Cluster { private final ClusterSpec.Id id; + private final boolean exclusive; private final ClusterResources min, max; private final Optional<ClusterResources> suggested; private final Optional<ClusterResources> target; public Cluster(ClusterSpec.Id id, + boolean exclusive, ClusterResources minResources, ClusterResources maxResources, Optional<ClusterResources> suggestedResources, Optional<ClusterResources> targetResources) { this.id = Objects.requireNonNull(id); + this.exclusive = exclusive; this.min = Objects.requireNonNull(minResources); this.max = Objects.requireNonNull(maxResources); this.suggested = Objects.requireNonNull(suggestedResources); @@ -47,6 +50,9 @@ public class Cluster { /** Returns the configured maximal resources in this cluster */ public ClusterResources maxResources() { return max; } + /** Returns whether the nodes allocated to this cluster must be on host exclusively dedicated to this application */ + public boolean exclusive() { return exclusive; } + /** * Returns the computed resources (between min and max, inclusive) this cluster should * have allocated at the moment (whether or not it actually has it), @@ -60,16 +66,16 @@ public class Cluster { */ public Optional<ClusterResources> suggestedResources() { return suggested; } - public Cluster withLimits(ClusterResources min, ClusterResources max) { - return new Cluster(id, min, max, suggested, target); + public Cluster withConfiguration(boolean exclusive, ClusterResources min, ClusterResources max) { + return new Cluster(id, exclusive, min, max, suggested, target); } public Cluster withSuggested(Optional<ClusterResources> suggested) { - return new Cluster(id, min, max, suggested, target); + return new Cluster(id, exclusive, min, max, suggested, target); } public Cluster withTarget(Optional<ClusterResources> target) { - return new Cluster(id, min, max, suggested, target); + return new Cluster(id, exclusive, min, max, suggested, target); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index 0fa04032146..267bfefa332 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -32,20 +32,22 @@ public class AllocatableClusterResources { private final double fulfilment; /** Fake allocatable resources from requested capacity */ - public AllocatableClusterResources(ClusterResources requested, ClusterSpec.Type clusterType) { - this.advertisedResources = requested.nodeResources(); - this.realResources = requested.nodeResources(); // we don't know + public AllocatableClusterResources(ClusterResources requested, + ClusterSpec.Type clusterType, + NodeRepository nodeRepository) { this.nodes = requested.nodes(); this.groups = requested.groups(); + this.realResources = nodeRepository.resourcesCalculator().requestToReal(requested.nodeResources()); + this.advertisedResources = requested.nodeResources(); this.clusterType = clusterType; this.fulfilment = 1; } public AllocatableClusterResources(List<Node> nodes, NodeRepository nodeRepository) { - this.advertisedResources = nodes.get(0).flavor().resources(); - this.realResources = nodeRepository.resourcesCalculator().realResourcesOf(nodes.get(0), nodeRepository); this.nodes = nodes.size(); this.groups = (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); + this.realResources = averageRealResourcesOf(nodes, nodeRepository); // Average since we average metrics over nodes + this.advertisedResources = nodes.get(0).resources(); this.clusterType = nodes.get(0).allocation().get().membership().cluster().type(); this.fulfilment = 1; } @@ -54,23 +56,10 @@ public class AllocatableClusterResources { NodeResources advertisedResources, NodeResources idealResources, ClusterSpec.Type clusterType) { - this.realResources = realResources.nodeResources(); - this.advertisedResources = advertisedResources; this.nodes = realResources.nodes(); this.groups = realResources.groups(); - this.clusterType = clusterType; - this.fulfilment = fulfilment(realResources.nodeResources(), idealResources); - } - - public AllocatableClusterResources(ClusterResources realResources, - Flavor flavor, - NodeResources idealResources, - ClusterSpec.Type clusterType, - HostResourcesCalculator calculator) { this.realResources = realResources.nodeResources(); - this.advertisedResources = calculator.advertisedResourcesOf(flavor); - this.nodes = realResources.nodes(); - this.groups = realResources.groups(); + this.advertisedResources = advertisedResources; this.clusterType = clusterType; this.fulfilment = fulfilment(realResources.nodeResources(), idealResources); } @@ -118,61 +107,76 @@ public class AllocatableClusterResources { } public boolean preferableTo(AllocatableClusterResources other) { - if (this.fulfilment > other.fulfilment) return true; // we always want to fulfil as much as possible + if (this.fulfilment < 1 || other.fulfilment < 1) + return this.fulfilment > other.fulfilment; // we always want to fulfil as much as possible return this.cost() < other.cost(); // otherwise, prefer lower cost } @Override public String toString() { - return nodes + " nodes with " + realResources() + + return nodes + " nodes " + + ( groups > 1 ? "(in " + groups + " groups) " : "" ) + + "with " + advertisedResources() + " at cost $" + cost() + (fulfilment < 1.0 ? " (fulfilment " + fulfilment + ")" : ""); } - /** - * Returns the best matching allocatable node resources given ideal node resources, - * or empty if none available within the limits. - */ - public static Optional<AllocatableClusterResources> from(ClusterResources resources, + private static NodeResources averageRealResourcesOf(List<Node> nodes, NodeRepository nodeRepository) { + NodeResources sum = new NodeResources(0, 0, 0, 0); + for (Node node : nodes) + sum = sum.add(nodeRepository.resourcesCalculator().realResourcesOf(node, nodeRepository).justNumbers()); + return nodes.get(0).resources().justNonNumbers() + .withVcpu(sum.vcpu() / nodes.size()) + .withMemoryGb(sum.memoryGb() / nodes.size()) + .withDiskGb(sum.diskGb() / nodes.size()) + .withBandwidthGbps(sum.bandwidthGbps() / nodes.size()); + } + + public static Optional<AllocatableClusterResources> from(ClusterResources wantedResources, + boolean exclusive, ClusterSpec.Type clusterType, - Limits limits, + Limits applicationLimits, NodeRepository nodeRepository) { - NodeResources cappedNodeResources = limits.cap(resources.nodeResources()); - cappedNodeResources = new NodeResourceLimits(nodeRepository.zone()).enlargeToLegal(cappedNodeResources, clusterType); - - if (nodeRepository.zone().getCloud().allowHostSharing()) { - // return the requested resources, or empty if they cannot fit on existing hosts + var systemLimits = new NodeResourceLimits(nodeRepository); + if ( !exclusive && nodeRepository.zone().getCloud().allowHostSharing()) { // Check if any flavor can fit these hosts + // We decide resources: Add overhead to what we'll request (advertised) to make sure real becomes (at least) cappedNodeResources + NodeResources advertisedResources = nodeRepository.resourcesCalculator().realToRequest(wantedResources.nodeResources()); + advertisedResources = systemLimits.enlargeToLegal(advertisedResources, clusterType); // Attempt to ask for something legal + advertisedResources = applicationLimits.cap(advertisedResources); // Overrides other conditions, even if it will then fail + NodeResources realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources); // ... thus, what we really get may change + if ( ! systemLimits.isWithinRealLimits(realResources, clusterType)) return Optional.empty(); for (Flavor flavor : nodeRepository.flavors().getFlavors()) { - if (flavor.resources().satisfies(cappedNodeResources)) - return Optional.of(new AllocatableClusterResources(resources.with(cappedNodeResources), - cappedNodeResources, - resources.nodeResources(), + if (flavor.resources().satisfies(advertisedResources)) + return Optional.of(new AllocatableClusterResources(wantedResources.with(realResources), + advertisedResources, + wantedResources.nodeResources(), clusterType)); } return Optional.empty(); } - else { - // return the cheapest flavor satisfying the target resources, if any + else { // Return the cheapest flavor satisfying the requested resources, if any + NodeResources cappedWantedResources = applicationLimits.cap(wantedResources.nodeResources()); Optional<AllocatableClusterResources> best = Optional.empty(); for (Flavor flavor : nodeRepository.flavors().getFlavors()) { + // Flavor decide resources: Real resources are the worst case real resources we'll get if we ask for these advertised resources NodeResources advertisedResources = nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor); - NodeResources realResources = flavor.resources(); + NodeResources realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources); // Adjust where we don't need exact match to the flavor if (flavor.resources().storageType() == NodeResources.StorageType.remote) { - advertisedResources = advertisedResources.withDiskGb(cappedNodeResources.diskGb()); - realResources = realResources.withDiskGb(cappedNodeResources.diskGb()); + advertisedResources = advertisedResources.withDiskGb(cappedWantedResources.diskGb()); + realResources = realResources.withDiskGb(cappedWantedResources.diskGb()); } - if (flavor.resources().bandwidthGbps() >= cappedNodeResources.bandwidthGbps()) { - advertisedResources = advertisedResources.withBandwidthGbps(cappedNodeResources.bandwidthGbps()); - realResources = realResources.withBandwidthGbps(cappedNodeResources.bandwidthGbps()); + if (flavor.resources().bandwidthGbps() >= advertisedResources.bandwidthGbps()) { + advertisedResources = advertisedResources.withBandwidthGbps(cappedWantedResources.bandwidthGbps()); + realResources = realResources.withBandwidthGbps(cappedWantedResources.bandwidthGbps()); } - if ( ! between(limits.min().nodeResources(), limits.max().nodeResources(), advertisedResources)) continue; - - var candidate = new AllocatableClusterResources(resources.with(realResources), + if ( ! between(applicationLimits.min().nodeResources(), applicationLimits.max().nodeResources(), advertisedResources)) continue; + if ( ! systemLimits.isWithinRealLimits(realResources, clusterType)) continue; + var candidate = new AllocatableClusterResources(wantedResources.with(realResources), advertisedResources, - resources.nodeResources(), + wantedResources.nodeResources(), clusterType); if (best.isEmpty() || candidate.preferableTo(best.get())) best = Optional.of(candidate); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 8d26bb89959..d2589b15421 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -14,6 +14,15 @@ import java.util.Optional; */ public class AllocationOptimizer { + // The min and max nodes to consider when not using application supplied limits + private static final int minimumNodes = 2; // Since this number includes redundancy it cannot be lower than 2 + private static final int maximumNodes = 150; + + // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component + // proportional to document count. We must account for this when comparing configurations with more or fewer nodes. + // TODO: Measure this, and only take it into account with queries + private static final double fixedCpuCostFraction = 0.1; + private final NodeRepository nodeRepository; public AllocationOptimizer(NodeRepository nodeRepository) { @@ -29,144 +38,68 @@ public class AllocationOptimizer { */ public Optional<AllocatableClusterResources> findBestAllocation(ResourceTarget target, AllocatableClusterResources current, - Limits limits) { + Limits limits, + boolean exclusive) { + if (limits.isEmpty()) + limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified()), + new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified())); Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); - for (ResourceIterator i = new ResourceIterator(target, current, limits); i.hasNext(); ) { - var allocatableResources = AllocatableClusterResources.from(i.next(), current.clusterType(), limits, nodeRepository); - if (allocatableResources.isEmpty()) continue; - if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) - bestAllocation = allocatableResources; + for (int groups = limits.min().groups(); groups <= limits.max().groups(); groups++) { + for (int nodes = limits.min().nodes(); nodes <= limits.max().nodes(); nodes++) { + if (nodes % groups != 0) continue; + int groupSize = nodes / groups; + + // Adjust for redundancy: Node in group if groups = 1, an extra group if multiple groups + // TODO: Make the best choice based on size and redundancy setting instead + int nodesAdjustedForRedundancy = target.adjustForRedundancy() ? (groups == 1 ? nodes - 1 : nodes - groupSize) : nodes; + int groupsAdjustedForRedundancy = target.adjustForRedundancy() ? (groups == 1 ? 1 : groups - 1) : groups; + + ClusterResources next = new ClusterResources(nodes, + groups, + nodeResourcesWith(nodesAdjustedForRedundancy, groupsAdjustedForRedundancy, limits, current, target)); + + var allocatableResources = AllocatableClusterResources.from(next, exclusive, current.clusterType(), limits, nodeRepository); + if (allocatableResources.isEmpty()) continue; + if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) + bestAllocation = allocatableResources; + } } + return bestAllocation; } /** - * Provides iteration over possible cluster resource allocations given a target total load - * and current groups/nodes allocation. + * For the observed load this instance is initialized with, returns the resources needed per node to be at + * ideal load given a target node count */ - private static class ResourceIterator { - - // The min and max nodes to consider when not using application supplied limits - private static final int minimumNodes = 3; // Since this number includes redundancy it cannot be lower than 2 - private static final int maximumNodes = 150; - - // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component - // proportional to document count. We must account for this when comparing configurations with more or fewer nodes. - // TODO: Measure this, and only take it into account with queries - private static final double fixedCpuCostFraction = 0.1; - - // Given state - private final Limits limits; - private final AllocatableClusterResources current; - private final ResourceTarget target; - - // Derived from the observed state - private final int nodeIncrement; - private final boolean singleGroupMode; - - // Iterator state - private int currentNodes; - - public ResourceIterator(ResourceTarget target, AllocatableClusterResources current, Limits limits) { - this.target = target; - this.current = current; - this.limits = limits; - - // What number of nodes is it effective to add or remove at the time from this cluster? - // This is the group size, since we (for now) assume the group size is decided by someone wiser than us - // and we decide the number of groups. - // The exception is when we only have one group, where we can add and remove single nodes in it. - singleGroupMode = current.groups() == 1; - nodeIncrement = singleGroupMode ? 1 : current.groupSize(); - - // Step to the right starting point - currentNodes = current.nodes(); - if (currentNodes < minNodes()) { // step up - while (currentNodes < minNodes() - && (singleGroupMode || currentNodes + nodeIncrement > current.groupSize())) // group level redundancy - currentNodes += nodeIncrement; - } - else { // step down - while (currentNodes - nodeIncrement >= minNodes() - && (singleGroupMode || currentNodes - nodeIncrement > current.groupSize())) // group level redundancy - currentNodes -= nodeIncrement; - } + private NodeResources nodeResourcesWith(int nodes, int groups, Limits limits, AllocatableClusterResources current, ResourceTarget target) { + // Cpu: Scales with cluster size (TODO: Only reads, writes scales with group size) + // Memory and disk: Scales with group size + double cpu, memory, disk; + + int groupSize = nodes / groups; + + if (current.clusterType().isContent()) { // load scales with node share of content + // The fixed cost portion of cpu does not scale with changes to the node count + // TODO: Only for the portion of cpu consumed by queries + double cpuPerGroup = fixedCpuCostFraction * target.nodeCpu() + + (1 - fixedCpuCostFraction) * target.nodeCpu() * current.groupSize() / groupSize; + cpu = cpuPerGroup * current.groups() / groups; + memory = target.nodeMemory() * current.groupSize() / groupSize; + disk = target.nodeDisk() * current.groupSize() / groupSize; } - - public ClusterResources next() { - ClusterResources next = resourcesWith(currentNodes); - currentNodes += nodeIncrement; - return next; - } - - public boolean hasNext() { - return currentNodes <= maxNodes(); - } - - private int minNodes() { - if (limits.isEmpty()) return minimumNodes; - if (singleGroupMode) return limits.min().nodes(); - return Math.max(limits.min().nodes(), limits.min().groups() * current.groupSize() ); - } - - private int maxNodes() { - if (limits.isEmpty()) return maximumNodes; - if (singleGroupMode) return limits.max().nodes(); - return Math.min(limits.max().nodes(), limits.max().groups() * current.groupSize() ); - } - - private ClusterResources resourcesWith(int nodes) { - int nodesAdjustedForRedundancy = nodes; - if (target.adjustForRedundancy()) - nodesAdjustedForRedundancy = nodes - (singleGroupMode ? 1 : current.groupSize()); - return new ClusterResources(nodes, - singleGroupMode ? 1 : nodes / current.groupSize(), - nodeResourcesWith(nodesAdjustedForRedundancy)); - } - - /** - * For the observed load this instance is initialized with, returns the resources needed per node to be at - * ideal load given a target node count - */ - private NodeResources nodeResourcesWith(int nodeCount) { - // Cpu: Scales with cluster size (TODO: Only reads, writes scales with group size) - // Memory and disk: Scales with group size - - double cpu, memory, disk; - if (singleGroupMode) { - // The fixed cost portion of cpu does not scale with changes to the node count - // TODO: Only for the portion of cpu consumed by queries - cpu = fixedCpuCostFraction * target.clusterCpu() / current.groupSize() + - (1 - fixedCpuCostFraction) * target.clusterCpu() / nodeCount; - - if (current.clusterType().isContent()) { // load scales with node share of content - memory = target.groupMemory() / nodeCount; - disk = target.groupDisk() / nodeCount; - } - else { - memory = target.nodeMemory(); - disk = target.nodeDisk(); - } - } - else { - cpu = target.clusterCpu() / nodeCount; - if (current.clusterType().isContent()) { // load scales with node share of content - memory = target.groupMemory() / current.groupSize(); - disk = target.groupDisk() / current.groupSize(); - } - else { - memory = target.nodeMemory(); - disk = target.nodeDisk(); - } - } - - // Combine the scaled resource values computed here - // with the currently configured non-scaled values, given in the limits, if any - NodeResources nonScaled = limits.isEmpty() || limits.min().nodeResources().isUnspecified() - ? current.toAdvertisedClusterResources().nodeResources() - : limits.min().nodeResources(); // min=max for non-scaled - return nonScaled.withVcpu(cpu).withMemoryGb(memory).withDiskGb(disk); + else { + cpu = target.nodeCpu() * current.nodes() / nodes; + memory = target.nodeMemory(); + disk = target.nodeDisk(); } + // Combine the scaled resource values computed here + // with the currently configured non-scaled values, given in the limits, if any + NodeResources nonScaled = limits.isEmpty() || limits.min().nodeResources().isUnspecified() + ? current.toAdvertisedClusterResources().nodeResources() + : limits.min().nodeResources(); // min=max for non-scaled + return nonScaled.withVcpu(cpu).withMemoryGb(memory).withDiskGb(disk); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 8930bf34f4a..27731159e9f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -53,7 +53,7 @@ public class Autoscaler { * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time */ public Optional<ClusterResources> suggest(Cluster cluster, List<Node> clusterNodes) { - return autoscale(clusterNodes, Limits.empty()) + return autoscale(clusterNodes, Limits.empty(), cluster.exclusive()) .map(AllocatableClusterResources::toAdvertisedClusterResources); } @@ -66,11 +66,11 @@ public class Autoscaler { */ public Optional<ClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes) { if (cluster.minResources().equals(cluster.maxResources())) return Optional.empty(); // Shortcut - return autoscale(clusterNodes, Limits.of(cluster)) + return autoscale(clusterNodes, Limits.of(cluster), cluster.exclusive()) .map(AllocatableClusterResources::toAdvertisedClusterResources); } - private Optional<AllocatableClusterResources> autoscale(List<Node> clusterNodes, Limits limits) { + private Optional<AllocatableClusterResources> autoscale(List<Node> clusterNodes, Limits limits, boolean exclusive) { if (unstable(clusterNodes)) return Optional.empty(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); @@ -82,7 +82,7 @@ public class Autoscaler { var target = ResourceTarget.idealLoad(cpuLoad.get(), memoryLoad.get(), diskLoad.get(), currentAllocation); Optional<AllocatableClusterResources> bestAllocation = - allocationOptimizer.findBestAllocation(target, currentAllocation, limits); + allocationOptimizer.findBestAllocation(target, currentAllocation, limits, exclusive); if (bestAllocation.isEmpty()) return Optional.empty(); if (similar(bestAllocation.get(), currentAllocation)) return Optional.empty(); return bestAllocation; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java index 7ca60c4c86d..15bf4427346 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java @@ -6,6 +6,8 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.applications.Cluster; +import java.util.Objects; + /** * Optional allocation limits * @@ -60,6 +62,10 @@ public class Limits { return new Limits(capacity.minResources(), capacity.maxResources()); } + public static Limits of(ClusterResources min, ClusterResources max) { + return new Limits(Objects.requireNonNull(min, "min"), Objects.requireNonNull(max, "max")); + } + @Override public String toString() { if (isEmpty()) return "no limits"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java index 232fee1df6a..acdd419c0de 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java @@ -58,7 +58,7 @@ public class NodeMetricsFetcher extends AbstractComponent implements NodeMetrics if (Autoscaler.unstable(applicationNodes.asList())) return Collections.emptyList(); Optional<Node> metricsV2Container = applicationNodes.container() - .filter(node -> expectedUp(node)) + .matching(node -> expectedUp(node)) .stream() .findFirst(); if (metricsV2Container.isEmpty()) return Collections.emptyList(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java index 287cd2ae86a..f7c2a51436a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java @@ -1,44 +1,29 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; -import com.yahoo.config.provision.NodeResources; - /** * A resource target to hit for the allocation optimizer. * The target is measured in cpu, memory and disk per node in the allocation given by current. + * + * @author bratseth */ public class ResourceTarget { private final boolean adjustForRedundancy; - /** The target resources per node, assuming the node assignment in current */ + /** The target resources per node, assuming the node assignment where this was decided */ private final double cpu, memory, disk; - /** The current allocation leading to this target */ - private final AllocatableClusterResources current; - - private ResourceTarget(double cpu, double memory, double disk, - boolean adjustForRedundancy, - AllocatableClusterResources current) { + private ResourceTarget(double cpu, double memory, double disk, boolean adjustForRedundancy) { this.cpu = cpu; this.memory = memory; this.disk = disk; this.adjustForRedundancy = adjustForRedundancy; - this.current = current; } /** Are the target resources given by this including redundancy or not */ public boolean adjustForRedundancy() { return adjustForRedundancy; } - /** Returns the target total cpu to allocate to the entire cluster */ - public double clusterCpu() { return nodeCpu() * current.nodes(); } - - /** Returns the target total memory to allocate to each group */ - public double groupMemory() { return nodeMemory() * current.groupSize(); } - - /** Returns the target total disk to allocate to each group */ - public double groupDisk() { return nodeDisk() * current.groupSize(); } - /** Returns the target cpu per node, in terms of the current allocation */ public double nodeCpu() { return cpu; } @@ -48,6 +33,13 @@ public class ResourceTarget { /** Returns the target disk per node, in terms of the current allocation */ public double nodeDisk() { return disk; } + @Override + public String toString() { + return "target " + + (adjustForRedundancy ? "(with redundancy adjustment) " : "") + + "[vcpu " + cpu + ", memoryGb " + memory + ", diskGb " + disk + "]"; + } + private static double nodeUsage(Resource resource, double load, AllocatableClusterResources current) { return load * resource.valueFrom(current.realResources()); } @@ -58,8 +50,7 @@ public class ResourceTarget { return new ResourceTarget(nodeUsage(Resource.cpu, currentCpuLoad, current) / Resource.cpu.idealAverageLoad(), nodeUsage(Resource.memory, currentMemoryLoad, current) / Resource.memory.idealAverageLoad(), nodeUsage(Resource.disk, currentDiskLoad, current) / Resource.disk.idealAverageLoad(), - true, - current); + true); } /** Crete a target of preserving a current allocation */ @@ -67,8 +58,7 @@ public class ResourceTarget { return new ResourceTarget(current.realResources().vcpu(), current.realResources().memoryGb(), current.realResources().diskGb(), - false, - current); + false); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index 09723d83e3e..edf2932ad6e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -3,9 +3,7 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.provision.NodeRepository; - -import java.util.Set; +import com.yahoo.config.provision.NodeType; /** * A managed load balance service. @@ -15,17 +13,14 @@ import java.util.Set; public interface LoadBalancerService { /** - * Create a load balancer for given application cluster. Implementations are expected to be idempotent + * Create a load balancer from the given specification. Implementations are expected to be idempotent * - * @param application Application owning the LB - * @param cluster Target cluster of the LB - * @param reals Reals that should be configured on the LB + * @param spec Load balancer specification * @param force Whether reconfiguration should be forced (e.g. allow configuring an empty set of reals on a * pre-existing load balancer). * @return The provisioned load balancer instance */ - LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository); + LoadBalancerInstance create(LoadBalancerSpec spec, boolean force); /** Permanently remove load balancer for given application cluster */ void remove(ApplicationId application, ClusterSpec.Id cluster); @@ -33,6 +28,12 @@ public interface LoadBalancerService { /** Returns the protocol supported by this load balancer service */ Protocol protocol(); + /** Returns whether load balancers created by this service can forward traffic to given node and cluster type */ + default boolean canForwardTo(NodeType nodeType, ClusterSpec.Type clusterType) { + return (nodeType == NodeType.tenant && clusterType.isContainer()) || + (nodeType == NodeType.config && clusterType == ClusterSpec.Type.admin); + } + /** Load balancer protocols */ enum Protocol { ipv4, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index 9bd1189420a..f4d689056c3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -5,13 +5,11 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Set; /** * @author mpolden @@ -30,19 +28,18 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository) { - var id = new LoadBalancerId(application, cluster); + public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { + var id = new LoadBalancerId(spec.application(), spec.cluster()); var oldInstance = instances.get(id); - if (!force && oldInstance != null && !oldInstance.reals().isEmpty() && reals.isEmpty()) { + if (!force && oldInstance != null && !oldInstance.reals().isEmpty() && spec.reals().isEmpty()) { throw new IllegalArgumentException("Refusing to remove all reals from load balancer " + id); } var instance = new LoadBalancerInstance( - HostName.from("lb-" + application.toShortString() + "-" + cluster.value()), + HostName.from("lb-" + spec.application().toShortString() + "-" + spec.cluster().value()), Optional.of(new DnsZone("zone-id-1")), Collections.singleton(4443), ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), - reals); + spec.reals()); instances.put(id, instance); return instance; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java new file mode 100644 index 00000000000..b76f5ba2bcf --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java @@ -0,0 +1,43 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.lb; + +import com.google.common.collect.ImmutableSortedSet; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; + +import java.util.Objects; +import java.util.Set; + +/** + * A specification for a load balancer. + * + * @author mpolden + */ +public class LoadBalancerSpec { + + private final ApplicationId application; + private final ClusterSpec.Id cluster; + private final Set<Real> reals; + + public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals) { + this.application = Objects.requireNonNull(application); + this.cluster = Objects.requireNonNull(cluster); + this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals)); + } + + /** Owner of the load balancer */ + public ApplicationId application() { + return application; + } + + /** The target cluster of this load balancer */ + public ClusterSpec.Id cluster() { + return cluster; + } + + /** Real servers to attach to this load balancer */ + public Set<Real> reals() { + return reals; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java index 07074bc45af..7667672e470 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerService.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.Comparator; import java.util.Optional; @@ -18,11 +17,10 @@ import java.util.Set; public class PassthroughLoadBalancerService implements LoadBalancerService { @Override - public LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository) { - var real = reals.stream() - .min(Comparator.naturalOrder()) - .orElseThrow(() -> new IllegalArgumentException("No reals given")); + public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { + var real = spec.reals().stream() + .min(Comparator.naturalOrder()) + .orElseThrow(() -> new IllegalArgumentException("No reals given")); return new LoadBalancerInstance(real.hostname(), Optional.empty(), Set.of(real.port()), Set.of(real.ipAddress() + "/32"), Set.of()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index a8faafc0bad..bc4381573c6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; -import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; @@ -27,13 +26,14 @@ public class SharedLoadBalancerService implements LoadBalancerService { private static final Comparator<Node> hostnameComparator = Comparator.comparing(Node::hostname); - @Inject - public SharedLoadBalancerService() { + private final NodeRepository nodeRepository; + + public SharedLoadBalancerService(NodeRepository nodeRepository) { + this.nodeRepository = Objects.requireNonNull(nodeRepository); } @Override - public LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, boolean force, - NodeRepository nodeRepository) { + public LoadBalancerInstance create(LoadBalancerSpec spec, boolean force) { var proxyNodes = new ArrayList<>(nodeRepository.getNodes(NodeType.proxy)); proxyNodes.sort(hostnameComparator); @@ -52,7 +52,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { Optional.empty(), Set.of(4080, 4443), networkNames, - reals + spec.reals() ); } @@ -66,6 +66,12 @@ public class SharedLoadBalancerService implements LoadBalancerService { return Protocol.dualstack; } + @Override + public boolean canForwardTo(NodeType nodeType, ClusterSpec.Type clusterType) { + // Shared routing layer only supports routing to tenant nodes + return nodeType == NodeType.tenant && clusterType.isContainer(); + } + private static String withPrefixLength(String address) { if (IP.isV6(address)) { return address + "/128"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 06300e45e9e..a762f718ab7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -74,12 +74,16 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer { /** Returns the applications that should be maintained by this now. */ protected abstract Set<ApplicationId> applicationsNeedingMaintenance(); - /** Redeploy this application. A lock will be taken for the duration of the deployment activation */ - protected final void deployWithLock(ApplicationId application) { + /** + * Redeploy this application. A lock will be taken for the duration of the deployment activation + * + * @return whether it was successfully deployed + */ + protected final boolean deployWithLock(ApplicationId application) { try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) { - if ( ! deployment.isValid()) return; // this will be done at another config server - if ( ! canDeployNow(application)) return; // redeployment is no longer needed - deployment.activate(); + if ( ! deployment.isValid()) return false; // this will be done at another config server + if ( ! canDeployNow(application)) return false; // redeployment is no longer needed + return deployment.activate(); } finally { pendingDeployments.remove(application); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index fa8e8375e23..c32b7854d4e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -85,7 +85,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { int currentGroups = (int)clusterNodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); log.info("Autoscaling " + application + " " + clusterType + " " + clusterId + ":" + - "\nfrom " + toString(clusterNodes.size(), currentGroups, clusterNodes.get(0).flavor().resources()) + + "\nfrom " + toString(clusterNodes.size(), currentGroups, clusterNodes.get(0).resources()) + "\nto " + toString(target.nodes(), target.groups(), target.nodeResources())); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index ca8399da629..f583728f9b8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -6,11 +6,15 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceComparator; import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; +/** + * @author mgimle + */ public class CapacityChecker { private List<Node> hosts; @@ -42,15 +46,15 @@ public class CapacityChecker { } public List<Node> nodesFromHostnames(List<String> hostnames) { - List<Node> nodes = hostnames.stream() - .filter(h -> nodeMap.containsKey(h)) - .map(h -> nodeMap.get(h)) - .collect(Collectors.toList()); + List<Node> nodes = hostnames.stream().filter(h -> nodeMap.containsKey(h)) + .map(h -> nodeMap.get(h)) + .collect(Collectors.toList()); + if (nodes.size() != hostnames.size()) { Set<String> notFoundNodes = new HashSet<>(hostnames); notFoundNodes.removeAll(nodes.stream().map(Node::hostname).collect(Collectors.toList())); throw new IllegalArgumentException(String.format("Host(s) not found: [ %s ]", - String.join(", ", notFoundNodes))); + String.join(", ", notFoundNodes))); } return nodes; @@ -92,9 +96,9 @@ public class CapacityChecker { if (hosts.size() == 0) return Optional.empty(); List<Node> parentRemovalPriorityList = heuristic.entrySet().stream() - .sorted(Comparator.comparingInt(Map.Entry::getValue)) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); + .sorted(this::hostMitigationOrder) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); for (int i = 1; i <= parentRemovalPriorityList.size(); i++) { List<Node> hostsToRemove = parentRemovalPriorityList.subList(0, i); @@ -110,18 +114,25 @@ public class CapacityChecker { throw new IllegalStateException("No path to failure found. This should be impossible!"); } + private int hostMitigationOrder(Map.Entry<Node, Integer> entry1, Map.Entry<Node, Integer> entry2) { + int result = Integer.compare(entry1.getValue(), entry2.getValue()); + if (result != 0) return result; + // Mitigate the largest hosts first + return NodeResourceComparator.defaultOrder().compare(entry2.getKey().resources(), entry1.getKey().resources()); + } + private Map<String, Node> constructHostnameToNodeMap(List<Node> nodes) { return nodes.stream().collect(Collectors.toMap(Node::hostname, n -> n)); } private Map<Node, List<Node>> constructNodeChildrenMap(List<Node> tenants, List<Node> hosts, Map<String, Node> hostnameToNode) { Map<Node, List<Node>> nodeChildren = tenants.stream() - .filter(n -> n.parentHostname().isPresent()) - .filter(n -> hostnameToNode.containsKey(n.parentHostname().get())) - .collect(Collectors.groupingBy( - n -> hostnameToNode.get(n.parentHostname().orElseThrow()))); + .filter(n -> n.parentHostname().isPresent()) + .filter(n -> hostnameToNode.containsKey(n.parentHostname().get())) + .collect(Collectors.groupingBy(n -> hostnameToNode.get(n.parentHostname().orElseThrow()))); - for (var host : hosts) nodeChildren.putIfAbsent(host, List.of()); + for (var host : hosts) + nodeChildren.putIfAbsent(host, List.of()); return nodeChildren; } @@ -133,7 +144,7 @@ public class CapacityChecker { int occupiedIps = 0; Set<String> ipPool = host.ipAddressPool().asSet(); for (var child : nodeChildren.get(host)) { - hostResources = hostResources.subtract(child.flavor().resources().justNumbers()); + hostResources = hostResources.subtract(child.resources().justNumbers()); occupiedIps += child.ipAddresses().stream().filter(ipPool::contains).count(); } availableResources.put(host, new AllocationResources(hostResources, host.ipAddressPool().asSet().size() - occupiedIps)); @@ -149,10 +160,8 @@ public class CapacityChecker { private Map<Node, Integer> computeMaximalRepeatedRemovals(List<Node> hosts, Map<Node, List<Node>> nodeChildren, Map<Node, AllocationResources> availableResources) { - Map<Node, Integer> timesNodeCanBeRemoved = hosts.stream().collect(Collectors.toMap( - Function.identity(), - __ -> Integer.MAX_VALUE - )); + Map<Node, Integer> timesNodeCanBeRemoved = hosts.stream().collect(Collectors.toMap(Function.identity(), + __ -> Integer.MAX_VALUE)); for (Node host : hosts) { List<Node> children = nodeChildren.get(host); if (children.size() == 0) continue; @@ -196,7 +205,8 @@ public class CapacityChecker { /** * Tests whether it's possible to remove the provided hosts. * Does not mutate any input variable. - * @return Empty optional if removal is possible, information on what caused the failure otherwise + * + * @return empty optional if removal is possible, information on what caused the failure otherwise */ private Optional<HostRemovalFailure> findHostRemovalFailure(List<Node> hostsToRemove, List<Node> allHosts, Map<Node, List<Node>> nodechildren, @@ -204,20 +214,24 @@ public class CapacityChecker { var containedAllocations = collateAllocations(nodechildren); var resourceMap = new HashMap<>(availableResources); List<Node> validAllocationTargets = allHosts.stream() - .filter(h -> !hostsToRemove.contains(h)) - .collect(Collectors.toList()); - if (validAllocationTargets.size() == 0) { + .filter(h -> !hostsToRemove.contains(h)) + .collect(Collectors.toList()); + if (validAllocationTargets.size() == 0) return Optional.of(HostRemovalFailure.none()); - } allocationHistory = new AllocationHistory(); for (var host : hostsToRemove) { Optional<Node> unallocatedNode = tryAllocateNodes(nodechildren.get(host), - validAllocationTargets, resourceMap, containedAllocations, true); + validAllocationTargets, + resourceMap, + containedAllocations, + true); if (unallocatedNode.isPresent()) { AllocationFailureReasonList failures = collateAllocationFailures(unallocatedNode.get(), - validAllocationTargets, resourceMap, containedAllocations); + validAllocationTargets, + resourceMap, + containedAllocations); return Optional.of(HostRemovalFailure.create(host, unallocatedNode.get(), failures)); } } @@ -248,7 +262,7 @@ public class CapacityChecker { long eligibleParents = hosts.stream().filter(h -> !violatesParentHostPolicy(node, h, containedAllocations) - && availableResources.get(h).satisfies(AllocationResources.from(node.flavor().resources()))).count(); + && availableResources.get(h).satisfies(AllocationResources.from(node.resources()))).count(); allocationHistory.addEntry(node, newParent.get(), eligibleParents + 1); } } @@ -300,7 +314,7 @@ public class CapacityChecker { reason.violatesParentHostPolicy = violatesParentHostPolicy(node, host, containedAllocations); NodeResources l = availableHostResources.nodeResources; - NodeResources r = node.allocation().map(Allocation::requestedResources).orElse(node.flavor().resources()); + NodeResources r = node.allocation().map(Allocation::requestedResources).orElse(node.resources()); if (l.vcpu() < r.vcpu()) reason.insufficientVcpu = true; @@ -326,8 +340,15 @@ public class CapacityChecker { * as well as the specific host and tenant which caused it. */ public static class HostFailurePath { + public List<Node> hostsCausingFailure; public HostRemovalFailure failureReason; + + @Override + public String toString() { + return "failure path: " + failureReason + " upon removing " + hostsCausingFailure; + } + } /** @@ -336,22 +357,21 @@ public class CapacityChecker { * will be empty. */ public static class HostRemovalFailure { + public Optional<Node> host; public Optional<Node> tenant; public AllocationFailureReasonList allocationFailures; public static HostRemovalFailure none() { - return new HostRemovalFailure( - Optional.empty(), - Optional.empty(), - new AllocationFailureReasonList(List.of())); + return new HostRemovalFailure(Optional.empty(), + Optional.empty(), + new AllocationFailureReasonList(List.of())); } public static HostRemovalFailure create(Node host, Node tenant, AllocationFailureReasonList failureReasons) { - return new HostRemovalFailure( - Optional.of(host), - Optional.of(tenant), - failureReasons); + return new HostRemovalFailure(Optional.of(host), + Optional.of(tenant), + failureReasons); } private HostRemovalFailure(Optional<Node> host, Optional<Node> tenant, AllocationFailureReasonList allocationFailures) { @@ -362,7 +382,7 @@ public class CapacityChecker { @Override public String toString() { - if (host.isEmpty() || tenant.isEmpty()) return "No removal candidates exists."; + if (host.isEmpty() || tenant.isEmpty()) return "No removal candidates exists"; return String.format( "Failure to remove host %s" + "\n\tNo new host found for tenant %s:" + @@ -386,7 +406,7 @@ public class CapacityChecker { if (node.allocation().isPresent()) return from(node.allocation().get().requestedResources()); else - return from(node.flavor().resources()); + return from(node.resources()); } public static AllocationResources from(NodeResources nodeResources) { @@ -406,6 +426,7 @@ public class CapacityChecker { public AllocationResources subtract(AllocationResources other) { return new AllocationResources(this.nodeResources.subtract(other.nodeResources), this.availableIPs - other.availableIPs); } + } /** @@ -449,6 +470,7 @@ public class CapacityChecker { return String.format("[%s]", String.join(", ", reasons)); } + } /** @@ -487,6 +509,7 @@ public class CapacityChecker { insufficientVcpu(), insufficientMemoryGb(), insufficientDiskGb(), incompatibleDiskSpeed(), incompatibleStorageType(), insufficientAvailableIps(), violatesParentHostPolicy()); } + } public static class AllocationHistory { @@ -506,7 +529,7 @@ public class CapacityChecker { public String toString() { return String.format("%-20s %-65s -> %15s [%3d valid]", tenant.hostname().replaceFirst("\\..+", ""), - tenant.flavor().resources(), + tenant.resources(), newParent == null ? "x" : newParent.hostname().replaceFirst("\\..+", ""), this.eligibleParents ); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java deleted file mode 100644 index f6cadabec54..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; - -import java.time.Duration; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Performs analysis on the node repository to produce metrics that pertain to the capacity of the node repository. - * These metrics include: - * Spare host capacity, or how many hosts the repository can stand to lose without ending up in a situation where it's - * unable to find a new home for orphaned tenants. - * Overcommitted hosts, which tracks if there are any hosts whose capacity is less than the sum of its children's. - * - * @author mgimle - */ -public class CapacityReportMaintainer extends NodeRepositoryMaintainer { - - private final Metric metric; - private final NodeRepository nodeRepository; - private static final Logger log = Logger.getLogger(CapacityReportMaintainer.class.getName()); - - CapacityReportMaintainer(NodeRepository nodeRepository, - Metric metric, - Duration interval) { - super(nodeRepository, interval); - this.nodeRepository = nodeRepository; - this.metric = Objects.requireNonNull(metric); - } - - @Override - protected void maintain() { - if (nodeRepository.zone().getCloud().dynamicProvisioning()) return; // Hosts and nodes are 1-1 - - CapacityChecker capacityChecker = new CapacityChecker(this.nodeRepository); - List<Node> overcommittedHosts = capacityChecker.findOvercommittedHosts(); - if (overcommittedHosts.size() != 0) { - log.log(Level.WARNING, String.format("%d nodes are overcommitted! [ %s ]", overcommittedHosts.size(), - overcommittedHosts.stream().map(Node::hostname).collect(Collectors.joining(", ")))); - } - metric.set("overcommittedHosts", overcommittedHosts.size(), null); - - Optional<CapacityChecker.HostFailurePath> failurePath = capacityChecker.worstCaseHostLossLeadingToFailure(); - if (failurePath.isPresent()) { - int worstCaseHostLoss = failurePath.get().hostsCausingFailure.size(); - metric.set("spareHostCapacity", worstCaseHostLoss - 1, null); - } - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 297285addc6..44bfed90106 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -10,7 +10,7 @@ import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; -import com.yahoo.vespa.flags.custom.PreprovisionCapacity; +import com.yahoo.vespa.flags.custom.HostCapacity; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -22,7 +22,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.Collection; +import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -36,6 +36,7 @@ import java.util.stream.IntStream; /** * @author freva + * @author mpolden */ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { @@ -43,7 +44,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision"); private final HostProvisioner hostProvisioner; - private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; + private final ListFlag<HostCapacity> targetCapacityFlag; DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, @@ -51,27 +52,25 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { FlagSource flagSource) { super(nodeRepository, interval); this.hostProvisioner = hostProvisioner; - this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); + this.targetCapacityFlag = Flags.TARGET_CAPACITY.bindTo(flagSource); } @Override protected void maintain() { - if (! nodeRepository().zone().getCloud().dynamicProvisioning()) return; - try (Mutex lock = nodeRepository().lockUnallocated()) { NodeList nodes = nodeRepository().list(); - - updateProvisioningNodes(nodes, lock); + resumeProvisioning(nodes, lock); convergeToCapacity(nodes); } } - void updateProvisioningNodes(NodeList nodes, Mutex lock) { + /** Resume provisioning of already provisioned hosts and their children */ + private void resumeProvisioning(NodeList nodes, Mutex lock) { Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant).asList().stream() - .filter(node -> node.parentHostname().isPresent()) - .collect(Collectors.groupingBy( - node -> node.parentHostname().get(), - Collectors.toSet())); + .filter(node -> node.parentHostname().isPresent()) + .collect(Collectors.groupingBy( + node -> node.parentHostname().get(), + Collectors.toSet())); nodes.state(Node.State.provisioned).nodeType(NodeType.host).forEach(host -> { Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of()); @@ -80,10 +79,10 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { nodeRepository().write(updatedNodes, lock); } catch (IllegalArgumentException | IllegalStateException e) { log.log(Level.INFO, "Failed to provision " + host.hostname() + " with " + children.size() + " children: " + - Exceptions.toMessageString(e)); + Exceptions.toMessageString(e)); } catch (FatalProvisioningException e) { log.log(Level.SEVERE, "Failed to provision " + host.hostname() + " with " + children.size() + - " children, failing out the host recursively", e); + " children, failing out the host recursively", e); // Fail out as operator to force a quick redeployment nodeRepository().failRecursively( host.hostname(), Agent.operator, "Failed by HostProvisioner due to provisioning failure"); @@ -93,31 +92,49 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { }); } - void convergeToCapacity(NodeList nodes) { - Collection<Node> removableHosts = getRemovableHosts(nodes); - List<NodeResources> preProvisionCapacity = preprovisionCapacityFlag.value().stream() - .flatMap(cap -> { - NodeResources resources = new NodeResources(cap.getVcpu(), cap.getMemoryGb(), cap.getDiskGb(), 1); - return IntStream.range(0, cap.getCount()).mapToObj(i -> resources); - }) - .sorted(NodeResourceComparator.memoryDiskCpuOrder().reversed()) - .collect(Collectors.toList()); + /** Converge zone to wanted capacity */ + private void convergeToCapacity(NodeList nodes) { + List<NodeResources> capacity = targetCapacity(); + List<Node> excessHosts = provision(capacity, nodes); + excessHosts.forEach(host -> { + try { + hostProvisioner.deprovision(host); + nodeRepository().removeRecursively(host, true); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to deprovision " + host.hostname() + ", will retry in " + interval(), e); + } + }); + } - for (Iterator<NodeResources> it = preProvisionCapacity.iterator(); it.hasNext() && !removableHosts.isEmpty();) { + /** + * Provision the nodes necessary to satisfy given capacity. + * + * @return Excess hosts that can safely be deprovisioned, if any. + */ + private List<Node> provision(List<NodeResources> capacity, NodeList nodes) { + List<Node> existingHosts = availableHostsOf(nodes); + if (nodeRepository().zone().getCloud().dynamicProvisioning()) { + existingHosts = removableHostsOf(existingHosts, nodes); + } else if (capacity.isEmpty()) { + return List.of(); + } + List<Node> excessHosts = new ArrayList<>(existingHosts); + for (Iterator<NodeResources> it = capacity.iterator(); it.hasNext() && !excessHosts.isEmpty(); ) { NodeResources resources = it.next(); - removableHosts.stream() - .filter(nodeRepository()::canAllocateTenantNodeTo) - .filter(host -> nodeRepository().resourcesCalculator().advertisedResourcesOf(host.flavor()).satisfies(resources)) - .min(Comparator.comparingInt(n -> n.flavor().cost())) - .ifPresent(host -> { - removableHosts.remove(host); - it.remove(); - }); + excessHosts.stream() + .filter(nodeRepository()::canAllocateTenantNodeTo) + .filter(host -> nodeRepository().resourcesCalculator() + .advertisedResourcesOf(host.flavor()) + .satisfies(resources)) + .min(Comparator.comparingInt(n -> n.flavor().cost())) + .ifPresent(host -> { + excessHosts.remove(host); + it.remove(); + }); } - - // pre-provisioning is best effort, do one host at a time - preProvisionCapacity.forEach(resources -> { + // Pre-provisioning is best effort, do one host at a time + capacity.forEach(resources -> { try { Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); List<Node> hosts = hostProvisioner.provisionHosts(nodeRepository().database().getProvisionIndexes(1), @@ -127,35 +144,47 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { .collect(Collectors.toList()); nodeRepository().addNodes(hosts, Agent.DynamicProvisioningMaintainer); } catch (OutOfCapacityException | IllegalArgumentException | IllegalStateException e) { - log.log(Level.WARNING, "Failed to pre-provision " + resources + ":" + e.getMessage()); + log.log(Level.WARNING, "Failed to pre-provision " + resources + ": " + e.getMessage()); } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to pre-provision " + resources + ", will retry in " + interval(), e); } }); + return removableHostsOf(excessHosts, nodes); + } - // Finally, deprovision excess hosts. - removableHosts.forEach(host -> { - try { - hostProvisioner.deprovision(host); - nodeRepository().removeRecursively(host, true); - } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to deprovision " + host.hostname() + ", will retry in " + interval(), e); - } - }); + + /** Reads node resources declared by target capacity flag */ + private List<NodeResources> targetCapacity() { + return targetCapacityFlag.value().stream() + .flatMap(cap -> { + NodeResources resources = new NodeResources(cap.getVcpu(), cap.getMemoryGb(), + cap.getDiskGb(), 1); + return IntStream.range(0, cap.getCount()).mapToObj(i -> resources); + }) + .sorted(NodeResourceComparator.memoryDiskCpuOrder().reversed()) + .collect(Collectors.toList()); } - private static Collection<Node> getRemovableHosts(NodeList nodes) { - Map<String, Node> hostsByHostname = nodes.nodeType(NodeType.host) - .asList().stream() - .filter(host -> host.state() != Node.State.parked || host.status().wantToDeprovision()) - .collect(Collectors.toMap(Node::hostname, Function.identity())); + /** Returns hosts that are considered available, i.e. not parked or flagged for deprovisioning */ + private static List<Node> availableHostsOf(NodeList nodes) { + return nodes.nodeType(NodeType.host) + .matching(host -> host.state() != Node.State.parked || host.status().wantToDeprovision()) + .asList(); + } + + /** Returns the subset of given hosts that have no containers and are thus removable */ + private static List<Node> removableHostsOf(List<Node> hosts, NodeList allNodes) { + Map<String, Node> hostsByHostname = hosts.stream() + .collect(Collectors.toMap(Node::hostname, + Function.identity())); - nodes.asList().stream() + allNodes.asList().stream() .filter(node -> node.allocation().isPresent()) .flatMap(node -> node.parentHostname().stream()) .distinct() .forEach(hostsByHostname::remove); - return hostsByHostname.values(); + return List.copyOf(hostsByHostname.values()); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java index 21746c96411..3cb7cc218a7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java @@ -5,6 +5,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.Status; import java.time.Clock; import java.time.Duration; @@ -14,15 +15,19 @@ import java.util.List; * Maintenance job which moves inactive nodes to dirty or parked after timeout. * * The timeout is in place for two reasons: - * <ul> - * <li>To ensure that the new application configuration has time to - * propagate before the node is used for something else - * <li>To provide a grace period in which nodes can be brought back to active - * if they were deactivated in error. As inactive nodes retain their state - * they can be brought back to active and correct state faster than a new node. - * </ul> * - * Nodes with the retired flag should not be reused and will be moved to parked instead of dirty. + * - To ensure that the new application configuration has time to + * propagate before the node is used for something else. + * + * - To provide a grace period in which nodes can be brought back to active + * if they were deactivated in error. As inactive nodes retain their state + * they can be brought back to active and correct state faster than a new node. + * + * Nodes with following flags set are not reusable and will be moved to parked + * instead of dirty: + * + * - {@link Status#wantToRetire()} (when set by an operator) + * - {@link Status#wantToDeprovision()} * * @author bratseth * @author mpolden @@ -39,8 +44,7 @@ public class InactiveExpirer extends Expirer { @Override protected void expire(List<Node> expired) { expired.forEach(node -> { - if (node.status().wantToRetire() && - node.history().event(History.Event.Type.wantToRetire).get().agent() == Agent.operator) { + if (node.status().wantToDeprovision() || retiredByOperator(node)) { nodeRepository.park(node.hostname(), false, Agent.InactiveExpirer, "Expired by InactiveExpirer"); } else { nodeRepository.setDirty(node, Agent.InactiveExpirer, "Expired by InactiveExpirer"); @@ -54,4 +58,11 @@ public class InactiveExpirer extends Expirer { || node.allocation().get().owner().instance().isTester(); } + private static boolean retiredByOperator(Node node) { + return node.status().wantToRetire() && node.history().event(History.Event.Type.wantToRetire) + .map(History.Event::agent) + .map(agent -> agent == Agent.operator) + .orElse(false); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 483b4dc8f84..6edd57de1c1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -1,13 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer.State; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import java.time.Duration; @@ -17,6 +17,7 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.logging.Level; import java.util.stream.Collectors; /** @@ -99,7 +100,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { // Remove any real no longer allocated to this application reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value())); try { - service.create(lb.id().application(), lb.id().cluster(), reals, true, nodeRepository()); + service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals), true); db.writeLoadBalancer(lb.with(lb.instance().withReals(reals))); } catch (Exception e) { failed.add(lb.id()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java index 3319db33730..4e1be9c486c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java @@ -7,10 +7,13 @@ import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.TransientException; import com.yahoo.jdisc.Metric; + +import java.util.Objects; import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.yolean.Exceptions; import java.io.Closeable; @@ -71,6 +74,11 @@ class MaintenanceDeployment implements Closeable { return doStep(() -> deployment.get().prepare()); } + /** + * Attempts to activate this deployment + * + * @return whether it was successfully activated + */ public boolean activate() { return doStep(() -> deployment.get().activate()); } @@ -123,4 +131,106 @@ class MaintenanceDeployment implements Closeable { return "deployment of " + application; } + public static class Move { + + private final Node node; + private final Node fromHost, toHost; + + Move(Node node, Node fromHost, Node toHost) { + this.node = node; + this.fromHost = fromHost; + this.toHost = toHost; + } + + public Node node() { return node; } + public Node fromHost() { return fromHost; } + public Node toHost() { return toHost; } + + /** + * Try to deploy to make this move. + * + * @param verifyTarget true to only make this move if the node ends up at the expected target host, + * false if we should perform it as long as it moves from the source host + * @return true if the move was done, false if it couldn't be + */ + public boolean execute(boolean verifyTarget, + Agent agent, Deployer deployer, Metric metric, NodeRepository nodeRepository) { + if (isEmpty()) return false; + ApplicationId application = node.allocation().get().owner(); + try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository)) { + if ( ! deployment.isValid()) return false; + + boolean couldMarkRetiredNow = markWantToRetire(node, true, agent, nodeRepository); + if ( ! couldMarkRetiredNow) return false; + + Optional<Node> expectedNewNode = Optional.empty(); + try { + if ( ! deployment.prepare()) return false; + if (verifyTarget) { + expectedNewNode = + nodeRepository.getNodes(application, Node.State.reserved).stream() + .filter(n -> !n.hostname().equals(node.hostname())) + .filter(n -> n.allocation().get().membership().cluster().id().equals(node.allocation().get().membership().cluster().id())) + .findAny(); + if (expectedNewNode.isEmpty()) return false; + if (!expectedNewNode.get().hasParent(toHost.hostname())) return false; + } + if ( ! deployment.activate()) return false; + + log.info(agent + " redeployed " + application + " to " + + ( verifyTarget ? this : "move " + (node.hostname() + " from " + fromHost))); + return true; + } + finally { + markWantToRetire(node, false, agent, nodeRepository); // Necessary if this failed, no-op otherwise + + // Immediately clean up if we reserved the node but could not activate or reserved a node on the wrong host + expectedNewNode.flatMap(node -> nodeRepository.getNode(node.hostname(), Node.State.reserved)) + .ifPresent(node -> nodeRepository.setDirty(node, agent, "Expired by " + agent)); + } + } + } + + /** Returns true only if this operation changes the state of the wantToRetire flag */ + private boolean markWantToRetire(Node node, boolean wantToRetire, Agent agent, NodeRepository nodeRepository) { + try (Mutex lock = nodeRepository.lock(node)) { + Optional<Node> nodeToMove = nodeRepository.getNode(node.hostname()); + if (nodeToMove.isEmpty()) return false; + if (nodeToMove.get().state() != Node.State.active) return false; + + if (nodeToMove.get().status().wantToRetire() == wantToRetire) return false; + + nodeRepository.write(nodeToMove.get().withWantToRetire(wantToRetire, agent, nodeRepository.clock().instant()), lock); + return true; + } + } + + public boolean isEmpty() { return node == null; } + + @Override + public int hashCode() { + return Objects.hash(node, fromHost, toHost); + } + + public boolean equals(Object o) { + if (o == this) return true; + if (o == null || o.getClass() != this.getClass()) return false; + + Move other = (Move)o; + if ( ! Objects.equals(other.node, this.node)) return false; + if ( ! Objects.equals(other.fromHost, this.fromHost)) return false; + if ( ! Objects.equals(other.toHost, this.toHost)) return false; + return true; + } + + @Override + public String toString() { + return "move " + + ( isEmpty() ? "none" : (node.hostname() + " from " + fromHost + " to " + toHost)); + } + + public static Move empty() { return new Move(null, null, null); } + + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 7ce674cbbbf..e918c1a815a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TransientException; import com.yahoo.jdisc.Metric; -import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceInstance; @@ -31,6 +30,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -138,7 +138,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { // We do this here ("lazily") to avoid writing to zk for each config request. for (Node node : nodeRepository().getNodes(Node.State.ready)) { Optional<Instant> lastLocalRequest = hostLivenessTracker.lastRequestFrom(node.hostname()); - if ( ! lastLocalRequest.isPresent()) continue; + if (lastLocalRequest.isEmpty()) continue; if (! node.history().hasEventAfter(History.Event.Type.requested, lastLocalRequest.get())) { History updatedHistory = node.history() @@ -189,11 +189,15 @@ public class NodeFailer extends NodeRepositoryMaintainer { .forEach((hostName, serviceInstances) -> { Node node = activeNodesByHostname.get(hostName.s()); if (node == null) return; - - if (badNode(serviceInstances)) { - recordAsDown(node); - } else { - clearDownRecord(node); + try (var lock = nodeRepository().lock(node.allocation().get().owner())) { + Optional<Node> currentNode = nodeRepository().getNode(node.hostname(), Node.State.active); // re-get inside lock + if (currentNode.isEmpty()) return; // Node disappeared since acquiring lock + node = currentNode.get(); + if (badNode(serviceInstances)) { + recordAsDown(node, lock); + } else { + clearDownRecord(node, lock); + } } }); } @@ -311,27 +315,16 @@ public class NodeFailer extends NodeRepositoryMaintainer { countsByStatus.getOrDefault(ServiceStatus.DOWN, 0L) > 0L; } - /** - * Record a node as down if not already recorded and returns the node in the new state. - * This assumes the node is found in the node - * repo and that the node is allocated. If we get here otherwise something is truly odd. - */ - private Node recordAsDown(Node node) { - if (node.history().event(History.Event.Type.down).isPresent()) return node; // already down: Don't change down timestamp - - try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) { - node = nodeRepository().getNode(node.hostname(), Node.State.active).get(); // re-get inside lock - return nodeRepository().write(node.downAt(clock.instant(), Agent.NodeFailer), lock); - } + /** Record a node as down if not already recorded */ + private void recordAsDown(Node node, Mutex lock) { + if (node.history().event(History.Event.Type.down).isPresent()) return; // already down: Don't change down timestamp + nodeRepository().write(node.downAt(clock.instant(), Agent.NodeFailer), lock); } - private void clearDownRecord(Node node) { - if ( ! node.history().event(History.Event.Type.down).isPresent()) return; - - try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) { - node = nodeRepository().getNode(node.hostname(), Node.State.active).get(); // re-get inside lock - nodeRepository().write(node.up(), lock); - } + /** Clear down record for node, if any */ + private void clearDownRecord(Node node, Mutex lock) { + if (node.history().event(History.Event.Type.down).isEmpty()) return; + nodeRepository().write(node.up(), lock); } /** @@ -344,7 +337,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { private boolean failActive(Node node, String reason) { Optional<Deployment> deployment = deployer.deployFromLocalActive(node.allocation().get().owner(), Duration.ofMinutes(30)); - if ( ! deployment.isPresent()) return false; // this will be done at another config server + if (deployment.isEmpty()) return false; // this will be done at another config server try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) { // If the active node that we are trying to fail is of type host, we need to successfully fail all @@ -386,6 +379,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { Instant startOfThrottleWindow = clock.instant().minus(throttlePolicy.throttleWindow); List<Node> nodes = nodeRepository().getNodes(); NodeList recentlyFailedNodes = nodes.stream() + .filter(n -> n.state() == Node.State.failed) .filter(n -> n.history().hasEventAfter(History.Event.Type.failed, startOfThrottleWindow)) .collect(collectingAndThen(Collectors.toList(), NodeList::copyOf)); @@ -393,7 +387,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { if (recentlyFailedNodes.size() < throttlePolicy.allowedToFailOf(nodes.size())) return false; // Always allow failing physical nodes up to minimum limit - if (!node.parentHostname().isPresent() && + if (node.parentHostname().isEmpty() && recentlyFailedNodes.parents().size() < throttlePolicy.minimumAllowedToFail) return false; log.info(String.format("Want to fail node %s, but throttling is in effect: %s", node.hostname(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index d388fb5a967..4323622df8b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -47,7 +47,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final InfrastructureProvisioner infrastructureProvisioner; private final Optional<LoadBalancerExpirer> loadBalancerExpirer; private final Optional<DynamicProvisioningMaintainer> dynamicProvisioningMaintainer; - private final CapacityReportMaintainer capacityReportMaintainer; + private final SpareCapacityMaintainer spareCapacityMaintainer; private final OsUpgradeActivator osUpgradeActivator; private final Rebalancer rebalancer; private final NodeMetricsDbMaintainer nodeMetricsDbMaintainer; @@ -84,13 +84,13 @@ public class NodeRepositoryMaintenance extends AbstractComponent { nodeRebooter = new NodeRebooter(nodeRepository, clock, flagSource); metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval, clock); infrastructureProvisioner = new InfrastructureProvisioner(nodeRepository, infraDeployer, defaults.infrastructureProvisionInterval); - loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService().map(lbService -> + loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService(nodeRepository).map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService)); dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner -> new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource)); - capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval); + spareCapacityMaintainer = new SpareCapacityMaintainer(deployer, nodeRepository, metric, defaults.spareCapacityMaintenanceInterval); osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval); - rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); + rebalancer = new Rebalancer(deployer, nodeRepository, metric, clock, defaults.rebalancerInterval); nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval); autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, nodeMetricsDb, deployer, metric, defaults.autoscalingInterval); scalingSuggestionsMaintainer = new ScalingSuggestionsMaintainer(nodeRepository, nodeMetricsDb, defaults.scalingSuggestionsInterval); @@ -110,7 +110,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { failedExpirer.close(); dirtyExpirer.close(); nodeRebooter.close(); - capacityReportMaintainer.close(); + spareCapacityMaintainer.close(); provisionedExpirer.close(); metricsReporter.close(); infrastructureProvisioner.close(); @@ -153,7 +153,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration failedExpirerInterval; private final Duration dirtyExpiry; private final Duration provisionedExpiry; - private final Duration capacityReportInterval; + private final Duration spareCapacityMaintenanceInterval; private final Duration metricsInterval; private final Duration retiredInterval; private final Duration infrastructureProvisionInterval; @@ -168,25 +168,24 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final NodeFailer.ThrottlePolicy throttlePolicy; DefaultTimes(Zone zone) { - failGrace = Duration.ofMinutes(30); - periodicRedeployInterval = Duration.ofMinutes(30); - // Don't redeploy in test environments - redeployMaintainerInterval = Duration.ofMinutes(1); - operatorChangeRedeployInterval = Duration.ofMinutes(1); + autoscalingInterval = Duration.ofMinutes(5); + dynamicProvisionerInterval = Duration.ofMinutes(5); failedExpirerInterval = Duration.ofMinutes(10); - provisionedExpiry = Duration.ofHours(4); - capacityReportInterval = Duration.ofMinutes(10); - metricsInterval = Duration.ofMinutes(1); + failGrace = Duration.ofMinutes(30); infrastructureProvisionInterval = Duration.ofMinutes(1); - throttlePolicy = NodeFailer.ThrottlePolicy.hosted; loadBalancerExpirerInterval = Duration.ofMinutes(5); - reservationExpiry = Duration.ofMinutes(15); // Need to be long enough for deployment to be finished for all config model versions - dynamicProvisionerInterval = Duration.ofMinutes(5); + metricsInterval = Duration.ofMinutes(1); + nodeMetricsCollectionInterval = Duration.ofMinutes(1); + operatorChangeRedeployInterval = Duration.ofMinutes(1); osUpgradeActivatorInterval = zone.system().isCd() ? Duration.ofSeconds(30) : Duration.ofMinutes(5); + periodicRedeployInterval = Duration.ofMinutes(30); + provisionedExpiry = Duration.ofHours(4); rebalancerInterval = Duration.ofMinutes(40); - nodeMetricsCollectionInterval = Duration.ofMinutes(1); - autoscalingInterval = Duration.ofMinutes(5); + redeployMaintainerInterval = Duration.ofMinutes(1); + reservationExpiry = Duration.ofMinutes(15); // Need to be long enough for deployment to be finished for all config model versions scalingSuggestionsInterval = Duration.ofMinutes(31); + spareCapacityMaintenanceInterval = Duration.ofMinutes(10); + throttlePolicy = NodeFailer.ThrottlePolicy.hosted; if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java index 9f829c095f4..b5933196ecc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java @@ -55,9 +55,10 @@ public class OperatorChangeApplicationMaintainer extends ApplicationMaintainer { */ @Override protected void deploy(ApplicationId application) { - deployWithLock(application); - log.info("Redeployed application " + application.toShortString() + - " as a manual change was made to its nodes"); + boolean deployed = deployWithLock(application); + if (deployed) + log.info("Redeployed application " + application.toShortString() + + " as a manual change was made to its nodes"); } private boolean hasNodesWithChanges(ApplicationId applicationId, List<Node> nodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java index b98b03671c4..a36ef8fda4d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java @@ -26,7 +26,7 @@ public class OsUpgradeActivator extends NodeRepositoryMaintainer { for (var nodeType : NodeType.values()) { if (!nodeType.isDockerHost()) continue; var active = canUpgradeOsOf(nodeType); - nodeRepository().osVersions().setActive(nodeType, active); + nodeRepository().osVersions().resumeUpgradeOf(nodeType, active); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java index d06d24872e1..06ecb1f4a01 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java @@ -64,9 +64,16 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer { return deploymentTimes.entrySet().stream() .sorted(Map.Entry.comparingByValue()) .map(Map.Entry::getKey) + .filter(id -> shouldMaintain(id)) .collect(Collectors.toCollection(LinkedHashSet::new)); } + private boolean shouldMaintain(ApplicationId id) { + if (id.tenant().value().equals("stream") && id.application().value().equals("stream-ranking")) return false; + if (id.tenant().value().equals("stream") && id.application().value().equals("stream-ranking-canary")) return false; + return true; + } + // TODO: Do not start deploying until some time has gone (ideally only until bootstrap of config server is finished) private boolean waitInitially() { return clock.instant().isBefore(start.plus(minTimeBetweenRedeployments)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index 7ffb541be2a..3df20fa9d08 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -6,62 +6,52 @@ import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; -import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.DockerHostCapacity; -import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; -import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; +import com.yahoo.vespa.hosted.provision.provisioning.HostCapacity; import java.time.Clock; import java.time.Duration; -import java.util.Optional; /** * @author bratseth */ public class Rebalancer extends NodeRepositoryMaintainer { + static final Duration waitTimeAfterPreviousDeployment = Duration.ofMinutes(10); + private final Deployer deployer; - private final Optional<HostProvisioner> hostProvisioner; private final Metric metric; private final Clock clock; public Rebalancer(Deployer deployer, NodeRepository nodeRepository, - Optional<HostProvisioner> hostProvisioner, Metric metric, Clock clock, Duration interval) { super(nodeRepository, interval); this.deployer = deployer; - this.hostProvisioner = hostProvisioner; this.metric = metric; this.clock = clock; } @Override protected void maintain() { - if (hostProvisioner.isPresent()) return; // All nodes will be allocated on new hosts, so rebalancing makes no sense - if (nodeRepository().zone().environment().isTest()) return; // Test zones have short lived deployments, no need to rebalance + if ( ! nodeRepository().zone().getCloud().allowHostSharing()) return; // Rebalancing not necessary + if (nodeRepository().zone().environment().isTest()) return; // Short lived deployments; no need to rebalance // Work with an unlocked snapshot as this can take a long time and full consistency is not needed NodeList allNodes = nodeRepository().list(); - updateSkewMetric(allNodes); - if ( ! zoneIsStable(allNodes)) return; - - Move bestMove = findBestMove(allNodes); - if (bestMove == Move.none) return; - deployTo(bestMove); + findBestMove(allNodes).execute(true, Agent.Rebalancer, deployer, metric, nodeRepository()); } /** We do this here rather than in MetricsReporter because it is expensive and frequent updates are unnecessary */ private void updateSkewMetric(NodeList allNodes) { - DockerHostCapacity capacity = new DockerHostCapacity(allNodes, nodeRepository().resourcesCalculator()); + HostCapacity capacity = new HostCapacity(allNodes, nodeRepository().resourcesCalculator()); double totalSkew = 0; int hostCount = 0; for (Node host : allNodes.nodeType((NodeType.host)).state(Node.State.active)) { @@ -83,111 +73,66 @@ public class Rebalancer extends NodeRepositoryMaintainer { * Returns Move.none if no moves can be made to reduce skew. */ private Move findBestMove(NodeList allNodes) { - DockerHostCapacity capacity = new DockerHostCapacity(allNodes, nodeRepository().resourcesCalculator()); - Move bestMove = Move.none; + HostCapacity capacity = new HostCapacity(allNodes, nodeRepository().resourcesCalculator()); + Move bestMove = Move.empty(); for (Node node : allNodes.nodeType(NodeType.tenant).state(Node.State.active)) { if (node.parentHostname().isEmpty()) continue; - if (node.allocation().get().owner().instance().isTester()) continue; - for (Node toHost : allNodes.filter(nodeRepository()::canAllocateTenantNodeTo)) { + ApplicationId applicationId = node.allocation().get().owner(); + if (applicationId.instance().isTester()) continue; + if (deployedRecently(applicationId)) continue; + for (Node toHost : allNodes.matching(nodeRepository()::canAllocateTenantNodeTo)) { if (toHost.hostname().equals(node.parentHostname().get())) continue; - if ( ! capacity.freeCapacityOf(toHost).satisfies(node.flavor().resources())) continue; + if ( ! capacity.freeCapacityOf(toHost).satisfies(node.resources())) continue; double skewReductionAtFromHost = skewReductionByRemoving(node, allNodes.parentOf(node).get(), capacity); double skewReductionAtToHost = skewReductionByAdding(node, toHost, capacity); double netSkewReduction = skewReductionAtFromHost + skewReductionAtToHost; if (netSkewReduction > bestMove.netSkewReduction) - bestMove = new Move(node, toHost, netSkewReduction); + bestMove = new Move(node, nodeRepository().getNode(node.parentHostname().get()).get(), toHost, netSkewReduction); } } return bestMove; } - /** Returns true only if this operation changes the state of the wantToRetire flag */ - private boolean markWantToRetire(Node node, boolean wantToRetire) { - try (Mutex lock = nodeRepository().lock(node)) { - Optional<Node> nodeToMove = nodeRepository().getNode(node.hostname()); - if (nodeToMove.isEmpty()) return false; - if (nodeToMove.get().state() != Node.State.active) return false; - - if (nodeToMove.get().status().wantToRetire() == wantToRetire) return false; - - nodeRepository().write(nodeToMove.get().withWantToRetire(wantToRetire, Agent.Rebalancer, clock.instant()), lock); - return true; - } - } - - /** - * Try a redeployment to effect the chosen move. - * If it can be done, that's ok; we'll try this or another move later. - * - * @return true if the move was done, false if it couldn't be - */ - private boolean deployTo(Move move) { - ApplicationId application = move.node.allocation().get().owner(); - try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) { - if ( ! deployment.isValid()) return false; - - boolean couldMarkRetiredNow = markWantToRetire(move.node, true); - if ( ! couldMarkRetiredNow) return false; - - Optional<Node> expectedNewNode = Optional.empty(); - try { - if ( ! deployment.prepare()) return false; - expectedNewNode = - nodeRepository().getNodes(application, Node.State.reserved).stream() - .filter(node -> !node.hostname().equals(move.node.hostname())) - .filter(node -> node.allocation().get().membership().cluster().id().equals(move.node.allocation().get().membership().cluster().id())) - .findAny(); - if (expectedNewNode.isEmpty()) return false; - if ( ! expectedNewNode.get().hasParent(move.toHost.hostname())) return false; - if ( ! deployment.activate()) return false; - - log.info("Rebalancer redeployed " + application + " to " + move); - return true; - } - finally { - markWantToRetire(move.node, false); // Necessary if this failed, no-op otherwise - - // Immediately clean up if we reserved the node but could not activate or reserved a node on the wrong host - expectedNewNode.flatMap(node -> nodeRepository().getNode(node.hostname(), Node.State.reserved)) - .ifPresent(node -> nodeRepository().setDirty(node, Agent.Rebalancer, "Expired by Rebalancer")); - } - } - } - - private double skewReductionByRemoving(Node node, Node fromHost, DockerHostCapacity capacity) { + private double skewReductionByRemoving(Node node, Node fromHost, HostCapacity capacity) { NodeResources freeHostCapacity = capacity.freeCapacityOf(fromHost); double skewBefore = Node.skew(fromHost.flavor().resources(), freeHostCapacity); double skewAfter = Node.skew(fromHost.flavor().resources(), freeHostCapacity.add(node.flavor().resources().justNumbers())); return skewBefore - skewAfter; } - private double skewReductionByAdding(Node node, Node toHost, DockerHostCapacity capacity) { + private double skewReductionByAdding(Node node, Node toHost, HostCapacity capacity) { NodeResources freeHostCapacity = capacity.freeCapacityOf(toHost); double skewBefore = Node.skew(toHost.flavor().resources(), freeHostCapacity); - double skewAfter = Node.skew(toHost.flavor().resources(), freeHostCapacity.subtract(node.flavor().resources().justNumbers())); + double skewAfter = Node.skew(toHost.flavor().resources(), freeHostCapacity.subtract(node.resources().justNumbers())); return skewBefore - skewAfter; } - private static class Move { + protected boolean deployedRecently(ApplicationId application) { + return deployer.lastDeployTime(application) + .map(lastDeployTime -> lastDeployTime.isAfter(clock.instant().minus(waitTimeAfterPreviousDeployment))) + // We only know last deploy time for applications that were deployed on this config server, + // the rest will be deployed on another config server + .orElse(true); + } - static final Move none = new Move(null, null, 0); + private static class Move extends MaintenanceDeployment.Move { - final Node node; - final Node toHost; final double netSkewReduction; - Move(Node node, Node toHost, double netSkewReduction) { - this.node = node; - this.toHost = toHost; + Move(Node node, Node fromHost, Node toHost, double netSkewReduction) { + super(node, fromHost, toHost); this.netSkewReduction = netSkewReduction; } @Override public String toString() { - return "move " + - ( node == null ? "none" : - (node.hostname() + " to " + toHost + " [skew reduction " + netSkewReduction + "]")); + if (isEmpty()) return "move none"; + return super.toString() + " [skew reduction " + netSkewReduction + "]"; + } + + public static Move empty() { + return new Move(null, null, null, 0); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java new file mode 100644 index 00000000000..54899372397 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java @@ -0,0 +1,337 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.Deployer; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.maintenance.MaintenanceDeployment.Move; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.provisioning.HostCapacity; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; +import java.util.stream.Collectors; + +/** + * A maintainer which attempts to ensure there is spare capacity available in chunks which can fit + * all node resource configuration in use, such that the system is able to quickly replace a failed node + * if necessary. + * + * This also emits the following metrics: + * - Overcommitted hosts: Hosts whose capacity is less than the sum of its children's + * - Spare host capacity, or how many hosts the repository can stand to lose without ending up in a situation where it's + * unable to find a new home for orphaned tenants. + * + * @author mgimle + * @author bratseth + */ +public class SpareCapacityMaintainer extends NodeRepositoryMaintainer { + + private final int maxIterations; + private final Deployer deployer; + private final Metric metric; + + public SpareCapacityMaintainer(Deployer deployer, + NodeRepository nodeRepository, + Metric metric, + Duration interval) { + this(deployer, nodeRepository, metric, interval, + 10_000 // Should take less than a few minutes + ); + } + + public SpareCapacityMaintainer(Deployer deployer, + NodeRepository nodeRepository, + Metric metric, + Duration interval, + int maxIterations) { + super(nodeRepository, interval); + this.deployer = deployer; + this.metric = metric; + this.maxIterations = maxIterations; + } + + @Override + protected void maintain() { + if ( ! nodeRepository().zone().getCloud().allowHostSharing()) return; + + CapacityChecker capacityChecker = new CapacityChecker(nodeRepository()); + + List<Node> overcommittedHosts = capacityChecker.findOvercommittedHosts(); + if (overcommittedHosts.size() != 0) { + log.log(Level.WARNING, String.format("%d nodes are overcommitted! [ %s ]", + overcommittedHosts.size(), + overcommittedHosts.stream().map(Node::hostname).collect(Collectors.joining(", ")))); + } + metric.set("overcommittedHosts", overcommittedHosts.size(), null); + + Optional<CapacityChecker.HostFailurePath> failurePath = capacityChecker.worstCaseHostLossLeadingToFailure(); + if (failurePath.isPresent()) { + int spareHostCapacity = failurePath.get().hostsCausingFailure.size() - 1; + if (spareHostCapacity == 0) { + Move move = findMitigatingMove(failurePath.get()); + if (moving(move)) { + // We succeeded or are in the process of taking a step to mitigate. + // Report with the assumption this will eventually succeed to avoid alerting before we're stuck + spareHostCapacity++; + } + } + metric.set("spareHostCapacity", spareHostCapacity, null); + } + } + + private boolean moving(Move move) { + if (move.isEmpty()) return false; + if (move.node().allocation().get().membership().retired()) return true; // Move already in progress + return move.execute(false, Agent.SpareCapacityMaintainer, deployer, metric, nodeRepository()); + } + + private Move findMitigatingMove(CapacityChecker.HostFailurePath failurePath) { + Optional<Node> nodeWhichCantMove = failurePath.failureReason.tenant; + if (nodeWhichCantMove.isEmpty()) return Move.empty(); + + Node node = nodeWhichCantMove.get(); + NodeList allNodes = nodeRepository().list(); + // Allocation will assign the two most empty nodes as "spares", which will not be allocated on + // unless needed for node failing. Our goal here is to make room on these spares for the given node + HostCapacity hostCapacity = new HostCapacity(allNodes, nodeRepository().resourcesCalculator()); + Set<Node> spareHosts = hostCapacity.findSpareHosts(allNodes.hosts().satisfies(node.resources()).asList(), 2); + List<Node> hosts = allNodes.hosts().except(spareHosts).asList(); + + CapacitySolver capacitySolver = new CapacitySolver(hostCapacity, maxIterations); + List<Move> shortestMitigation = null; + for (Node spareHost : spareHosts) { + List<Move> mitigation = capacitySolver.makeRoomFor(node, spareHost, hosts, List.of(), List.of()); + if (mitigation == null) continue; + if (shortestMitigation == null || shortestMitigation.size() > mitigation.size()) + shortestMitigation = mitigation; + } + if (shortestMitigation == null || shortestMitigation.isEmpty()) return Move.empty(); + return shortestMitigation.get(0); + } + + private static class CapacitySolver { + + private final HostCapacity hostCapacity; + private final int maxIterations; + + private int iterations = 0; + + CapacitySolver(HostCapacity hostCapacity, int maxIterations) { + this.hostCapacity = hostCapacity; + this.maxIterations = maxIterations; + } + + /** The map of subproblem solutions already found. The value is null when there is no solution. */ + private Map<SolutionKey, List<Move>> solutions = new HashMap<>(); + + /** + * Finds the shortest sequence of moves which makes room for the given node on the given host, + * assuming the given moves already made over the given hosts' current allocation. + * + * @param node the node to make room for + * @param host the target host to make room on + * @param hosts the hosts onto which we can move nodes + * @param movesConsidered the moves already being considered to add as part of this scenario + * (after any moves made by this) + * @param movesMade the moves already made in this scenario + * @return the list of movesMade with the moves needed for this appended, in the order they should be performed, + * or null if no sequence could be found + */ + List<Move> makeRoomFor(Node node, Node host, List<Node> hosts, List<Move> movesConsidered, List<Move> movesMade) { + SolutionKey solutionKey = new SolutionKey(node, host, movesConsidered, movesMade); + List<Move> solution = solutions.get(solutionKey); + if (solution == null) { + solution = findRoomFor(node, host, hosts, movesConsidered, movesMade); + solutions.put(solutionKey, solution); + } + return solution; + } + + private List<Move> findRoomFor(Node node, Node host, List<Node> hosts, + List<Move> movesConsidered, List<Move> movesMade) { + if (iterations++ > maxIterations) + return null; + + if ( ! host.resources().satisfies(node.resources())) return null; + NodeResources freeCapacity = freeCapacityWith(movesMade, host); + if (freeCapacity.satisfies(node.resources())) return List.of(); + + List<Move> shortest = null; + for (var i = subsets(hostCapacity.allNodes().childrenOf(host), 5); i.hasNext(); ) { + List<Node> childrenToMove = i.next(); + if ( ! addResourcesOf(childrenToMove, freeCapacity).satisfies(node.resources())) continue; + List<Move> moves = move(childrenToMove, host, hosts, movesConsidered, movesMade); + if (moves == null) continue; + + if (shortest == null || moves.size() < shortest.size()) + shortest = moves; + } + if (shortest == null) return null; + return append(movesMade, shortest); + } + + private List<Move> move(List<Node> nodes, Node host, List<Node> hosts, List<Move> movesConsidered, List<Move> movesMade) { + List<Move> moves = new ArrayList<>(); + for (Node childToMove : nodes) { + List<Move> childMoves = move(childToMove, host, hosts, movesConsidered, append(movesMade, moves)); + if (childMoves == null) return null; + moves.addAll(childMoves); + } + return moves; + } + + private List<Move> move(Node node, Node host, List<Node> hosts, List<Move> movesConsidered, List<Move> movesMade) { + if (contains(node, movesConsidered)) return null; + if (contains(node, movesMade)) return null; + List<Move> shortest = null; + for (Node target : hosts) { + if (target.equals(host)) continue; + Move move = new Move(node, host, target); + List<Move> childMoves = makeRoomFor(node, target, hosts, append(movesConsidered, move), movesMade); + if (childMoves == null) continue; + if (shortest == null || shortest.size() > childMoves.size() + 1) { + shortest = new ArrayList<>(childMoves); + shortest.add(move); + } + } + return shortest; + } + + private boolean contains(Node node, List<Move> moves) { + return moves.stream().anyMatch(move -> move.node().equals(node)); + } + + private NodeResources addResourcesOf(List<Node> nodes, NodeResources resources) { + for (Node node : nodes) + resources = resources.add(node.resources()); + return resources; + } + + private Iterator<List<Node>> subsets(NodeList nodes, int maxSize) { + return new SubsetIterator(nodes.asList(), maxSize); + } + + private List<Move> append(List<Move> a, List<Move> b) { + List<Move> list = new ArrayList<>(); + list.addAll(a); + list.addAll(b); + return list; + } + + private List<Move> append(List<Move> moves, Move move) { + List<Move> list = new ArrayList<>(moves); + list.add(move); + return list; + } + + private NodeResources freeCapacityWith(List<Move> moves, Node host) { + NodeResources resources = hostCapacity.freeCapacityOf(host); + for (Move move : moves) { + if ( ! move.toHost().equals(host)) continue; + resources = resources.subtract(move.node().resources()); + } + for (Move move : moves) { + if ( ! move.fromHost().equals(host)) continue; + resources = resources.add(move.node().resources()); + } + return resources; + } + + } + + private static class SolutionKey { + + private final Node node; + private final Node host; + private final List<Move> movesConsidered; + private final List<Move> movesMade; + + private final int hash; + + public SolutionKey(Node node, Node host, List<Move> movesConsidered, List<Move> movesMade) { + this.node = node; + this.host = host; + this.movesConsidered = movesConsidered; + this.movesMade = movesMade; + + hash = Objects.hash(node, host, movesConsidered, movesMade); + } + + @Override + public int hashCode() { return hash; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (o == null || o.getClass() != this.getClass()) return false; + + SolutionKey other = (SolutionKey)o; + if ( ! other.node.equals(this.node)) return false; + if ( ! other.host.equals(this.host)) return false; + if ( ! other.movesConsidered.equals(this.movesConsidered)) return false; + if ( ! other.movesMade.equals(this.movesMade)) return false; + return true; + } + + } + + private static class SubsetIterator implements Iterator<List<Node>> { + + private final List<Node> nodes; + private final int maxLength; + + // A number whose binary representation determines which items of list we'll include + private int i = 0; // first "previous" = 0 -> skip the empty set + private List<Node> next = null; + + public SubsetIterator(List<Node> nodes, int maxLength) { + this.nodes = new ArrayList<>(nodes.subList(0, Math.min(nodes.size(), 31))); + this.maxLength = maxLength; + } + + @Override + public boolean hasNext() { + if (next != null) return true; + + // find next + while (++i < 1<<nodes.size()) { + int ones = Integer.bitCount(i); + if (ones > maxLength) continue; + + next = new ArrayList<>(ones); + for (int position = 0; position < nodes.size(); position++) { + if (hasOneAtPosition(position, i)) + next.add(nodes.get(position)); + } + return true; + } + return false; + } + + @Override + public List<Node> next() { + if ( ! hasNext()) throw new IllegalStateException("No more elements"); + var current = next; + next = null; + return current; + } + + private boolean hasOneAtPosition(int position, int number) { + return (number & (1 << position)) > 0; + } + + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java index 18a8fe7be6a..eba9e4a1ac9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java @@ -7,9 +7,11 @@ package com.yahoo.vespa.hosted.provision.node; * @author bratseth */ public enum Agent { + operator, // A hosted Vespa operator. Some logic recognizes these events. application, // An application package change deployment system, // An unspecified system agent + // Specific system agents: NodeFailer, Rebalancer, @@ -18,5 +20,8 @@ public enum Agent { InactiveExpirer, ProvisionedExpirer, ReservationExpirer, - DynamicProvisioningMaintainer + DynamicProvisioningMaintainer, + RetiringUpgrader, + SpareCapacityMaintainer + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index aa824e7a930..3210de18b7d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -259,7 +259,7 @@ public class IP { */ public Set<String> findUnused(NodeList nodes) { var unusedAddresses = new LinkedHashSet<>(asSet()); - nodes.filter(node -> node.ipConfig().primary().stream().anyMatch(ip -> asSet().contains(ip))) + nodes.matching(node -> node.ipConfig().primary().stream().anyMatch(ip -> asSet().contains(ip))) .forEach(node -> unusedAddresses.removeAll(node.ipConfig().primary())); return Collections.unmodifiableSet(unusedAddresses); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java index 6b52bd68e73..3e5ef81e614 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java @@ -36,6 +36,9 @@ public class Status { this.vespaVersion = Objects.requireNonNull(vespaVersion, "Vespa version must be non-null").filter(v -> !Version.emptyVersion.equals(v)); this.dockerImage = Objects.requireNonNull(dockerImage, "Docker image must be non-null").filter(d -> !DockerImage.EMPTY.equals(d)); this.failCount = failCount; + if (wantToDeprovision && !wantToRetire) { + throw new IllegalArgumentException("Node cannot be marked wantToDeprovision unless it's also marked wantToRetire"); + } this.wantToRetire = wantToRetire; this.wantToDeprovision = wantToDeprovision; this.osVersion = Objects.requireNonNull(osVersion, "OS version must be non-null"); @@ -69,8 +72,8 @@ public class Status { /** Returns how many times this node has been moved to the failed state. */ public int failCount() { return failCount; } - /** Returns a copy of this with the want to retire flag changed */ - public Status withWantToRetire(boolean wantToRetire) { + /** Returns a copy of this with the want to retire/deprovision flags changed */ + public Status withWantToRetire(boolean wantToRetire, boolean wantToDeprovision) { return new Status(reboot, vespaVersion, dockerImage, failCount, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); } @@ -82,11 +85,6 @@ public class Status { return wantToRetire; } - /** Returns a copy of this with the want to de-provision flag changed */ - public Status withWantToDeprovision(boolean wantToDeprovision) { - return new Status(reboot, vespaVersion, dockerImage, failCount, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); - } - /** * Returns whether this node should be de-provisioned when possible. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java index 5778800d02c..a2590e8c727 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java @@ -1,11 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node.filter; -import com.google.common.collect.ImmutableSet; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.hosted.provision.Node; -import java.util.Collections; +import java.util.EnumSet; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -23,22 +22,27 @@ public class StateFilter extends NodeFilter { private StateFilter(Set<Node.State> states, NodeFilter next) { super(next); Objects.requireNonNull(states, "state cannot be null, use an empty set"); - this.states = ImmutableSet.copyOf(states); + this.states = EnumSet.copyOf(states); } @Override public boolean matches(Node node) { - if ( ! states.isEmpty() && ! states.contains(node.state())) return false; + if ( ! states.contains(node.state())) return false; return nextMatches(node); } /** Returns a copy of the given filter which only matches for the given state */ public static StateFilter from(Node.State state, NodeFilter filter) { - return new StateFilter(Collections.singleton(state), filter); + return new StateFilter(EnumSet.of(state), filter); } /** Returns a node filter which matches a comma or space-separated list of states */ - public static StateFilter from(String states, NodeFilter next) { + public static StateFilter from(String states, boolean includeDeprovisioned, NodeFilter next) { + if (states == null) { + return new StateFilter(includeDeprovisioned ? + EnumSet.allOf(Node.State.class) : EnumSet.complementOf(EnumSet.of(Node.State.deprovisioned)), next); + } + return new StateFilter(StringUtilities.split(states).stream().map(Node.State::valueOf).collect(Collectors.toSet()), next); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingUpgrader.java new file mode 100644 index 00000000000..03d04a5f6cf --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingUpgrader.java @@ -0,0 +1,60 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.os; + +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; + +import java.util.Objects; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * An upgrader that delegates the upgrade to the node itself, triggered by changing its wanted OS version. This + * implementation limits the number of parallel upgrades to avoid overloading the orchestrator with suspension requests. + * + * Used in clouds where nodes can upgrade themselves in-place, without data loss. + * + * @author mpolden + */ +public class DelegatingUpgrader implements Upgrader { + + private static final Logger LOG = Logger.getLogger(DelegatingUpgrader.class.getName()); + + private final NodeRepository nodeRepository; + + /** The maximum number of nodes, within a single node type, that can upgrade in parallel. */ + private final int maxActiveUpgrades; + + public DelegatingUpgrader(NodeRepository nodeRepository, int maxActiveUpgrades) { + this.nodeRepository = Objects.requireNonNull(nodeRepository); + this.maxActiveUpgrades = maxActiveUpgrades; + } + + @Override + public void upgradeTo(OsVersionTarget target) { + NodeList activeNodes = nodeRepository.list().nodeType(target.nodeType()).state(Node.State.active); + int numberToUpgrade = Math.max(0, maxActiveUpgrades - activeNodes.changingOsVersionTo(target.version()).size()); + NodeList nodesToUpgrade = activeNodes.not().changingOsVersionTo(target.version()) + .not().onOsVersion(target.version()) + .byIncreasingOsVersion() + .first(numberToUpgrade); + if (nodesToUpgrade.size() == 0) return; + LOG.info("Upgrading " + nodesToUpgrade.size() + " nodes of type " + target.nodeType() + " to OS version " + + target.version().toFullString()); + nodeRepository.upgradeOs(NodeListFilter.from(nodesToUpgrade.asList()), Optional.of(target.version())); + } + + @Override + public void disableUpgrade(NodeType type) { + NodeList nodesUpgrading = nodeRepository.list() + .nodeType(type) + .changingOsVersion(); + if (nodesUpgrading.size() == 0) return; + LOG.info("Disabling OS upgrade of all " + type + " nodes"); + nodeRepository.upgradeOs(NodeListFilter.from(nodesUpgrading.asList()), Optional.empty()); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionChange.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionChange.java new file mode 100644 index 00000000000..053bd7c4feb --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionChange.java @@ -0,0 +1,77 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.os; + +import com.google.common.collect.ImmutableSortedMap; +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; + +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * The OS version change being deployed in a {@link com.yahoo.vespa.hosted.provision.NodeRepository}. + * + * @author mpolden + */ +public class OsVersionChange { + + public static final OsVersionChange NONE = new OsVersionChange(Map.of()); + + private final Map<NodeType, OsVersionTarget> targets; + + public OsVersionChange(Map<NodeType, OsVersionTarget> targets) { + this.targets = ImmutableSortedMap.copyOf(Objects.requireNonNull(targets)); + } + + /** Version targets in this */ + public Map<NodeType, OsVersionTarget> targets() { + return targets; + } + + /** Returns a copy of this with target for given node type removed */ + public OsVersionChange withoutTarget(NodeType nodeType) { + var targets = new HashMap<>(this.targets); + targets.remove(nodeType); + return new OsVersionChange(targets); + } + + /** Returns a copy of this with given target added */ + public OsVersionChange withTarget(Version version, NodeType nodeType, Optional<Duration> upgradeBudget) { + var targets = new HashMap<>(this.targets); + targets.compute(nodeType, (key, prevTarget) -> { + Optional<Instant> lastRetiredAt = Optional.ofNullable(prevTarget).flatMap(OsVersionTarget::lastRetiredAt); + return new OsVersionTarget(nodeType, version, upgradeBudget, lastRetiredAt); + }); + return new OsVersionChange(targets); + } + + /** Returns a copy of this with last retirement for given node type changed */ + public OsVersionChange withRetirementAt(Instant instant, NodeType nodeType) { + requireTarget(nodeType); + var targets = new HashMap<>(this.targets); + targets.computeIfPresent(nodeType, (key, target) -> new OsVersionTarget(nodeType, target.version(), target.upgradeBudget(), Optional.of(instant))); + return new OsVersionChange(targets); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OsVersionChange change = (OsVersionChange) o; + return targets.equals(change.targets); + } + + @Override + public int hashCode() { + return Objects.hash(targets); + } + + private void requireTarget(NodeType nodeType) { + if (!targets.containsKey(nodeType)) throw new IllegalArgumentException("No target set for " + nodeType); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionTarget.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionTarget.java new file mode 100644 index 00000000000..89c49447f17 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersionTarget.java @@ -0,0 +1,74 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.os; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; + +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * The target OS version for a {@link NodeType}. + * + * @author mpolden + */ +public class OsVersionTarget { + + private final NodeType nodeType; + private final Version version; + private final Optional<Duration> upgradeBudget; + private final Optional<Instant> lastRetiredAt; + + public OsVersionTarget(NodeType nodeType, Version version, Optional<Duration> upgradeBudget, Optional<Instant> lastRetiredAt) { + this.nodeType = Objects.requireNonNull(nodeType); + this.version = Objects.requireNonNull(version); + this.upgradeBudget = requireNotNegative(upgradeBudget); + this.lastRetiredAt = Objects.requireNonNull(lastRetiredAt); + } + + /** The node type this applies to */ + public NodeType nodeType() { + return nodeType; + } + + /** The OS version of this target */ + public Version version() { + return version; + } + + /** The upgrade budget for this. All nodes targeting this must upgrade within this budget */ + public Optional<Duration> upgradeBudget() { + return upgradeBudget; + } + + /** The most recent time a node was retired to apply a version upgrade */ + public Optional<Instant> lastRetiredAt() { + return lastRetiredAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OsVersionTarget target = (OsVersionTarget) o; + return nodeType == target.nodeType && + version.equals(target.version) && + upgradeBudget.equals(target.upgradeBudget) && + lastRetiredAt.equals(target.lastRetiredAt); + } + + @Override + public int hashCode() { + return Objects.hash(nodeType, version, upgradeBudget, lastRetiredAt); + } + + private static Optional<Duration> requireNotNegative(Optional<Duration> duration) { + Objects.requireNonNull(duration); + if (duration.isEmpty()) return duration; + if (duration.get().isNegative()) throw new IllegalArgumentException("Duration cannot be negative"); + return duration; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java index be474eddf97..54586105720 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java @@ -1,21 +1,21 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; import com.yahoo.component.Version; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; -import java.util.Map; +import java.time.Duration; import java.util.Objects; import java.util.Optional; +import java.util.function.UnaryOperator; import java.util.logging.Logger; /** - * Thread-safe class that manages target OS versions for nodes in this repository. + * Thread-safe class that manages an OS version change for nodes in this repository. An {@link Upgrader} decides how a + * {@link OsVersionTarget} is applied to nodes. * * A version target is initially inactive. Activation decision is taken by * {@link com.yahoo.vespa.hosted.provision.maintenance.OsUpgradeActivator}. @@ -28,39 +28,41 @@ public class OsVersions { private static final Logger log = Logger.getLogger(OsVersions.class.getName()); - /** - * The maximum number of nodes, within a single node type, that can upgrade in parallel. We limit the number of - * concurrent upgrades to avoid overloading the orchestrator. - */ - private static final int MAX_ACTIVE_UPGRADES = 30; - - private final NodeRepository nodeRepository; private final CuratorDatabaseClient db; - private final int maxActiveUpgrades; + private final Upgrader upgrader; public OsVersions(NodeRepository nodeRepository) { - this(nodeRepository, MAX_ACTIVE_UPGRADES); + this(nodeRepository, upgraderIn(nodeRepository)); } - OsVersions(NodeRepository nodeRepository, int maxActiveUpgrades) { - this.nodeRepository = Objects.requireNonNull(nodeRepository, "nodeRepository must be non-null"); - this.db = nodeRepository.database(); - this.maxActiveUpgrades = maxActiveUpgrades; + OsVersions(NodeRepository nodeRepository, Upgrader upgrader) { + this.db = Objects.requireNonNull(nodeRepository).database(); + this.upgrader = Objects.requireNonNull(upgrader); // Read and write all versions to make sure they are stored in the latest version of the serialized format - try (var lock = db.lockOsVersions()) { - db.writeOsVersions(db.readOsVersions()); + try (var lock = db.lockOsVersionChange()) { + db.writeOsVersionChange(db.readOsVersionChange()); } } - /** Returns the current target versions for each node type */ - public Map<NodeType, Version> targets() { - return db.readOsVersions(); + /** Returns the current OS version change */ + public OsVersionChange readChange() { + return db.readOsVersionChange(); + } + + /** Write the current OS version change with the result of the given operation applied */ + public void writeChange(UnaryOperator<OsVersionChange> operation) { + try (var lock = db.lockOsVersionChange()) { + OsVersionChange change = readChange(); + OsVersionChange newChange = operation.apply(change); + if (newChange.equals(change)) return; // Nothing changed + db.writeOsVersionChange(newChange); + } } /** Returns the current target version for given node type, if any */ public Optional<Version> targetFor(NodeType type) { - return Optional.ofNullable(targets().get(type)); + return Optional.ofNullable(readChange().targets().get(type)).map(OsVersionTarget::version); } /** @@ -69,26 +71,21 @@ public class OsVersions { */ public void removeTarget(NodeType nodeType) { require(nodeType); - try (Lock lock = db.lockOsVersions()) { - var osVersions = db.readOsVersions(); - osVersions.remove(nodeType); - disableUpgrade(nodeType); - db.writeOsVersions(osVersions); - } + writeChange((change) -> { + upgrader.disableUpgrade(nodeType); + return change.withoutTarget(nodeType); + }); } - /** Set the target OS version for nodes of given type */ - public void setTarget(NodeType nodeType, Version newTarget, boolean force) { + /** Set the target OS version and upgrade budget for nodes of given type */ + public void setTarget(NodeType nodeType, Version newTarget, Optional<Duration> upgradeBudget, boolean force) { require(nodeType); - if (newTarget.isEmpty()) { - throw new IllegalArgumentException("Invalid target version: " + newTarget.toFullString()); - } - try (Lock lock = db.lockOsVersions()) { - var osVersions = db.readOsVersions(); - var oldTarget = Optional.ofNullable(osVersions.get(nodeType)); - + requireNonZero(newTarget); + requireUpgradeBudget(upgradeBudget); + writeChange((change) -> { + var oldTarget = targetFor(nodeType); if (oldTarget.filter(v -> v.equals(newTarget)).isPresent()) { - return; // Old target matches new target, nothing to do + return change; // Old target matches new target, nothing to do } if (!force && oldTarget.filter(v -> v.isAfter(newTarget)).isPresent()) { @@ -97,48 +94,35 @@ public class OsVersions { + oldTarget.get()); } - osVersions.put(nodeType, newTarget); - db.writeOsVersions(osVersions); log.info("Set OS target version for " + nodeType + " nodes to " + newTarget.toFullString()); - } + return change.withTarget(newTarget, nodeType, upgradeBudget); + }); } - /** Activate or deactivate upgrade of given node type. This is used for resuming or pausing an OS upgrade. */ - public void setActive(NodeType nodeType, boolean active) { + /** Resume or halt upgrade of given node type */ + public void resumeUpgradeOf(NodeType nodeType, boolean resume) { require(nodeType); - try (Lock lock = db.lockOsVersions()) { - var osVersions = db.readOsVersions(); - var currentVersion = osVersions.get(nodeType); - if (currentVersion == null) return; // No target version set for this type - if (active) { - upgrade(nodeType, currentVersion); + try (Lock lock = db.lockOsVersionChange()) { + var target = readChange().targets().get(nodeType); + if (target == null) return; // No target set for this type + if (resume) { + upgrader.upgradeTo(target); } else { - disableUpgrade(nodeType); + upgrader.disableUpgrade(nodeType); } } } - /** Trigger upgrade of nodes of given type*/ - private void upgrade(NodeType type, Version version) { - var activeNodes = nodeRepository.list().nodeType(type).state(Node.State.active); - var numberToUpgrade = Math.max(0, maxActiveUpgrades - activeNodes.changingOsVersionTo(version).size()); - var nodesToUpgrade = activeNodes.not().changingOsVersionTo(version) - .not().onOsVersion(version) - .byIncreasingOsVersion() - .first(numberToUpgrade); - if (nodesToUpgrade.size() == 0) return; - log.info("Upgrading " + nodesToUpgrade.size() + " nodes of type " + type + " to OS version " + version.toFullString()); - nodeRepository.upgradeOs(NodeListFilter.from(nodesToUpgrade.asList()), Optional.of(version)); + private void requireUpgradeBudget(Optional<Duration> upgradeBudget) { + if (upgrader instanceof RetiringUpgrader && upgradeBudget.isEmpty()) { + throw new IllegalArgumentException("Zone requires a time budget for OS upgrades"); + } } - /** Disable OS upgrade for all nodes of given type */ - private void disableUpgrade(NodeType type) { - var nodesUpgrading = nodeRepository.list() - .nodeType(type) - .changingOsVersion(); - if (nodesUpgrading.size() == 0) return; - log.info("Disabling OS upgrade of all " + type + " nodes"); - nodeRepository.upgradeOs(NodeListFilter.from(nodesUpgrading.asList()), Optional.empty()); + private static void requireNonZero(Version version) { + if (version.isEmpty()) { + throw new IllegalArgumentException("Invalid target version: " + version.toFullString()); + } } private static void require(NodeType nodeType) { @@ -147,4 +131,11 @@ public class OsVersions { } } + private static Upgrader upgraderIn(NodeRepository nodeRepository) { + if (nodeRepository.zone().getCloud().reprovisionToUpgradeOs()) { + return new RetiringUpgrader(nodeRepository); + } + return new DelegatingUpgrader(nodeRepository, 30); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java new file mode 100644 index 00000000000..b2b83b6d064 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java @@ -0,0 +1,84 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.os; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * An upgrader that retires and deprovisions nodes on stale OS versions. Retirement of each node is spread out in time, + * according to a time budget, to avoid potential service impact of retiring too many nodes close together. + * + * Used in clouds where nodes must be re-provisioned to upgrade their OS. + * + * @author mpolden + */ +public class RetiringUpgrader implements Upgrader { + + private static final Logger LOG = Logger.getLogger(RetiringUpgrader.class.getName()); + + private final NodeRepository nodeRepository; + + public RetiringUpgrader(NodeRepository nodeRepository) { + this.nodeRepository = nodeRepository; + } + + @Override + public void upgradeTo(OsVersionTarget target) { + NodeList activeNodes = nodeRepository.list().nodeType(target.nodeType()).state(Node.State.active); + if (activeNodes.size() == 0) return; // No nodes eligible for upgrade + + Instant now = nodeRepository.clock().instant(); + Duration nodeBudget = target.upgradeBudget() + .orElseThrow(() -> new IllegalStateException("OS upgrades in this zone requires " + + "a time budget, but none is set")) + .dividedBy(activeNodes.size()); + Instant retiredAt = target.lastRetiredAt().orElse(Instant.EPOCH); + if (now.isBefore(retiredAt.plus(nodeBudget))) return; // Budget has not been spent yet + + activeNodes.not().onOsVersion(target.version()) + .not().deprovisioning() + .byIncreasingOsVersion() + .first(1) + .forEach(node -> retire(node, target.version(), now)); + } + + @Override + public void disableUpgrade(NodeType type) { + // No action needed in this implementation. + } + + /** Retire and deprovision given host and its children */ + private void retire(Node host, Version target, Instant now) { + if (!host.type().isDockerHost()) throw new IllegalArgumentException("Cannot retire non-host " + host); + try (var lock = nodeRepository.lock(host)) { + Optional<Node> currentNode = nodeRepository.getNode(host.hostname()); + if (currentNode.isEmpty()) return; + host = currentNode.get(); + NodeType nodeType = host.type(); + List<Node> nodesToRetire = nodeRepository.list().childrenOf(host).stream() + .map(child -> child.withWantToRetire(true, Agent.RetiringUpgrader, now)) + .collect(Collectors.toList()); + LOG.info("Retiring and deprovisioning " + host + ": On stale OS version " + + host.status().osVersion().current().map(Version::toFullString).orElse("<unset>") + + ", want " + target); + + host = host.withWantToRetire(true, true, Agent.RetiringUpgrader, now); + host = host.with(host.status().withOsVersion(host.status().osVersion().withWanted(Optional.of(target)))); + nodesToRetire.add(host); + nodeRepository.write(nodesToRetire, lock); + nodeRepository.osVersions().writeChange((change) -> change.withRetirementAt(now, nodeType)); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/Upgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/Upgrader.java new file mode 100644 index 00000000000..e5e68cd258e --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/Upgrader.java @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.os; + +import com.yahoo.config.provision.NodeType; + +/** + * Interface for an OS upgrader. + * + * @author mpolden + */ +public interface Upgrader { + + /** Trigger upgrade to given target */ + void upgradeTo(OsVersionTarget target); + + /** Disable OS upgrade for all nodes of given type */ + void disableUpgrade(NodeType type); + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index b1453a9729a..3464e9dd881 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -35,6 +35,7 @@ public class ApplicationSerializer { private static final String idKey = "id"; private static final String clustersKey = "clusters"; + private static final String exclusiveKey = "exclusive"; private static final String minResourcesKey = "min"; private static final String maxResourcesKey = "max"; private static final String suggestedResourcesKey = "suggested"; @@ -80,6 +81,7 @@ public class ApplicationSerializer { } private static void toSlime(Cluster cluster, Cursor clusterObject) { + clusterObject.setBool(exclusiveKey, cluster.exclusive()); toSlime(cluster.minResources(), clusterObject.setObject(minResourcesKey)); toSlime(cluster.maxResources(), clusterObject.setObject(maxResourcesKey)); cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject(suggestedResourcesKey))); @@ -88,6 +90,7 @@ public class ApplicationSerializer { private static Cluster clusterFromSlime(String id, Inspector clusterObject) { return new Cluster(ClusterSpec.Id.from(id), + clusterObject.field(exclusiveKey).asBool(), clusterResourcesFromSlime(clusterObject.field(minResourcesKey)), clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)), optionalClusterResourcesFromSlime(clusterObject.field(suggestedResourcesKey)), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 4defbb55485..367271564ea 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.google.common.util.concurrent.UncheckedTimeoutException; @@ -12,7 +12,6 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -27,6 +26,7 @@ import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Status; +import com.yahoo.vespa.hosted.provision.os.OsVersionChange; import java.time.Clock; import java.time.Duration; @@ -488,19 +488,19 @@ public class CuratorDatabaseClient implements JobControl.Db { // OS versions ----------------------------------------------------------- - public Map<NodeType, Version> readOsVersions() { - return read(osVersionsPath, OsVersionsSerializer::fromJson).orElseGet(TreeMap::new); + public OsVersionChange readOsVersionChange() { + return read(osVersionsPath, OsVersionChangeSerializer::fromJson).orElse(OsVersionChange.NONE); } - public void writeOsVersions(Map<NodeType, Version> versions) { + public void writeOsVersionChange(OsVersionChange change) { NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(osVersionsPath.getAbsolute(), - OsVersionsSerializer.toJson(versions))); + OsVersionChangeSerializer.toJson(change))); transaction.commit(); } - public Lock lockOsVersions() { + public Lock lockOsVersionChange() { return db.lock(lockPath.append("osVersionsLock"), defaultLockTimeout); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 81fc542afcc..37842115949 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -19,17 +19,17 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.slime.Type; import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.IP; +import com.yahoo.vespa.hosted.provision.node.OsVersion; import com.yahoo.vespa.hosted.provision.node.Reports; import com.yahoo.vespa.hosted.provision.node.Status; -import com.yahoo.vespa.hosted.provision.node.OsVersion; import java.io.IOException; import java.time.Instant; @@ -386,14 +386,16 @@ public class NodeSerializer { case "operator" : return Agent.operator; case "application" : return Agent.application; case "system" : return Agent.system; - case "NodeFailer" : return Agent.NodeFailer; - case "Rebalancer" : return Agent.Rebalancer; case "DirtyExpirer" : return Agent.DirtyExpirer; + case "DynamicProvisioningMaintainer" : return Agent.DynamicProvisioningMaintainer; case "FailedExpirer" : return Agent.FailedExpirer; case "InactiveExpirer" : return Agent.InactiveExpirer; + case "NodeFailer" : return Agent.NodeFailer; case "ProvisionedExpirer" : return Agent.ProvisionedExpirer; + case "Rebalancer" : return Agent.Rebalancer; case "ReservationExpirer" : return Agent.ReservationExpirer; - case "DynamicProvisioningMaintainer" : return Agent.DynamicProvisioningMaintainer; + case "RetiringUpgrader" : return Agent.RetiringUpgrader; + case "SpareCapacityMaintainer": return Agent.SpareCapacityMaintainer; } throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'"); } @@ -402,14 +404,16 @@ public class NodeSerializer { case operator : return "operator"; case application : return "application"; case system : return "system"; - case NodeFailer : return "NodeFailer"; - case Rebalancer : return "Rebalancer"; case DirtyExpirer : return "DirtyExpirer"; + case DynamicProvisioningMaintainer : return "DynamicProvisioningMaintainer"; case FailedExpirer : return "FailedExpirer"; case InactiveExpirer : return "InactiveExpirer"; + case NodeFailer : return "NodeFailer"; case ProvisionedExpirer : return "ProvisionedExpirer"; + case Rebalancer : return "Rebalancer"; case ReservationExpirer : return "ReservationExpirer"; - case DynamicProvisioningMaintainer : return "DynamicProvisioningMaintainer"; + case RetiringUpgrader: return "RetiringUpgrader"; + case SpareCapacityMaintainer: return "SpareCapacityMaintainer"; } throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionChangeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionChangeSerializer.java new file mode 100644 index 00000000000..c9890194d73 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionChangeSerializer.java @@ -0,0 +1,70 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.provision.os.OsVersionChange; +import com.yahoo.vespa.hosted.provision.os.OsVersionTarget; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Optional; + +/** + * Serializer for {@link OsVersionChange}. + * + * @author mpolden + */ +public class OsVersionChangeSerializer { + + private static final String TARGETS_FIELD = "targets"; + private static final String NODE_TYPE_FIELD = "nodeType"; + private static final String VERSION_FIELD = "version"; + private static final String UPGRADE_BUDGET_FIELD = "upgradeBudget"; + private static final String LAST_RETIRED_AT_FIELD = "lastRetiredAt"; + + private OsVersionChangeSerializer() {} + + public static byte[] toJson(OsVersionChange change) { + var slime = new Slime(); + var object = slime.setObject(); + var targetsObject = object.setArray(TARGETS_FIELD); + change.targets().forEach((nodeType, target) -> { + var targetObject = targetsObject.addObject(); + targetObject.setString(NODE_TYPE_FIELD, NodeSerializer.toString(nodeType)); + targetObject.setString(VERSION_FIELD, target.version().toFullString()); + target.upgradeBudget().ifPresent(duration -> targetObject.setLong(UPGRADE_BUDGET_FIELD, duration.toMillis())); + target.lastRetiredAt().ifPresent(instant -> targetObject.setLong(LAST_RETIRED_AT_FIELD, instant.toEpochMilli())); + }); + try { + return SlimeUtils.toJsonBytes(slime); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static OsVersionChange fromJson(byte[] data) { + var targets = new HashMap<NodeType, OsVersionTarget>(); + var inspector = SlimeUtils.jsonToSlime(data).get(); + inspector.field(TARGETS_FIELD).traverse((ArrayTraverser) (idx, arrayInspector) -> { + var version = Version.fromString(arrayInspector.field(VERSION_FIELD).asString()); + var nodeType = NodeSerializer.nodeTypeFromString(arrayInspector.field(NODE_TYPE_FIELD).asString()); + Optional<Duration> budget = optionalLong(arrayInspector.field(UPGRADE_BUDGET_FIELD)).map(Duration::ofMillis); + Optional<Instant> lastRetiredAt = optionalLong(arrayInspector.field(LAST_RETIRED_AT_FIELD)).map(Instant::ofEpochMilli); + targets.put(nodeType, new OsVersionTarget(nodeType, version, budget, lastRetiredAt)); + }); + return new OsVersionChange(targets); + } + + private static Optional<Long> optionalLong(Inspector field) { + return field.valid() ? Optional.of(field.asLong()) : Optional.empty(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java deleted file mode 100644 index fd430350b5c..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.persistence; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.NodeType; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.provision.node.OsVersion; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Map; -import java.util.TreeMap; - -/** - * Serializer for a map of {@link NodeType} and {@link OsVersion}. - * - * @author mpolden - */ -public class OsVersionsSerializer { - - private static final String VERSION_FIELD = "version"; - - private OsVersionsSerializer() {} - - public static byte[] toJson(Map<NodeType, Version> versions) { - var slime = new Slime(); - var object = slime.setObject(); - versions.forEach((nodeType, osVersion) -> { - var versionObject = object.setObject(NodeSerializer.toString(nodeType)); - versionObject.setString(VERSION_FIELD, osVersion.toFullString()); - }); - try { - return SlimeUtils.toJsonBytes(slime); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static Map<NodeType, Version> fromJson(byte[] data) { - var versions = new TreeMap<NodeType, Version>(); // Use TreeMap to sort by node type - var inspector = SlimeUtils.jsonToSlime(data).get(); - inspector.traverse((ObjectTraverser) (key, value) -> { - if (isNodeType(key)) { - var version = Version.fromString(value.field(VERSION_FIELD).asString()); - versions.put(NodeSerializer.nodeTypeFromString(key), version); - } - }); - return versions; - } - - private static boolean isNodeType(String name) { - try { - NodeType.valueOf(name); - return true; - } catch (IllegalArgumentException ignored) { - return false; - } - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 36034b62cfb..7158ccc57e3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.transaction.Mutex; @@ -110,10 +111,10 @@ class Activator { /** Activate load balancers */ private void activateLoadBalancers(ApplicationId application, Collection<HostSpec> hosts, NestedTransaction transaction, @SuppressWarnings("unused") Mutex applicationLock) { - loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(application, clustersOf(hosts), applicationLock, transaction)); + loadBalancerProvisioner.ifPresent(provisioner -> provisioner.activate(application, allClustersOf(hosts), applicationLock, transaction)); } - private static Set<ClusterSpec> clustersOf(Collection<HostSpec> hosts) { + private static Set<ClusterSpec> allClustersOf(Collection<HostSpec> hosts) { return hosts.stream() .map(HostSpec::membership) .flatMap(Optional::stream) @@ -183,12 +184,12 @@ class Activator { for (Node node : nodes) { HostSpec hostSpec = getHost(node.hostname(), hosts); node = hostSpec.membership().get().retired() ? node.retire(nodeRepository.clock().instant()) : node.unretire(); - if (hostSpec.flavor().isPresent()) // Docker nodes may change flavor - node = node.with(hostSpec.flavor().get()); + if (! hostSpec.advertisedResources().equals(node.resources())) // A resized node + node = node.with(new Flavor(hostSpec.advertisedResources())); Allocation allocation = node.allocation().get() .with(hostSpec.membership().get()) .withRequestedResources(hostSpec.requestedResources() - .orElse(node.flavor().resources())); + .orElse(node.resources())); if (hostSpec.networkPorts().isPresent()) allocation = allocation.withNetworkPorts(hostSpec.networkPorts().get()); node = node.with(allocation); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index e9e6fd30286..5c990f0a3f3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -8,8 +8,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; - -import java.util.Locale; +import com.yahoo.vespa.hosted.provision.NodeRepository; /** * Defines the policies for assigning cluster capacity in various environments @@ -20,14 +19,11 @@ public class CapacityPolicies { private final Zone zone; - private final NodeResourceLimits nodeResourceLimits; - /* Deployments must match 1-to-1 the advertised resources of a physical host */ private final boolean isUsingAdvertisedResources; - public CapacityPolicies(Zone zone) { - this.zone = zone; - this.nodeResourceLimits = new NodeResourceLimits(zone); + public CapacityPolicies(NodeRepository nodeRepository) { + this.zone = nodeRepository.zone(); this.isUsingAdvertisedResources = zone.getCloud().dynamicProvisioning(); } @@ -45,31 +41,21 @@ public class CapacityPolicies { } } - public NodeResources decideNodeResources(NodeResources requested, Capacity capacity, ClusterSpec cluster) { - if (requested.isUnspecified()) - requested = defaultNodeResources(cluster.type()); - ensureSufficientResources(requested, cluster); + public NodeResources decideNodeResources(NodeResources target, Capacity capacity, ClusterSpec cluster) { + if (target.isUnspecified()) + target = defaultNodeResources(cluster.type()); - if (capacity.isRequired()) return requested; + if (capacity.isRequired()) return target; // Allow slow storage in zones which are not performance sensitive if (zone.system().isCd() || zone.environment() == Environment.dev || zone.environment() == Environment.test) - requested = requested.with(NodeResources.DiskSpeed.any).with(NodeResources.StorageType.any); + target = target.with(NodeResources.DiskSpeed.any).with(NodeResources.StorageType.any); // Dev does not cap the cpu of containers since usage is spotty: Allocate just a small amount exclusively - // Do not cap in AWS as hosts are allocated on demand and 1-to-1, so the node can use the entire host - if (zone.environment() == Environment.dev && !zone.region().value().contains("aws-")) - requested = requested.withVcpu(0.1); - - return requested; - } + if (zone.environment() == Environment.dev && zone.getCloud().allowHostSharing()) + target = target.withVcpu(0.1); - private void ensureSufficientResources(NodeResources resources, ClusterSpec cluster) { - double minMemoryGb = nodeResourceLimits.minMemoryGb(cluster.type()); - if (resources.memoryGb() >= minMemoryGb) return; - throw new IllegalArgumentException(String.format(Locale.ENGLISH, - "Must specify at least %.2f Gb of memory for %s cluster '%s', was: %.2f Gb", - minMemoryGb, cluster.type().name(), cluster.id().value(), resources.memoryGb())); + return target; } private NodeResources defaultNodeResources(ClusterSpec.Type clusterType) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index da54d89b4e5..38dd9f29873 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -17,7 +17,7 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { private final HostResourcesCalculator hostResourcesCalculator = new IdentityHostResourcesCalculator(); @Override - public Optional<LoadBalancerService> getLoadBalancerService() { + public Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository) { return Optional.empty(); } @@ -34,14 +34,17 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { @Override - public NodeResources realResourcesOf(Node node, NodeRepository repository) { - return node.flavor().resources(); - } + public NodeResources realResourcesOf(Node node, NodeRepository repository) { return node.resources(); } @Override - public NodeResources advertisedResourcesOf(Flavor flavor) { - return flavor.resources(); - } + public NodeResources advertisedResourcesOf(Flavor flavor) { return flavor.resources(); } + + @Override + public NodeResources requestToReal(NodeResources resources) { return resources; } + + @Override + public NodeResources realToRequest(NodeResources resources) { return resources; } } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java index 78f851fbc9e..e04c1aa208d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/FlavorConfigBuilder.java @@ -47,15 +47,21 @@ public class FlavorConfigBuilder { FlavorConfigBuilder flavorConfigBuilder = new FlavorConfigBuilder(); for (String flavorName : flavors) { if (flavorName.equals("docker")) - flavorConfigBuilder.addFlavor(flavorName, 1., 3., 2., 1.5, Flavor.Type.DOCKER_CONTAINER); + flavorConfigBuilder.addFlavor(flavorName, 1., 30., 20., 1.5, Flavor.Type.DOCKER_CONTAINER); else if (flavorName.equals("docker2")) - flavorConfigBuilder.addFlavor(flavorName, 2., 4., 4., 0.5, Flavor.Type.DOCKER_CONTAINER); + flavorConfigBuilder.addFlavor(flavorName, 2., 40., 40., 0.5, Flavor.Type.DOCKER_CONTAINER); else if (flavorName.equals("host")) - flavorConfigBuilder.addFlavor(flavorName, 7., 10., 12., 5, Flavor.Type.BARE_METAL); + flavorConfigBuilder.addFlavor(flavorName, 7., 100., 120., 5, Flavor.Type.BARE_METAL); + else if (flavorName.equals("host2")) + flavorConfigBuilder.addFlavor(flavorName, 16, 24, 100, 1, Flavor.Type.BARE_METAL); + else if (flavorName.equals("host3")) + flavorConfigBuilder.addFlavor(flavorName, 24, 64, 100, 1, Flavor.Type.BARE_METAL); + else if (flavorName.equals("host4")) + flavorConfigBuilder.addFlavor(flavorName, 48, 128, 1000, 1, Flavor.Type.BARE_METAL); else if (flavorName.equals("devhost")) - flavorConfigBuilder.addFlavor(flavorName, 4., 8., 10, 10, Flavor.Type.BARE_METAL); + flavorConfigBuilder.addFlavor(flavorName, 4., 80., 100, 10, Flavor.Type.BARE_METAL); else - flavorConfigBuilder.addFlavor(flavorName, 1., 3., 2., 3, Flavor.Type.BARE_METAL); + flavorConfigBuilder.addFlavor(flavorName, 1., 30., 20., 3, Flavor.Type.BARE_METAL); } return new NodeFlavors(flavorConfigBuilder.build()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index f8774073ce8..d3e5f60599f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -11,7 +11,7 @@ import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; -import com.yahoo.vespa.flags.custom.PreprovisionCapacity; +import com.yahoo.vespa.flags.custom.HostCapacity; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -30,14 +30,14 @@ public class GroupPreparer { private final NodeRepository nodeRepository; private final Optional<HostProvisioner> hostProvisioner; - private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag; + private final ListFlag<HostCapacity> preprovisionCapacityFlag; public GroupPreparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, FlagSource flagSource) { this.nodeRepository = nodeRepository; this.hostProvisioner = hostProvisioner; - this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource); + this.preprovisionCapacityFlag = Flags.TARGET_CAPACITY.bindTo(flagSource); } /** @@ -58,7 +58,7 @@ public class GroupPreparer { // active config model which is changed on activate public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, List<Node> surplusActiveNodes, MutableInteger highestIndex, int spareCount, int wantedGroups) { - boolean dynamicProvisioningEnabled = hostProvisioner.isPresent() && nodeRepository.zone().getCloud().dynamicProvisioning(); + boolean dynamicProvisioningEnabled = nodeRepository.canProvisionHosts() && nodeRepository.zone().getCloud().dynamicProvisioning(); boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty(); try (Mutex lock = nodeRepository.lock(application)) { @@ -66,25 +66,22 @@ public class GroupPreparer { try (Mutex allocationLock = nodeRepository.lockUnallocated()) { // Create a prioritized set of nodes - LockedNodeList nodeList = nodeRepository.list(allocationLock); - NodePrioritizer prioritizer = new NodePrioritizer(nodeList, + LockedNodeList allNodes = nodeRepository.list(allocationLock); + NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes, + highestIndex, nodeRepository); + + NodePrioritizer prioritizer = new NodePrioritizer(allNodes, application, cluster, requestedNodes, spareCount, wantedGroups, - nodeRepository.nameResolver(), - nodeRepository.resourcesCalculator(), - allocateFully); - + allocateFully, + nodeRepository); prioritizer.addApplicationNodes(); prioritizer.addSurplusNodes(surplusActiveNodes); prioritizer.addReadyNodes(); - prioritizer.addNewDockerNodes(nodeRepository::canAllocateTenantNodeTo); - // Allocate from the prioritized list - NodeAllocation allocation = new NodeAllocation(nodeList, application, cluster, requestedNodes, - highestIndex, nodeRepository.flavors(), - nodeRepository.zone(), nodeRepository.clock()); + prioritizer.addNewDockerNodes(); allocation.offer(prioritizer.prioritize()); if (dynamicProvisioningEnabled) { @@ -114,28 +111,17 @@ public class GroupPreparer { } if (! allocation.fulfilled() && requestedNodes.canFail()) - throw new OutOfCapacityException("Could not satisfy " + requestedNodes + " for " + cluster + - " in " + application.toShortString() + - outOfCapacityDetails(allocation)); + throw new OutOfCapacityException((cluster.group().isPresent() ? "Out of capacity on " + cluster.group().get() :"") + + allocation.outOfCapacityDetails()); // Carry out and return allocation nodeRepository.reserve(allocation.reservableNodes()); nodeRepository.addDockerNodes(new LockedNodeList(allocation.newNodes(), allocationLock)); - surplusActiveNodes.removeAll(allocation.surplusNodes()); - return allocation.finalNodes(surplusActiveNodes); + List<Node> acceptedNodes = allocation.finalNodes(); + surplusActiveNodes.removeAll(acceptedNodes); + return acceptedNodes; } } } - private static String outOfCapacityDetails(NodeAllocation allocation) { - if (allocation.wouldBeFulfilledWithoutExclusivity()) - return ": Not enough nodes available due to host exclusivity constraints."; - else if (allocation.wouldBeFulfilledWithClashingParentHost()) - return ": Not enough nodes available on separate physical hosts."; - else if (allocation.wouldBeFulfilledWithRetiredNodes()) - return ": Not enough nodes available due to retirement."; - else - return "."; - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java index 672be25c5be..fd16e61417f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java @@ -3,34 +3,59 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; /** * Capacity calculation for docker hosts. * <p> - * The calculations is based on an immutable copy of nodes that represents - * all capacities in the system - i.e. all nodes in the node repo give or take. + * The calculations are based on an immutable copy of nodes that represents + * all capacities in the system - i.e. all nodes in the node repo. * * @author smorgrav */ -public class DockerHostCapacity { +public class HostCapacity { private final NodeList allNodes; private final HostResourcesCalculator hostResourcesCalculator; - public DockerHostCapacity(NodeList allNodes, HostResourcesCalculator hostResourcesCalculator) { + public HostCapacity(NodeList allNodes, HostResourcesCalculator hostResourcesCalculator) { this.allNodes = Objects.requireNonNull(allNodes, "allNodes must be non-null"); this.hostResourcesCalculator = Objects.requireNonNull(hostResourcesCalculator, "hostResourcesCalculator must be non-null"); } - int compareWithoutInactive(Node hostA, Node hostB) { + public NodeList allNodes() { return allNodes; } + + /** + * Spare hosts are the hosts in the system with the most free capacity. + * + * We do not count retired or inactive nodes as used capacity (as they could have been + * moved to create space for the spare node in the first place). + * + * @param candidates the candidates to consider. This list may contain all kinds of nodes. + * @param count the max number of spare hosts to return + */ + public Set<Node> findSpareHosts(List<Node> candidates, int count) { + return candidates.stream() + .filter(node -> node.type() == NodeType.host) + .filter(dockerHost -> dockerHost.state() == Node.State.active) + .filter(dockerHost -> freeIPs(dockerHost) > 0) + .sorted(this::compareWithoutInactive) + .limit(count) + .collect(Collectors.toSet()); + } + + private int compareWithoutInactive(Node hostA, Node hostB) { int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true)); if (result != 0) return result; - // If resources are equal we want to assign to the one with the most IPaddresses free + // If resources are equal we want to assign to the one with the most IP addresses free return freeIPs(hostB) - freeIPs(hostA); } @@ -65,9 +90,9 @@ public class DockerHostCapacity { NodeResources freeCapacityOf(Node host, boolean excludeInactive) { // Only hosts have free capacity - if (!host.type().canRun(NodeType.tenant)) return new NodeResources(0, 0, 0, 0); - NodeResources hostResources = hostResourcesCalculator.advertisedResourcesOf(host.flavor()); + if ( ! host.type().canRun(NodeType.tenant)) return new NodeResources(0, 0, 0, 0); + NodeResources hostResources = hostResourcesCalculator.advertisedResourcesOf(host.flavor()); return allNodes.childrenOf(host).asList().stream() .filter(node -> !(excludeInactive && isInactiveOrRetired(node))) .map(node -> node.flavor().resources().justNumbers()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java index 096e58e963e..a071fb25da2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -16,10 +16,22 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; */ public interface HostResourcesCalculator { - /** Nodes use advertised resources. This returns the real resources for the node. */ + /** Returns the real resources available on a node */ NodeResources realResourcesOf(Node node, NodeRepository nodeRepository); - /** Flavors use real resources. This returns the advertised resources of the flavor. */ + /** Returns the advertised resources of a flavor */ NodeResources advertisedResourcesOf(Flavor flavor); + /** + * Used with exclusive hosts: + * Returns the lowest possible real resources we'll get if requesting the given advertised resources + */ + NodeResources requestToReal(NodeResources advertisedResources); + + /** + * Used with shared hosts: + * Returns the advertised resources we need to request to be sure to get at least the given real resources. + */ + NodeResources realToRequest(NodeResources realResources); + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 026788e9d6c..460e1e71e65 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -6,9 +6,11 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.exception.LoadBalancerServiceException; -import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -16,6 +18,7 @@ import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec; import com.yahoo.vespa.hosted.provision.lb.Real; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; @@ -24,8 +27,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -45,11 +50,13 @@ public class LoadBalancerProvisioner { private final NodeRepository nodeRepository; private final CuratorDatabaseClient db; private final LoadBalancerService service; + private final BooleanFlag provisionConfigServerLoadBalancer; - public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service) { + public LoadBalancerProvisioner(NodeRepository nodeRepository, LoadBalancerService service, FlagSource flagSource) { this.nodeRepository = nodeRepository; this.db = nodeRepository.database(); this.service = service; + this.provisionConfigServerLoadBalancer = Flags.CONFIGSERVER_PROVISION_LB.bindTo(flagSource); // Read and write all load balancers to make sure they are stored in the latest version of the serialization format for (var id : db.readLoadBalancerIds()) { try (var lock = db.lock(id.application())) { @@ -70,11 +77,12 @@ public class LoadBalancerProvisioner { * Calling this for irrelevant node or cluster types is a no-op. */ public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { - if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type - if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type + if (!canForwardTo(requestedNodes.type(), cluster)) return; // Nothing to provision for this node and cluster type if (application.instance().isTester()) return; // Do not provision for tester instances try (var lock = db.lock(application)) { - provision(application, effectiveId(cluster), false, lock); + ClusterSpec.Id clusterId = effectiveId(cluster); + List<Node> nodes = nodesOf(clusterId, application); + provision(application, clusterId, nodes,false, lock); } } @@ -91,13 +99,14 @@ public class LoadBalancerProvisioner { public void activate(ApplicationId application, Set<ClusterSpec> clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { try (var lock = db.lock(application)) { - var containerClusters = containerClustersOf(clusters); - for (var clusterId : containerClusters) { + for (var cluster : loadBalancedClustersOf(application).entrySet()) { // Provision again to ensure that load balancer instance is re-configured with correct nodes - provision(application, clusterId, true, lock); + provision(application, cluster.getKey(), cluster.getValue(), true, lock); } // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed - var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters); + var surplusLoadBalancers = surplusLoadBalancersOf(application, clusters.stream() + .map(LoadBalancerProvisioner::effectiveId) + .collect(Collectors.toSet())); deactivate(surplusLoadBalancers, transaction); } } @@ -138,9 +147,17 @@ public class LoadBalancerProvisioner { db.writeLoadBalancers(deactivatedLoadBalancers, transaction); } + // TODO(mpolden): Inline when feature flag is removed + private boolean canForwardTo(NodeType type, ClusterSpec cluster) { + boolean canForwardTo = service.canForwardTo(type, cluster.type()); + if (canForwardTo && type == NodeType.config) { + return provisionConfigServerLoadBalancer.value(); + } + return canForwardTo; + } /** Idempotently provision a load balancer for given application and cluster */ - private void provision(ApplicationId application, ClusterSpec.Id clusterId, boolean activate, + private void provision(ApplicationId application, ClusterSpec.Id clusterId, List<Node> nodes, boolean activate, @SuppressWarnings("unused") Mutex loadBalancersLock) { var id = new LoadBalancerId(application, clusterId); var now = nodeRepository.clock().instant(); @@ -148,7 +165,7 @@ public class LoadBalancerProvisioner { if (loadBalancer.isEmpty() && activate) return; // Nothing to activate as this load balancer was never prepared var force = loadBalancer.isPresent() && loadBalancer.get().state() != LoadBalancer.State.active; - var instance = create(application, clusterId, allocatedContainers(application, clusterId), force); + var instance = provisionInstance(application, clusterId, nodes, force); LoadBalancer newLoadBalancer; if (loadBalancer.isEmpty()) { newLoadBalancer = new LoadBalancer(id, instance, LoadBalancer.State.reserved, now); @@ -159,7 +176,8 @@ public class LoadBalancerProvisioner { db.writeLoadBalancer(newLoadBalancer); } - private LoadBalancerInstance create(ApplicationId application, ClusterSpec.Id cluster, List<Node> nodes, boolean force) { + private LoadBalancerInstance provisionInstance(ApplicationId application, ClusterSpec.Id cluster, List<Node> nodes, + boolean force) { var reals = new LinkedHashSet<Real>(); for (var node : nodes) { for (var ip : reachableIpAddresses(node)) { @@ -169,7 +187,7 @@ public class LoadBalancerProvisioner { log.log(Level.FINE, "Creating load balancer for " + cluster + " in " + application.toShortString() + ", targeting: " + reals); try { - return service.create(application, cluster, reals, force, nodeRepository); + return service.create(new LoadBalancerSpec(application, cluster, reals), force); } catch (Exception e) { throw new LoadBalancerServiceException("Failed to (re)configure load balancer for " + cluster + " in " + application + ", targeting: " + reals + ". The operation will be " + @@ -177,14 +195,21 @@ public class LoadBalancerProvisioner { } } - /** Returns a list of active and reserved nodes of type container in given cluster */ - private List<Node> allocatedContainers(ApplicationId application, ClusterSpec.Id clusterId) { - return NodeList.copyOf(nodeRepository.getNodes(NodeType.tenant, Node.State.reserved, Node.State.active)) - .owner(application) - .filter(node -> node.state().isAllocated()) - .container() - .filter(node -> effectiveId(node.allocation().get().membership().cluster()).equals(clusterId)) - .asList(); + /** Returns the nodes allocated to the given load balanced cluster */ + private List<Node> nodesOf(ClusterSpec.Id loadBalancedCluster, ApplicationId application) { + return loadBalancedClustersOf(application).getOrDefault(loadBalancedCluster, List.of()); + } + + /** Returns the load balanced clusters of given application and their nodes */ + private Map<ClusterSpec.Id, List<Node>> loadBalancedClustersOf(ApplicationId application) { + NodeList nodes = NodeList.copyOf(nodeRepository.getNodes(Node.State.reserved, Node.State.active)) + .owner(application); + if (nodes.stream().anyMatch(node -> node.type() == NodeType.config)) { + nodes = nodes.nodeType(NodeType.config).type(ClusterSpec.Type.admin); + } else { + nodes = nodes.nodeType(NodeType.tenant).container(); + } + return nodes.stream().collect(Collectors.groupingBy(node -> effectiveId(node.allocation().get().membership().cluster()))); } /** Find IP addresses reachable by the load balancer service */ @@ -202,14 +227,6 @@ public class LoadBalancerProvisioner { return reachable; } - /** Returns the container cluster IDs of the given clusters */ - private static Set<ClusterSpec.Id> containerClustersOf(Set<ClusterSpec> clusters) { - return clusters.stream() - .filter(c -> c.type().isContainer()) - .map(LoadBalancerProvisioner::effectiveId) - .collect(Collectors.toUnmodifiableSet()); - } - private static ClusterSpec.Id effectiveId(ClusterSpec cluster) { return cluster.combinedId().orElse(cluster.id()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 71e8259665b..df8a7e45917 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -6,18 +6,16 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; import com.yahoo.lang.MutableInteger; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; -import java.time.Clock; import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; @@ -60,11 +58,13 @@ class NodeAllocation { private int acceptedWithoutResizingRetired = 0; /** The number of nodes rejected because of clashing parentHostname */ - private int rejectedWithClashingParentHost = 0; + private int rejectedDueToClashingParentHost = 0; /** The number of nodes rejected due to exclusivity constraints */ private int rejectedDueToExclusivity = 0; + private int rejectedDueToInsufficientRealResources = 0; + /** The number of nodes that just now was changed to retired */ private int wasRetiredJustNow = 0; @@ -74,20 +74,18 @@ class NodeAllocation { /** The next membership index to assign to a new node */ private final MutableInteger highestIndex; - private final NodeFlavors flavors; - private final Zone zone; - private final Clock clock; + private final NodeRepository nodeRepository; + private final NodeResourceLimits nodeResourceLimits; NodeAllocation(NodeList allNodes, ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, - MutableInteger highestIndex, NodeFlavors flavors, Zone zone, Clock clock) { + MutableInteger highestIndex, NodeRepository nodeRepository) { this.allNodes = allNodes; this.application = application; this.cluster = cluster; this.requestedNodes = requestedNodes; this.highestIndex = highestIndex; - this.flavors = flavors; - this.zone = zone; - this.clock = clock; + this.nodeRepository = nodeRepository; + nodeResourceLimits = new NodeResourceLimits(nodeRepository); } /** @@ -104,6 +102,7 @@ class NodeAllocation { List<Node> accepted = new ArrayList<>(); for (PrioritizableNode node : nodesPrioritized) { Node offered = node.node; + if (offered.allocation().isPresent()) { ClusterMembership membership = offered.allocation().get().membership(); if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application @@ -114,6 +113,7 @@ class NodeAllocation { if (requestedNodes.considerRetiring()) { boolean wantToRetireNode = false; + if ( ! nodeResourceLimits.isWithinRealLimits(offered, cluster)) wantToRetireNode = true; if (violatesParentHostPolicy(this.nodes, offered)) wantToRetireNode = true; if ( ! hasCompatibleFlavor(node)) wantToRetireNode = true; if (offered.status().wantToRetire()) wantToRetireNode = true; @@ -127,8 +127,12 @@ class NodeAllocation { } } else if (! saturated() && hasCompatibleFlavor(node)) { + if ( ! nodeResourceLimits.isWithinRealLimits(offered, cluster)) { + ++rejectedDueToInsufficientRealResources; + continue; + } if ( violatesParentHostPolicy(this.nodes, offered)) { - ++rejectedWithClashingParentHost; + ++rejectedDueToClashingParentHost; continue; } if ( ! exclusiveTo(application.tenant(), application.application(), offered.parentHostname())) { @@ -144,8 +148,8 @@ class NodeAllocation { } node.node = offered.allocate(application, ClusterMembership.from(cluster, highestIndex.add(1)), - requestedNodes.resources().orElse(node.node.flavor().resources()), - clock.instant()); + requestedNodes.resources().orElse(node.node.resources()), + nodeRepository.clock().instant()); accepted.add(acceptNode(node, false, false)); } } @@ -159,7 +163,9 @@ class NodeAllocation { } private boolean checkForClashingParentHost() { - return zone.system() == SystemName.main && zone.environment().isProduction() && ! application.instance().isTester(); + return nodeRepository.zone().system() == SystemName.main && + nodeRepository.zone().environment().isProduction() && + ! application.instance().isTester(); } private boolean offeredNodeHasParentHostnameAlreadyAccepted(Collection<PrioritizableNode> accepted, Node offered) { @@ -229,14 +235,14 @@ class NodeAllocation { } private boolean hasCompatibleFlavor(PrioritizableNode node) { - return requestedNodes.isCompatible(node.node.flavor(), flavors) || node.isResizable; + return requestedNodes.isCompatible(node.node.flavor(), nodeRepository.flavors()) || node.isResizable; } private Node acceptNode(PrioritizableNode prioritizableNode, boolean wantToRetire, boolean resizeable) { Node node = prioritizableNode.node; if (node.allocation().isPresent()) // Record the currently requested resources - node = node.with(node.allocation().get().withRequestedResources(requestedNodes.resources().orElse(node.flavor().resources()))); + node = node.with(node.allocation().get().withRequestedResources(requestedNodes.resources().orElse(node.resources()))); if (! wantToRetire) { accepted++; @@ -255,7 +261,7 @@ class NodeAllocation { } else { ++wasRetiredJustNow; // Retire nodes which are of an unwanted flavor, retired flavor or have an overlapping parent host - node = node.retire(clock.instant()); + node = node.retire(nodeRepository.clock().instant()); } if ( ! node.allocation().get().membership().cluster().equals(cluster)) { // group may be different @@ -290,18 +296,6 @@ class NodeAllocation { return requestedNodes.fulfilledBy(accepted); } - boolean wouldBeFulfilledWithRetiredNodes() { - return requestedNodes.fulfilledBy(accepted + wasRetiredJustNow); - } - - boolean wouldBeFulfilledWithClashingParentHost() { - return requestedNodes.fulfilledBy(accepted + rejectedWithClashingParentHost); - } - - boolean wouldBeFulfilledWithoutExclusivity() { - return requestedNodes.fulfilledBy(accepted + rejectedDueToExclusivity); - } - /** * Returns {@link FlavorCount} describing the docker node deficit for the given {@link NodeSpec}. * @@ -322,18 +316,16 @@ class NodeAllocation { * Prefer to retire nodes of the wrong flavor. * Make as few changes to the retired set as possible. * - * @param surplusNodes this will add nodes not any longer needed by this group to this list * @return the final list of nodes */ - List<Node> finalNodes(List<Node> surplusNodes) { + List<Node> finalNodes() { int currentRetiredCount = (int) nodes.stream().filter(node -> node.node.allocation().get().membership().retired()).count(); int deltaRetiredCount = requestedNodes.idealRetiredCount(nodes.size(), currentRetiredCount) - currentRetiredCount; if (deltaRetiredCount > 0) { // retire until deltaRetiredCount is 0, prefer to retire higher indexes to minimize redistribution for (PrioritizableNode node : byDecreasingIndex(nodes)) { if ( ! node.node.allocation().get().membership().retired() && node.node.state() == Node.State.active) { - node.node = node.node.retire(Agent.application, clock.instant()); - surplusNodes.add(node.node); // offer this node to other groups + node.node = node.node.retire(Agent.application, nodeRepository.clock().instant()); if (--deltaRetiredCount == 0) break; } } @@ -396,6 +388,21 @@ class NodeAllocation { return Comparator.comparing((PrioritizableNode n) -> n.node.allocation().get().membership().index()); } + public String outOfCapacityDetails() { + List<String> reasons = new ArrayList<>(); + if (rejectedDueToExclusivity > 0) + reasons.add("host exclusivity constraints"); + if (rejectedDueToClashingParentHost > 0) + reasons.add("insufficient nodes available on separate physical hosts"); + if (wasRetiredJustNow > 0) + reasons.add("retirement of allocated nodes"); + if (rejectedDueToInsufficientRealResources > 0) + reasons.add("insufficient real resources on hosts"); + + if (reasons.isEmpty()) return ""; + return ": Not enough nodes available due to " + reasons.stream().collect(Collectors.joining(", ")); + } + static class FlavorCount { private final NodeResources flavor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index a7d83bbfad9..8560dd424e7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -11,8 +11,8 @@ import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import java.util.EnumSet; import java.util.HashMap; @@ -20,7 +20,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -38,11 +37,11 @@ public class NodePrioritizer { private final Map<Node, PrioritizableNode> nodes = new HashMap<>(); private final LockedNodeList allNodes; - private final DockerHostCapacity capacity; + private final HostCapacity capacity; private final NodeSpec requestedNodes; private final ApplicationId application; private final ClusterSpec clusterSpec; - private final NameResolver nameResolver; + private final NodeRepository nodeRepository; private final boolean isDocker; private final boolean isAllocatingForReplacement; private final boolean isTopologyChange; @@ -52,16 +51,15 @@ public class NodePrioritizer { private final Set<Node> spareHosts; NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, - int spares, int wantedGroups, NameResolver nameResolver, HostResourcesCalculator hostResourcesCalculator, - boolean allocateFully) { + int spares, int wantedGroups, boolean allocateFully, NodeRepository nodeRepository) { this.allNodes = allNodes; - this.capacity = new DockerHostCapacity(allNodes, hostResourcesCalculator); + this.capacity = new HostCapacity(allNodes, nodeRepository.resourcesCalculator()); this.requestedNodes = nodeSpec; this.clusterSpec = clusterSpec; this.application = application; - this.nameResolver = nameResolver; - this.spareHosts = findSpareHosts(allNodes, capacity, spares); + this.spareHosts = capacity.findSpareHosts(allNodes.asList(), spares); this.allocateFully = allocateFully; + this.nodeRepository = nodeRepository; NodeList nodesInCluster = allNodes.owner(application).type(clusterSpec.type()).cluster(clusterSpec.id()); NodeList nonRetiredNodesInCluster = nodesInCluster.not().retired(); @@ -78,28 +76,11 @@ public class NodePrioritizer { .filter(clusterSpec.group()::equals) .count(); - this.isAllocatingForReplacement = isReplacement( - nodesInCluster.size(), - nodesInCluster.state(Node.State.failed).size()); + this.isAllocatingForReplacement = isReplacement(nodesInCluster.size(), + nodesInCluster.state(Node.State.failed).size()); this.isDocker = resources(requestedNodes) != null; } - /** - * Spare hosts are the two hosts in the system with the most free capacity. - * - * We do not count retired or inactive nodes as used capacity (as they could have been - * moved to create space for the spare node in the first place). - */ - private static Set<Node> findSpareHosts(LockedNodeList nodes, DockerHostCapacity capacity, int spares) { - return nodes.asList().stream() - .filter(node -> node.type() == NodeType.host) - .filter(dockerHost -> dockerHost.state() == Node.State.active) - .filter(dockerHost -> capacity.freeIPs(dockerHost) > 0) - .sorted(capacity::compareWithoutInactive) - .limit(spares) - .collect(Collectors.toSet()); - } - /** Returns the list of nodes sorted by PrioritizableNode::compare */ List<PrioritizableNode> prioritize() { return nodes.values().stream().sorted().collect(Collectors.toList()); @@ -119,11 +100,11 @@ public class NodePrioritizer { } /** Add a node on each docker host with enough capacity for the requested flavor */ - void addNewDockerNodes(Predicate<Node> canAllocateTenantNodeTo) { + void addNewDockerNodes() { if ( ! isDocker) return; LockedNodeList candidates = allNodes - .filter(node -> node.type() != NodeType.host || canAllocateTenantNodeTo.test(node)) + .filter(node -> node.type() != NodeType.host || nodeRepository.canAllocateTenantNodeTo(node)) .filter(node -> node.reservedTo().isEmpty() || node.reservedTo().get().equals(application.tenant())); if (allocateFully) { @@ -142,25 +123,20 @@ public class NodePrioritizer { } private void addNewDockerNodesOn(LockedNodeList candidates) { - NodeResources wantedResources = resources(requestedNodes); - for (Node host : candidates) { - boolean hostHasCapacityForWantedFlavor = capacity.hasCapacity(host, wantedResources); - boolean conflictingCluster = allNodes.childrenOf(host).owner(application).asList().stream() - .anyMatch(child -> child.allocation().get().membership().cluster().id().equals(clusterSpec.id())); - - if (!hostHasCapacityForWantedFlavor || conflictingCluster) continue; + if ( ! capacity.hasCapacity(host, resources(requestedNodes))) continue; + if ( ! allNodes.childrenOf(host).owner(application).cluster(clusterSpec.id()).isEmpty()) continue; - log.log(Level.FINE, "Trying to add new Docker node on " + host); Optional<IP.Allocation> allocation; try { - allocation = host.ipConfig().pool().findAllocation(allNodes, nameResolver); + allocation = host.ipConfig().pool().findAllocation(allNodes, nodeRepository.nameResolver()); if (allocation.isEmpty()) continue; // No free addresses in this pool } catch (Exception e) { log.log(Level.WARNING, "Failed allocating IP address on " + host.hostname(), e); continue; } + log.log(Level.FINE, "Creating new docker node on " + host); Node newNode = Node.createDockerNode(allocation.get().addresses(), allocation.get().hostname(), host.hostname(), @@ -204,17 +180,16 @@ public class NodePrioritizer { * parameters to the priority sorting procedure. */ private PrioritizableNode toPrioritizable(Node node, boolean isSurplusNode, boolean isNewNode) { - PrioritizableNode.Builder builder = new PrioritizableNode.Builder(node) - .surplusNode(isSurplusNode) - .newNode(isNewNode); + PrioritizableNode.Builder builder = new PrioritizableNode.Builder(node).surplusNode(isSurplusNode) + .newNode(isNewNode); allNodes.parentOf(node).ifPresent(parent -> { NodeResources parentCapacity = capacity.freeCapacityOf(parent, false); builder.parent(parent).freeParentCapacity(parentCapacity); if (!isNewNode) - builder.resizable(!allocateFully - && requestedNodes.canResize(node.flavor().resources(), parentCapacity, isTopologyChange, currentClusterSize)); + builder.resizable(! allocateFully + && requestedNodes.canResize(node.resources(), parentCapacity, isTopologyChange, currentClusterSize)); if (spareHosts.contains(parent)) builder.violatesSpares(true); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index df35cabab36..f9d8213072e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -14,7 +14,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; -import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.FlagSource; @@ -35,6 +34,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -56,23 +56,21 @@ public class NodeRepositoryProvisioner implements Provisioner { private final Preparer preparer; private final Activator activator; private final Optional<LoadBalancerProvisioner> loadBalancerProvisioner; - - int getSpareCapacityProd() { - return SPARE_CAPACITY_PROD; - } + private final NodeResourceLimits nodeResourceLimits; @Inject public NodeRepositoryProvisioner(NodeRepository nodeRepository, Zone zone, ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource) { this.nodeRepository = nodeRepository; this.allocationOptimizer = new AllocationOptimizer(nodeRepository); - this.capacityPolicies = new CapacityPolicies(zone); + this.capacityPolicies = new CapacityPolicies(nodeRepository); this.zone = zone; - this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService().map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService)); + this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService(nodeRepository) + .map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService, flagSource)); + this.nodeResourceLimits = new NodeResourceLimits(nodeRepository); this.preparer = new Preparer(nodeRepository, zone.environment() == Environment.prod ? SPARE_CAPACITY_PROD : SPARE_CAPACITY_NONPROD, provisionServiceProvider.getHostProvisioner(), - provisionServiceProvider.getHostResourcesCalculator(), flagSource, loadBalancerProvisioner); this.activator = new Activator(nodeRepository, loadBalancerProvisioner); @@ -93,7 +91,10 @@ public class NodeRepositoryProvisioner implements Provisioner { if ( ! hasQuota(application, requested.maxResources().nodes())) throw new IllegalArgumentException(requested + " requested for " + cluster + - ". Max value exceeds your quota. Resolve this at https://cloud.vespa.ai/quota"); + ". Max value exceeds your quota. Resolve this at https://cloud.vespa.ai/pricing"); + + nodeResourceLimits.ensureWithinAdvertisedLimits("Min", requested.minResources().nodeResources(), cluster); + nodeResourceLimits.ensureWithinAdvertisedLimits("Max", requested.maxResources().nodeResources(), cluster); int groups; NodeResources resources; @@ -132,6 +133,8 @@ public class NodeRepositoryProvisioner implements Provisioner { loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.deactivate(application, transaction)); } + int getSpareCapacityProd() { return SPARE_CAPACITY_PROD; } + /** * Returns the target cluster resources, a value between the min and max in the requested capacity, * and updates the application store with the received min and max. @@ -139,7 +142,7 @@ public class NodeRepositoryProvisioner implements Provisioner { private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity requested) { try (Mutex lock = nodeRepository.lock(applicationId)) { Application application = nodeRepository.applications().get(applicationId).orElse(new Application(applicationId)); - application = application.withClusterLimits(clusterSpec.id(), requested.minResources(), requested.maxResources()); + application = application.withCluster(clusterSpec.id(), clusterSpec.isExclusive(), requested.minResources(), requested.maxResources()); nodeRepository.applications().put(application, lock); return application.clusters().get(clusterSpec.id()).targetResources() .orElseGet(() -> currentResources(applicationId, clusterSpec, requested)); @@ -156,17 +159,19 @@ public class NodeRepositoryProvisioner implements Provisioner { .not().removable() .asList(); AllocatableClusterResources currentResources = - nodes.isEmpty() ? new AllocatableClusterResources(requested.minResources(), clusterSpec.type()) // new deployment: Use min + nodes.isEmpty() ? new AllocatableClusterResources(requested.minResources(), + clusterSpec.type(), + nodeRepository) // new deployment: Use min : new AllocatableClusterResources(nodes, nodeRepository); - return ensureWithin(Limits.of(requested), currentResources); + return within(Limits.of(requested), clusterSpec.isExclusive(), currentResources); } /** Make the minimal adjustments needed to the current resources to stay within the limits */ - private ClusterResources ensureWithin(Limits limits, AllocatableClusterResources current) { + private ClusterResources within(Limits limits, boolean exclusive, AllocatableClusterResources current) { if (limits.isEmpty()) return current.toAdvertisedClusterResources(); if (limits.min().equals(limits.max())) return limits.min(); - return allocationOptimizer.findBestAllocation(ResourceTarget.preserve(current), current, limits) + return allocationOptimizer.findBestAllocation(ResourceTarget.preserve(current), current, limits, exclusive) .orElseThrow(() -> new IllegalArgumentException("No allocation possible within " + limits)) .toAdvertisedClusterResources(); } @@ -192,12 +197,12 @@ public class NodeRepositoryProvisioner implements Provisioner { log.log(Level.FINE, () -> "Prepared node " + node.hostname() + " - " + node.flavor()); Allocation nodeAllocation = node.allocation().orElseThrow(IllegalStateException::new); hosts.add(new HostSpec(node.hostname(), - List.of(), - Optional.of(node.flavor()), - Optional.of(nodeAllocation.membership()), + nodeRepository.resourcesCalculator().realResourcesOf(node, nodeRepository), + node.flavor().resources(), + requestedResources, + nodeAllocation.membership(), node.status().vespaVersion(), nodeAllocation.networkPorts(), - requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources), node.status().dockerImage())); if (nodeAllocation.networkPorts().isPresent()) { log.log(Level.FINE, () -> "Prepared node " + node.hostname() + " has port allocations"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index ca04bf66ce3..f9eaad072de 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -5,6 +5,10 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; + +import java.util.Locale; /** * Defines the resource limits for nodes in various zones @@ -13,20 +17,76 @@ import com.yahoo.config.provision.Zone; */ public class NodeResourceLimits { - private final Zone zone; + private final NodeRepository nodeRepository; + + public NodeResourceLimits(NodeRepository nodeRepository) { + this.nodeRepository = nodeRepository; + } + + /** Validates the resources applications ask for (which are in "advertised" resource space) */ + public void ensureWithinAdvertisedLimits(String type, NodeResources requested, ClusterSpec cluster) { + if (requested.isUnspecified()) return; + + if (requested.memoryGb() < minAdvertisedMemoryGb(cluster.type())) + illegal(type, "memory", cluster, requested.memoryGb(), minAdvertisedMemoryGb(cluster.type())); + if (requested.diskGb() < minAdvertisedDiskGb(requested)) + illegal(type, "disk", cluster, requested.diskGb(), minAdvertisedDiskGb(requested)); + } - public NodeResourceLimits(Zone zone) { - this.zone = zone; + /** Returns whether the real resources we'll end up with on a given tenant node are within limits */ + public boolean isWithinRealLimits(Node candidateTenantNode, ClusterSpec cluster) { + return isWithinRealLimits(nodeRepository.resourcesCalculator().realResourcesOf(candidateTenantNode, nodeRepository), + cluster.type()); } - public int minMemoryGb(ClusterSpec.Type clusterType) { - if (zone.system() == SystemName.dev) return 1; // Allow small containers in dev system + /** Returns whether the real resources we'll end up with on a given tenant node are within limits */ + public boolean isWithinRealLimits(NodeResources realResources, ClusterSpec.Type clusterType) { + if (realResources.isUnspecified()) return true; + + if (realResources.memoryGb() < minRealMemoryGb(clusterType)) return false; + if (realResources.diskGb() < minRealDiskGb()) return false; + return true; + } + + public NodeResources enlargeToLegal(NodeResources requested, ClusterSpec.Type clusterType) { + if (requested.isUnspecified()) return requested; + + return requested.withMemoryGb(Math.max(minAdvertisedMemoryGb(clusterType), requested.memoryGb())) + .withDiskGb(Math.max(minAdvertisedDiskGb(requested), requested.diskGb())); + } + + private double minAdvertisedMemoryGb(ClusterSpec.Type clusterType) { + if (nodeRepository.zone().system() == SystemName.dev) return 1; // Allow small containers in dev system if (clusterType == ClusterSpec.Type.admin) return 2; return 4; } - public NodeResources enlargeToLegal(NodeResources resources, ClusterSpec.Type clusterType) { - return resources.withMemoryGb(Math.max(minMemoryGb(clusterType), resources.memoryGb())); + private double minRealMemoryGb(ClusterSpec.Type clusterType) { + return minAdvertisedMemoryGb(clusterType) - 1.7; + } + + private double minAdvertisedDiskGb(NodeResources requested) { + + if (requested.storageType() == NodeResources.StorageType.local + && nodeRepository.zone().getCloud().dynamicProvisioning()) { + if (nodeRepository.zone().system() == SystemName.Public) + return 10 + minRealDiskGb(); + else + return 55 + minRealDiskGb(); + } + return 4 + minRealDiskGb(); + } + + private double minRealDiskGb() { + return 6; + } + + private void illegal(String type, String resource, ClusterSpec cluster, double requested, double minAllowed) { + String message = String.format(Locale.ENGLISH, + "%s cluster '%s': " + type + " " + resource + + " size is %.2f Gb but must be at least %.2f Gb", + cluster.type().name(), cluster.id().value(), requested, minAllowed); + throw new IllegalArgumentException(message); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index a8abdc3f38a..9971aae1714 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -139,7 +139,7 @@ public interface NodeSpec { @Override public boolean needsResize(Node node) { - return ! node.flavor().resources().compatibleWith(requestedNodeResources); + return ! node.resources().compatibleWith(requestedNodeResources); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index f88caffa6c6..ad3dbc7eaa5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.lang.MutableInteger; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.provision.Node; @@ -28,7 +29,7 @@ class Preparer { private final int spareCount; public Preparer(NodeRepository nodeRepository, int spareCount, Optional<HostProvisioner> hostProvisioner, - HostResourcesCalculator hostResourcesCalculator, FlagSource flagSource, + FlagSource flagSource, Optional<LoadBalancerProvisioner> loadBalancerProvisioner) { this.nodeRepository = nodeRepository; this.spareCount = spareCount; @@ -38,9 +39,17 @@ class Preparer { /** Prepare all required resources for the given application and cluster */ public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups) { - var nodes = prepareNodes(application, cluster, requestedNodes, wantedGroups); - prepareLoadBalancer(application, cluster, requestedNodes); - return nodes; + try { + var nodes = prepareNodes(application, cluster, requestedNodes, wantedGroups); + prepareLoadBalancer(application, cluster, requestedNodes); + return nodes; + } + catch (OutOfCapacityException e) { + throw new OutOfCapacityException("Could not satisfy " + requestedNodes + + ( wantedGroups > 1 ? " (in " + wantedGroups + " groups)" : "") + + " in " + application + " " + cluster + + ": " + e.getMessage()); + } } /** @@ -64,7 +73,7 @@ class Preparer { replace(acceptedNodes, accepted); } moveToActiveGroup(surplusNodes, wantedGroups, cluster.group()); - replace(acceptedNodes, retire(surplusNodes)); + acceptedNodes.removeAll(surplusNodes); return acceptedNodes; } @@ -131,13 +140,4 @@ class Preparer { return highestIndex; } - /** Returns retired copies of the given nodes, unless they are removable */ - private List<Node> retire(List<Node> nodes) { - List<Node> retired = new ArrayList<>(nodes.size()); - for (Node node : nodes) { - if ( ! node.allocation().get().isRemovable()) - retired.add(node.retire(Agent.application, nodeRepository.clock().instant())); - } - return retired; - } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java index 8cfcbcb3797..0c1b396c40c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java @@ -33,7 +33,7 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { /** True if the node is allocated to a host that should be dedicated as a spare */ final boolean violatesSpares; - /** True if this is a node that has been retired earlier in the allocation process */ + /** True if this node belongs to a group which will not be needed after this deployment */ final boolean isSurplusNode; /** This node does not exist in the node repository yet */ @@ -74,14 +74,14 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { if (!this.isSurplusNode && other.isSurplusNode) return -1; if (!other.isSurplusNode && this.isSurplusNode) return 1; - // Choose inactive nodes - if (this.node.state() == Node.State.inactive && other.node.state() != Node.State.inactive) return -1; - if (other.node.state() == Node.State.inactive && this.node.state() != Node.State.inactive) return 1; - // Choose reserved nodes from a previous allocation attempt (the exist in node repo) if (this.isInNodeRepoAndReserved() && ! other.isInNodeRepoAndReserved()) return -1; if (other.isInNodeRepoAndReserved() && ! this.isInNodeRepoAndReserved()) return 1; + // Choose inactive nodes + if (this.node.state() == Node.State.inactive && other.node.state() != Node.State.inactive) return -1; + if (other.node.state() == Node.State.inactive && this.node.state() != Node.State.inactive) return 1; + // Choose ready nodes if (this.node.state() == Node.State.ready && other.node.state() != Node.State.ready) return -1; if (other.node.state() == Node.State.ready && this.node.state() != Node.State.ready) return 1; @@ -126,7 +126,7 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { double skewWithoutThis() { return skewWith(zeroResources); } /** Returns the allocation skew of the parent of this after adding this node to it */ - double skewWithThis() { return skewWith(node.flavor().resources()); } + double skewWithThis() { return skewWith(node.resources()); } private double skewWith(NodeResources resources) { if (parent.isEmpty()) return 0; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index a86bd581516..563fc50e697 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import java.util.Optional; @@ -12,7 +13,7 @@ import java.util.Optional; */ public interface ProvisionServiceProvider { - Optional<LoadBalancerService> getLoadBalancerService(); + Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository); Optional<HostProvisioner> getHostProvisioner(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index 2393d2c92ab..a4161a318ab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -40,8 +40,10 @@ public class ApplicationSerializer { private static void toSlime(Cluster cluster, List<Node> applicationNodes, Cursor clusterObject) { List<Node> nodes = NodeList.copyOf(applicationNodes).not().retired().cluster(cluster.id()).asList(); + if (nodes.isEmpty()) return; + int groups = (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); - ClusterResources currentResources = new ClusterResources(nodes.size(), groups, nodes.get(0).flavor().resources()); + ClusterResources currentResources = new ClusterResources(nodes.size(), groups, nodes.get(0).resources()); toSlime(cluster.minResources(), clusterObject.setObject("min")); toSlime(cluster.maxResources(), clusterObject.setObject("max")); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java index 12a29707303..e28b03d7517 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java @@ -20,6 +20,7 @@ import java.util.Optional; * @author mgimle */ public class HostCapacityResponse extends HttpResponse { + private final StringBuilder text; private final Slime slime; private final CapacityChecker capacityChecker; @@ -128,7 +129,7 @@ public class HostCapacityResponse extends HttpResponse { ); failurePath.failureReason.tenant.ifPresent(tenant -> { object.setString("failedTenant", tenant.hostname()); - object.setString("failedTenantResources", tenant.flavor().resources().toString()); + object.setString("failedTenantResources", tenant.resources().toString()); tenant.allocation().ifPresent(allocation -> object.setString("failedTenantAllocation", allocation.toString()) ); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 8b5639cc514..897af634d49 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -6,13 +6,12 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Type; import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -48,6 +47,7 @@ import static com.yahoo.config.provision.NodeResources.StorageType.remote; public class NodePatcher { private static final String WANT_TO_RETIRE = "wantToRetire"; + private static final String WANT_TO_DEPROVISION = "wantToDeprovision"; private final NodeFlavors nodeFlavors; private final Inspector inspector; @@ -77,13 +77,13 @@ public class NodePatcher { List<Node> patchedNodes = new ArrayList<>(); inspector.traverse((String name, Inspector value) -> { try { - node = applyField(node, name, value); + node = applyField(node, name, value, inspector); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Could not set field '" + name + "'", e); } try { - patchedNodes.addAll(applyFieldRecursive(name, value)); + patchedNodes.addAll(applyFieldRecursive(name, value, inspector)); } catch (IllegalArgumentException e) { // Non recursive field, ignore } @@ -93,12 +93,12 @@ public class NodePatcher { return patchedNodes; } - private List<Node> applyFieldRecursive(String name, Inspector value) { + private List<Node> applyFieldRecursive(String name, Inspector value, Inspector root) { switch (name) { case WANT_TO_RETIRE: List<Node> childNodes = node.type().isDockerHost() ? nodes.get().childrenOf(node).asList() : List.of(); return childNodes.stream() - .map(child -> applyField(child, name, value)) + .map(child -> applyField(child, name, value, root)) .collect(Collectors.toList()); default : @@ -106,7 +106,7 @@ public class NodePatcher { } } - private Node applyField(Node node, String name, Inspector value) { + private Node applyField(Node node, String name, Inspector value, Inspector root) { switch (name) { case "currentRebootGeneration" : return node.withCurrentRebootGeneration(asLong(value), clock.instant()); @@ -134,11 +134,10 @@ public class NodePatcher { case "additionalIpAddresses" : return IP.Config.verify(node.with(node.ipConfig().with(IP.Pool.of(asStringSet(value)))), nodes.get()); case WANT_TO_RETIRE : - return node.withWantToRetire(asBoolean(value), Agent.operator, clock.instant()); - case "wantToDeprovision" : - if (node.type() != NodeType.host && asBoolean(value)) - throw new IllegalArgumentException("wantToDeprovision can only be set for hosts"); - return node.with(node.status().withWantToDeprovision(asBoolean(value))); + case WANT_TO_DEPROVISION : + boolean wantToRetire = asOptionalBoolean(root.field(WANT_TO_RETIRE)).orElse(node.status().wantToRetire()); + boolean wantToDeprovision = asOptionalBoolean(root.field(WANT_TO_DEPROVISION)).orElse(node.status().wantToDeprovision()); + return node.withWantToRetire(wantToRetire, wantToDeprovision, Agent.operator, clock.instant()); case "reports" : return nodeWithPatchedReports(node, value); case "openStackId" : @@ -202,7 +201,7 @@ public class NodePatcher { if ((hasHardFailReports && node.state() == Node.State.failed) || node.state() == Node.State.parked) return patchedNode; - patchedNode = patchedNode.with(patchedNode.status().withWantToDeprovision(hasHardFailReports)); + patchedNode = patchedNode.withWantToRetire(hasHardFailReports, hasHardFailReports, Agent.system, clock.instant()); } return patchedNode; @@ -252,19 +251,14 @@ public class NodePatcher { return field.asString(); } - private Optional<String> asOptionalString(Inspector field) { - return field.type().equals(Type.NIX) ? Optional.empty() : Optional.of(asString(field)); - } - - // Allows us to clear optional flags by passing "null" as slime does not have an empty (but present) representation - private Optional<String> removeQuotedNulls(Optional<String> value) { - return value.filter(v -> !v.equals("null")); - } - private boolean asBoolean(Inspector field) { if ( ! field.type().equals(Type.BOOL)) throw new IllegalArgumentException("Expected a BOOL value, got a " + field.type()); return field.asBool(); } + private Optional<Boolean> asOptionalBoolean(Inspector field) { + return Optional.of(field).filter(Inspector::valid).map(this::asBoolean); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 9d02d2907cc..ce9a461e17f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -130,6 +130,7 @@ class NodesResponse extends HttpResponse { toSlime(node, true, object); } + @SuppressWarnings("deprecation") private void toSlime(Node node, boolean allFields, Cursor object) { object.setString("url", nodeParentUrl + node.hostname()); if ( ! allFields) return; @@ -145,7 +146,7 @@ class NodesResponse extends HttpResponse { object.setString("flavor", node.flavor().name()); node.reservedTo().ifPresent(reservedTo -> object.setString("reservedTo", reservedTo.value())); if (node.flavor().isConfigured()) - object.setDouble("cpuCores", node.flavor().getMinCpuCores()); + object.setDouble("cpuCores", node.flavor().resources().vcpu()); NodeResourcesSerializer.toSlime(node.flavor().resources(), object.setObject("resources")); if (node.flavor().cost() > 0) object.setLong("cost", node.flavor().cost()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 175091fa729..80136de6f03 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -49,6 +49,7 @@ import java.io.InputStream; import java.io.UncheckedIOException; import java.net.URI; import java.net.URISyntaxException; +import java.time.Duration; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -321,7 +322,7 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { request.getProperty("clusterType"), request.getProperty("clusterId"))); filter = ApplicationFilter.from(request.getProperty("application"), filter); - filter = StateFilter.from(request.getProperty("state"), filter); + filter = StateFilter.from(request.getProperty("state"), request.getBooleanProperty("includeDeprovisioned"), filter); filter = NodeTypeFilter.from(request.getProperty("type"), filter); filter = ParentHostFilter.from(request.getProperty("parentHost"), filter); filter = NodeOsVersionFilter.from(request.getProperty("osVersion"), filter); @@ -360,12 +361,13 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { private MessageResponse setTargetVersions(HttpRequest request) { NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase()); Inspector inspector = toSlime(request.getData()).get(); - List<String> messageParts = new ArrayList<>(3); + List<String> messageParts = new ArrayList<>(4); boolean force = inspector.field("force").asBool(); Inspector versionField = inspector.field("version"); Inspector osVersionField = inspector.field("osVersion"); Inspector dockerImageField = inspector.field("dockerImage"); + Inspector upgradeBudgetField = inspector.field("upgradeBudget"); if (versionField.valid()) { Version version = Version.fromString(versionField.asString()); @@ -380,8 +382,19 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { messageParts.add("osVersion to null"); } else { Version osVersion = Version.fromString(v); - nodeRepository.osVersions().setTarget(nodeType, osVersion, force); + Optional<Duration> upgradeBudget = Optional.of(upgradeBudgetField) + .filter(Inspector::valid) + .map(Inspector::asString) + .map(s -> { + try { + return Duration.parse(s); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid duration '" + s + "'", e); + } + }); + nodeRepository.osVersions().setTarget(nodeType, osVersion, upgradeBudget, force); messageParts.add("osVersion to " + osVersion.toFullString()); + upgradeBudget.ifPresent(d -> messageParts.add("upgradeBudget to " + d)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java index 16858ec6963..1082e9cce60 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java @@ -39,7 +39,7 @@ public class UpgradeResponse extends HttpResponse { infrastructureVersions.getTargetVersions().forEach((nodeType, version) -> versionsObject.setString(nodeType.name(), version.toFullString())); Cursor osVersionsObject = root.setObject("osVersions"); - osVersions.targets().forEach((nodeType, osVersion) -> osVersionsObject.setString(nodeType.name(), osVersion.toFullString())); + osVersions.readChange().targets().forEach((nodeType, target) -> osVersionsObject.setString(nodeType.name(), target.version().toFullString())); Cursor dockerImagesObject = root.setObject("dockerImages"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java index 5ec5c2c08e8..ae3d6ebf815 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java @@ -10,6 +10,9 @@ import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import java.time.Clock; @@ -27,18 +30,22 @@ import java.util.stream.Collectors; */ public class MockDeployer implements Deployer { + // For actual deploy mode private final NodeRepositoryProvisioner provisioner; private final Map<ApplicationId, ApplicationContext> applications; - private final Map<ApplicationId, Instant> lastDeployTimes = new HashMap<>(); + // For mock deploy anything, changing wantToRetire to retired only + private final NodeRepository nodeRepository; /** The number of redeployments done to this */ public int redeployments = 0; + private final Map<ApplicationId, Instant> lastDeployTimes = new HashMap<>(); private final Clock clock; private final ReentrantLock lock = new ReentrantLock(); private boolean failActivate = false; + /** Create a mock deployer which returns empty on every deploy request. */ @Inject @SuppressWarnings("unused") public MockDeployer() { @@ -46,15 +53,30 @@ public class MockDeployer implements Deployer { } /** - * Create a mock deployer which contains a substitute for an application repository, fullfilled to + * Create a mock deployer which returns a deployment on every request, + * and fulfills it by not actually deploying but only changing any wantToRetire nodes + * for the application to retired. + */ + public MockDeployer(NodeRepository nodeRepository) { + this.provisioner = null; + this.applications = Map.of(); + this.nodeRepository = nodeRepository; + + this.clock = nodeRepository.clock(); + } + + /** + * Create a mock deployer which contains a substitute for an application repository, filled to * be able to call provision with the right parameters. */ public MockDeployer(NodeRepositoryProvisioner provisioner, Clock clock, Map<ApplicationId, ApplicationContext> applications) { this.provisioner = provisioner; - this.clock = clock; this.applications = new HashMap<>(applications); + this.nodeRepository = null; + + this.clock = clock; } public ReentrantLock lock() { return lock; } @@ -74,9 +96,13 @@ public class MockDeployer implements Deployer { throw new RuntimeException(e); } try { - return Optional.ofNullable(applications.get(id)) - .map(application -> new MockDeployment(provisioner, application)); - } finally { + if (provisioner != null) + return Optional.ofNullable(applications.get(id)) + .map(application -> new MockDeployment(provisioner, application)); + else + return Optional.of(new RetiringOnlyMockDeployment(nodeRepository, id)); + } + finally { lock.unlock(); } } @@ -135,6 +161,33 @@ public class MockDeployer implements Deployer { } + public class RetiringOnlyMockDeployment implements Deployment { + + private final NodeRepository nodeRepository; + private final ApplicationId applicationId; + + private RetiringOnlyMockDeployment(NodeRepository nodeRepository, ApplicationId applicationId) { + this.nodeRepository = nodeRepository; + this.applicationId = applicationId; + } + + @Override + public void prepare() { } + + @Override + public void activate() { + redeployments++; + lastDeployTimes.put(applicationId, clock.instant()); + + for (Node node : nodeRepository.list().owner(applicationId).state(Node.State.active).wantToRetire().asList()) + nodeRepository.write(node.retire(nodeRepository.clock().instant()), nodeRepository.lock(node)); + } + + @Override + public void restart(HostFilter filter) {} + + } + /** An application context which substitutes for an application repository */ public static class ApplicationContext { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index c79c6b45247..151bd80a7b7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -29,7 +29,6 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; -import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; import java.time.Clock; import java.time.Instant; @@ -65,7 +64,7 @@ public class MockNodeRepository extends NodeRepository { Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); this.flavors = flavors; curator.setZooKeeperEnsembleConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234"); @@ -118,7 +117,7 @@ public class MockNodeRepository extends NodeRepository { Node node55 = createNode("node55", "host55.yahoo.com", ipConfig(55), Optional.empty(), new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant); - nodes.add(node55.with(node55.status().withWantToRetire(true).withWantToDeprovision(true))); + nodes.add(node55.with(node55.status().withWantToRetire(true, true))); /* Setup docker hosts (two of these will be reserved for spares */ nodes.add(createNode("dockerhost1", "dockerhost1.yahoo.com", ipConfig(100, 1, 3), Optional.empty(), @@ -131,6 +130,8 @@ public class MockNodeRepository extends NodeRepository { flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); nodes.add(createNode("dockerhost5", "dockerhost5.yahoo.com", ipConfig(104, 1, 3), Optional.empty(), flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); + nodes.add(createNode("dockerhost6", "dockerhost6.yahoo.com", ipConfig(105, 1, 3), Optional.empty(), + flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host)); // Config servers nodes.add(createNode("cfg1", "cfg1.yahoo.com", ipConfig(201), Optional.empty(), @@ -148,6 +149,9 @@ public class MockNodeRepository extends NodeRepository { fail(node5.hostname(), Agent.system, getClass().getSimpleName()); dirtyRecursively(node55.hostname(), Agent.system, getClass().getSimpleName()); + fail("dockerhost6.yahoo.com", Agent.operator, getClass().getSimpleName()); + removeRecursively("dockerhost6.yahoo.com"); + ApplicationId zoneApp = ApplicationId.from(TenantName.from("zoneapp"), ApplicationName.from("zoneapp"), InstanceName.from("zoneapp")); ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build(); activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), null), zoneApp, provisioner); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java index 9a02f65daf9..20538732c7a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisionServiceProvider.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.google.inject.Inject; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; @@ -37,7 +38,7 @@ public class MockProvisionServiceProvider implements ProvisionServiceProvider { } @Override - public Optional<LoadBalancerService> getLoadBalancerService() { + public Optional<LoadBalancerService> getLoadBalancerService(NodeRepository nodeRepository) { return loadBalancerService; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index 7bb80fe2a21..9d5e7691fe8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision; import com.yahoo.config.provision.NodeType; -import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.IP; @@ -13,7 +12,6 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.Arrays; -import java.util.EnumSet; import java.util.HashSet; import java.util.Set; import java.util.function.Predicate; @@ -34,7 +32,7 @@ import static org.junit.Assert.fail; public class NodeRepositoryTest { @Test - public void nodeRepositoryTest() { + public void add_and_remove() { NodeRepositoryTester tester = new NodeRepositoryTester(); assertEquals(0, tester.nodeRepository().getNodes().size()); @@ -122,13 +120,8 @@ public class NodeRepositoryTest { tester.addNode("node11", "node11", "host1", "docker", NodeType.tenant); tester.addNode("node12", "node12", "host1", "docker", NodeType.tenant); tester.addNode("node20", "node20", "host2", "docker", NodeType.tenant); - assertEquals(6, tester.nodeRepository().getNodes().size()); - - Node node = tester.nodeRepository().getNode("host1").get(); - IP.Config cfg = new IP.Config(Set.of("127.0.0.1"), Set.of()); - node = node.with(cfg); - tester.setNodeState("node11", Node.State.active); + assertEquals(6, tester.nodeRepository().getNodes().size()); try { tester.nodeRepository().removeRecursively("host1"); @@ -154,6 +147,32 @@ public class NodeRepositoryTest { } @Test + public void delete_config_host() { + NodeRepositoryTester tester = new NodeRepositoryTester(); + + String cfghost1 = "cfghost1"; + String cfg1 = "cfg1"; + tester.addNode("id1", cfghost1, "default", NodeType.confighost); + tester.addNode("id2", cfg1, cfghost1, "docker", NodeType.config); + tester.setNodeState(cfghost1, Node.State.active); + tester.setNodeState(cfg1, Node.State.active); + assertEquals(2, tester.nodeRepository().getNodes().size()); + + try { + tester.nodeRepository().removeRecursively(cfghost1); + fail("Should not be able to delete host node, one of the children is in state active"); + } catch (IllegalArgumentException ignored) { } + assertEquals(2, tester.nodeRepository().getNodes().size()); + + // Fail host and container + tester.nodeRepository().failRecursively(cfghost1, Agent.system, getClass().getSimpleName()); + + // Remove recursively + tester.nodeRepository().removeRecursively(cfghost1); + assertEquals(0, tester.nodeRepository().list().not().state(Node.State.deprovisioned).size()); + } + + @Test public void relevant_information_from_deprovisioned_hosts_are_merged_into_readded_host() { NodeRepositoryTester tester = new NodeRepositoryTester(); Instant testStart = tester.nodeRepository().clock().instant(); @@ -165,8 +184,7 @@ public class NodeRepositoryTest { // Set host 1 properties and deprovision it Node host1 = tester.nodeRepository().getNode("host1").get(); - host1 = host1.withWantToRetire(true, Agent.system, tester.nodeRepository().clock().instant()); - host1 = host1.with(host1.status().withWantToDeprovision(true)); + host1 = host1.withWantToRetire(true, true, Agent.system, tester.nodeRepository().clock().instant()); host1 = host1.withFirmwareVerifiedAt(tester.clock().instant()); host1 = host1.with(host1.status().withIncreasedFailCount()); host1 = host1.with(host1.reports().withReport(Report.basicReport("id", Report.Type.HARD_FAIL, tester.clock().instant(), "Test report"))); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 88804576310..52c68cd74b2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvid import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import java.time.Clock; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -41,7 +40,7 @@ public class NodeRepositoryTester { Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); } public NodeRepository nodeRepository() { return nodeRepository; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java index 032375943c8..e65abaa21ec 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -56,7 +56,7 @@ public class AutoscalingIntegrationTest { ClusterResources max = new ClusterResources(2, 1, nodes); Application application = tester.nodeRepository().applications().get(application1).orElse(new Application(application1)) - .withClusterLimits(cluster1.id(), min, max); + .withCluster(cluster1.id(), false, min, max); tester.nodeRepository().applications().put(application, tester.nodeRepository().lock(application1)); var scaledResources = autoscaler.suggest(application.clusters().get(cluster1.id()), tester.nodeRepository().getNodes(application1)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 8b01868e75e..a0a44e4f342 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -11,12 +11,16 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import org.junit.Test; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import static com.yahoo.config.provision.NodeResources.StorageType.local; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -26,7 +30,7 @@ import static org.junit.Assert.assertTrue; public class AutoscalingTest { @Test - public void testAutoscalingSingleContentGroup() { + public void test_autoscaling_single_content_group() { NodeResources hostResources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1, NodeResources.DiskSpeed.any)); @@ -68,7 +72,7 @@ public class AutoscalingTest { } @Test - public void testAutoscalingHandlesDiskSettingChanges() { + public void autoscaling_handles_disk_setting_changes() { NodeResources hostResources = new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.slow); AutoscalingTester tester = new AutoscalingTester(hostResources); @@ -98,7 +102,7 @@ public class AutoscalingTest { /** We prefer fewer nodes for container clusters as (we assume) they all use the same disk and memory */ @Test - public void testAutoscalingSingleContainerGroup() { + public void test_autoscaling_single_container_group() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); @@ -112,7 +116,7 @@ public class AutoscalingTest { tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); ClusterResources scaledResources = tester.assertResources("Scaling up since cpu usage is too high", - 7, 1, 2.6, 80.0, 80.0, + 7, 1, 2.5, 80.0, 80.0, tester.autoscale(application1, cluster1.id(), min, max)); tester.deploy(application1, cluster1, scaledResources); @@ -120,12 +124,12 @@ public class AutoscalingTest { tester.addMeasurements(Resource.cpu, 0.1f, 1f, 120, application1); tester.assertResources("Scaling down since cpu usage has gone down", - 4, 1, 2.4, 68.6, 68.6, + 4, 1, 2.5, 68.6, 68.6, tester.autoscale(application1, cluster1.id(), min, max)); } @Test - public void testAutoscalingRespectsUpperLimit() { + public void autoscaling_respects_upper_limit() { NodeResources hostResources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); @@ -146,7 +150,7 @@ public class AutoscalingTest { } @Test - public void testAutoscalingRespectsLowerLimit() { + public void autoscaling_respects_lower_limit() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 4, 1, new NodeResources(1.8, 7.4, 8.5, 1)); ClusterResources max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); @@ -161,12 +165,12 @@ public class AutoscalingTest { tester.addMeasurements(Resource.memory, 0.05f, 120, application1); tester.addMeasurements(Resource.disk, 0.05f, 120, application1); tester.assertResources("Scaling down to limit since resource usage is low", - 4, 1, 1.8, 7.4, 8.5, + 4, 1, 1.8, 7.4, 10.0, tester.autoscale(application1, cluster1.id(), min, max)); } @Test - public void testAutoscalingRespectsGroupLimit() { + public void autoscaling_respects_group_limit() { NodeResources hostResources = new NodeResources(30.0, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources(18, 6, new NodeResources(100, 1000, 1000, 1)); @@ -179,12 +183,12 @@ public class AutoscalingTest { tester.deploy(application1, cluster1, 5, 5, new NodeResources(3.0, 10, 10, 1)); tester.addMeasurements(Resource.cpu, 0.3f, 1f, 240, application1); tester.assertResources("Scaling up since resource usage is too high", - 6, 6, 3.6, 8.0, 8.0, + 6, 6, 3.6, 8.0, 10.0, tester.autoscale(application1, cluster1.id(), min, max)); } @Test - public void testAutoscalingLimitsWhenMinEqualsMax() { + public void test_autoscaling_limits_when_min_equals_xax() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = min; @@ -200,7 +204,7 @@ public class AutoscalingTest { } @Test - public void testSuggestionsIgnoresLimits() { + public void suggestions_ignores_limits() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = min; @@ -213,12 +217,12 @@ public class AutoscalingTest { tester.deploy(application1, cluster1, 5, 1, resources); tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); tester.assertResources("Scaling up since resource usage is too high", - 7, 1, 2.6, 80.0, 80.0, + 7, 1, 2.5, 80.0, 80.0, tester.suggest(application1, cluster1.id(), min, max)); } @Test - public void testAutoscalingGroupSize1() { + public void test_autoscaling_group_size_1() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources(20, 20, new NodeResources(100, 1000, 1000, 1)); @@ -236,7 +240,7 @@ public class AutoscalingTest { } @Test - public void testAutoscalingGroupSize3() { + public void test_autoscalinggroupsize_by_cpu() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1)); @@ -247,24 +251,42 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 6, 2, resources); - tester.addMeasurements(Resource.cpu, 0.22f, 1f, 120, application1); - tester.assertResources("Scaling up since resource usage is too high", - 9, 3, 2.7, 83.3, 83.3, + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); + tester.assertResources("Scaling up since resource usage is too high, changing to 1 group is cheaper", + 8, 1, 2.7, 83.3, 83.3, tester.autoscale(application1, cluster1.id(), min, max)); } @Test - public void testAutoscalingAvoidsIllegalConfigurations() { - NodeResources resources = new NodeResources(3, 100, 100, 1); + public void test_autoscaling_group_size() { + NodeResources hostResources = new NodeResources(100, 1000, 1000, 100); + ClusterResources min = new ClusterResources( 3, 2, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(30, 30, new NodeResources(100, 100, 1000, 1)); + AutoscalingTester tester = new AutoscalingTester(hostResources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 6, 2, new NodeResources(10, 100, 100, 1)); + tester.addMeasurements(Resource.memory, 1.0f, 1f, 1000, application1); + tester.assertResources("Increase group size to reduce memory load", + 8, 2, 12.9, 89.3, 62.5, + tester.autoscale(application1, cluster1.id(), min, max)); + } + + @Test + public void autoscaling_avoids_illegal_configurations() { + NodeResources hostResources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); - AutoscalingTester tester = new AutoscalingTester(resources); + AutoscalingTester tester = new AutoscalingTester(hostResources); ApplicationId application1 = tester.applicationId("application1"); ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1"); // deploy - tester.deploy(application1, cluster1, 6, 1, resources); + tester.deploy(application1, cluster1, 6, 1, hostResources); tester.addMeasurements(Resource.memory, 0.02f, 0.95f, 120, application1); tester.assertResources("Scaling down", 6, 1, 2.8, 4.0, 95.0, @@ -272,7 +294,44 @@ public class AutoscalingTest { } @Test - public void testAutoscalingAws() { + public void real_resources_are_taken_into_account() { + NodeResources hostResources = new NodeResources(60, 100, 1000, 10); + ClusterResources min = new ClusterResources(2, 1, new NodeResources( 2, 20, 200, 1)); + ClusterResources max = new ClusterResources(4, 1, new NodeResources(60, 100, 1000, 1)); + + { // No memory tax + AutoscalingTester tester = new AutoscalingTester(hostResources, new OnlySubtractingWhenForecastingCalculator(0)); + + ApplicationId application1 = tester.applicationId("app1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1"); + + tester.deploy(application1, cluster1, min); + tester.addMeasurements(Resource.cpu, 1.0f, 1000, application1); + tester.addMeasurements(Resource.memory, 1.0f, 1000, application1); + tester.addMeasurements(Resource.disk, 0.7f, 1000, application1); + tester.assertResources("Scaling up", + 4, 1, 7.0, 20, 200, + tester.autoscale(application1, cluster1.id(), min, max)); + } + + { // 15 Gb memory tax + AutoscalingTester tester = new AutoscalingTester(hostResources, new OnlySubtractingWhenForecastingCalculator(15)); + + ApplicationId application1 = tester.applicationId("app1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1"); + + tester.deploy(application1, cluster1, min); + tester.addMeasurements(Resource.cpu, 1.0f, 1000, application1); + tester.addMeasurements(Resource.memory, 1.0f, 1000, application1); + tester.addMeasurements(Resource.disk, 0.7f, 1000, application1); + tester.assertResources("Scaling up", + 4, 1, 7.0, 34, 200, + tester.autoscale(application1, cluster1.id(), min, max)); + } + } + + @Test + public void test_autoscaling_without_host_sharing() { ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); List<Flavor> flavors = new ArrayList<>(); @@ -280,9 +339,10 @@ public class AutoscalingTest { flavors.add(new Flavor("aws-large", new NodeResources(3, 150, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); flavors.add(new Flavor("aws-medium", new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); flavors.add(new Flavor("aws-small", new NodeResources(3, 80, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote))); - AutoscalingTester tester = new AutoscalingTester(new Zone(Cloud.defaultCloud() - .withDynamicProvisioning(true) - .withAllowHostSharing(false), + AutoscalingTester tester = new AutoscalingTester(new Zone(Cloud.builder() + .dynamicProvisioning(true) + .allowHostSharing(false) + .build(), SystemName.main, Environment.prod, RegionName.from("us-east")), flavors); @@ -307,4 +367,40 @@ public class AutoscalingTest { tester.autoscale(application1, cluster1.id(), min, max)); } + /** + * This calculator subtracts the memory tax when forecasting overhead, but not when actually + * returning information about nodes. This is allowed because the forecast is a *worst case*. + * It is useful here because it ensures that we end up with the same real (and therefore target) + * resources regardless of tax which makes it easier to compare behavior with different tax levels. + */ + private static class OnlySubtractingWhenForecastingCalculator implements HostResourcesCalculator { + + private final int memoryTaxGb; + + public OnlySubtractingWhenForecastingCalculator(int memoryTaxGb) { + this.memoryTaxGb = memoryTaxGb; + } + + @Override + public NodeResources realResourcesOf(Node node, NodeRepository nodeRepository) { + return node.resources(); + } + + @Override + public NodeResources advertisedResourcesOf(Flavor flavor) { + return flavor.resources(); + } + + @Override + public NodeResources requestToReal(NodeResources resources) { + return resources.withMemoryGb(resources.memoryGb() - memoryTaxGb); + } + + @Override + public NodeResources realToRequest(NodeResources resources) { + return resources.withMemoryGb(resources.memoryGb() + memoryTaxGb); + } + + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 79d6b20dce8..1137ae5ce2c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import static com.yahoo.config.provision.NodeResources.StorageType.local; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -44,15 +45,24 @@ class AutoscalingTester { /** Creates an autoscaling tester with a single host type ready */ public AutoscalingTester(NodeResources hostResources) { - this(new Zone(Environment.prod, RegionName.from("us-east")), List.of(new Flavor("hostFlavor", hostResources))); + this(hostResources, null); + } + + public AutoscalingTester(NodeResources hostResources, HostResourcesCalculator resourcesCalculator) { + this(new Zone(Environment.prod, RegionName.from("us-east")), List.of(new Flavor("hostFlavor", hostResources)), resourcesCalculator); provisioningTester.makeReadyNodes(20, "hostFlavor", NodeType.host, 8); provisioningTester.deployZoneApp(); } public AutoscalingTester(Zone zone, List<Flavor> flavors) { + this(zone, flavors, new MockHostResourcesCalculator(zone)); + } + + private AutoscalingTester(Zone zone, List<Flavor> flavors, + HostResourcesCalculator resourcesCalculator) { provisioningTester = new ProvisioningTester.Builder().zone(zone) .flavors(flavors) - .resourcesCalculator(new MockHostResourcesCalculator(zone)) + .resourcesCalculator(resourcesCalculator) .hostProvisioner(new MockHostProvisioner(flavors)) .build(); @@ -147,7 +157,7 @@ class AutoscalingTester { public Optional<ClusterResources> autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, ClusterResources min, ClusterResources max) { Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)) - .withClusterLimits(clusterId, min, max); + .withCluster(clusterId, false, min, max); nodeRepository().applications().put(application, nodeRepository().lock(applicationId)); return autoscaler.autoscale(application.clusters().get(clusterId), nodeRepository().getNodes(applicationId, Node.State.active)); @@ -156,7 +166,7 @@ class AutoscalingTester { public Optional<ClusterResources> suggest(ApplicationId applicationId, ClusterSpec.Id clusterId, ClusterResources min, ClusterResources max) { Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)) - .withClusterLimits(clusterId, min, max); + .withCluster(clusterId, false, min, max); nodeRepository().applications().put(application, nodeRepository().lock(applicationId)); return autoscaler.suggest(application.clusters().get(clusterId), nodeRepository().getNodes(applicationId, Node.State.active)); @@ -168,11 +178,12 @@ class AutoscalingTester { Optional<ClusterResources> resources) { double delta = 0.0000000001; assertTrue(message, resources.isPresent()); - assertEquals("Node count: " + message, nodeCount, resources.get().nodes()); - assertEquals("Group count: " + message, groupCount, resources.get().groups()); - assertEquals("Cpu: " + message, approxCpu, Math.round(resources.get().nodeResources().vcpu() * 10) / 10.0, delta); - assertEquals("Memory: " + message, approxMemory, Math.round(resources.get().nodeResources().memoryGb() * 10) / 10.0, delta); - assertEquals("Disk: " + message, approxDisk, Math.round(resources.get().nodeResources().diskGb() * 10) / 10.0, delta); + NodeResources nodeResources = resources.get().nodeResources(); + assertEquals("Node count in " + resources.get() + ": " + message, nodeCount, resources.get().nodes()); + assertEquals("Group count in " + resources.get() + ": " + message, groupCount, resources.get().groups()); + assertEquals("Cpu in " + resources.get() + ": " + message, approxCpu, Math.round(nodeResources.vcpu() * 10) / 10.0, delta); + assertEquals("Memory in " + resources.get() + ": " + message, approxMemory, Math.round(nodeResources.memoryGb() * 10) / 10.0, delta); + assertEquals("Disk in: " + resources.get() + ": " + message, approxDisk, Math.round(nodeResources.diskGb() * 10) / 10.0, delta); return resources.get(); } @@ -197,9 +208,9 @@ class AutoscalingTester { @Override public NodeResources realResourcesOf(Node node, NodeRepository nodeRepository) { if (zone.getCloud().dynamicProvisioning()) - return node.flavor().resources().withMemoryGb(node.flavor().resources().memoryGb() - 3); + return node.resources().withMemoryGb(node.resources().memoryGb() - 3); else - return node.flavor().resources(); + return node.resources(); } @Override @@ -210,6 +221,16 @@ class AutoscalingTester { return flavor.resources(); } + @Override + public NodeResources requestToReal(NodeResources resources) { + return resources.withMemoryGb(resources.memoryGb() - 3); + } + + @Override + public NodeResources realToRequest(NodeResources resources) { + return resources.withMemoryGb(resources.memoryGb() + 3); + } + } private class MockHostProvisioner implements HostProvisioner { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java index e70fc184b87..997aec8a156 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/PassthroughLoadBalancerServiceTest.java @@ -20,8 +20,8 @@ public class PassthroughLoadBalancerServiceTest { var lbService = new PassthroughLoadBalancerService(); var real = new Real(HostName.from("host1.example.com"), "192.0.2.10"); var reals = Set.of(real, new Real(HostName.from("host2.example.com"), "192.0.2.11")); - var instance = lbService.create(ApplicationId.from("tenant1", "app1", "default"), - ClusterSpec.Id.from("c1"), reals, false, null); + var instance = lbService.create(new LoadBalancerSpec(ApplicationId.from("tenant1", "app1", "default"), + ClusterSpec.Id.from("c1"), reals), false); assertEquals(real.hostname(), instance.hostname()); assertEquals(Set.of(real.port()), instance.ports()); assertEquals(Set.of(real.ipAddress() + "/32"), instance.networks()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index 64d189b9111..06f18d94c5f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; public class SharedLoadBalancerServiceTest { private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); - private final SharedLoadBalancerService loadBalancerService = new SharedLoadBalancerService(); + private final SharedLoadBalancerService loadBalancerService = new SharedLoadBalancerService(tester.nodeRepository()); private final ApplicationId applicationId = ApplicationId.from("tenant1", "application1", "default"); private final ClusterSpec.Id clusterId = ClusterSpec.Id.from("qrs1"); private final Set<Real> reals = Set.of( @@ -30,7 +30,7 @@ public class SharedLoadBalancerServiceTest { @Test public void test_create_lb() { tester.makeReadyNodes(2, "default", NodeType.proxy); - var lb = loadBalancerService.create(applicationId, clusterId, reals, false, tester.nodeRepository()); + var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals), false); assertEquals(HostName.from("host-1.yahoo.com"), lb.hostname()); assertEquals(Optional.empty(), lb.dnsZone()); @@ -40,7 +40,7 @@ public class SharedLoadBalancerServiceTest { @Test(expected = IllegalStateException.class) public void test_exception_on_missing_proxies() { - loadBalancerService.create(applicationId, clusterId, reals, false, tester.nodeRepository()); + loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals), false); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 4dab064ce1e..c34db3210c1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -41,10 +41,10 @@ public class AutoscalingMaintainerTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east3"))).flavorsConfig(flavorsConfig()).build(); ApplicationId app1 = tester.makeApplicationId("app1"); - ClusterSpec cluster1 = tester.clusterSpec(); + ClusterSpec cluster1 = tester.containerClusterSpec(); ApplicationId app2 = tester.makeApplicationId("app2"); - ClusterSpec cluster2 = tester.clusterSpec(); + ClusterSpec cluster2 = tester.containerClusterSpec(); NodeResources lowResources = new NodeResources(4, 4, 10, 0.1); NodeResources highResources = new NodeResources(6.5, 9, 20, 0.1); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java index 5813585554d..5e72cfc53ac 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java @@ -29,6 +29,7 @@ public class CapacityCheckerTest { var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure(); assertTrue(failurePath.isPresent()); assertTrue(tester.nodeRepository.getNodes(NodeType.host).containsAll(failurePath.get().hostsCausingFailure)); + assertEquals(5, failurePath.get().hostsCausingFailure.size()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index b1f6eaea502..62e9a227109 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -45,6 +45,7 @@ import java.util.stream.IntStream; * @author mgimle */ public class CapacityCheckerTester { + public static final Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); // Components with state @@ -62,7 +63,7 @@ public class CapacityCheckerTester { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); } private void updateCapacityChecker() { @@ -129,7 +130,7 @@ public class CapacityCheckerTester { childModel.parentHostname = Optional.of(hostname); Node childNode = createNodeFromModel(childModel); - childResources.add(childNode.flavor().resources()); + childResources.add(childNode.resources()); hosts.add(childNode); } @@ -138,8 +139,7 @@ public class CapacityCheckerTester { .mapToObj(n -> String.format("%04X::%04X", hostindex, n)) .collect(Collectors.toSet()); - NodeResources nr = containingNodeResources(childResources, - excessCapacity); + NodeResources nr = containingNodeResources(childResources, excessCapacity); Node node = nodeRepository.createNode(hostname, hostname, new IP.Config(Set.of("::"), availableIps), Optional.empty(), new Flavor(nr), Optional.empty(), NodeType.host); @@ -159,7 +159,8 @@ public class CapacityCheckerTester { Set<String> availableIps = IntStream.range(0, ips) .mapToObj(n -> String.format("%04X::%04X", hostid, n)) .collect(Collectors.toSet()); - Node node = nodeRepository.createNode(hostname, hostname, + Node node = nodeRepository.createNode(hostname, + hostname, new IP.Config(Set.of("::"), availableIps), Optional.empty(), new Flavor(capacity), Optional.empty(), NodeType.host); hosts.add(node); @@ -175,8 +176,8 @@ public class CapacityCheckerTester { ); createNodes(childrenPerHost, numDistinctChildren, childResources, - numHosts, hostExcessCapacity, hostExcessIps, - numEmptyHosts, emptyHostExcessCapacity, emptyHostExcessIps); + numHosts, hostExcessCapacity, hostExcessIps, + numEmptyHosts, emptyHostExcessCapacity, emptyHostExcessIps); } void createNodes(int childrenPerHost, int numDistinctChildren, List<NodeResources> childResources, int numHosts, NodeResources hostExcessCapacity, int hostExcessIps, @@ -264,10 +265,11 @@ public class CapacityCheckerTester { owner = ApplicationId.from(nodeModel.owner.tenant, nodeModel.owner.application, nodeModel.owner.instance); } - NodeResources.DiskSpeed diskSpeed; - NodeResources nr = new NodeResources(nodeModel.minCpuCores, nodeModel.minMainMemoryAvailableGb, - nodeModel.minDiskAvailableGb, nodeModel.bandwidth * 1000, - nodeModel.fastDisk ? NodeResources.DiskSpeed.fast : NodeResources.DiskSpeed.slow); + NodeResources nr = new NodeResources(nodeModel.minCpuCores, + nodeModel.minMainMemoryAvailableGb, + nodeModel.minDiskAvailableGb, + nodeModel.bandwidth * 1000, + nodeModel.fastDisk ? NodeResources.DiskSpeed.fast : NodeResources.DiskSpeed.slow); Flavor f = new Flavor(nr); Node node = nodeRepository.createNode(nodeModel.id, nodeModel.hostname, @@ -275,7 +277,7 @@ public class CapacityCheckerTester { nodeModel.parentHostname, f, Optional.empty(), nodeModel.type); if (membership != null) { - return node.allocate(owner, membership, node.flavor().resources(), Instant.now()); + return node.allocate(owner, membership, node.resources(), Instant.now()); } else { return node; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 6eba517fde2..ba859655ab7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -3,9 +3,10 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.Cloud; import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; @@ -14,13 +15,12 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; -import com.yahoo.test.ManualClock; -import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.flags.custom.PreprovisionCapacity; +import com.yahoo.vespa.flags.custom.HostCapacity; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.History; @@ -30,205 +30,239 @@ import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; -import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import org.hamcrest.BaseMatcher; -import org.hamcrest.Description; -import org.junit.Before; import org.junit.Test; import java.time.Duration; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.createNode; -import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.proxyApp; -import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.proxyHostApp; -import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.tenantApp; -import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.tenantHostApp; +import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.MockHostProvisioner.Behaviour; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; -import static org.mockito.hamcrest.MockitoHamcrest.argThat; /** * @author freva + * @author mpolden */ public class DynamicProvisioningMaintainerTest { - private final HostProvisionerTester tester = new HostProvisionerTester(); - private final HostProvisioner hostProvisioner = mock(HostProvisioner.class); - private static final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource() - .withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(), PreprovisionCapacity.class); - private final DynamicProvisioningMaintainer maintainer = - new DynamicProvisioningMaintainer(tester.nodeRepository, - Duration.ofDays(1), - hostProvisioner, - flagSource); - @Test public void delegates_to_host_provisioner_and_writes_back_result() { - addNodes(); + var tester = new DynamicProvisioningTester().addInitialNodes(); + tester.hostProvisioner.with(Behaviour.failDeprovisioning); // To avoid deleting excess nodes + Node host3 = tester.nodeRepository.getNode("host3").orElseThrow(); Node host4 = tester.nodeRepository.getNode("host4").orElseThrow(); Node host41 = tester.nodeRepository.getNode("host4-1").orElseThrow(); - assertTrue(Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); - - Node host3new = host3.with(host3.ipConfig().with(Set.of("::5"))); - when(hostProvisioner.provision(eq(host3), eq(Set.of()))).thenReturn(List.of(host3new)); + assertTrue("No IP addresses assigned", + Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); - Node host4new = host4.with(host4.ipConfig().with(Set.of("::2"))); - Node host41new = host41.with(host4.ipConfig().with(Set.of("::4", "10.0.0.1"))); - when(hostProvisioner.provision(eq(host4), eq(Set.of(host41)))).thenReturn(List.of(host4new, host41new)); + Node host3new = host3.with(host3.ipConfig().with(Set.of("::3:0"))); + Node host4new = host4.with(host4.ipConfig().with(Set.of("::4:0"))); + Node host41new = host41.with(host41.ipConfig().with(Set.of("::4:1", "::4:2"))); - maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {}); - verify(hostProvisioner).provision(eq(host4), eq(Set.of(host41))); - verify(hostProvisioner).provision(eq(host3), eq(Set.of())); - verifyNoMoreInteractions(hostProvisioner); - - assertEquals(Optional.of(host3new), tester.nodeRepository.getNode("host3")); - assertEquals(Optional.of(host4new), tester.nodeRepository.getNode("host4")); - assertEquals(Optional.of(host41new), tester.nodeRepository.getNode("host4-1")); + tester.maintainer.maintain(); + assertEquals(host3new, tester.nodeRepository.getNode("host3").get()); + assertEquals(host4new, tester.nodeRepository.getNode("host4").get()); + assertEquals(host41new, tester.nodeRepository.getNode("host4-1").get()); } @Test public void correctly_fails_if_irrecoverable_failure() { - Node host4 = tester.addNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()); - Node host41 = tester.addNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)); - - assertTrue(Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); - when(hostProvisioner.provision(eq(host4), eq(Set.of(host41)))).thenThrow(new FatalProvisioningException("Fatal")); - - maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {}); + var tester = new DynamicProvisioningTester(); + tester.hostProvisioner.with(Behaviour.failProvisioning); + Node host4 = tester.addNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned); + Node host41 = tester.addNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, DynamicProvisioningTester.tenantApp); + assertTrue("No IP addresses assigned", Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty)); + tester.maintainer.maintain(); assertEquals(Set.of("host4", "host4-1"), - tester.nodeRepository.getNodes(Node.State.failed).stream().map(Node::hostname).collect(Collectors.toSet())); + tester.nodeRepository.getNodes(Node.State.failed).stream().map(Node::hostname).collect(Collectors.toSet())); } @Test public void finds_nodes_that_need_deprovisioning_without_pre_provisioning() { - addNodes(); + var tester = new DynamicProvisioningTester().addInitialNodes(); + assertTrue(tester.nodeRepository.getNode("host2").isPresent()); + assertTrue(tester.nodeRepository.getNode("host3").isPresent()); - maintainer.convergeToCapacity(tester.nodeRepository.list()); - verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host2"))); - verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host3"))); - verifyNoMoreInteractions(hostProvisioner); + tester.maintainer.maintain(); assertTrue(tester.nodeRepository.getNode("host2").isEmpty()); assertTrue(tester.nodeRepository.getNode("host3").isEmpty()); } @Test public void does_not_deprovision_when_preprovisioning_enabled() { - flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(1, 3, 2, 1)), PreprovisionCapacity.class); - addNodes(); - - maintainer.convergeToCapacity(tester.nodeRepository.list()); - verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host2"))); // host2 because it is failed - verifyNoMoreInteractions(hostProvisioner); + var tester = new DynamicProvisioningTester().addInitialNodes(); + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(1, 3, 2, 1)), HostCapacity.class); + Optional<Node> failedHost = tester.nodeRepository.getNode("host2"); + assertTrue(failedHost.isPresent()); + + tester.maintainer.maintain(); + assertTrue("Failed host is deprovisioned", tester.nodeRepository.getNode(failedHost.get().hostname()).isEmpty()); + assertEquals(1, tester.hostProvisioner.deprovisionedHosts); } @Test public void provision_deficit_and_deprovision_excess() { - flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), - List.of(new PreprovisionCapacity(2, 4, 8, 1), - new PreprovisionCapacity(2, 3, 2, 2)), - PreprovisionCapacity.class); - addNodes(); - - maintainer.convergeToCapacity(tester.nodeRepository.list()); - assertTrue(tester.nodeRepository.getNode("host2").isEmpty()); - assertTrue(tester.nodeRepository.getNode("host3").isPresent()); - verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host2"))); - verify(hostProvisioner, times(2)).provisionHosts(argThatLambda(list -> list.size() == 1), eq(new NodeResources(2, 3, 2, 1)), any(), any()); - verifyNoMoreInteractions(hostProvisioner); + var tester = new DynamicProvisioningTester().addInitialNodes(); + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), + List.of(new HostCapacity(24, 64, 100, 2), + new HostCapacity(16, 24, 100, 1)), + HostCapacity.class); + assertTrue(tester.nodeRepository.getNode("host2").isPresent()); + assertEquals(0 ,tester.hostProvisioner.provisionedHosts.size()); + + // failed host2 is removed + Optional<Node> failedHost = tester.nodeRepository.getNode("host2"); + assertTrue(failedHost.isPresent()); + tester.maintainer.maintain(); + assertTrue("Failed host is deprovisioned", tester.nodeRepository.getNode(failedHost.get().hostname()).isEmpty()); + assertTrue("Host with matching resources is kept", tester.nodeRepository.getNode("host3").isPresent()); + + // Two more hosts are provisioned with expected resources + NodeResources resources = new NodeResources(24, 64, 100, 1); + assertEquals(2, tester.provisionedHostsMatching(resources)); } @Test public void does_not_remove_if_host_provisioner_failed() { - Node host2 = tester.addNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)); - doThrow(new RuntimeException()).when(hostProvisioner).deprovision(eq(host2)); - - maintainer.convergeToCapacity(tester.nodeRepository.list()); + var tester = new DynamicProvisioningTester(); + Node host2 = tester.addNode("host2", Optional.empty(), NodeType.host, Node.State.failed, DynamicProvisioningTester.tenantApp); + tester.hostProvisioner.with(Behaviour.failDeprovisioning); - assertEquals(1, tester.nodeRepository.getNodes().size()); - verify(hostProvisioner).deprovision(eq(host2)); - verifyNoMoreInteractions(hostProvisioner); + tester.maintainer.maintain(); + assertTrue(tester.nodeRepository.getNode(host2.hostname()).isPresent()); } - @Before - public void setup() { - doAnswer(invocation -> { - Flavor flavor = invocation.getArgument(0, Flavor.class); - if ("default".equals(flavor.name())) return new NodeResources(2, 4, 8, 1); - return invocation.getArguments()[1]; - }).when(hostResourcesCalculator).advertisedResourcesOf(any()); + @Test + public void provision_exact_capacity() { + var tester = new DynamicProvisioningTester(Cloud.builder().dynamicProvisioning(false).build()); + NodeResources resources1 = new NodeResources(24, 64, 100, 1); + NodeResources resources2 = new NodeResources(16, 24, 100, 1); + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1), + new HostCapacity(resources2.vcpu(), resources2.memoryGb(), resources2.diskGb(), 2)), + HostCapacity.class); + tester.maintainer.maintain(); + + // Hosts are provisioned + assertEquals(1, tester.provisionedHostsMatching(resources1)); + assertEquals(2, tester.provisionedHostsMatching(resources2)); + + // Next maintenance run does nothing + tester.assertNodesUnchanged(); + + // Target capacity is changed + NodeResources resources3 = new NodeResources(48, 128, 1000, 1); + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1), + new HostCapacity(resources3.vcpu(), resources3.memoryGb(), resources3.diskGb(), 1)), + HostCapacity.class); + + // Excess hosts are deprovisioned + tester.maintainer.maintain(); + assertEquals(1, tester.provisionedHostsMatching(resources1)); + assertEquals(0, tester.provisionedHostsMatching(resources2)); + assertEquals(1, tester.provisionedHostsMatching(resources3)); + assertEquals(2, tester.nodeRepository.getNodes(Node.State.deprovisioned).size()); + + // Activate hosts + tester.maintainer.maintain(); // Resume provisioning of new hosts + List<Node> provisioned = tester.nodeRepository.list().state(Node.State.provisioned).asList(); + tester.nodeRepository.setReady(provisioned, Agent.system, this.getClass().getSimpleName()); + tester.provisioningTester.deployZoneApp(); + + // Allocating nodes to a host does not result in provisioning of additional capacity + ApplicationId application = tester.provisioningTester.makeApplicationId(); + tester.provisioningTester.deploy(application, + Capacity.from(new ClusterResources(2, 1, new NodeResources(4, 8, 50, 0.1)))); + assertEquals(2, tester.nodeRepository.list().owner(application).size()); + tester.assertNodesUnchanged(); + + // Clearing flag does nothing + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(), HostCapacity.class); + tester.assertNodesUnchanged(); + + // Capacity reduction does not remove host with children + tester.flagSource.withListFlag(Flags.TARGET_CAPACITY.id(), List.of(new HostCapacity(resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), 1)), + HostCapacity.class); + tester.assertNodesUnchanged(); } - public void addNodes() { - List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)), - createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), - createNode("host1-2", Optional.of("host1"), NodeType.tenant, Node.State.failed, Optional.empty()), + private static class DynamicProvisioningTester { - createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)), - createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()), + private static final ApplicationId tenantApp = ApplicationId.from("mytenant", "myapp", "default"); + private static final ApplicationId tenantHostApp = ApplicationId.from("vespa", "tenant-host", "default"); + private static final ApplicationId proxyHostApp = ApplicationId.from("vespa", "proxy-host", "default"); + private static final ApplicationId proxyApp = ApplicationId.from("vespa", "proxy", "default"); + private static final NodeFlavors flavors = FlavorConfigBuilder.createDummies("default", "docker", "host2", "host3", "host4"); - createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), + private final InMemoryFlagSource flagSource = new InMemoryFlagSource().withListFlag(Flags.TARGET_CAPACITY.id(), + List.of(), + HostCapacity.class); - createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), - createNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), + private final NodeRepository nodeRepository; + private final MockHostProvisioner hostProvisioner; + private final DynamicProvisioningMaintainer maintainer; + private final ProvisioningTester provisioningTester; - createNode("proxyhost1", Optional.empty(), NodeType.proxyhost, Node.State.provisioned, Optional.empty()), + public DynamicProvisioningTester() { + this(Cloud.builder().dynamicProvisioning(true).build()); + } - createNode("proxyhost2", Optional.empty(), NodeType.proxyhost, Node.State.active, Optional.of(proxyHostApp)), - createNode("proxy2", Optional.of("proxyhost2"), NodeType.proxy, Node.State.active, Optional.of(proxyApp))) - .forEach(node -> tester.nodeRepository.database().addNodesInState(List.of(node), node.state())); - } + public DynamicProvisioningTester(Cloud cloud) { + MockNameResolver nameResolver = new MockNameResolver().mockAnyLookup(); + this.hostProvisioner = new MockHostProvisioner(flavors, nameResolver); + this.provisioningTester = new ProvisioningTester.Builder().zone(new Zone(cloud, SystemName.defaultSystem(), + Environment.defaultEnvironment(), + RegionName.defaultName())) + .flavors(flavors.getFlavors()) + .nameResolver(nameResolver) + .hostProvisioner(hostProvisioner) + .build(); + this.nodeRepository = provisioningTester.nodeRepository(); + this.maintainer = new DynamicProvisioningMaintainer(nodeRepository, + Duration.ofDays(1), + hostProvisioner, + flagSource); + } - @SuppressWarnings("unchecked") - private static <T> T argThatLambda(Predicate<T> predicate) { - return argThat(new BaseMatcher<T>() { - @Override public boolean matches(Object item) { return predicate.test((T) item); } - @Override public void describeTo(Description description) { } - }); - } + private DynamicProvisioningTester addInitialNodes() { + List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)), + createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), + createNode("host1-2", Optional.of("host1"), NodeType.tenant, Node.State.failed, Optional.empty()), + createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)), + createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()), + createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), + createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()), + createNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)), + createNode("proxyhost1", Optional.empty(), NodeType.proxyhost, Node.State.provisioned, Optional.empty()), + createNode("proxyhost2", Optional.empty(), NodeType.proxyhost, Node.State.active, Optional.of(proxyHostApp)), + createNode("proxy2", Optional.of("proxyhost2"), NodeType.proxy, Node.State.active, Optional.of(proxyApp))) + .forEach(node -> nodeRepository.database().addNodesInState(List.of(node), node.state())); + return this; + } - static class HostProvisionerTester { - private static final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "docker"); - static final ApplicationId tenantApp = ApplicationId.from("mytenant", "myapp", "default"); - static final ApplicationId tenantHostApp = ApplicationId.from("vespa", "tenant-host", "default"); - static final ApplicationId proxyHostApp = ApplicationId.from("vespa", "proxy-host", "default"); - static final ApplicationId proxyApp = ApplicationId.from("vespa", "proxy", "default"); - - private final ManualClock clock = new ManualClock(); - private final Zone zone = new Zone(Cloud.defaultCloud().withDynamicProvisioning(true), SystemName.defaultSystem(), Environment.defaultEnvironment(), RegionName.defaultName()); - private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, - hostResourcesCalculator, - new MockCurator(), - clock, - zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-image"), true); - - Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) { - Node node = createNode(hostname, parentHostname, nodeType, state, application); + private Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state) { + return addNode(hostname, parentHostname, nodeType, state, null); + } + + private Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, ApplicationId application) { + Node node = createNode(hostname, parentHostname, nodeType, state, Optional.ofNullable(application)); return nodeRepository.database().addNodesInState(List.of(node), node.state()).get(0); } - static Node createNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) { - Flavor flavor = nodeFlavors.getFlavor(parentHostname.isPresent() ? "docker" : "default").orElseThrow(); + private Node createNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) { + Flavor flavor = nodeRepository.flavors().getFlavor(parentHostname.isPresent() ? "docker" : "host2").orElseThrow(); Optional<Allocation> allocation = application .map(app -> new Allocation( app, @@ -238,7 +272,101 @@ public class DynamicProvisioningMaintainerTest { false)); var ipConfig = new IP.Config(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of()); return new Node("fake-id-" + hostname, ipConfig, hostname, parentHostname, flavor, Status.initial(), - state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty()); + state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty()); } + + private long provisionedHostsMatching(NodeResources resources) { + return hostProvisioner.provisionedHosts.stream() + .filter(host -> host.nodeResources().equals(resources)) + .count(); + } + + private void assertNodesUnchanged() { + List<Node> nodes = nodeRepository.getNodes(); + maintainer.maintain(); + assertEquals("Nodes are unchanged after maintenance run", nodes, nodeRepository.getNodes()); + } + + } + + static class MockHostProvisioner implements HostProvisioner { + + private final List<ProvisionedHost> provisionedHosts = new ArrayList<>(); + private final NodeFlavors flavors; + private final MockNameResolver nameResolver; + + private int deprovisionedHosts = 0; + private EnumSet<Behaviour> behaviours = EnumSet.noneOf(Behaviour.class); + + public MockHostProvisioner(NodeFlavors flavors, MockNameResolver nameResolver) { + this.flavors = flavors; + this.nameResolver = nameResolver; + } + + @Override + public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId, Version osVersion) { + Flavor hostFlavor = flavors.getFlavors().stream() + .filter(f -> !f.isDocker()) + .filter(f -> f.resources().compatibleWith(resources)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No host flavor found satisfying " + resources)); + List<ProvisionedHost> hosts = new ArrayList<>(); + for (int index : provisionIndexes) { + hosts.add(new ProvisionedHost("host" + index, + "hostname" + index, + hostFlavor, + "nodename" + index, + resources, + osVersion)); + } + provisionedHosts.addAll(hosts); + return hosts; + } + + @Override + public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException { + if (behaviours.contains(Behaviour.failProvisioning)) throw new FatalProvisioningException("Failed to provision node(s)"); + assertSame(Node.State.provisioned, host.state()); + List<Node> result = new ArrayList<>(); + result.add(withIpAssigned(host)); + for (var child : children) { + assertSame(Node.State.reserved, child.state()); + result.add(withIpAssigned(child)); + } + return result; + } + + @Override + public void deprovision(Node host) { + if (behaviours.contains(Behaviour.failDeprovisioning)) throw new FatalProvisioningException("Failed to deprovision node"); + provisionedHosts.removeIf(provisionedHost -> provisionedHost.hostHostname().equals(host.hostname())); + deprovisionedHosts++; + } + + private MockHostProvisioner with(Behaviour first, Behaviour... rest) { + this.behaviours = EnumSet.of(first, rest); + return this; + } + + private Node withIpAssigned(Node node) { + if (node.parentHostname().isPresent()) return node; + int hostIndex = Integer.parseInt(node.hostname().replaceAll("^[a-z]+|-\\d+$", "")); + Set<String> addresses = Set.of("::" + hostIndex + ":0"); + nameResolver.addRecord(node.hostname(), addresses.iterator().next()); + Set<String> pool = new HashSet<>(); + for (int i = 1; i <= 2; i++) { + String ip = "::" + hostIndex + ":" + i; + pool.add(ip); + nameResolver.addRecord(node.hostname() + "-" + i, ip); + } + return node.with(node.ipConfig().with(addresses).with(IP.Pool.of(pool))); + } + + enum Behaviour { + failProvisioning, + failDeprovisioning, + } + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 575d84c0129..24a0020df4f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -258,8 +258,8 @@ public class FailedExpirerTest { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), - true); - this.provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); + true, false); + this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30)); } @@ -328,12 +328,13 @@ public class FailedExpirerTest { public FailureScenario allocate(ClusterSpec.Type clusterType, NodeResources flavor, String... hostname) { ClusterSpec clusterSpec = ClusterSpec.request(clusterType, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); - Capacity capacity = Capacity.from(new ClusterResources(hostname.length, 1, flavor), false, true); + Capacity capacity = Capacity.from(new ClusterResources(hostname.length, 1, flavor), true, true); return allocate(applicationId, clusterSpec, capacity); } public FailureScenario allocate(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity capacity) { - List<HostSpec> preparedNodes = provisioner.prepare(applicationId, clusterSpec, capacity, null); + List<HostSpec> preparedNodes = provisioner.prepare(applicationId, clusterSpec, capacity, + (level, message) -> System.out.println(level + ": " + message) ); NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); provisioner.activate(transaction, applicationId, Set.copyOf(preparedNodes)); transaction.commit(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index 4fcd5793b8a..89e43f80479 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; -import com.yahoo.vespa.hosted.provision.testutils.MockNodeMetrics; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; import org.junit.Test; @@ -52,7 +51,7 @@ public class InactiveAndFailedExpirerTest { @Test public void inactive_and_failed_times_out() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - List<Node> nodes = tester.makeReadyNodes(2, nodeResources); + tester.makeReadyNodes(2, nodeResources); // Allocate then deallocate 2 nodes ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); @@ -90,7 +89,7 @@ public class InactiveAndFailedExpirerTest { @Test public void reboot_generation_is_increased_when_node_moves_to_dirty() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - List<Node> nodes = tester.makeReadyNodes(2, nodeResources); + tester.makeReadyNodes(2, nodeResources); // Allocate and deallocate a single node ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); @@ -164,7 +163,7 @@ public class InactiveAndFailedExpirerTest { } @Test - public void testersExpireImmediately() { + public void tester_applications_expire_immediately() { ApplicationId testerId = ApplicationId.from(applicationId.tenant().value(), applicationId.application().value(), applicationId.instance().value() + "-t"); @@ -189,4 +188,27 @@ public class InactiveAndFailedExpirerTest { } + @Test + public void nodes_marked_for_deprovisioning_move_to_parked() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + tester.makeReadyHosts(2, nodeResources); + + // Activate and deallocate + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); + List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.fromRequiredNodeType(NodeType.host)); + tester.activate(applicationId, new HashSet<>(preparedNodes)); + assertEquals(2, tester.getNodes(applicationId, Node.State.active).size()); + tester.deactivate(applicationId); + List<Node> inactiveNodes = tester.getNodes(applicationId, Node.State.inactive).asList(); + assertEquals(2, inactiveNodes.size()); + + // Nodes marked for deprovisioning are moved to parked + tester.nodeRepository().write(inactiveNodes.stream() + .map(node -> node.withWantToRetire(true, true, Agent.system, tester.clock().instant())) + .collect(Collectors.toList()), () -> {}); + tester.advanceTime(Duration.ofMinutes(11)); + new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10)).run(); + assertEquals(2, tester.nodeRepository().getNodes(Node.State.parked).size()); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java index 11df6146b06..754870e798e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java @@ -3,21 +3,17 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; -import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import java.time.Instant; @@ -44,7 +40,7 @@ public class MaintenanceTester { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); public MaintenanceTester() { curator.setZooKeeperEnsembleConnectionSpec("zk1.host:1,zk2.host:2,zk3.host:3"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 6dfb404d81a..727232e5c7c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -84,7 +84,7 @@ public class MetricsReporterTest { Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); nodeRepository.addNodes(List.of(node), Agent.system); Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy); @@ -150,7 +150,7 @@ public class MetricsReporterTest { Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); // Allow 4 containers Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5"); @@ -193,13 +193,13 @@ public class MetricsReporterTest { assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts assertEquals(2, metric.values.get("hostedVespa.reservedHosts")); - assertEquals(12.0, metric.values.get("hostedVespa.docker.totalCapacityDisk")); - assertEquals(10.0, metric.values.get("hostedVespa.docker.totalCapacityMem")); - assertEquals(7.0, metric.values.get("hostedVespa.docker.totalCapacityCpu")); + assertEquals(120.0, metric.values.get("hostedVespa.docker.totalCapacityDisk")); + assertEquals(100.0, metric.values.get("hostedVespa.docker.totalCapacityMem")); + assertEquals( 7.0, metric.values.get("hostedVespa.docker.totalCapacityCpu")); - assertEquals(6.0, metric.values.get("hostedVespa.docker.freeCapacityDisk")); - assertEquals(3.0, metric.values.get("hostedVespa.docker.freeCapacityMem")); - assertEquals(4.0, metric.values.get("hostedVespa.docker.freeCapacityCpu")); + assertEquals(114.0, metric.values.get("hostedVespa.docker.freeCapacityDisk")); + assertEquals( 93.0, metric.values.get("hostedVespa.docker.freeCapacityMem")); + assertEquals( 4.0, metric.values.get("hostedVespa.docker.freeCapacityCpu")); Metric.Context app1context = metric.createContext(Map.of("app", "test.default", "tenantName", "app1", "applicationId", "app1.test.default")); assertEquals(2.0, metric.sumDoubleValues("hostedVespa.docker.allocatedCapacityDisk", app1context), 0.01d); @@ -223,7 +223,7 @@ public class MetricsReporterTest { if (tenant.isPresent()) { Allocation allocation = new Allocation(app(tenant.get()), ClusterMembership.from("container/id1/0/3", new Version(), Optional.empty()), - owner.flavor().resources(), + owner.resources(), Generation.initial(), false); return Optional.of(allocation); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index e33cc9e655e..9cf03c6f33b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -50,7 +50,7 @@ import static org.junit.Assert.assertEquals; */ public class NodeFailTester { - public static final NodeResources nodeResources = new NodeResources(2, 8, 50, 1); + public static final NodeResources nodeResources = new NodeResources(2, 8, 500, 1); // Immutable components public static final ApplicationId tenantHostApp = ApplicationId.from("hosted-vespa", "tenant-host", "default"); @@ -82,7 +82,7 @@ public class NodeFailTester { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); hostLivenessTracker = new TestHostLivenessTracker(clock); orchestrator = new OrchestratorMock(); @@ -122,7 +122,7 @@ public class NodeFailTester { List<Node> hosts = tester.createHostNodes(numberOfHosts); for (int i = 0; i < hosts.size(); i++) { tester.createReadyNodes(nodesPerHost, i * nodesPerHost, Optional.of("parent" + i), - new NodeResources(1, 4, 10, 0.3), NodeType.tenant); + new NodeResources(1, 4, 100, 0.3), NodeType.tenant); } // Create applications @@ -130,8 +130,8 @@ public class NodeFailTester { ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("6.75.0").build(); ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.75.0").build(); Capacity allHosts = Capacity.fromRequiredNodeType(NodeType.host); - Capacity capacity1 = Capacity.from(new ClusterResources(3, 1, new NodeResources(1, 4, 10, 0.3)), false, true); - Capacity capacity2 = Capacity.from(new ClusterResources(5, 1, new NodeResources(1, 4, 10, 0.3)), false, true); + Capacity capacity1 = Capacity.from(new ClusterResources(3, 1, new NodeResources(1, 4, 100, 0.3)), false, true); + Capacity capacity2 = Capacity.from(new ClusterResources(5, 1, new NodeResources(1, 4, 100, 0.3)), false, true); tester.activate(tenantHostApp, clusterNodeAdminApp, allHosts); tester.activate(app1, clusterApp1, capacity1); tester.activate(app2, clusterApp2, capacity2); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java index e2fd8a8721c..51f70e8b640 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -301,9 +301,9 @@ public class NodeFailerTest { // Two ready nodes and a ready docker node die, but only 2 of those are failed out tester.clock.advance(Duration.ofMinutes(180)); - Node dockerNode = ready.stream().filter(node -> node.flavor().resources().equals(newNodeResources)).findFirst().get(); + Node dockerNode = ready.stream().filter(node -> node.resources().equals(newNodeResources)).findFirst().get(); List<Node> otherNodes = ready.stream() - .filter(node -> ! node.flavor().resources().equals(newNodeResources)) + .filter(node -> ! node.resources().equals(newNodeResources)) .collect(Collectors.toList()); tester.allNodesMakeAConfigRequestExcept(otherNodes.get(0), otherNodes.get(2), dockerNode); tester.failer.run(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java index d143253a4b1..bae6de5a095 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java @@ -10,6 +10,7 @@ import org.junit.Test; import java.time.Duration; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -108,7 +109,7 @@ public class NodeRebooterTest { /** Schedule OS upgrade for all host nodes */ private void scheduleOsUpgrade(MaintenanceTester tester) { - tester.nodeRepository.osVersions().setTarget(NodeType.host, Version.fromString("7.0"), false); + tester.nodeRepository.osVersions().setTarget(NodeType.host, Version.fromString("7.0"), Optional.empty(), false); } /** Simulate completion of an OS upgrade */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index e57d57d4c4c..b9c57e013c3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -61,7 +61,7 @@ public class OperatorChangeApplicationMaintainerTest { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); this.fixture = new Fixture(zone, nodeRepository); createReadyNodes(15, this.fixture.nodeResources, nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java index c30b49ac97a..65c7bf13b42 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivatorTest.java @@ -57,8 +57,8 @@ public class OsUpgradeActivatorTest { // New OS target version is set var osVersion0 = Version.fromString("8.0"); - osVersions.setTarget(NodeType.host, osVersion0, false); - osVersions.setTarget(NodeType.confighost, osVersion0, false); + osVersions.setTarget(NodeType.host, osVersion0, Optional.empty(), false); + osVersions.setTarget(NodeType.confighost, osVersion0, Optional.empty(), false); // New OS version is activated as there is no ongoing Vespa upgrade osUpgradeActivator.maintain(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index 8a2a69bb437..b6ffe4ebe26 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -67,7 +67,7 @@ public class PeriodicApplicationMaintainerTest { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); this.fixture = new Fixture(zone, nodeRepository); createReadyNodes(15, fixture.nodeResources, nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index e6609caa4bc..05e5b4829e9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -7,14 +7,15 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.test.ManualClock; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; @@ -22,11 +23,13 @@ import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; import org.junit.Test; import java.time.Duration; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.hosted.provision.maintenance.Rebalancer.waitTimeAfterPreviousDeployment; +import static com.yahoo.vespa.hosted.provision.maintenance.RebalancerTest.RebalancerTester.cpuApp; +import static com.yahoo.vespa.hosted.provision.maintenance.RebalancerTest.RebalancerTester.memoryApp; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -38,68 +41,43 @@ public class RebalancerTest { @Test public void testRebalancing() { - // --- Setup - ApplicationId cpuApp = makeApplicationId("t1", "a1"); - ApplicationId memApp = makeApplicationId("t2", "a2"); - NodeResources cpuResources = new NodeResources(8, 4, 1, 0.1); - NodeResources memResources = new NodeResources(4, 9, 1, 0.1); - - ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); - TestMetric metric = new TestMetric(); - - Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( - cpuApp, new MockDeployer.ApplicationContext(cpuApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, cpuResources))), - memApp, new MockDeployer.ApplicationContext(memApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, memResources)))); - MockDeployer deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps); - - Rebalancer rebalancer = new Rebalancer(deployer, - tester.nodeRepository(), - Optional.empty(), - metric, - tester.clock(), - Duration.ofMinutes(1)); - - - tester.makeReadyNodes(3, "flt", NodeType.host, 8); - tester.deployZoneApp(); + RebalancerTester tester = new RebalancerTester(); // --- Deploying a cpu heavy application - causing 1 of these nodes to be skewed - deployApp(cpuApp, clusterSpec("c"), cpuResources, tester, 1); - Node cpuSkewedNode = tester.nodeRepository().getNodes(cpuApp).get(0); + tester.deployApp(cpuApp); + Node cpuSkewedNode = tester.getNode(cpuApp); - rebalancer.maintain(); + tester.maintain(); assertFalse("No better place to move the skewed node, so no action is taken", - tester.nodeRepository().getNode(cpuSkewedNode.hostname()).get().status().wantToRetire()); - assertEquals(0.00325, metric.values.get("hostedVespa.docker.skew").doubleValue(), 0.00001); + tester.getNode(cpuSkewedNode.hostname()).get().status().wantToRetire()); + assertEquals(0.00325, tester.metric().values.get("hostedVespa.docker.skew").doubleValue(), 0.00001); // --- Making a more suitable node configuration available causes rebalancing - Node newCpuHost = tester.makeReadyNodes(1, "cpu", NodeType.host, 8).get(0); + Node newCpuHost = tester.makeReadyNode("cpu"); tester.deployZoneApp(); - rebalancer.maintain(); - assertTrue("Rebalancer retired the node we wanted to move away from", - tester.nodeRepository().getNode(cpuSkewedNode.hostname()).get().allocation().get().membership().retired()); + tester.maintain(); + assertTrue("Rebalancer retired the node we wanted to move away from", tester.isNodeRetired(cpuSkewedNode)); assertTrue("... and added a node on the new host instead", - tester.nodeRepository().getNodes(cpuApp, Node.State.active).stream().anyMatch(node -> node.hasParent(newCpuHost.hostname()))); + tester.getNodes(cpuApp, Node.State.active).stream().anyMatch(node -> node.hasParent(newCpuHost.hostname()))); assertEquals("Skew is reduced", - 0.00244, metric.values.get("hostedVespa.docker.skew").doubleValue(), 0.00001); + 0.00244, tester.metric().values.get("hostedVespa.docker.skew").doubleValue(), 0.00001); // --- Deploying a mem heavy application - allocated to the best option and causing increased skew - deployApp(memApp, clusterSpec("c"), memResources, tester, 1); - assertEquals("Assigned to a flat node as that causes least skew", "flt", - tester.nodeRepository().list().parentOf(tester.nodeRepository().getNodes(memApp).get(0)).get().flavor().name()); - rebalancer.maintain(); + tester.deployApp(memoryApp); + assertEquals("Assigned to a flat node as that causes least skew", "flat", + tester.nodeRepository().list().parentOf(tester.getNode(memoryApp)).get().flavor().name()); + tester.maintain(); assertEquals("Deploying the mem skewed app increased skew", - 0.00734, metric.values.get("hostedVespa.docker.skew").doubleValue(), 0.00001); + 0.00734, tester.metric().values.get("hostedVespa.docker.skew").doubleValue(), 0.00001); // --- Adding a more suitable node reconfiguration causes no action as the system is not stable - Node memSkewedNode = tester.nodeRepository().getNodes(memApp).get(0); - Node newMemHost = tester.makeReadyNodes(1, "mem", NodeType.host, 8).get(0); + Node memSkewedNode = tester.getNode(memoryApp); + Node newMemHost = tester.makeReadyNode("mem"); tester.deployZoneApp(); - rebalancer.maintain(); - assertFalse("No rebalancing happens because cpuSkewedNode is still retired", - tester.nodeRepository().getNode(memSkewedNode.hostname()).get().allocation().get().membership().retired()); + tester.maintain(); + assertFalse("No rebalancing happens because cpuSkewedNode is still retired", tester.isNodeRetired(memSkewedNode)); // --- Making the system stable enables rebalancing NestedTransaction tx = new NestedTransaction(); @@ -107,44 +85,127 @@ public class RebalancerTest { tx.commit(); // ... if activation fails when trying, we clean up the state - deployer.setFailActivate(true); - rebalancer.maintain(); - assertTrue("Want to retire is reset", tester.nodeRepository().getNodes(Node.State.active).stream().noneMatch(node -> node.status().wantToRetire())); - assertEquals("Reserved node was moved to dirty", 1, tester.nodeRepository().getNodes(Node.State.dirty).size()); - String reservedHostname = tester.nodeRepository().getNodes(Node.State.dirty).get(0).hostname(); + tester.deployer().setFailActivate(true); + tester.maintain(); + assertTrue("Want to retire is reset", tester.getNodes(Node.State.active).stream().noneMatch(node -> node.status().wantToRetire())); + assertEquals("Reserved node was moved to dirty", 1, tester.getNodes(Node.State.dirty).size()); + String reservedHostname = tester.getNodes(Node.State.dirty).get(0).hostname(); tester.nodeRepository().setReady(reservedHostname, Agent.system, "Cleanup"); tester.nodeRepository().removeRecursively(reservedHostname); // ... otherwise we successfully rebalance, again reducing skew - deployer.setFailActivate(false); - rebalancer.maintain(); - assertTrue("Rebalancer retired the node we wanted to move away from", - tester.nodeRepository().getNode(memSkewedNode.hostname()).get().allocation().get().membership().retired()); + tester.deployer().setFailActivate(false); + tester.maintain(); + assertTrue("Rebalancer retired the node we wanted to move away from", tester.isNodeRetired(memSkewedNode)); assertTrue("... and added a node on the new host instead", - tester.nodeRepository().getNodes(memApp, Node.State.active).stream().anyMatch(node -> node.hasParent(newMemHost.hostname()))); + tester.getNodes(memoryApp, Node.State.active).stream().anyMatch(node -> node.hasParent(newMemHost.hostname()))); assertEquals("Skew is reduced", - 0.00587, metric.values.get("hostedVespa.docker.skew").doubleValue(), 0.00001); + 0.00587, tester.metric().values.get("hostedVespa.docker.skew").doubleValue(), 0.00001); } - private ClusterSpec clusterSpec(String clusterId) { - return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId)).vespaVersion("6.42").build(); - } - private ApplicationId makeApplicationId(String tenant, String appName) { - return ApplicationId.from(tenant, appName, "default"); - } + @Test + public void testNoRebalancingIfRecentlyDeployed() { + RebalancerTester tester = new RebalancerTester(); - private void deployApp(ApplicationId id, ClusterSpec spec, NodeResources flavor, ProvisioningTester tester, int nodeCount) { - List<HostSpec> hostSpec = tester.prepare(id, spec, nodeCount, 1, flavor); - tester.activate(id, new HashSet<>(hostSpec)); + // --- Deploying a cpu heavy application - causing 1 of these nodes to be skewed + tester.deployApp(cpuApp); + Node cpuSkewedNode = tester.getNode(cpuApp); + tester.maintain(); + assertFalse("No better place to move the skewed node, so no action is taken", + tester.getNode(cpuSkewedNode.hostname()).get().status().wantToRetire()); + + // --- Making a more suitable node configuration available causes rebalancing + Node newCpuHost = tester.makeReadyNode("cpu"); + tester.deployZoneApp(); + + tester.deployApp(cpuApp, false /* skip advancing clock after deployment */); + tester.maintain(); + assertFalse("No action, since app was recently deployed", tester.isNodeRetired(cpuSkewedNode)); + + tester.clock().advance(waitTimeAfterPreviousDeployment); + tester.maintain(); + assertTrue("Rebalancer retired the node we wanted to move away from", tester.isNodeRetired(cpuSkewedNode)); + assertTrue("... and added a node on the new host instead", + tester.getNodes(cpuApp, Node.State.active).stream().anyMatch(node -> node.hasParent(newCpuHost.hostname()))); } - private FlavorsConfig flavorsConfig() { - FlavorConfigBuilder b = new FlavorConfigBuilder(); - b.addFlavor("flt", 30, 30, 40, 3, Flavor.Type.BARE_METAL); - b.addFlavor("cpu", 40, 20, 40, 3, Flavor.Type.BARE_METAL); - b.addFlavor("mem", 20, 40, 40, 3, Flavor.Type.BARE_METAL); - return b.build(); + + static class RebalancerTester { + + static ApplicationId cpuApp = makeApplicationId("t1", "a1"); + static ApplicationId memoryApp = makeApplicationId("t2", "a2"); + private static NodeResources cpuResources = new NodeResources(8, 4, 10, 0.1); + private static NodeResources memResources = new NodeResources(4, 9, 10, 0.1); + private TestMetric metric = new TestMetric(); + private ProvisioningTester tester = new ProvisioningTester.Builder() + .zone(new Zone(Environment.perf, RegionName.from("us-east"))) + .flavorsConfig(flavorsConfig()) + .build(); + private final MockDeployer deployer; + private final Rebalancer rebalancer; + + RebalancerTester() { + Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( + cpuApp, new MockDeployer.ApplicationContext(cpuApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, cpuResources))), + memoryApp, new MockDeployer.ApplicationContext(memoryApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, memResources)))); + deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps); + rebalancer = new Rebalancer(deployer, tester.nodeRepository(), metric, tester.clock(), Duration.ofMinutes(1)); + tester.makeReadyNodes(3, "flat", NodeType.host, 8); + tester.deployZoneApp(); + } + + void maintain() { rebalancer.maintain(); } + + Node makeReadyNode(String flavor) { return tester.makeReadyNodes(1, flavor, NodeType.host, 8).get(0); } + + TestMetric metric() { return metric; } + + NodeRepository nodeRepository() { return tester.nodeRepository(); } + + void deployZoneApp() { tester.deployZoneApp(); } + + void deployApp(ApplicationId id) { deployApp(id, true); } + + void deployApp(ApplicationId id, boolean advanceClock) { + deployer.deployFromLocalActive(id).get().activate(); + if (advanceClock) tester.clock().advance(waitTimeAfterPreviousDeployment); + } + + List<Node> getNodes(ApplicationId applicationId, Node.State nodeState) { + return tester.nodeRepository().getNodes(applicationId, nodeState); + } + + boolean isNodeRetired(Node node) { + return getNode(node.hostname()).get().allocation().get().membership().retired(); + } + + Optional<Node> getNode(String hostname) { return tester.nodeRepository().getNode(hostname); } + + List<Node> getNodes(Node.State nodeState) { return tester.nodeRepository().getNodes(nodeState); } + + Node getNode(ApplicationId applicationId) { return tester.nodeRepository().getNodes(applicationId).get(0); } + + ManualClock clock() { return tester.clock(); } + + MockDeployer deployer() { return deployer; } + + private FlavorsConfig flavorsConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("flat", 30, 30, 400, 3, Flavor.Type.BARE_METAL); + b.addFlavor("cpu", 40, 20, 400, 3, Flavor.Type.BARE_METAL); + b.addFlavor("mem", 20, 40, 400, 3, Flavor.Type.BARE_METAL); + return b.build(); + } + + private static ClusterSpec clusterSpec(String clusterId) { + return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId)).vespaVersion("6.42").build(); + } + + private static ApplicationId makeApplicationId(String tenant, String appName) { + return ApplicationId.from(tenant, appName, "default"); + } + } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 59514cb3c95..2c0dac10f9f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -52,7 +52,7 @@ public class ReservationExpirerTest { Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); List<Node> nodes = new ArrayList<>(2); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index e57bae09280..29451d7b690 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -71,7 +71,7 @@ public class RetiredExpirerTest { zone, new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, false); private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); private final Orchestrator orchestrator = mock(Orchestrator.class); @@ -119,37 +119,6 @@ public class RetiredExpirerTest { } @Test - public void ensure_retired_groups_time_out() { - createReadyNodes(8, nodeResources, nodeRepository); - createHostNodes(4, nodeRepository, nodeFlavors); - - ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); - - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); - activate(applicationId, cluster, 8, 8, provisioner); - activate(applicationId, cluster, 2, 2, provisioner); - assertEquals(8, nodeRepository.getNodes(applicationId, Node.State.active).size()); - assertEquals(0, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); - - // Cause inactivation of retired nodes - clock.advance(Duration.ofHours(30)); // Retire period spent - MockDeployer deployer = - new MockDeployer(provisioner, - clock, - Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, - cluster, - Capacity.from(new ClusterResources(2, 1, nodeResources))))); - createRetiredExpirer(deployer).run(); - assertEquals(2, nodeRepository.getNodes(applicationId, Node.State.active).size()); - assertEquals(6, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); - assertEquals(1, deployer.redeployments); - - // inactivated nodes are not retired - for (Node node : nodeRepository.getNodes(applicationId, Node.State.inactive)) - assertFalse(node.allocation().get().membership().retired()); - } - - @Test public void ensure_early_inactivation() throws OrchestrationException { createReadyNodes(7, nodeResources, nodeRepository); createHostNodes(4, nodeRepository, nodeFlavors); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 59da88790d7..b7f21eb3114 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -19,12 +19,10 @@ import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; import com.yahoo.vespa.hosted.provision.autoscale.Resource; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; -import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; import org.junit.Test; import java.time.Duration; import java.util.List; -import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -42,10 +40,10 @@ public class ScalingSuggestionsMaintainerTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east3"))).flavorsConfig(flavorsConfig()).build(); ApplicationId app1 = tester.makeApplicationId("app1"); - ClusterSpec cluster1 = tester.clusterSpec(); + ClusterSpec cluster1 = tester.containerClusterSpec(); ApplicationId app2 = tester.makeApplicationId("app2"); - ClusterSpec cluster2 = tester.clusterSpec(); + ClusterSpec cluster2 = tester.contentClusterSpec(); NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(); @@ -71,9 +69,9 @@ public class ScalingSuggestionsMaintainerTest { Duration.ofMinutes(1)); maintainer.maintain(); - assertEquals("7 nodes with [vcpu: 15.3, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps, storage type: remote]", + assertEquals("14 nodes with [vcpu: 6.9, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps, storage type: remote]", tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestedResources().get().toString()); - assertEquals("7 nodes with [vcpu: 16.8, memory: 5.7 Gb, disk 16.5 Gb, bandwidth: 0.1 Gbps, storage type: remote]", + assertEquals("8 nodes with [vcpu: 14.7, memory: 4.0 Gb, disk 11.8 Gb, bandwidth: 0.1 Gbps, storage type: remote]", tester.nodeRepository().applications().get(app2).get().cluster(cluster2.id()).get().suggestedResources().get().toString()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java new file mode 100644 index 00000000000..fb84dc0a32a --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java @@ -0,0 +1,327 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.test.ManualClock; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.IP; +import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; +import org.junit.Ignore; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SpareCapacityMaintainerTest { + + @Test + public void testEmpty() { + var tester = new SpareCapacityMaintainerTester(); + tester.maintainer.maintain(); + assertEquals(0, tester.deployer.redeployments); + assertEquals(0, tester.nodeRepository.list().retired().size()); + } + + @Test + public void testOneSpare() { + var tester = new SpareCapacityMaintainerTester(); + tester.addHosts(2, new NodeResources(10, 100, 1000, 1)); + tester.addNodes(0, 1, new NodeResources(10, 100, 1000, 1), 0); + tester.maintainer.maintain(); + assertEquals(0, tester.deployer.redeployments); + assertEquals(0, tester.nodeRepository.list().retired().size()); + assertEquals(1, tester.metric.values.get("spareHostCapacity")); + } + + @Test + public void testTwoSpares() { + var tester = new SpareCapacityMaintainerTester(); + tester.addHosts(3, new NodeResources(10, 100, 1000, 1)); + tester.addNodes(0, 1, new NodeResources(10, 100, 1000, 1), 0); + tester.maintainer.maintain(); + assertEquals(0, tester.deployer.redeployments); + assertEquals(0, tester.nodeRepository.list().retired().size()); + assertEquals(2, tester.metric.values.get("spareHostCapacity")); + } + + @Test + public void testNoSpares() { + var tester = new SpareCapacityMaintainerTester(); + tester.addHosts(2, new NodeResources(10, 100, 1000, 1)); + tester.addNodes(0, 2, new NodeResources(10, 100, 1000, 1), 0); + tester.maintainer.maintain(); + assertEquals(0, tester.deployer.redeployments); + assertEquals(0, tester.nodeRepository.list().retired().size()); + assertEquals(0, tester.metric.values.get("spareHostCapacity")); + } + + @Test + public void testAllWorksAsSpares() { + var tester = new SpareCapacityMaintainerTester(); + tester.addHosts(4, new NodeResources(10, 100, 1000, 1)); + tester.addNodes(0, 2, new NodeResources(5, 50, 500, 0.5), 0); + tester.addNodes(1, 2, new NodeResources(5, 50, 500, 0.5), 2); + tester.maintainer.maintain(); + assertEquals(0, tester.deployer.redeployments); + assertEquals(0, tester.nodeRepository.list().retired().size()); + assertEquals(2, tester.metric.values.get("spareHostCapacity")); + } + + @Test + public void testMoveIsNeeded() { + // Moving application id 1 and 2 to the same nodes frees up spares for application 0 + var tester = new SpareCapacityMaintainerTester(); + tester.addHosts(6, new NodeResources(10, 100, 1000, 1)); + tester.addNodes(0, 2, new NodeResources(10, 100, 1000, 1), 0); + tester.addNodes(1, 2, new NodeResources(5, 50, 500, 0.5), 2); + tester.addNodes(2, 2, new NodeResources(5, 50, 500, 0.5), 4); + tester.maintainer.maintain(); + assertEquals(1, tester.deployer.redeployments); + assertEquals(1, tester.nodeRepository.list().retired().size()); + assertEquals(1, tester.metric.values.get("spareHostCapacity")); + + // Maintaining again is a no-op since the node to move is already retired + tester.maintainer.maintain(); + assertEquals(1, tester.deployer.redeployments); + assertEquals(1, tester.nodeRepository.list().retired().size()); + assertEquals(1, tester.metric.values.get("spareHostCapacity")); + } + + @Test + public void testMultipleMovesAreNeeded() { + // Moving application id 1 and 2 to the same nodes frees up spares for application 0 + // so that it can be moved from size 12 to size 10 hosts, clearing up spare room for the size 12 application + var tester = new SpareCapacityMaintainerTester(); + tester.addHosts(4, new NodeResources(12, 120, 1200, 1.2)); + tester.addHosts(4, new NodeResources(10, 100, 1000, 1)); + tester.addNodes(0, 2, new NodeResources(10, 100, 1000, 1.0), 0); + tester.addNodes(1, 2, new NodeResources(12, 120, 1200, 1.2), 2); + tester.addNodes(2, 2, new NodeResources(5, 50, 500, 0.5), 4); + tester.addNodes(3, 2, new NodeResources(5, 50, 500, 0.5), 6); + tester.maintainer.maintain(); + assertEquals(1, tester.deployer.redeployments); + assertEquals(1, tester.nodeRepository.list().retired().size()); + assertEquals(1, tester.metric.values.get("spareHostCapacity")); + } + + @Test + public void testMultipleNodesMustMoveFromOneHost() { + // By moving the 4 small nodes from host 2 we free up sufficient space on the third host to act as a spare for + // application 0 + var tester = new SpareCapacityMaintainerTester(); + setupMultipleHosts(tester, 5); + + tester.maintainer.maintain(); + assertEquals(1, tester.deployer.redeployments); + assertEquals(1, tester.nodeRepository.list().retired().size()); + assertEquals(1, tester.metric.values.get("spareHostCapacity")); + } + + @Test + public void testMultipleNodesMustMoveFromOneHostButInsufficientCapacity() { + var tester = new SpareCapacityMaintainerTester(); + setupMultipleHosts(tester, 4); + + tester.maintainer.maintain(); + assertEquals(0, tester.deployer.redeployments); + assertEquals(0, tester.nodeRepository.list().retired().size()); + assertEquals(0, tester.metric.values.get("spareHostCapacity")); + } + + private void setupMultipleHosts(SpareCapacityMaintainerTester tester, int smallNodeCount) { + tester.addHosts(2, new NodeResources(10, 100, 1000, 1)); + tester.addNodes(0, 2, new NodeResources(10, 100, 1000, 1.0), 0); + + tester.addHosts(1, new NodeResources(16, 160, 1600, 1.6)); + tester.addNodes(1, 1, new NodeResources(1, 10, 100, 0.1), 2); + tester.addNodes(2, 1, new NodeResources(1, 10, 100, 0.1), 2); + tester.addNodes(3, 1, new NodeResources(1, 10, 100, 0.1), 2); + tester.addNodes(4, 1, new NodeResources(1, 10, 100, 0.1), 2); + tester.addNodes(5, 1, new NodeResources(2, 20, 200, 2.0), 2); + tester.addNodes(6, 1, new NodeResources(2, 20, 200, 2.0), 2); + tester.addNodes(7, 1, new NodeResources(2, 20, 200, 2.0), 2); + + tester.addHosts(smallNodeCount, new NodeResources(2, 20, 200, 2.0)); + } + + @Test + public void testTooManyIterationsAreNeeded() { + // 6 nodes must move to the next host, which is more than the max limit + var tester = new SpareCapacityMaintainerTester(5); + + tester.addHosts(2, new NodeResources(10, 100, 1000, 1)); + tester.addHosts(1, new NodeResources(9, 90, 900, 0.9)); + tester.addHosts(1, new NodeResources(8, 80, 800, 0.8)); + tester.addHosts(1, new NodeResources(7, 70, 700, 0.7)); + tester.addHosts(1, new NodeResources(6, 60, 600, 0.6)); + tester.addHosts(1, new NodeResources(5, 50, 500, 0.5)); + tester.addHosts(1, new NodeResources(4, 40, 400, 0.4)); + + tester.addNodes(0, 1, new NodeResources(10, 100, 1000, 1.0), 0); + tester.addNodes(1, 1, new NodeResources( 9, 90, 900, 0.9), 1); + tester.addNodes(2, 1, new NodeResources( 8, 80, 800, 0.8), 2); + tester.addNodes(3, 1, new NodeResources( 7, 70, 700, 0.7), 3); + tester.addNodes(4, 1, new NodeResources( 6, 60, 600, 0.6), 4); + tester.addNodes(5, 1, new NodeResources( 5, 50, 500, 0.5), 5); + tester.addNodes(6, 1, new NodeResources( 4, 40, 400, 0.4), 6); + + tester.maintainer.maintain(); + assertEquals(0, tester.deployer.redeployments); + assertEquals(0, tester.nodeRepository.list().retired().size()); + assertEquals(0, tester.metric.values.get("spareHostCapacity")); + } + + /** Microbenchmark */ + @Test + @Ignore + public void testLargeNodeRepo() { + // Completely fill 200 hosts with 2000 nodes + int hosts = 200; + var tester = new SpareCapacityMaintainerTester(); + tester.addHosts(hosts, new NodeResources(100, 1000, 10000, 10)); + int hostOffset = 0; + for (int i = 0; i < 200; i++) { + int applicationSize = 10; + int resourceSize = 10; + tester.addNodes(i, applicationSize, new NodeResources(resourceSize, resourceSize * 10, resourceSize * 100, 0.1), hostOffset); + hostOffset = (hostOffset + applicationSize) % hosts; + } + long startTime = System.currentTimeMillis(); + tester.maintainer.maintain(); + long totalTime = System.currentTimeMillis() - startTime; + System.out.println("Complete in " + ( totalTime / 1000) + " seconds"); + assertEquals(0, tester.deployer.redeployments); + assertEquals(0, tester.nodeRepository.list().retired().size()); + assertEquals(0, tester.metric.values.get("spareHostCapacity")); + } + + private static class SpareCapacityMaintainerTester { + + NodeRepository nodeRepository; + MockDeployer deployer; + TestMetric metric = new TestMetric(); + SpareCapacityMaintainer maintainer; + private int hostIndex = 0; + private int nodeIndex = 0; + + private SpareCapacityMaintainerTester() { + this(1000); + } + + private SpareCapacityMaintainerTester(int maxIterations) { + NodeFlavors flavors = new NodeFlavors(new FlavorConfigBuilder().build()); + nodeRepository = new NodeRepository(flavors, + new EmptyProvisionServiceProvider().getHostResourcesCalculator(), + new MockCurator(), + new ManualClock(), + new Zone(Environment.prod, RegionName.from("us-east-3")), + new MockNameResolver().mockAnyLookup(), + DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true, false); + deployer = new MockDeployer(nodeRepository); + maintainer = new SpareCapacityMaintainer(deployer, nodeRepository, metric, Duration.ofDays(1), maxIterations); + } + + private void addHosts(int count, NodeResources resources) { + List<Node> hosts = new ArrayList<>(); + for (int i = 0; i < count; i++) { + Node host = nodeRepository.createNode("host" + hostIndex, + "host" + hostIndex + ".yahoo.com", + ipConfig(hostIndex + nodeIndex, true), + Optional.empty(), + new Flavor(resources), + Optional.empty(), + NodeType.host); + hosts.add(host); + hostIndex++; + } + hosts = nodeRepository.addNodes(hosts, Agent.system); + hosts = nodeRepository.setReady(hosts, Agent.system, "Test"); + var transaction = new NestedTransaction(); + nodeRepository.activate(hosts, transaction); + transaction.commit(); + } + + private void addNodes(int id, int count, NodeResources resources, int hostOffset) { + List<Node> nodes = new ArrayList<>(); + ApplicationId application = ApplicationId.from("tenant" + id, "application" + id, "default"); + for (int i = 0; i < count; i++) { + ClusterMembership membership = ClusterMembership.from(ClusterSpec.specification(ClusterSpec.Type.content, ClusterSpec.Id.from("cluster" + id)) + .group(ClusterSpec.Group.from(0)) + .vespaVersion("7") + .build(), + i); + Node node = nodeRepository.createNode("node" + nodeIndex, + "node" + nodeIndex + ".yahoo.com", + ipConfig(hostIndex + nodeIndex, false), + Optional.of("host" + ( hostOffset + i) + ".yahoo.com"), + new Flavor(resources), + Optional.empty(), + NodeType.tenant); + node = node.allocate(application, membership, node.resources(), Instant.now()); + nodes.add(node); + nodeIndex++; + } + nodes = nodeRepository.addNodes(nodes, Agent.system); + for (int i = 0; i < count; i++) { + Node node = nodes.get(i); + ClusterMembership membership = ClusterMembership.from(ClusterSpec.specification(ClusterSpec.Type.content, ClusterSpec.Id.from("cluster" + id)) + .group(ClusterSpec.Group.from(0)) + .vespaVersion("7") + .build(), + i); + node = node.allocate(application, membership, node.resources(), Instant.now()); + nodes.set(i, node); + } + nodes = nodeRepository.reserve(nodes); + var transaction = new NestedTransaction(); + nodes = nodeRepository.activate(nodes, transaction); + transaction.commit(); + } + + private IP.Config ipConfig(int id, boolean host) { + return new IP.Config(Set.of(String.format("%04X::%04X", id, 0)), + host ? IntStream.range(0, 10) + .mapToObj(n -> String.format("%04X::%04X", id, n)) + .collect(Collectors.toSet()) + : Set.of()); + } + + private void dumpState() { + for (Node host : nodeRepository.list().hosts().asList()) { + System.out.println("Host " + host.hostname() + " " + host.resources()); + for (Node node : nodeRepository.list().childrenOf(host).asList()) + System.out.println(" Node " + node.hostname() + " " + node.resources() + " allocation " +node.allocation()); + } + } + + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 5f4bde85c88..914008af227 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -1,9 +1,11 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -12,6 +14,8 @@ import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -31,43 +35,42 @@ import static org.junit.Assert.fail; public class OsVersionsTest { private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); - private final ApplicationId infraApplication = ApplicationId.from("hosted-vespa", "tenant-host", "default"); + private final ApplicationId infraApplication = ApplicationId.from("hosted-vespa", "infra", "default"); @Test - public void test_versions() { - var versions = new OsVersions(tester.nodeRepository(), Integer.MAX_VALUE); - tester.makeReadyNodes(10, "default", NodeType.host); - tester.prepareAndActivateInfraApplication(infraApplication, NodeType.host); + public void versions() { + var versions = new OsVersions(tester.nodeRepository(), new DelegatingUpgrader(tester.nodeRepository(), Integer.MAX_VALUE)); + provisionInfraApplication(10); Supplier<List<Node>> hostNodes = () -> tester.nodeRepository().getNodes(NodeType.host); // Upgrade OS - assertTrue("No versions set", versions.targets().isEmpty()); + assertTrue("No versions set", versions.readChange().targets().isEmpty()); var version1 = Version.fromString("7.1"); - versions.setTarget(NodeType.host, version1, false); + versions.setTarget(NodeType.host, version1, Optional.empty(), false); assertEquals(version1, versions.targetFor(NodeType.host).get()); assertTrue("Per-node wanted OS version remains unset", hostNodes.get().stream().allMatch(node -> node.status().osVersion().wanted().isEmpty())); // Upgrade OS again var version2 = Version.fromString("7.2"); - versions.setTarget(NodeType.host, version2, false); + versions.setTarget(NodeType.host, version2, Optional.empty(), false); assertEquals(version2, versions.targetFor(NodeType.host).get()); // Target can be (de)activated - versions.setActive(NodeType.host, true); + versions.resumeUpgradeOf(NodeType.host, true); assertTrue("Target version activated", hostNodes.get().stream() .allMatch(node -> node.status().osVersion().wanted().isPresent())); - versions.setActive(NodeType.host, false); + versions.resumeUpgradeOf(NodeType.host, false); assertTrue("Target version deactivated", hostNodes.get().stream() .allMatch(node -> node.status().osVersion().wanted().isEmpty())); // Downgrading fails try { - versions.setTarget(NodeType.host, version1, false); + versions.setTarget(NodeType.host, version1, Optional.empty(), false); fail("Expected exception"); } catch (IllegalArgumentException ignored) {} // Forcing downgrade succeeds - versions.setTarget(NodeType.host, version1, true); + versions.setTarget(NodeType.host, version1, Optional.empty(), true); assertEquals(version1, versions.targetFor(NodeType.host).get()); // Target can be removed @@ -77,13 +80,12 @@ public class OsVersionsTest { } @Test - public void test_max_active_upgrades() { + public void max_active_upgrades() { int totalNodes = 20; int maxActiveUpgrades = 5; - var versions = new OsVersions(tester.nodeRepository(), maxActiveUpgrades); - tester.makeReadyNodes(totalNodes, "default", NodeType.host); + var versions = new OsVersions(tester.nodeRepository(), new DelegatingUpgrader(tester.nodeRepository(), maxActiveUpgrades)); + provisionInfraApplication(totalNodes); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().list().state(Node.State.active).nodeType(NodeType.host); - tester.prepareAndActivateInfraApplication(infraApplication, NodeType.host); // 5 nodes have no version. The other 15 are spread across different versions var hostNodesList = hostNodes.get().asList(); @@ -100,12 +102,12 @@ public class OsVersionsTest { // Set target var version1 = Version.fromString("7.1"); - versions.setTarget(NodeType.host, version1, false); + versions.setTarget(NodeType.host, version1, Optional.empty(), false); assertEquals(version1, versions.targetFor(NodeType.host).get()); // Activate target for (int i = 0; i < totalNodes; i += maxActiveUpgrades) { - versions.setActive(NodeType.host, true); + versions.resumeUpgradeOf(NodeType.host, true); var nodes = hostNodes.get(); var nodesUpgrading = nodes.changingOsVersion(); assertEquals("Target is changed for a subset of nodes", maxActiveUpgrades, nodesUpgrading.size()); @@ -121,15 +123,14 @@ public class OsVersionsTest { } // Activating again after all nodes have upgraded does nothing - versions.setActive(NodeType.host, true); + versions.resumeUpgradeOf(NodeType.host, true); assertEquals("All nodes upgraded", version1, minVersion(hostNodes.get(), OsVersion::current)); } @Test - public void test_newer_upgrade_aborts_upgrade_to_stale_version() { - var versions = new OsVersions(tester.nodeRepository(), Integer.MAX_VALUE); - tester.makeReadyNodes(10, "default", NodeType.host); - tester.prepareAndActivateInfraApplication(infraApplication, NodeType.host); + public void newer_upgrade_aborts_upgrade_to_stale_version() { + var versions = new OsVersions(tester.nodeRepository(), new DelegatingUpgrader(tester.nodeRepository(), Integer.MAX_VALUE)); + provisionInfraApplication(10); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().list().nodeType(NodeType.host); // Some nodes are targeting an older version @@ -138,13 +139,118 @@ public class OsVersionsTest { // Trigger upgrade to next version var version2 = Version.fromString("7.2"); - versions.setTarget(NodeType.host, version2, false); - versions.setActive(NodeType.host, true); + versions.setTarget(NodeType.host, version2, Optional.empty(), false); + versions.resumeUpgradeOf(NodeType.host, true); // Wanted version is changed to newest target for all nodes assertEquals(version2, minVersion(hostNodes.get(), OsVersion::wanted)); } + @Test + public void upgrade_by_retiring() { + var versions = new OsVersions(tester.nodeRepository(), new RetiringUpgrader(tester.nodeRepository())); + var clock = (ManualClock) tester.nodeRepository().clock(); + int hostCount = 10; + // Provision hosts and children + List<Node> hosts = provisionInfraApplication(hostCount); + NodeResources resources = new NodeResources(2, 4, 8, 1); + for (var host : hosts) { + tester.makeReadyVirtualDockerNodes(2, resources, host.hostname()); + } + Supplier<NodeList> hostNodes = () -> tester.nodeRepository().list() + .nodeType(NodeType.host) + .not().state(Node.State.deprovisioned); + + // Target is set and upgrade started + var version1 = Version.fromString("7.1"); + Duration totalBudget = Duration.ofHours(12); + Duration nodeBudget = totalBudget.dividedBy(hostCount); + versions.setTarget(NodeType.host, version1, Optional.of(totalBudget),false); + versions.resumeUpgradeOf(NodeType.host, true); + + // One host is deprovisioning + assertEquals(1, hostNodes.get().deprovisioning().size()); + + // Nothing happens on next resume as first host has not spent its budget + versions.resumeUpgradeOf(NodeType.host, true); + NodeList nodesDeprovisioning = hostNodes.get().deprovisioning(); + assertEquals(1, nodesDeprovisioning.size()); + assertEquals(2, retiringChildrenOf(nodesDeprovisioning.asList().get(0)).size()); + + // Budget has been spent and another host is retired + clock.advance(nodeBudget); + versions.resumeUpgradeOf(NodeType.host, true); + assertEquals(2, hostNodes.get().deprovisioning().size()); + + // Two nodes complete their upgrade by being reprovisioned + completeUpgradeOf(hostNodes.get().deprovisioning().asList()); + assertEquals(2, hostNodes.get().onOsVersion(version1).size()); + // The remaining hosts complete their upgrade + for (int i = 0; i < hostCount - 2; i++) { + clock.advance(nodeBudget); + versions.resumeUpgradeOf(NodeType.host, true); + nodesDeprovisioning = hostNodes.get().deprovisioning(); + assertEquals(1, nodesDeprovisioning.size()); + assertEquals(2, retiringChildrenOf(nodesDeprovisioning.asList().get(0)).size()); + completeUpgradeOf(nodesDeprovisioning.asList()); + } + + // All hosts upgraded and none are deprovisioning + assertEquals(hostCount, hostNodes.get().onOsVersion(version1).not().deprovisioning().size()); + assertEquals(hostCount, tester.nodeRepository().list().state(Node.State.deprovisioned).size()); + var lastRetiredAt = clock.instant().truncatedTo(ChronoUnit.MILLIS); + + // Resuming after everything has upgraded does nothing + versions.resumeUpgradeOf(NodeType.host, true); + assertEquals(0, hostNodes.get().deprovisioning().size()); + + // Another upgrade is triggered. Last retirement time is preserved + clock.advance(Duration.ofDays(1)); + var version2 = Version.fromString("7.2"); + versions.setTarget(NodeType.host, version2, Optional.of(totalBudget), false); + assertEquals(lastRetiredAt, versions.readChange().targets().get(NodeType.host).lastRetiredAt().get()); + } + + @Test + public void upgrade_by_retiring_everything_at_once() { + var versions = new OsVersions(tester.nodeRepository(), new RetiringUpgrader(tester.nodeRepository())); + int hostCount = 3; + provisionInfraApplication(hostCount, NodeType.confighost); + Supplier<NodeList> hostNodes = () -> tester.nodeRepository().list() + .nodeType(NodeType.confighost) + .not().state(Node.State.deprovisioned); + + // Target is set with zero budget and upgrade started + var version1 = Version.fromString("7.1"); + versions.setTarget(NodeType.confighost, version1, Optional.of(Duration.ZERO),false); + for (int i = 0; i < hostCount; i++) { + versions.resumeUpgradeOf(NodeType.confighost, true); + } + + // All hosts are deprovisioning + assertEquals(hostCount, hostNodes.get().deprovisioning().size()); + // Nodes complete their upgrade by being reprovisioned + completeUpgradeOf(hostNodes.get().deprovisioning().asList(), NodeType.confighost); + assertEquals(hostCount, hostNodes.get().onOsVersion(version1).size()); + } + + private NodeList retiringChildrenOf(Node parent) { + return tester.nodeRepository().list().childrenOf(parent).matching(child -> child.status().wantToRetire()); + } + + private List<Node> provisionInfraApplication(int nodeCount) { + return provisionInfraApplication(nodeCount, NodeType.host); + } + + private List<Node> provisionInfraApplication(int nodeCount, NodeType nodeType) { + var nodes = tester.makeReadyNodes(nodeCount, "default", nodeType); + tester.prepareAndActivateInfraApplication(infraApplication, nodeType); + return nodes.stream() + .map(Node::hostname) + .flatMap(hostname -> tester.nodeRepository().getNode(hostname).stream()) + .collect(Collectors.toList()); + } + private Version minVersion(NodeList nodes, Function<OsVersion, Optional<Version>> versionField) { return nodes.asList().stream() .map(Node::status) @@ -174,13 +280,21 @@ public class OsVersionsTest { } private void completeUpgradeOf(List<Node> nodes) { - for (var node : nodes) { - try (var lock = tester.nodeRepository().lock(node)) { - node = tester.nodeRepository().getNode(node.hostname()).get(); - node = node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(node.status().osVersion().wanted()))); - tester.nodeRepository().write(node, lock); + completeUpgradeOf(nodes, NodeType.host); + } + + private void completeUpgradeOf(List<Node> nodes, NodeType nodeType) { + writeNode(nodes, (node) -> { + Optional<Version> wantedOsVersion = node.status().osVersion().wanted(); + if (node.status().wantToDeprovision()) { + // Complete upgrade by deprovisioning stale hosts and provisioning new ones + tester.nodeRepository().park(node.hostname(), false, Agent.system, + OsVersionsTest.class.getSimpleName()); + tester.nodeRepository().removeRecursively(node.hostname()); + node = provisionInfraApplication(1, nodeType).get(0); } - } + return node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(wantedOsVersion))); + }); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index c50805eebb8..598831d1eeb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -26,11 +26,13 @@ public class ApplicationSerializerTest { public void testApplicationSerialization() { List<Cluster> clusters = new ArrayList<>(); clusters.add(new Cluster(ClusterSpec.Id.from("c1"), + false, new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), new ClusterResources(12, 6, new NodeResources(3, 6, 21, 24)), Optional.empty(), Optional.empty())); clusters.add(new Cluster(ClusterSpec.Id.from("c2"), + true, new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), new ClusterResources(14, 7, new NodeResources(3, 6, 21, 24)), Optional.of(new ClusterResources(20, 10, new NodeResources(0.5, 4, 14, 16))), @@ -49,6 +51,7 @@ public class ApplicationSerializerTest { assertNotSame(originalCluster, serializedCluster); assertEquals(originalCluster, serializedCluster); assertEquals(originalCluster.id(), serializedCluster.id()); + assertEquals(originalCluster.exclusive(), serializedCluster.exclusive()); assertEquals(originalCluster.minResources(), serializedCluster.minResources()); assertEquals(originalCluster.maxResources(), serializedCluster.maxResources()); assertEquals(originalCluster.suggestedResources(), serializedCluster.suggestedResources()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 15c73d15e8a..f3fe1fc4915 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -275,12 +275,12 @@ public class NodeSerializerTest { @Test public void flavor_overrides_serialization() { Node node = createNode(); - assertEquals(2, node.flavor().getMinDiskAvailableGb(), 0); + assertEquals(20, node.flavor().resources().diskGb(), 0); node = node.with(node.flavor().with(FlavorOverrides.ofDisk(1234))); - assertEquals(1234, node.flavor().getMinDiskAvailableGb(), 0); + assertEquals(1234, node.flavor().resources().diskGb(), 0); Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); - assertEquals(1234, copy.flavor().getMinDiskAvailableGb(), 0); + assertEquals(1234, copy.flavor().resources().diskGb(), 0); assertEquals(node, copy); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionChangeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionChangeSerializerTest.java new file mode 100644 index 00000000000..15b9cfec4ca --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionChangeSerializerTest.java @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.os.OsVersionChange; +import com.yahoo.vespa.hosted.provision.os.OsVersionTarget; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class OsVersionChangeSerializerTest { + + @Test + public void serialization() { + var change = new OsVersionChange(Map.of( + NodeType.host, new OsVersionTarget(NodeType.host, Version.fromString("1.2.3"), Optional.of(Duration.ofHours(1)), Optional.of(Instant.ofEpochMilli(123))), + NodeType.proxyhost, new OsVersionTarget(NodeType.proxyhost, Version.fromString("4.5.6"), Optional.empty(), Optional.empty()), + NodeType.confighost, new OsVersionTarget(NodeType.confighost, Version.fromString("7.8.9"), Optional.of(Duration.ZERO), Optional.of(Instant.ofEpochMilli(456))) + )); + var serialized = OsVersionChangeSerializer.fromJson(OsVersionChangeSerializer.toJson(change)); + assertEquals(serialized, change); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializerTest.java deleted file mode 100644 index 36dbf26c0d3..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializerTest.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.persistence; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.NodeType; -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * @author mpolden - */ -public class OsVersionsSerializerTest { - - @Test - public void serialization() { - var versions = Map.of( - NodeType.host, Version.fromString("1.2.3"), - NodeType.proxyhost, Version.fromString("4.5.6"), - NodeType.confighost, Version.fromString("7.8.9") - ); - var serialized = OsVersionsSerializer.fromJson(OsVersionsSerializer.toJson(versions)); - assertEquals(serialized, versions); - } - - @Test - public void ignores_unknown_keys() { - var jsonWithUnknownKeys = "{\n" + - " \"foo\": \"bar\",\n" + - " " + - "\"host\": {\n" + - " \"version\": \"1.2.3\"\n" + - " },\n" + - " " + - "\"proxyhost\": {\n" + - " \"version\": \"4.5.6\"\n" + - " },\n" + - " " + - "\"confighost\": {\n" + - " \"version\": \"7.8.9\"\n" + - " }\n" + - "}"; - var versions = Map.of( - NodeType.host, Version.fromString("1.2.3"), - NodeType.proxyhost, Version.fromString("4.5.6"), - NodeType.confighost, Version.fromString("7.8.9") - ); - assertEquals(versions, OsVersionsSerializer.fromJson(jsonWithUnknownKeys.getBytes(StandardCharsets.UTF_8))); - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java index fd2374c55b7..644b2338a5a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java @@ -102,13 +102,13 @@ public class AllocationVisualizer extends JPanel { if (isHost) { g.setColor(Color.GRAY); - for (int i = 0; i < node.flavor().getMinMainMemoryAvailableGb(); i++) { + for (int i = 0; i < node.resources().memoryGb(); i++) { g.fillRect(x, y - nodeHeight, nodeWidth, nodeHeight); y = y - (nodeHeight + 2); } } else { g.setColor(Color.YELLOW); - int multi = (int) node.flavor().getMinMainMemoryAvailableGb(); + int multi = (int) node.resources().memoryGb(); int height = multi * nodeHeight + ((multi - 1) * 2); g.fillRect(x, y - height, nodeWidth, height); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java index 5079fce4418..0c5a682c3c5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; @@ -39,8 +40,8 @@ import static org.junit.Assert.fail; */ public class DockerProvisioningTest { - private static final NodeResources dockerFlavor = new NodeResources(1, 4, 10, 1, - NodeResources.DiskSpeed.fast, NodeResources.StorageType.local); + private static final NodeResources dockerResources = new NodeResources(1, 4, 100, 1, + NodeResources.DiskSpeed.fast, NodeResources.StorageType.local); @Test public void docker_application_deployment() { @@ -48,28 +49,28 @@ public class DockerProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); for (int i = 1; i < 10; i++) - tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost" + i); + tester.makeReadyVirtualDockerNodes(1, dockerResources, "dockerHost" + i); Version wantedVespaVersion = Version.fromString("6.39"); int nodeCount = 7; List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(), - nodeCount, 1, dockerFlavor); + nodeCount, 1, dockerResources); tester.activate(application1, new HashSet<>(hosts)); NodeList nodes = tester.getNodes(application1, Node.State.active); assertEquals(nodeCount, nodes.size()); - assertEquals(dockerFlavor, nodes.asList().get(0).flavor().resources()); + assertEquals(dockerResources, nodes.asList().get(0).resources()); // Upgrade Vespa version on nodes Version upgradedWantedVespaVersion = Version.fromString("6.40"); List<HostSpec> upgradedHosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(upgradedWantedVespaVersion).build(), - nodeCount, 1, dockerFlavor); + nodeCount, 1, dockerResources); tester.activate(application1, new HashSet<>(upgradedHosts)); NodeList upgradedNodes = tester.getNodes(application1, Node.State.active); assertEquals(nodeCount, upgradedNodes.size()); - assertEquals(dockerFlavor, upgradedNodes.asList().get(0).flavor().resources()); + assertEquals(dockerResources, upgradedNodes.asList().get(0).resources()); assertEquals(hosts, upgradedHosts); } @@ -78,16 +79,16 @@ public class DockerProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId zoneApplication = tester.makeApplicationId(); - List<Node> parents = tester.makeReadyNodes(10, new NodeResources(2, 2, 2, 2), NodeType.host, 1); + List<Node> parents = tester.makeReadyNodes(10, new NodeResources(2, 4, 20, 2), NodeType.host, 1); for (Node parent : parents) - tester.makeReadyVirtualDockerNodes(1, dockerFlavor, parent.hostname()); + tester.makeReadyVirtualDockerNodes(1, dockerResources, parent.hostname()); ApplicationId application1 = tester.makeApplicationId(); Version wantedVespaVersion = Version.fromString("6.39"); int nodeCount = 7; List<HostSpec> nodes = tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(), - nodeCount, 1, dockerFlavor); + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(), + nodeCount, 1, dockerResources); try { tester.activate(application1, new HashSet<>(nodes)); fail("Expected the allocation to fail due to parent hosts not being active yet"); @@ -101,8 +102,8 @@ public class DockerProvisioningTest { // Try allocating tenants again nodes = tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(), - nodeCount, 1, dockerFlavor); + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(), + nodeCount, 1, dockerResources); tester.activate(application1, new HashSet<>(nodes)); NodeList activeNodes = tester.getNodes(application1, Node.State.active); @@ -111,7 +112,7 @@ public class DockerProvisioningTest { @Test public void reservations_are_respected() { - NodeResources resources = new NodeResources(10, 10, 10, 10); + NodeResources resources = new NodeResources(10, 10, 100, 10); ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); TenantName tenant1 = TenantName.from("tenant1"); TenantName tenant2 = TenantName.from("tenant2"); @@ -153,13 +154,13 @@ public class DockerProvisioningTest { public void docker_application_deployment_with_exclusive_app_first() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); for (int i = 1; i <= 4; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host1"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host1"); for (int i = 5; i <= 8; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host2"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host2"); for (int i = 9; i <= 12; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host3"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host3"); for (int i = 13; i <= 16; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host4"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host4"); ApplicationId application1 = tester.makeApplicationId(); prepareAndActivate(application1, 2, true, tester); @@ -176,13 +177,13 @@ public class DockerProvisioningTest { public void docker_application_deployment_with_exclusive_app_last() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); for (int i = 1; i <= 4; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host1"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host1"); for (int i = 5; i <= 8; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host2"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host2"); for (int i = 9; i <= 12; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host3"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host3"); for (int i = 13; i <= 16; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host4"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host4"); ApplicationId application1 = tester.makeApplicationId(); prepareAndActivate(application1, 2, false, tester); @@ -199,13 +200,13 @@ public class DockerProvisioningTest { public void docker_application_deployment_change_to_exclusive_and_back() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); for (int i = 1; i <= 4; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host1"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host1"); for (int i = 5; i <= 8; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host2"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host2"); for (int i = 9; i <= 12; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host3"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host3"); for (int i = 13; i <= 16; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host4"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host4"); ApplicationId application1 = tester.makeApplicationId(); prepareAndActivate(application1, 2, false, tester); @@ -228,13 +229,13 @@ public class DockerProvisioningTest { public void docker_application_deployment_with_exclusive_app_causing_allocation_failure() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); for (int i = 1; i <= 4; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host1"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host1"); for (int i = 5; i <= 8; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host2"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host2"); for (int i = 9; i <= 12; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host3"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host3"); for (int i = 13; i <= 16; i++) - tester.makeReadyVirtualDockerNode(i, dockerFlavor, "host4"); + tester.makeReadyVirtualDockerNode(i, dockerResources, "host4"); ApplicationId application1 = tester.makeApplicationId(); prepareAndActivate(application1, 2, true, tester); @@ -247,7 +248,12 @@ public class DockerProvisioningTest { } catch (Exception e) { assertEquals("No room for 3 nodes as 2 of 4 hosts are exclusive", - "Could not satisfy request for 3 nodes with [vcpu: 1.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps, storage type: local] for container cluster 'myContainer' group 0 6.39 in tenant1.app1: Not enough nodes available due to host exclusivity constraints.", + "Could not satisfy request for 3 nodes with " + + "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local] " + + "in tenant1.app1 container cluster 'myContainer' 6.39: " + + "Out of capacity on group 0: " + + "Not enough nodes available due to host exclusivity constraints, " + + "insufficient nodes available on separate physical hosts", e.getMessage()); } @@ -261,16 +267,16 @@ public class DockerProvisioningTest { public void get_specified_flavor_not_default_flavor_for_docker() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("corp-us-east-1"))).build(); ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost"); + tester.makeReadyVirtualDockerNodes(1, dockerResources, "dockerHost"); List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion("6.42").build(), - 1, 1, dockerFlavor); + 1, 1, dockerResources); tester.activate(application1, new HashSet<>(hosts)); NodeList nodes = tester.getNodes(application1, Node.State.active); assertEquals(1, nodes.size()); - assertEquals("[vcpu: 1.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", nodes.asList().get(0).flavor().name()); + assertEquals("[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", nodes.asList().get(0).flavor().name()); } @Test @@ -278,17 +284,65 @@ public class DockerProvisioningTest { try { ProvisioningTester tester = new ProvisioningTester.Builder() .zone(new Zone(Environment.prod, RegionName.from("us-east-1"))).build(); - ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost1"); - tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost2"); - - List<HostSpec> hosts = tester.prepare(application1, - ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion("6.42").build(), - 2, 1, - dockerFlavor.with(NodeResources.StorageType.remote)); + ApplicationId application1 = tester.makeApplicationId("app1"); + tester.makeReadyVirtualDockerNodes(1, dockerResources, "dockerHost1"); + tester.makeReadyVirtualDockerNodes(1, dockerResources, "dockerHost2"); + + tester.prepare(application1, + ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion("6.42").build(), + 2, 1, + dockerResources.with(NodeResources.StorageType.remote)); } catch (OutOfCapacityException e) { - assertTrue(e.getMessage().startsWith("Could not satisfy request for 2 nodes with [vcpu: 1.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps, storage type: remote]")); + assertEquals("Could not satisfy request for 2 nodes with " + + "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote] " + + "in tenant.app1 content cluster 'myContent'" + + " 6.42: Out of capacity on group 0", + e.getMessage()); + } + } + + @Test + public void initial_allocation_is_within_limits() { + Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 100, 4)); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) + .resourcesCalculator(3, 0) + .flavors(List.of(hostFlavor)) + .build(); + tester.makeReadyHosts(2, hostFlavor.resources()).deployZoneApp(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + var resources = new NodeResources(1, 8, 10, 1); + tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, resources), + new ClusterResources(4, 1, resources))); + tester.assertNodes("Initial allocation at min with default resources", + 2, 1, 1, 8, 10, 1.0, + app1, cluster1); + } + + @Test + public void too_few_real_resources_causes_failure() { + try { + Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 100, 4)); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) + .resourcesCalculator(3, 0) + .flavors(List.of(hostFlavor)) + .build(); + tester.makeReadyHosts(2, hostFlavor.resources()).deployZoneApp(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + // 5 Gb requested memory becomes 5-3=2 Gb real memory, which is an illegally small amount + var resources = new NodeResources(1, 5, 10, 1); + tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, resources), + new ClusterResources(4, 1, resources))); + } + catch (IllegalArgumentException e) { + assertEquals("No allocation possible within limits: from 2 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps] to 4 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps]", + e.getMessage()); } } @@ -299,7 +353,7 @@ public class DockerProvisioningTest { private void prepareAndActivate(ApplicationId application, int nodeCount, boolean exclusive, ProvisioningTester tester) { Set<HostSpec> hosts = new HashSet<>(tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer")).vespaVersion("6.39").exclusive(exclusive).build(), - Capacity.from(new ClusterResources(nodeCount, 1, dockerFlavor), false, true))); + Capacity.from(new ClusterResources(nodeCount, 1, dockerResources), false, true))); tester.activate(application, hosts); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 4eca0542992..98ec01e8e95 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -36,9 +36,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; /** @@ -66,7 +64,7 @@ public class DynamicDockerAllocationTest { tester.makeReadyNodes(4, "host-small", NodeType.host, 32); tester.deployZoneApp(); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - NodeResources flavor = new NodeResources(1, 4, 10, 1); + NodeResources flavor = new NodeResources(1, 4, 100, 1); // Application 1 ApplicationId application1 = makeApplicationId("t1", "a1"); @@ -109,7 +107,7 @@ public class DynamicDockerAllocationTest { tester.makeReadyNodes(5, "host-small", NodeType.host, 32); tester.deployZoneApp(); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - NodeResources resources = new NodeResources(1, 4, 10, 0.3); + NodeResources resources = new NodeResources(1, 4, 100, 0.3); // Application 1 ApplicationId application1 = makeApplicationId("t1", "a1"); @@ -157,9 +155,9 @@ public class DynamicDockerAllocationTest { tester.makeReadyNodes(3, "cpu", NodeType.host, 8); // cpu: 40, mem: 20 tester.makeReadyNodes(3, "mem", NodeType.host, 8); // cpu: 20, mem: 40 tester.deployZoneApp(); - NodeResources fltResources = new NodeResources(6, 6, 1, 0.1); - NodeResources cpuResources = new NodeResources(8, 4, 1, 0.1); - NodeResources memResources = new NodeResources(4, 8, 1, 0.1); + NodeResources fltResources = new NodeResources(6, 6, 10, 0.1); + NodeResources cpuResources = new NodeResources(8, 4, 10, 0.1); + NodeResources memResources = new NodeResources(4, 8, 10, 0.1); // Cpu heavy application ApplicationId application1 = makeApplicationId("t1", "a1"); @@ -201,7 +199,7 @@ public class DynamicDockerAllocationTest { tester.makeReadyNodes(2, "host-small", NodeType.host, 32); tester.deployZoneApp(); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); - NodeResources flavor = new NodeResources(1, 4, 10, 1); + NodeResources flavor = new NodeResources(1, 4, 100, 1); // Application 1 ApplicationId application1 = makeApplicationId("t1", "a1"); @@ -230,7 +228,7 @@ public class DynamicDockerAllocationTest { //Deploy an application having 6 nodes (3 nodes in 2 groups). We only have 5 docker hosts available ApplicationId application1 = tester.makeApplicationId(); - tester.prepare(application1, clusterSpec("myContent.t1.a1"), 6, 2, new NodeResources(1, 4, 10, 1)); + tester.prepare(application1, clusterSpec("myContent.t1.a1"), 6, 2, new NodeResources(1, 4, 100, 1)); fail("Two groups have been allocated to the same parent host"); } @@ -248,7 +246,7 @@ public class DynamicDockerAllocationTest { ApplicationId application1 = tester.makeApplicationId(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); tester.deployZoneApp(); - NodeResources flavor = new NodeResources(1, 4, 10, 1); + NodeResources flavor = new NodeResources(1, 4, 100, 1); // Deploy initial state (can max deploy 3 nodes due to redundancy requirements) ClusterSpec clusterSpec = clusterSpec("myContent.t1.a1"); @@ -256,7 +254,7 @@ public class DynamicDockerAllocationTest { tester.activate(application1, ImmutableSet.copyOf(hosts)); List<Node> initialSpareCapacity = findSpareCapacity(tester); - assertThat(initialSpareCapacity.size(), is(2)); + assertEquals(2, initialSpareCapacity.size()); try { hosts = tester.prepare(application1, clusterSpec, 4, 1, flavor); @@ -269,7 +267,7 @@ public class DynamicDockerAllocationTest { List<Node> finalSpareCapacity = findSpareCapacity(tester); - assertThat(finalSpareCapacity.size(), is(1)); + assertEquals(1, finalSpareCapacity.size()); } @Test @@ -278,7 +276,7 @@ public class DynamicDockerAllocationTest { tester.makeReadyNodes(3, "host-small", NodeType.host, 32); tester.deployZoneApp(); ApplicationId application1 = tester.makeApplicationId(); - List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 10, 1)); + List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1)); tester.activate(application1, ImmutableSet.copyOf(hosts)); List<Node> initialSpareCapacity = findSpareCapacity(tester); @@ -288,10 +286,10 @@ public class DynamicDockerAllocationTest { @Test public void cd_uses_slow_disk_nodes_for_docker_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(SystemName.cd, Environment.test, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); - tester.makeReadyNodes(4, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); + tester.makeReadyNodes(4, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); tester.deployZoneApp(); ApplicationId application1 = tester.makeApplicationId(); - List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 10, 1)); + List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1)); tester.activate(application1, ImmutableSet.copyOf(hosts)); } @@ -302,7 +300,7 @@ public class DynamicDockerAllocationTest { tester.nodeRepository().fail(node.hostname(), Agent.system, getClass().getSimpleName())); ApplicationId application = tester.makeApplicationId(); - tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, new NodeResources(1, 4, 10, 1)); + tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, new NodeResources(1, 40, 100, 1)); } @Test @@ -312,7 +310,7 @@ public class DynamicDockerAllocationTest { tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); - List<HostSpec> hosts = tester.prepare(application, clusterSpec("myContent.t1.a1"), 2, 1, new NodeResources(1, 4, 10, 1)); + List<HostSpec> hosts = tester.prepare(application, clusterSpec("myContent.t1.a1"), 2, 1, new NodeResources(1, 4, 100, 1)); tester.activate(application, hosts); List<Node> activeNodes = tester.nodeRepository().getNodes(application); @@ -338,30 +336,30 @@ public class DynamicDockerAllocationTest { @Test public void slow_disk_nodes_are_preferentially_allocated() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); - tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); - tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); + tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); + tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); - NodeResources resources = new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any); + NodeResources resources = new NodeResources(1, 4, 100, 1, NodeResources.DiskSpeed.any); List<HostSpec> hosts = tester.prepare(application, cluster, 2, 1, resources); assertEquals(2, hosts.size()); - assertEquals(NodeResources.DiskSpeed.slow, hosts.get(0).flavor().get().resources().diskSpeed()); - assertEquals(NodeResources.DiskSpeed.slow, hosts.get(1).flavor().get().resources().diskSpeed()); + assertEquals(NodeResources.DiskSpeed.slow, hosts.get(0).advertisedResources().diskSpeed()); + assertEquals(NodeResources.DiskSpeed.slow, hosts.get(1).advertisedResources().diskSpeed()); tester.activate(application, hosts); } private void provisionFastAndSlowThenDeploy(NodeResources.DiskSpeed requestDiskSpeed, boolean expectOutOfCapacity) { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); - tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); - tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); + tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); + tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); - NodeResources resources = new NodeResources(1, 4, 10, 1, requestDiskSpeed); + NodeResources resources = new NodeResources(1, 4, 100, 1, requestDiskSpeed); try { List<HostSpec> hosts = tester.prepare(application, cluster, 4, 1, resources); @@ -377,35 +375,35 @@ public class DynamicDockerAllocationTest { @Test public void nodeResourcesAreRelaxedInDev() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); - tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); - tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); + tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); + tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); - NodeResources resources = new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.fast); + NodeResources resources = new NodeResources(1, 4, 100, 1, NodeResources.DiskSpeed.fast); List<HostSpec> hosts = tester.prepare(application, cluster, 4, 1, resources); assertEquals(1, hosts.size()); tester.activate(application, hosts); - assertEquals(0.1, hosts.get(0).flavor().get().resources().vcpu(), 0.000001); + assertEquals(0.1, hosts.get(0).advertisedResources().vcpu(), 0.000001); assertEquals("Slow nodes are allowed in dev and preferred because they are cheaper", - NodeResources.DiskSpeed.slow, hosts.get(0).flavor().get().resources().diskSpeed()); + NodeResources.DiskSpeed.slow, hosts.get(0).advertisedResources().diskSpeed()); } @Test public void testSwitchingFromLegacyFlavorSyntaxToResourcesDoesNotCauseReallocation() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); - tester.makeReadyNodes(2, new Flavor(new NodeResources(5, 20, 140, 3)), NodeType.host, 10, true); + tester.makeReadyNodes(2, new Flavor(new NodeResources(5, 20, 1400, 3)), NodeType.host, 10, true); tester.deployZoneApp(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build(); - List<HostSpec> hosts1 = tester.prepare(application, cluster, Capacity.from(new ClusterResources(2, 1, NodeResources.fromLegacyName("d-2-8-50")), false, true)); + List<HostSpec> hosts1 = tester.prepare(application, cluster, Capacity.from(new ClusterResources(2, 1, NodeResources.fromLegacyName("d-2-8-500")), false, true)); tester.activate(application, hosts1); - NodeResources resources = new NodeResources(1.5, 8, 50, 0.3); + NodeResources resources = new NodeResources(1.5, 8, 500, 0.3); List<HostSpec> hosts2 = tester.prepare(application, cluster, Capacity.from(new ClusterResources(2, 1, resources))); tester.activate(application, hosts2); @@ -427,7 +425,7 @@ public class DynamicDockerAllocationTest { ); ClusterMembership clusterMembership1 = ClusterMembership.from( clusterSpec.with(Optional.of(ClusterSpec.Group.from(0))), index); // Need to add group here so that group is serialized in node allocation - Node node1aAllocation = node1a.allocate(id, clusterMembership1, node1a.flavor().resources(), Instant.now()); + Node node1aAllocation = node1a.allocate(id, clusterMembership1, node1a.resources(), Instant.now()); tester.nodeRepository().addNodes(Collections.singletonList(node1aAllocation), Agent.system); NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(tester.getCurator())); @@ -446,11 +444,11 @@ public class DynamicDockerAllocationTest { private FlavorsConfig flavorsConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); - b.addFlavor("host-large", 6, 24, 80, 6, Flavor.Type.BARE_METAL); - b.addFlavor("host-small", 3, 12, 40, 3, Flavor.Type.BARE_METAL); - b.addFlavor("flt", 30, 30, 40, 3, Flavor.Type.BARE_METAL); - b.addFlavor("cpu", 40, 20, 40, 3, Flavor.Type.BARE_METAL); - b.addFlavor("mem", 20, 40, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("host-large", 6, 24, 800, 6, Flavor.Type.BARE_METAL); + b.addFlavor("host-small", 3, 12, 400, 3, Flavor.Type.BARE_METAL); + b.addFlavor("flt", 30, 30, 400, 3, Flavor.Type.BARE_METAL); + b.addFlavor("cpu", 40, 20, 400, 3, Flavor.Type.BARE_METAL); + b.addFlavor("mem", 20, 40, 400, 3, Flavor.Type.BARE_METAL); return b.build(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 59507f277e4..db6d75d724e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -19,7 +19,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; @@ -33,7 +32,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; -import static com.yahoo.config.provision.NodeResources.DiskSpeed.any; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.StorageType.local; import static com.yahoo.config.provision.NodeResources.StorageType.remote; @@ -47,11 +45,12 @@ import static org.mockito.Mockito.verify; /** * @author freva + * @author bratseth */ public class DynamicDockerProvisionTest { private static final Zone zone = new Zone( - Cloud.defaultCloud().withDynamicProvisioning(true).withAllowHostSharing(false), + Cloud.builder().dynamicProvisioning(true).allowHostSharing(false).build(), SystemName.main, Environment.prod, RegionName.from("us-east")); @@ -159,7 +158,7 @@ public class DynamicDockerProvisionTest { } @Test - public void test_capacity_is_in_advertised_amounts_on_aws() { + public void test_capacity_is_in_advertised_amounts() { int memoryTax = 3; List<Flavor> flavors = List.of(new Flavor("2x", new NodeResources(2, 17, 200, 10, fast, remote))); @@ -168,7 +167,7 @@ public class DynamicDockerProvisionTest { .flavors(flavors) .hostProvisioner(new MockHostProvisioner(flavors, memoryTax)) .nameResolver(nameResolver) - .resourcesCalculator(new MockResourcesCalculator(memoryTax)) + .resourcesCalculator(memoryTax, 0) .build(); tester.deployZoneApp(); @@ -192,19 +191,19 @@ public class DynamicDockerProvisionTest { tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 2, 20, 40), resources(4, 1, 2, 20, 40))); tester.assertNodes("Allocation specifies memory in the advertised amount", - 3, 1, 2, 20, 40, + 2, 1, 2, 20, 40, app1, cluster1); // Redeploy the same tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 2, 20, 40), resources(4, 1, 2, 20, 40))); tester.assertNodes("Allocation specifies memory in the advertised amount", - 3, 1, 2, 20, 40, + 2, 1, 2, 20, 40, app1, cluster1); } @Test - public void test_changing_limits_on_aws() { + public void test_changing_limits() { int memoryTax = 3; List<Flavor> flavors = List.of(new Flavor("1x", new NodeResources(1, 10 - memoryTax, 100, 0.1, fast, remote)), new Flavor("2x", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, remote)), @@ -214,7 +213,7 @@ public class DynamicDockerProvisionTest { .flavors(flavors) .hostProvisioner(new MockHostProvisioner(flavors, memoryTax)) .nameResolver(nameResolver) - .resourcesCalculator(new MockResourcesCalculator(memoryTax)) + .resourcesCalculator(memoryTax, 0) .build(); tester.deployZoneApp(); @@ -251,33 +250,34 @@ public class DynamicDockerProvisionTest { tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 20), resources(6, 3, 3, 25, 25))); tester.assertNodes("New allocation at new max", - 6, 3, 2, 20, 25, + 6, 2, 2, 20, 25, app1, cluster1); - // Widening window lets us find a cheaper alternative + // Widening window does not change allocation tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 1, 5, 15), resources(8, 4, 4, 20, 30))); - tester.assertNodes("Cheaper allocation", - 8, 4, 1, 10, 25, + tester.assertNodes("No change", + 6, 2, 2, 20, 25, app1, cluster1); - // Changing group size - tester.activate(app1, cluster1, Capacity.from(resources(6, 3, 0.5, 5, 5), + // Force 1 more groups: Reducing to 2 nodes per group to preserve node count is rejected + // since it will reduce total group memory from 60 to 40. + tester.activate(app1, cluster1, Capacity.from(resources(6, 3, 0.5, 5, 10), resources(9, 3, 5, 20, 15))); - tester.assertNodes("Groups changed", - 6, 3, 1, 10, 15, + tester.assertNodes("Group size is preserved", + 9, 3, 2, 20, 15, app1, cluster1); // Stop specifying node resources - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(6, 3, NodeResources.unspecified), - new ClusterResources(9, 3, NodeResources.unspecified))); - tester.assertNodes("Minimal allocation", - 6, 3, 1, 10, 15, + tester.activate(app1, cluster1, Capacity.from(new ClusterResources(6, 3, NodeResources.unspecified()), + new ClusterResources(9, 3, NodeResources.unspecified()))); + tester.assertNodes("Existing allocation is preserved", + 9, 3, 2, 20, 15, app1, cluster1); } @Test - public void test_changing_storage_type_on_aws() { + public void test_changing_storage_type() { int memoryTax = 3; List<Flavor> flavors = List.of(new Flavor("2x", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, remote)), new Flavor("2xl", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, local)), @@ -288,7 +288,7 @@ public class DynamicDockerProvisionTest { .flavors(flavors) .hostProvisioner(new MockHostProvisioner(flavors, memoryTax)) .nameResolver(nameResolver) - .resourcesCalculator(new MockResourcesCalculator(memoryTax)) + .resourcesCalculator(memoryTax, 0) .build(); tester.deployZoneApp(); @@ -309,6 +309,34 @@ public class DynamicDockerProvisionTest { app1, cluster1); } + @Test + public void test_any_disk_prefers_remote() { + int memoryTax = 3; + int localDiskTax = 55; + // Disk tax is not included in flavor resources but memory tax is + List<Flavor> flavors = List.of(new Flavor("2x", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, local)), + new Flavor("4x", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, local)), + new Flavor("2xl", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, remote)), + new Flavor("4xl", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, remote))); + + ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone) + .flavors(flavors) + .hostProvisioner(new MockHostProvisioner(flavors, memoryTax)) + .nameResolver(nameResolver) + .resourcesCalculator(memoryTax, localDiskTax) + .build(); + + tester.deployZoneApp(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 200, fast, StorageType.any), + resources(6, 3, 3, 25, 400, fast, StorageType.any))); + tester.assertNodes("'any' selects a flavor with remote storage since it produces higher fulfilment", + 4, 2, 2, 20, 200, fast, remote, + app1, cluster1); + } private static ClusterSpec clusterSpec(String clusterId) { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId)).vespaVersion("6.42").build(); @@ -335,28 +363,6 @@ public class DynamicDockerProvisionTest { }).when(hostProvisioner).provisionHosts(any(), any(), any(), any()); } - private static class MockResourcesCalculator implements HostResourcesCalculator { - - private final int memoryTaxGb; - - public MockResourcesCalculator(int memoryTaxGb) { - this.memoryTaxGb = memoryTaxGb; - } - - @Override - public NodeResources realResourcesOf(Node node, NodeRepository nodeRepository) { - if (node.type() == NodeType.host) return node.flavor().resources(); - return node.flavor().resources().withMemoryGb(node.flavor().resources().memoryGb() - memoryTaxGb); - } - - @Override - public NodeResources advertisedResourcesOf(Flavor flavor) { - if ( ! flavor.isConfigured()) return flavor.resources(); - return flavor.resources().withMemoryGb(flavor.resources().memoryGb() + memoryTaxGb); - } - - } - private static class MockHostProvisioner implements HostProvisioner { private final List<Flavor> hostFlavors; @@ -377,12 +383,13 @@ public class DynamicDockerProvisionTest { .collect(Collectors.toList()); } - private boolean compatible(Flavor hostFlavor, NodeResources nodeResources) { - NodeResources resourcesToVerify = nodeResources.withMemoryGb(nodeResources.memoryGb() - memoryTaxGb); + private boolean compatible(Flavor hostFlavor, NodeResources resources) { + NodeResources resourcesToVerify = resources.withMemoryGb(resources.memoryGb() - memoryTaxGb); + if (hostFlavor.resources().storageType() == NodeResources.StorageType.remote - && hostFlavor.resources().diskGb() >= nodeResources.diskGb()) + && hostFlavor.resources().diskGb() >= resources.diskGb()) resourcesToVerify = resourcesToVerify.withDiskGb(hostFlavor.resources().diskGb()); - if (hostFlavor.resources().bandwidthGbps() >= nodeResources.bandwidthGbps()) + if (hostFlavor.resources().bandwidthGbps() >= resources.bandwidthGbps()) resourcesToVerify = resourcesToVerify.withBandwidthGbps(hostFlavor.resources().bandwidthGbps()); return hostFlavor.resources().compatibleWith(resourcesToVerify); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java index b3cda5bb3df..da78aff493e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java @@ -27,14 +27,14 @@ import static org.mockito.Mockito.mock; /** * @author smorgrav */ -public class DockerHostCapacityTest { +public class HostCapacityTest { private final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class); - private DockerHostCapacity capacity; + private HostCapacity capacity; private List<Node> nodes; private Node host1, host2, host3; - private final NodeResources resources1 = new NodeResources(1, 3, 2, 1.5); - private final NodeResources resources2 = new NodeResources(2, 4, 4, 0.5); + private final NodeResources resources1 = new NodeResources(1, 30, 20, 1.5); + private final NodeResources resources2 = new NodeResources(2, 40, 40, 0.5); @Before public void setup() { @@ -61,7 +61,7 @@ public class DockerHostCapacityTest { // init docker host capacity nodes = new ArrayList<>(List.of(host1, host2, host3, nodeA, nodeB, nodeC, nodeD, nodeE)); - capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); + capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); } @Test @@ -76,7 +76,7 @@ public class DockerHostCapacityTest { // Add a new node to host1 to deplete the memory resource Node nodeF = Node.createDockerNode(Set.of("::6"), "nodeF", "host1", resources1, NodeType.tenant); nodes.add(nodeF); - capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); + capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); assertFalse(capacity.hasCapacity(host1, resources1)); assertFalse(capacity.hasCapacity(host1, resources2)); } @@ -90,9 +90,9 @@ public class DockerHostCapacityTest { @Test public void freeCapacityOf() { - assertEquals(new NodeResources(5, 4, 8, 2, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + assertEquals(new NodeResources(5, 40, 80, 2, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), capacity.freeCapacityOf(host1, false)); - assertEquals(new NodeResources(5, 6, 8, 4.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + assertEquals(new NodeResources(5, 60, 80, 4.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), capacity.freeCapacityOf(host3, false)); doAnswer(invocation -> { @@ -100,9 +100,9 @@ public class DockerHostCapacityTest { return totalHostResources.subtract(new NodeResources(1, 2, 3, 0.5, NodeResources.DiskSpeed.any)); }).when(hostResourcesCalculator).advertisedResourcesOf(any()); - assertEquals(new NodeResources(4, 2, 5, 1.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + assertEquals(new NodeResources(4, 38, 77, 1.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), capacity.freeCapacityOf(host1, false)); - assertEquals(new NodeResources(4, 4, 5, 4, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + assertEquals(new NodeResources(4, 58, 77, 4, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), capacity.freeCapacityOf(host3, false)); } @@ -116,12 +116,12 @@ public class DockerHostCapacityTest { var cfg = Node.createDockerNode(Set.of("::2"), "cfg", "devhost", resources1, NodeType.config); var nodes = new ArrayList<>(List.of(cfg)); - var capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); + var capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); assertTrue(capacity.hasCapacity(devHost, resources1)); var container1 = Node.createDockerNode(Set.of("::3"), "container1", "devhost", resources1, NodeType.tenant); nodes = new ArrayList<>(List.of(cfg, container1)); - capacity = new DockerHostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); + capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); assertFalse(capacity.hasCapacity(devHost, resources1)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java index aab5f8b0078..71c3ec37d65 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java @@ -48,9 +48,9 @@ import static org.junit.Assert.fail; */ public class InPlaceResizeProvisionTest { - private static final NodeResources smallResources = new NodeResources(2, 4, 8, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); - private static final NodeResources mediumResources = new NodeResources(4, 8, 16, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); - private static final NodeResources largeResources = new NodeResources(8, 16, 32, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); + private static final NodeResources smallResources = new NodeResources(2, 4, 80, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); + private static final NodeResources mediumResources = new NodeResources(4, 8, 160, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); + private static final NodeResources largeResources = new NodeResources(8, 16, 320, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any); private static final ClusterSpec container1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1")).vespaVersion("7.157.9").build(); private static final ClusterSpec container2 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container2")).vespaVersion("7.157.9").build(); @@ -68,10 +68,10 @@ public class InPlaceResizeProvisionTest { addParentHosts(4, largeResources.with(fast).with(local)); new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate(); - assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local)); + assertSizeAndResources(container1, 4, new NodeResources(4, 8, 160, 1, fast, local)); new PrepareHelper(tester, app).prepare(container1, 4, 1, largeResources).activate(); - assertSizeAndResources(container1, 4, new NodeResources(8, 16, 32, 1, fast, local)); + assertSizeAndResources(container1, 4, new NodeResources(8, 16, 320, 1, fast, local)); assertEquals("No nodes are retired", 0, tester.getNodes(app, Node.State.active).retired().size()); } @@ -80,10 +80,10 @@ public class InPlaceResizeProvisionTest { addParentHosts(4, mediumResources.with(fast).with(local)); new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate(); - assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local)); + assertSizeAndResources(container1, 4, new NodeResources(4, 8, 160, 1, fast, local)); new PrepareHelper(tester, app).prepare(container1, 4, 1, smallResources).activate(); - assertSizeAndResources(container1, 4, new NodeResources(2, 4, 8, 1, fast, local)); + assertSizeAndResources(container1, 4, new NodeResources(2, 4, 80, 1, fast, local)); assertEquals("No nodes are retired", 0, tester.getNodes(app, Node.State.active).retired().size()); } @@ -96,16 +96,16 @@ public class InPlaceResizeProvisionTest { .prepare(container2, 4, 1, mediumResources) .activate(); Set<String> container1Hostnames = listCluster(container1).stream().map(Node::hostname).collect(Collectors.toSet()); - assertSizeAndResources(container1, 4, new NodeResources(2, 4, 8, 1, fast, local)); - assertSizeAndResources(container2, 4, new NodeResources(4, 8, 16, 1, fast, local)); + assertSizeAndResources(container1, 4, new NodeResources(2, 4, 80, 1, fast, local)); + assertSizeAndResources(container2, 4, new NodeResources(4, 8, 160, 1, fast, local)); new PrepareHelper(tester, app) .prepare(container1, 4, 1, mediumResources) .prepare(container2, 4, 1, smallResources) .activate(); assertEquals(container1Hostnames, listCluster(container1).stream().map(Node::hostname).collect(Collectors.toSet())); - assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local)); - assertSizeAndResources(container2, 4, new NodeResources(2, 4, 8, 1, fast, local)); + assertSizeAndResources(container1, 4, new NodeResources(4, 8, 160, 1, fast, local)); + assertSizeAndResources(container2, 4, new NodeResources(2, 4, 80, 1, fast, local)); assertEquals("No nodes are retired", 0, tester.getNodes(app, Node.State.active).retired().size()); } @@ -119,13 +119,13 @@ public class InPlaceResizeProvisionTest { new PrepareHelper(tester, app) .prepare(container1, 6, 1, largeResources).activate(); assertTrue(listCluster(container1).stream().map(Node::hostname).collect(Collectors.toSet()).containsAll(initialHostnames)); - assertSizeAndResources(container1, 6, new NodeResources(8, 16, 32, 1, fast, local)); + assertSizeAndResources(container1, 6, new NodeResources(8, 16, 320, 1, fast, local)); assertEquals("No nodes are retired", 0, tester.getNodes(app, Node.State.active).retired().size()); } @Test public void partial_in_place_resource_increase() { - addParentHosts(4, new NodeResources(8, 16, 32, 8, fast, local)); + addParentHosts(4, new NodeResources(8, 16, 320, 8, fast, local)); // Allocate 2 nodes for one app that leaves exactly enough capacity for mediumResources left on the host new PrepareHelper(tester, tester.makeApplicationId()).prepare(container1, 2, 1, mediumResources).activate(); @@ -141,7 +141,7 @@ public class InPlaceResizeProvisionTest { // Add 2 more parent host, now we should be able to do the same deployment that failed earlier // 2 of the nodes will be increased in-place and 2 will be allocated to the new hosts. - addParentHosts(2, new NodeResources(8, 16, 32, 8, fast, local)); + addParentHosts(2, new NodeResources(8, 16, 320, 8, fast, local)); Set<String> initialHostnames = listCluster(container1).stream().map(Node::hostname) .collect(Collectors.collectingAndThen(Collectors.toSet(), HashSet::new)); @@ -150,9 +150,9 @@ public class InPlaceResizeProvisionTest { assertEquals(6, appNodes.size()); // 4 nodes with large resources + 2 retired nodes with medium resources appNodes.forEach(node -> { if (node.allocation().get().membership().retired()) - assertEquals(new NodeResources(4, 8, 16, 1, fast, local), node.flavor().resources()); + assertEquals(new NodeResources(4, 8, 160, 1, fast, local), node.resources()); else - assertEquals(new NodeResources(8, 16, 32, 1, fast, local), node.flavor().resources()); + assertEquals(new NodeResources(8, 16, 320, 1, fast, local), node.resources()); initialHostnames.remove(node.hostname()); }); assertTrue("All initial nodes should still be allocated to the application", initialHostnames.isEmpty()); @@ -160,13 +160,13 @@ public class InPlaceResizeProvisionTest { @Test public void in_place_resource_decrease() { - addParentHosts(30, new NodeResources(10, 100, 1000, 8, fast, local)); + addParentHosts(30, new NodeResources(10, 100, 10000, 8, fast, local)); - var largeResources = new NodeResources(6, 64, 800, 1); + var largeResources = new NodeResources(6, 64, 8000, 1); new PrepareHelper(tester, app).prepare(content1, 12, 1, largeResources).activate(); assertSizeAndResources(content1, 12, largeResources.with(local)); - var smallerResources = new NodeResources(6, 48, 500, 1); + var smallerResources = new NodeResources(6, 48, 5000, 1); new PrepareHelper(tester, app).prepare(content1, 12, 1, smallerResources).activate(); assertSizeAndResources(content1, 12, smallerResources.with(local)); assertEquals(0, listCluster(content1).retired().size()); @@ -178,8 +178,8 @@ public class InPlaceResizeProvisionTest { public void increase_size_decrease_resources() { addParentHosts(14, largeResources.with(fast)); - NodeResources resources = new NodeResources(4, 8, 16, 1); - NodeResources halvedResources = new NodeResources(2, 4, 8, 1); + NodeResources resources = new NodeResources(4, 8, 160, 1); + NodeResources halvedResources = new NodeResources(2, 4, 80, 1); new PrepareHelper(tester, app).prepare(content1, 4, 1, resources).activate(); assertSizeAndResources(content1, 4, resources); @@ -205,7 +205,7 @@ public class InPlaceResizeProvisionTest { // ... same with setting a node to want to retire Node nodeToWantoToRetire = listCluster(content1).not().retired().asList().get(0); - tester.nodeRepository().write(nodeToWantoToRetire.with(nodeToWantoToRetire.status().withWantToRetire(true)), + tester.nodeRepository().write(nodeToWantoToRetire.withWantToRetire(true, Agent.system, tester.clock().instant()), tester.nodeRepository().lock(nodeToWantoToRetire)); new PrepareHelper(tester, app).prepare(content1, 8, 1, halvedResources).activate(); assertTrue(listCluster(content1).retired().stream().anyMatch(n -> n.equals(nodeToWantoToRetire))); @@ -218,7 +218,7 @@ public class InPlaceResizeProvisionTest { addParentHosts(6, mediumResources.with(fast).with(local)); new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate(); - assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local)); + assertSizeAndResources(container1, 4, new NodeResources(4, 8, 160, 1, fast, local)); new PrepareHelper(tester, app).prepare(container1, 6, 1, smallResources); } @@ -228,7 +228,7 @@ public class InPlaceResizeProvisionTest { addParentHosts(4, largeResources.with(fast).with(local)); new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate(); - assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local)); + assertSizeAndResources(container1, 4, new NodeResources(4, 8, 160, 1, fast, local)); new PrepareHelper(tester, app).prepare(container1, 2, 1, smallResources); } @@ -238,7 +238,7 @@ public class InPlaceResizeProvisionTest { addParentHosts(4, largeResources.with(fast).with(local)); new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate(); - assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local)); + assertSizeAndResources(container1, 4, new NodeResources(4, 8, 160, 1, fast, local)); new PrepareHelper(tester, app).prepare(container1, 4, 2, smallResources); } @@ -254,12 +254,12 @@ public class InPlaceResizeProvisionTest { private void assertSizeAndResources(NodeList nodes, int size, NodeResources resources) { assertEquals(size, nodes.size()); - nodes.forEach(n -> assertEquals(resources, n.flavor().resources())); + nodes.forEach(n -> assertEquals(resources, n.resources())); } private NodeList listCluster(ClusterSpec cluster) { return tester.getNodes(app, Node.State.active) - .filter(node -> node.allocation().get().membership().cluster().satisfies(cluster)); + .matching(node -> node.allocation().get().membership().cluster().satisfies(cluster)); } private static class PrepareHelper { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java index 48bd091011e..e45ea09d372 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java @@ -140,7 +140,7 @@ public class InfraDeployerImplTest { Optional<Node> nodeWithAllocation = wantedVespaVersion.map(version -> { ClusterSpec clusterSpec = application.getClusterSpecWithVersion(version).with(Optional.of(ClusterSpec.Group.from(0))); ClusterMembership membership = ClusterMembership.from(clusterSpec, 1); - Allocation allocation = new Allocation(application.getApplicationId(), membership, node.flavor().resources(), Generation.initial(), false); + Allocation allocation = new Allocation(application.getApplicationId(), membership, node.resources(), Generation.initial(), false); return node.with(allocation); }); return nodeRepository.database().writeTo(state, nodeWithAllocation.orElse(node), Agent.system, Optional.empty()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index ad9d13355dc..f48127f650d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -11,6 +11,8 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; @@ -43,7 +45,8 @@ public class LoadBalancerProvisionerTest { private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default"); private final ApplicationId infraApp1 = ApplicationId.from("vespa", "tenant-host", "default"); - private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private final ProvisioningTester tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); @Test public void provision_load_balancer() { @@ -200,29 +203,31 @@ public class LoadBalancerProvisionerTest { assertEquals(List.of(), tester.nodeRepository().loadBalancers(app1).asList()); } - // TODO(mpolden): Remove when ClusterSpec with combined type rejects empty combinedId @Test - public void provision_load_balancer_combined_cluster_without_id() { + public void provision_load_balancer_combined_cluster() { Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(app1).asList(); - ClusterSpec.Id cluster = ClusterSpec.Id.from("foo"); - - var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, cluster)); + var combinedId = ClusterSpec.Id.from("container1"); + var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, ClusterSpec.Id.from("content1"), Optional.of(combinedId))); assertEquals(1, lbs.get().size()); assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().reals().size()); tester.activate(app1, nodes); assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); + assertEquals(combinedId, lbs.get().get(0).id().cluster()); } @Test - public void provision_load_balancer_combined_cluster() { - Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(app1).asList(); - var combinedId = ClusterSpec.Id.from("container1"); - var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, ClusterSpec.Id.from("content1"), Optional.of(combinedId))); + public void provision_load_balancer_config_server_cluster() { + flagSource.withBooleanFlag(Flags.CONFIGSERVER_PROVISION_LB.id(), true); + ApplicationId configServerApp = ApplicationId.from("hosted-vespa", "zone-config-servers", "default"); + Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(configServerApp).asList(); + var cluster = ClusterSpec.Id.from("zone-config-servers"); + var nodes = prepare(configServerApp, Capacity.fromRequiredNodeType(NodeType.config), false, + clusterRequest(ClusterSpec.Type.admin, cluster)); assertEquals(1, lbs.get().size()); assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().reals().size()); - tester.activate(app1, nodes); + tester.activate(configServerApp, nodes); assertSame(LoadBalancer.State.active, lbs.get().get(0).state()); - assertEquals(combinedId, lbs.get().get(0).id().cluster()); + assertEquals(cluster, lbs.get().get(0).id().cluster()); } private void dirtyNodesOf(ApplicationId application) { @@ -241,7 +246,7 @@ public class LoadBalancerProvisionerTest { } Set<HostSpec> allNodes = new LinkedHashSet<>(); for (ClusterSpec spec : specs) { - allNodes.addAll(tester.prepare(application, spec, capacity, false)); + allNodes.addAll(tester.prepare(application, spec, capacity)); } return allNodes; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index 050f3b7e865..6e609d13d3b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.RegionName; @@ -21,6 +22,7 @@ import java.time.Duration; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -40,9 +42,9 @@ public class MultigroupProvisioningTest { public void test_provisioning_of_multiple_groups() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - ApplicationId application1 = tester.makeApplicationId(); + ApplicationId application1 = tester.makeApplicationId("app1"); - tester.makeReadyNodes(21, small); + tester.makeReadyNodes(31, small); deploy(application1, 6, 1, small, tester); deploy(application1, 6, 2, small, tester); @@ -86,10 +88,10 @@ public class MultigroupProvisioningTest { public void test_provisioning_of_multiple_groups_after_flavor_migration() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - ApplicationId application1 = tester.makeApplicationId(); + ApplicationId application1 = tester.makeApplicationId("app1"); tester.makeReadyNodes(10, small); - tester.makeReadyNodes(10, large); + tester.makeReadyNodes(16, large); deploy(application1, 8, 1, small, tester); deploy(application1, 8, 1, large, tester); @@ -121,14 +123,107 @@ public class MultigroupProvisioningTest { deploy(application1, Capacity.from(new ClusterResources(2, 2, large), true, true), tester); } + /** + * When increasing the number of groups without changing node count, we need to provison new nodes for + * the new groups since although we can remove nodes from existing groups without losing data we + * cannot do so without losing coverage. + */ + @Test + public void test_split_to_groups() { + Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 100, 4)); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) + .flavors(List.of(hostFlavor)) + .build(); + tester.makeReadyHosts(6, hostFlavor.resources()).deployZoneApp(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + // Deploy with 1 group + tester.activate(app1, cluster1, Capacity.from(resources(4, 1, 10, 30, 10))); + assertEquals(4, tester.getNodes(app1, Node.State.active).size()); + assertEquals(4, tester.getNodes(app1, Node.State.active).group(0).size()); + assertEquals(0, tester.getNodes(app1, Node.State.active).group(0).retired().size()); + + // Split into 2 groups + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 10, 30, 10))); + assertEquals(6, tester.getNodes(app1, Node.State.active).size()); + assertEquals(4, tester.getNodes(app1, Node.State.active).group(0).size()); + assertEquals(2, tester.getNodes(app1, Node.State.active).group(0).retired().size()); + assertEquals(2, tester.getNodes(app1, Node.State.active).group(1).size()); + assertEquals(0, tester.getNodes(app1, Node.State.active).group(1).retired().size()); + } + + @Test + public void test_remove_group() { + Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 100, 4)); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) + .flavors(List.of(hostFlavor)) + .build(); + tester.makeReadyHosts(6, hostFlavor.resources()).deployZoneApp(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + // Deploy with 3 groups + tester.activate(app1, cluster1, Capacity.from(resources(6, 3, 10, 30, 10))); + assertEquals(6, tester.getNodes(app1, Node.State.active).size()); + assertEquals(0, tester.getNodes(app1, Node.State.active).retired().size()); + assertEquals(0, tester.getNodes(app1, Node.State.inactive).size()); + assertEquals(2, tester.getNodes(app1, Node.State.active).group(0).size()); + assertEquals(2, tester.getNodes(app1, Node.State.active).group(1).size()); + assertEquals(2, tester.getNodes(app1, Node.State.active).group(2).size()); + + // Remove a group + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 10, 30, 10))); + assertEquals(4, tester.getNodes(app1, Node.State.active).size()); + assertEquals(0, tester.getNodes(app1, Node.State.active).retired().size()); + assertEquals(2, tester.getNodes(app1, Node.State.inactive).size()); + assertEquals(2, tester.getNodes(app1, Node.State.active).group(0).size()); + assertEquals(2, tester.getNodes(app1, Node.State.active).group(1).size()); + assertEquals(0, tester.getNodes(app1, Node.State.active).group(2).size()); + } + + @Test + public void test_layout_change_to_fewer_groups() { + Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 100, 4)); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) + .flavors(List.of(hostFlavor)) + .build(); + tester.makeReadyHosts(12, hostFlavor.resources()).deployZoneApp(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + // Deploy with 3 groups + tester.activate(app1, cluster1, Capacity.from(resources(12, 4, 10, 30, 10))); + assertEquals(12, tester.getNodes(app1, Node.State.active).size()); + assertEquals( 0, tester.getNodes(app1, Node.State.active).retired().size()); + assertEquals( 0, tester.getNodes(app1, Node.State.inactive).size()); + assertEquals( 3, tester.getNodes(app1, Node.State.active).group(0).size()); + assertEquals( 3, tester.getNodes(app1, Node.State.active).group(1).size()); + assertEquals( 3, tester.getNodes(app1, Node.State.active).group(2).size()); + assertEquals( 3, tester.getNodes(app1, Node.State.active).group(3).size()); + + // Remove a group + tester.activate(app1, cluster1, Capacity.from(resources(12, 3, 10, 30, 10))); + assertEquals(12, tester.getNodes(app1, Node.State.active).size()); + assertEquals( 0, tester.getNodes(app1, Node.State.active).retired().size()); + assertEquals( 0, tester.getNodes(app1, Node.State.inactive).size()); + assertEquals( 4, tester.getNodes(app1, Node.State.active).group(0).size()); + assertEquals( 4, tester.getNodes(app1, Node.State.active).group(1).size()); + assertEquals( 4, tester.getNodes(app1, Node.State.active).group(2).size()); + assertEquals( 0, tester.getNodes(app1, Node.State.active).group(3).size()); + } + @Test public void test_provisioning_of_multiple_groups_after_flavor_migration_and_exiration() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - ApplicationId application1 = tester.makeApplicationId(); + ApplicationId application1 = tester.makeApplicationId("app1"); tester.makeReadyNodes(10, small); - tester.makeReadyNodes(10, large); + tester.makeReadyNodes(16, large); deploy(application1, 8, 1, small, tester); deploy(application1, 8, 1, large, tester); @@ -164,13 +259,8 @@ public class MultigroupProvisioningTest { int nodeCount = capacity.minResources().nodes(); NodeResources nodeResources = capacity.minResources().nodeResources(); - int previousActiveNodeCount = tester.getNodes(application, Node.State.active).resources(nodeResources).size(); - tester.activate(application, prepare(application, capacity, tester)); - assertEquals("Superfluous nodes are retired, but no others - went from " + previousActiveNodeCount + " to " + nodeCount + " nodes", - Math.max(0, previousActiveNodeCount - capacity.minResources().nodes()), - tester.getNodes(application, Node.State.active).retired().resources(nodeResources).size()); - assertEquals("Other flavors are retired", + assertEquals("Nodes of wrong size are retired", 0, tester.getNodes(application, Node.State.active).not().retired().not().resources(nodeResources).size()); // Check invariants for all nodes @@ -216,4 +306,8 @@ public class MultigroupProvisioningTest { return new HashSet<>(tester.prepare(application, cluster(), capacity)); } + private ClusterResources resources(int nodes, int groups, double vcpu, double memory, double disk) { + return new ClusterResources(nodes, groups, new NodeResources(vcpu, memory, disk, 0.1)); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java index 98133898cf6..11e7af512c3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java @@ -113,7 +113,7 @@ public class NodeTypeProvisioningTest { Node nodeToRetire = tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).get(5); { // Pick out a node and retire it - tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true)), () -> {}); + tester.nodeRepository().write(nodeToRetire.withWantToRetire(true, Agent.system, tester.clock().instant()), () -> {}); List<HostSpec> hosts = deployProxies(application, tester); assertEquals(11, hosts.size()); @@ -186,7 +186,7 @@ public class NodeTypeProvisioningTest { String currentyRetiringHostname; { nodesToRetire.forEach(nodeToRetire -> - tester.nodeRepository().write(nodeToRetire.with(nodeToRetire.status().withWantToRetire(true)), () -> {})); + tester.nodeRepository().write(nodeToRetire.withWantToRetire(true, Agent.system, tester.clock().instant()), () -> {})); List<HostSpec> hosts = deployProxies(application, tester); assertEquals(11, hosts.size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java index 1d9c135b4b5..3865baa51c1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNodeTest.java @@ -29,8 +29,8 @@ public class PrioritizableNodeTest { List<PrioritizableNode> expected = List.of( new PrioritizableNode(node("01", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), false, true, false, false), new PrioritizableNode(node("02", Node.State.active), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), - new PrioritizableNode(node("03", Node.State.inactive), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), new PrioritizableNode(node("04", Node.State.reserved), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), + new PrioritizableNode(node("03", Node.State.inactive), new NodeResources(2, 2, 2, 2), Optional.empty(), true, false, false, false), new PrioritizableNode(node("05", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.active)), true, false, true, false), new PrioritizableNode(node("06", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.ready)), true, false, true, false), new PrioritizableNode(node("07", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.provisioned)), true, false, true, false), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index c9eb6466cd7..607ca963cef 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -408,12 +408,30 @@ public class ProvisioningTest { } @Test + public void test_node_limits_only() { + Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 100, 4)); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) + .flavors(List.of(hostFlavor)) + .build(); + tester.makeReadyHosts(4, hostFlavor.resources()).deployZoneApp(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, NodeResources.unspecified()), + new ClusterResources(4, 1, NodeResources.unspecified()))); + tester.assertNodes("Initial allocation at min with default resources", + 2, 1, 1.5, 8, 50, 0.3, + app1, cluster1); + } + + @Test public void test_changing_limits() { Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 100, 4)); ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); - tester.makeReadyHosts(30, hostFlavor.resources()).deployZoneApp(); + tester.makeReadyHosts(31, hostFlavor.resources()).deployZoneApp(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -426,45 +444,45 @@ public class ProvisioningTest { app1, cluster1); // Move window above current allocation - tester.activate(app1, cluster1, Capacity.from(resources(8, 4, 4, 20, 40), + tester.activate(app1, cluster1, Capacity.from(resources(8, 4, 4, 21, 40), resources(10, 5, 5, 25, 50))); tester.assertNodes("New allocation at new min", - 8, 4, 4, 20, 40, + 8, 4, 4, 21, 40, app1, cluster1); // Move window below current allocation tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 20), resources(6, 3, 3, 15, 25))); - tester.assertNodes("New allocation at new max", - 6, 3, 3, 15, 25, + tester.assertNodes("Allocation preserving resources within new limits", + 6, 2, 3, 8.0/4*21 / (6.0/2), 25, app1, cluster1); // Widening window does not change allocation - tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 1, 5, 15), - resources(8, 4, 4, 20, 30))); + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 1, 5, 15), + resources(8, 4, 4, 21, 30))); tester.assertNodes("Same allocation", - 6, 3, 3, 15, 25, + 6, 2, 3, 8.0/4*21 / (6.0/2), 25, app1, cluster1); // Changing limits in opposite directions cause a mixture of min and max - tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 10, 30, 5), - resources(4, 2, 14, 40, 10))); + tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 10, 30, 10), + resources(4, 2, 14, 40, 13))); tester.assertNodes("A mix of min and max", - 4, 2, 10, 30, 10, + 4, 1, 10, 30, 13, app1, cluster1); // Changing group size - tester.activate(app1, cluster1, Capacity.from(resources(6, 3, 8, 25, 5), + tester.activate(app1, cluster1, Capacity.from(resources(6, 3, 8, 25, 10), resources(9, 3, 12, 35, 15))); tester.assertNodes("Groups changed", - 6, 3, 8, 30, 10, + 9, 3, 8, 35, 15, app1, cluster1); // Stop specifying node resources - tester.activate(app1, cluster1, Capacity.from(new ClusterResources(6, 3, NodeResources.unspecified), - new ClusterResources(9, 3, NodeResources.unspecified))); + tester.activate(app1, cluster1, Capacity.from(new ClusterResources(6, 3, NodeResources.unspecified()), + new ClusterResources(9, 3, NodeResources.unspecified()))); tester.assertNodes("No change", - 6, 3, 8, 30, 10, + 9, 3, 8, 35, 15, app1, cluster1); } @@ -488,7 +506,7 @@ public class ProvisioningTest { new NodeResources(2, 2, 10, 2), tester); } catch (IllegalArgumentException e) { - assertEquals("Must specify at least 4.00 Gb of memory for container cluster 'container0', was: 2.00 Gb", e.getMessage()); + assertEquals("container cluster 'container0': Min memory size is 2.00 Gb but must be at least 4.00 Gb", e.getMessage()); } } @@ -582,7 +600,7 @@ public class ProvisioningTest { fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("6 nodes with [vcpu: 1.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 4.0 Gbps] requested for content cluster 'content0' 6.42. Max value exceeds your quota. Resolve this at https://cloud.vespa.ai/quota", + assertEquals("6 nodes with [vcpu: 1.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 4.0 Gbps] requested for content cluster 'content0' 6.42. Max value exceeds your quota. Resolve this at https://cloud.vespa.ai/pricing", e.getMessage()); } } @@ -604,7 +622,7 @@ public class ProvisioningTest { tester.makeReadyHosts(4, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); - tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified), false, false)); + tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified()), false, false)); // No exception; Success } @@ -615,7 +633,7 @@ public class ProvisioningTest { ApplicationId application = tester.makeApplicationId(); // Flag all nodes for retirement List<Node> readyNodes = tester.makeReadyNodes(5, defaultResources); - readyNodes.forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true)))); + readyNodes.forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); try { prepare(application, 2, 0, 2, 0, defaultResources, tester); @@ -643,7 +661,7 @@ public class ProvisioningTest { assertEquals(0, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size()); // Mark the nodes as want to retire - tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true)))); + tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); // redeploy without allow failing tester.activate(application, tester.prepare(application, cluster, capacityFORCED)); @@ -706,7 +724,7 @@ public class ProvisioningTest { // Retire some nodes and redeploy { List<Node> nodesToRetire = tester.getNodes(application, Node.State.active).asList().subList(0, 2); - nodesToRetire.forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true)))); + nodesToRetire.forEach(node -> tester.patchNode(node.withWantToRetire(true, Agent.system, tester.clock().instant()))); SystemState state = prepare(application, 2, 0, 2, 0, defaultResources, tester); tester.activate(application, state.allHosts); @@ -811,7 +829,10 @@ public class ProvisioningTest { Capacity.from(new ClusterResources(2, 1, defaultResources), false, false))); // Application is redeployed with cluster type combined - cluster = ClusterSpec.request(ClusterSpec.Type.combined, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); + cluster = ClusterSpec.request(ClusterSpec.Type.combined, ClusterSpec.Id.from("music")) + .vespaVersion("1.2.3") + .combinedId(Optional.of(ClusterSpec.Id.from("qrs"))) + .build(); var newNodes = tester.activate(application, tester.prepare(application, cluster, Capacity.from(new ClusterResources(2, 1, defaultResources), false, false))); @@ -860,7 +881,7 @@ public class ProvisioningTest { allHosts.addAll(content0); allHosts.addAll(content1); - Function<Integer, Capacity> capacity = count -> Capacity.from(new ClusterResources(count, 1, NodeResources.unspecified), required, true); + Function<Integer, Capacity> capacity = count -> Capacity.from(new ClusterResources(count, 1, NodeResources.unspecified()), required, true); int expectedContainer0Size = tester.capacityPolicies().decideSize(container0Size, capacity.apply(container0Size), containerCluster0, application); int expectedContainer1Size = tester.capacityPolicies().decideSize(container1Size, capacity.apply(container1Size), containerCluster1, application); int expectedContent0Size = tester.capacityPolicies().decideSize(content0Size, capacity.apply(content0Size), contentCluster0, application); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 806a9984da3..e73aeb05ce3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -56,6 +56,7 @@ import java.util.function.Function; import java.util.logging.Level; import java.util.stream.Collectors; +import static com.yahoo.config.provision.NodeResources.StorageType.local; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -94,6 +95,7 @@ public class ProvisioningTester { this.curator = curator; this.nodeFlavors = nodeFlavors; this.clock = new ManualClock(); + ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner); this.nodeRepository = new NodeRepository(nodeFlavors, resourcesCalculator, curator, @@ -101,23 +103,23 @@ public class ProvisioningTester { zone, nameResolver, DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - true); + true, + provisionServiceProvider.getHostProvisioner().isPresent()); this.orchestrator = orchestrator; - ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource); - this.capacityPolicies = new CapacityPolicies(zone); + this.capacityPolicies = new CapacityPolicies(nodeRepository); this.provisionLogger = new NullProvisionLogger(); this.loadBalancerService = loadBalancerService; } public static FlavorsConfig createConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); - b.addFlavor("default", 2., 4., 100, 10, Flavor.Type.BARE_METAL).cost(3); - b.addFlavor("small", 1., 2., 50, 5, Flavor.Type.BARE_METAL).cost(2); - b.addFlavor("dockerSmall", 1., 1., 10, 1, Flavor.Type.DOCKER_CONTAINER).cost(1); - b.addFlavor("dockerLarge", 2., 1., 20, 1, Flavor.Type.DOCKER_CONTAINER).cost(3); - b.addFlavor("v-4-8-100", 4., 8., 100, 10, Flavor.Type.VIRTUAL_MACHINE).cost(4); - b.addFlavor("large", 4., 8., 100, 10, Flavor.Type.BARE_METAL).cost(10); + b.addFlavor("default", 2., 40., 100, 10, Flavor.Type.BARE_METAL).cost(3); + b.addFlavor("small", 1., 20., 50, 5, Flavor.Type.BARE_METAL).cost(2); + b.addFlavor("dockerSmall", 1., 10., 10, 1, Flavor.Type.DOCKER_CONTAINER).cost(1); + b.addFlavor("dockerLarge", 2., 10., 20, 1, Flavor.Type.DOCKER_CONTAINER).cost(3); + b.addFlavor("v-4-8-100", 4., 80., 100, 10, Flavor.Type.VIRTUAL_MACHINE).cost(4); + b.addFlavor("large", 4., 80., 100, 10, Flavor.Type.BARE_METAL).cost(10); return b.build(); } @@ -145,17 +147,12 @@ public class ProvisioningTester { } public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity) { - return prepare(application, cluster, capacity, true); - } - - public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, boolean idempotentPrepare) { Set<String> reservedBefore = toHostNames(nodeRepository.getNodes(application, Node.State.reserved)); Set<String> inactiveBefore = toHostNames(nodeRepository.getNodes(application, Node.State.inactive)); List<HostSpec> hosts1 = provisioner.prepare(application, cluster, capacity, provisionLogger); - if (idempotentPrepare) { // prepare twice to ensure idempotence - List<HostSpec> hosts2 = provisioner.prepare(application, cluster, capacity, provisionLogger); - assertEquals(hosts1, hosts2); - } + // prepare twice to ensure idempotence + List<HostSpec> hosts2 = provisioner.prepare(application, cluster, capacity, provisionLogger); + assertEquals(hosts1, hosts2); Set<String> newlyActivated = toHostNames(nodeRepository.getNodes(application, Node.State.reserved)); newlyActivated.removeAll(reservedBefore); newlyActivated.removeAll(inactiveBefore); @@ -163,7 +160,7 @@ public class ProvisioningTester { } public Collection<HostSpec> activate(ApplicationId application, ClusterSpec cluster, Capacity capacity) { - List<HostSpec> preparedNodes = prepare(application, cluster, capacity, true); + List<HostSpec> preparedNodes = prepare(application, cluster, capacity); // Add ip addresses and activate parent host if necessary for (HostSpec prepared : preparedNodes) { @@ -197,7 +194,7 @@ public class ProvisioningTester { public void prepareAndActivateInfraApplication(ApplicationId application, NodeType nodeType, Version version) { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString())).vespaVersion(version).build(); Capacity capacity = Capacity.fromRequiredNodeType(nodeType); - List<HostSpec> hostSpecs = prepare(application, cluster, capacity, true); + List<HostSpec> hostSpecs = prepare(application, cluster, capacity); activate(application, hostSpecs); } @@ -236,13 +233,27 @@ public class ProvisioningTester { /** Assert on the current *non retired* nodes */ public void assertNodes(String explanation, int nodes, int groups, double vcpu, double memory, double disk, ApplicationId app, ClusterSpec cluster) { - assertNodes(explanation, nodes, groups, vcpu, memory, disk, DiskSpeed.getDefault(), StorageType.getDefault(), app, cluster); + assertNodes(explanation, nodes, groups, vcpu, memory, disk, 0.1, app, cluster); } - public void assertNodes(String explanation, int nodes, int groups, double vcpu, double memory, double disk, + /** Assert on the current *non retired* nodes */ + public void assertNodes(String explanation, int nodes, int groups, double vcpu, double memory, double disk, double bandwidth, + ApplicationId app, ClusterSpec cluster) { + assertNodes(explanation, nodes, groups, vcpu, memory, disk, bandwidth, DiskSpeed.getDefault(), StorageType.getDefault(), app, cluster); + } + + public void assertNodes(String explanation, int nodes, int groups, + double vcpu, double memory, double disk, + DiskSpeed diskSpeed, StorageType storageType, + ApplicationId app, ClusterSpec cluster) { + assertNodes(explanation, nodes, groups, vcpu, memory, disk, 0.1, diskSpeed, storageType, app, cluster); + } + + public void assertNodes(String explanation, int nodes, int groups, + double vcpu, double memory, double disk, double bandwidth, DiskSpeed diskSpeed, StorageType storageType, ApplicationId app, ClusterSpec cluster) { - List<Node> nodeList = nodeRepository.list().owner(app).cluster(cluster.id()).not().retired().asList(); + List<Node> nodeList = nodeRepository.list().owner(app).cluster(cluster.id()).state(Node.State.active).not().retired().asList(); assertEquals(explanation + ": Node count", nodes, nodeList.size()); @@ -250,9 +261,9 @@ public class ProvisioningTester { groups, nodeList.stream().map(n -> n.allocation().get().membership().cluster().group().get()).distinct().count()); for (Node node : nodeList) { - var expected = new NodeResources(vcpu, memory, disk, 0.1, diskSpeed, storageType); - assertTrue(explanation + ": Resources: Expected " + expected + " but was " + node.flavor().resources(), - expected.compatibleWith(node.flavor().resources())); + var expected = new NodeResources(vcpu, memory, disk, bandwidth, diskSpeed, storageType); + assertTrue(explanation + ": Resources: Expected " + expected + " but was " + node.resources(), + expected.compatibleWith(node.resources())); } } @@ -482,12 +493,16 @@ public class ProvisioningTester { activate(applicationId, Set.copyOf(list)); } - public ClusterSpec clusterSpec() { + public ClusterSpec containerClusterSpec() { return ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); } + public ClusterSpec contentClusterSpec() { + return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); + } + public List<Node> deploy(ApplicationId application, Capacity capacity) { - return deploy(application, clusterSpec(), capacity); + return deploy(application, containerClusterSpec(), capacity); } public List<Node> deploy(ApplicationId application, ClusterSpec cluster, Capacity capacity) { @@ -543,7 +558,13 @@ public class ProvisioningTester { } public Builder resourcesCalculator(HostResourcesCalculator resourcesCalculator) { - this.resourcesCalculator = resourcesCalculator; + if (resourcesCalculator != null) + this.resourcesCalculator = resourcesCalculator; + return this; + } + + public Builder resourcesCalculator(int memoryTax, int diskTax) { + this.resourcesCalculator = new MockResourcesCalculator(memoryTax, diskTax); return this; } @@ -625,4 +646,43 @@ public class ProvisioningTester { @Override public void log(Level level, String message) { } } + static class MockResourcesCalculator implements HostResourcesCalculator { + + private final int memoryTaxGb; + private final int localDiskTax; + + public MockResourcesCalculator(int memoryTaxGb, int localDiskTax) { + this.memoryTaxGb = memoryTaxGb; + this.localDiskTax = localDiskTax; + } + + @Override + public NodeResources realResourcesOf(Node node, NodeRepository nodeRepository) { + NodeResources resources = node.resources(); + if (node.type() == NodeType.host) return resources; + return resources.withMemoryGb(resources.memoryGb() - memoryTaxGb) + .withDiskGb(resources.diskGb() - ( resources.storageType() == local ? localDiskTax : 0)); + } + + @Override + public NodeResources advertisedResourcesOf(Flavor flavor) { + NodeResources resources = flavor.resources(); + if ( ! flavor.isConfigured()) return resources; + return resources.withMemoryGb(resources.memoryGb() + memoryTaxGb); + } + + @Override + public NodeResources requestToReal(NodeResources resources) { + return resources.withMemoryGb(resources.memoryGb() - memoryTaxGb) + .withDiskGb(resources.diskGb() - ( resources.storageType() == local ? localDiskTax : 0) ); + } + + @Override + public NodeResources realToRequest(NodeResources resources) { + return resources.withMemoryGb(resources.memoryGb() + memoryTaxGb) + .withDiskGb(resources.diskGb() + ( resources.storageType() == local ? localDiskTax : 0) ); + } + + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 414b12e77b9..1e788e2c70e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -1,36 +1,28 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; -import com.yahoo.application.Networking; -import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; import com.yahoo.text.Utf8; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.OsUpgradeActivator; -import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import org.junit.After; import org.junit.Before; -import org.junit.ComparisonFailure; import org.junit.Test; -import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; /** @@ -64,6 +56,7 @@ public class NodesV2ApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/state/active?recursive=true"), "active-nodes.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/"), "nodes.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true"), "nodes-recursive.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&includeDeprovisioned=true"), "nodes-recursive-include-deprovisioned.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "node2.json"); // GET with filters @@ -214,8 +207,17 @@ public class NodesV2ApiTest { Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), + Utf8.toBytes("{\"wantToRetire\": true}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantToDeprovision\": false, \"wantToRetire\": false}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantToDeprovision\": true, \"wantToRetire\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"modelName\": null}"), Request.Method.PATCH), @@ -386,7 +388,7 @@ public class NodesV2ApiTest { @Test public void fails_to_ready_node_with_hard_fail() throws Exception { assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("host12.yahoo.com", "default") + "]"). + ("[" + asHostJson("host12.yahoo.com", "default", Optional.empty()) + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); @@ -570,7 +572,7 @@ public class NodesV2ApiTest { @Test public void test_reports_patching() throws IOException { // Add report - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{" + " \"reports\": {" + " \"actualCpuCores\": {" + @@ -591,19 +593,19 @@ public class NodesV2ApiTest { " }" + "}"), Request.Method.PATCH), - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports.json"); // Patching with an empty reports is no-op - tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"reports\": {}}"), Request.Method.PATCH), 200, - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports.json"); // Patching existing report overwrites - tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{" + " \"reports\": {" + " \"actualCpuCores\": {" + @@ -613,22 +615,22 @@ public class NodesV2ApiTest { "}"), Request.Method.PATCH), 200, - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports-2.json"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-2.json"); // Clearing one report - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"reports\": { \"diskSpace\": null } }"), Request.Method.PATCH), - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports-3.json"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-3.json"); // Clearing all reports - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"reports\": null }"), Request.Method.PATCH), - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6.json"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-4.json"); } @Test @@ -807,6 +809,19 @@ public class NodesV2ApiTest { "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com\"}," + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}" + "]}"); + + // Schedule OS upgrade with budget + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host", + Utf8.toBytes("{\"osVersion\": \"7.42.1\", \"upgradeBudget\": \"PT24H\"}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.42.1, upgradeBudget to PT24H for nodes of type host\"}"); + + // Invalid budget + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host", + Utf8.toBytes("{\"osVersion\": \"7.42.1\", \"upgradeBudget\": \"foo\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid duration 'foo': Text cannot be parsed to a Duration\"}"); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-2.json index a3d53798d7c..220fdbd8654 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-2.json @@ -1,35 +1,50 @@ { - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "cpuCores": 4.0, + "resources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "environment": "BARE_METAL", "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" }, "membership": { - "clustertype": "content", - "clusterid": "id2", + "clustertype": "container", + "clusterid": "node-admin", "group": "0", - "index": 1, + "index": 0, "retired": false }, "restartGeneration": 0, "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "requestedResources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, "allowedToBeDown": false, - "rebootGeneration": 1, + "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, - "wantToRetire": false, + "wantToRetire": true, "wantToDeprovision": true, "history": [ { @@ -51,13 +66,22 @@ "event": "activated", "at": 123, "agent": "application" + }, + { + "event": "wantToRetire", + "at": 123, + "agent": "system" } ], "ipAddresses": [ - "127.0.6.1", - "::6:1" + "127.0.100.1", + "::100:1" + ], + "additionalIpAddresses": [ + "::100:2", + "::100:3", + "::100:4" ], - "additionalIpAddresses": [], "reports": { "actualCpuCores": { "createdMillis": 3 @@ -65,7 +89,7 @@ "diskSpace": { "createdMillis": 2, "description": "Actual disk space (2TB) differs from spec (3TB)", - "type":"HARD_FAIL", + "type": "HARD_FAIL", "details": { "inGib": 3, "disks": [ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-3.json new file mode 100644 index 00000000000..d2474f21c55 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-3.json @@ -0,0 +1,90 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "cpuCores": 4.0, + "resources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + }, + { + "event": "wantToRetire", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.100.1", + "::100:1" + ], + "additionalIpAddresses": [ + "::100:2", + "::100:3", + "::100:4" + ], + "reports": { + "actualCpuCores": { + "createdMillis": 3 + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-4.json index a1775dc794e..cbf02795d73 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports-4.json @@ -7,7 +7,14 @@ "openStackId": "dockerhost1", "flavor": "large", "cpuCores": 4.0, - "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast"}, + "resources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, "environment": "BARE_METAL", "owner": { "tenant": "zoneapp", @@ -25,12 +32,17 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"any" }, + "requestedResources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, - "currentOsVersion": "7.5.2", - "wantedOsVersion": "7.5.2", "failCount": 0, "wantToRetire": false, "wantToDeprovision": false, @@ -54,6 +66,11 @@ "event": "activated", "at": 123, "agent": "application" + }, + { + "event": "wantToRetire", + "at": 123, + "agent": "system" } ], "ipAddresses": [ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports.json index 67b8d67c7f1..c00c06634b5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-reports.json @@ -1,35 +1,50 @@ { - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "cpuCores": 4.0, + "resources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "environment": "BARE_METAL", "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" }, "membership": { - "clustertype": "content", - "clusterid": "id2", + "clustertype": "container", + "clusterid": "node-admin", "group": "0", - "index": 1, + "index": 0, "retired": false }, "restartGeneration": 0, "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "requestedResources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, "allowedToBeDown": false, - "rebootGeneration": 1, + "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, - "wantToRetire": false, + "wantToRetire": true, "wantToDeprovision": true, "history": [ { @@ -51,24 +66,33 @@ "event": "activated", "at": 123, "agent": "application" + }, + { + "event": "wantToRetire", + "at": 123, + "agent": "system" } ], "ipAddresses": [ - "127.0.6.1", - "::6:1" + "127.0.100.1", + "::100:1" + ], + "additionalIpAddresses": [ + "::100:2", + "::100:3", + "::100:4" ], - "additionalIpAddresses": [], "reports": { "actualCpuCores": { "createdMillis": 1, "description": "Actual number of CPU cores (2) differs from spec (4)", - "type":"HARD_FAIL", + "type": "HARD_FAIL", "value": 2 }, "diskSpace": { "createdMillis": 2, "description": "Actual disk space (2TB) differs from spec (3TB)", - "type":"HARD_FAIL", + "type": "HARD_FAIL", "details": { "inGib": 3, "disks": [ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost6.json new file mode 100644 index 00000000000..8d437b3e47c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost6.json @@ -0,0 +1,48 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost6.yahoo.com", + "id": "dockerhost6.yahoo.com", + "state": "deprovisioned", + "type": "host", + "hostname": "dockerhost6.yahoo.com", + "openStackId": "dockerhost6", + "flavor": "large", + "cpuCores": 4.0, + "resources": { + "vcpu": 4.0, + "memoryGb": 32.0, + "diskGb": 1600.0, + "bandwidthGbps": 20.0, + "diskSpeed": "fast", + "storageType": "remote" + }, + "environment": "BARE_METAL", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 1, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "failed", + "at": 123, + "agent": "operator" + }, + { + "event": "deprovisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags1.json deleted file mode 100644 index a606777e9fd..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags1.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "flags": [ - { - "id": "exclusive-load-balancer", - "enabled": false, - "enabledHostnames": [], - "enabledApplications": ["zoneapp:zoneapp:zoneapp"] - } - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags2.json deleted file mode 100644 index 4baf75f2169..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags2.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "flags": [ - { - "id": "exclusive-load-balancer", - "enabled": false, - "enabledHostnames": [ - "host1" - ], - "enabledApplications": [ - "zoneapp:zoneapp:zoneapp", - "foo:bar:default" - ] - } - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json index e041a7b8b54..6bb30d90218 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json @@ -4,9 +4,6 @@ "name": "AutoscalingMaintainer" }, { - "name": "CapacityReportMaintainer" - }, - { "name": "DirtyExpirer" }, { @@ -56,6 +53,9 @@ }, { "name":"ScalingSuggestionsMaintainer" + }, + { + "name": "SpareCapacityMaintainer" } ], "inactive": [ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-after-changes.json deleted file mode 100644 index 65c7e9db6cd..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-after-changes.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":1.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" - }, - "membership": { - "clustertype": "content", - "clusterid": "id2", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.6.1", - "::6:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json deleted file mode 100644 index 7f0c3a5f706..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" - }, - "membership": { - "clustertype": "content", - "clusterid": "id2", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.6.1", - "::6:1" - ], - "additionalIpAddresses": [], - "reports": { - "actualCpuCores": { - "createdMillis": 3 - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json new file mode 100644 index 00000000000..2b650bad39b --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive-include-deprovisioned.json @@ -0,0 +1,24 @@ +{ + "nodes": [ + @include(node7.json), + @include(cfg1.json), + @include(node3.json), + @include(cfg2.json), + @include(node10.json), + @include(docker-node3.json), + @include(node14.json), + @include(node4.json), + @include(docker-node4.json), + @include(node6.json), + @include(docker-container1.json), + @include(docker-node5.json), + @include(docker-node2.json), + @include(node13.json), + @include(node2.json), + @include(docker-node1.json), + @include(node1.json), + @include(node55.json), + @include(node5.json), + @include(dockerhost6.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent1.json deleted file mode 100644 index 40f4a3b5160..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent1.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/parent1.yahoo.com", - "id": "parent1.yahoo.com", - "state": "ready", - "type": "host", - "hostname": "parent1.yahoo.com", - "openStackId": "parent1", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb]", - "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":1.0,"diskSpeed":"fast"}, - "environment": "BARE_METAL", - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.0.1", - "::1" - ], - "additionalIpAddresses": [] -} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java index cbb6902066e..d5cf0b39fda 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.orchestrator.status; import com.yahoo.config.provision.ApplicationId; import com.yahoo.jdisc.Timer; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; @@ -14,12 +13,12 @@ import com.yahoo.vespa.orchestrator.status.json.WireHostInfo; import org.apache.zookeeper.KeeperException.NoNodeException; import java.time.Instant; +import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; /** * Handles all ZooKeeper data structures related to each active application, including HostInfo. @@ -42,18 +41,32 @@ public class HostInfosServiceImpl implements HostInfosService { public HostInfos getHostInfos(ApplicationInstanceReference reference) { ApplicationId application = OrchestratorUtil.toApplicationId(reference); String hostsRootPath = hostsPath(application); - if (uncheck(() -> curator.framework().checkExists().forPath(hostsRootPath)) == null) { + + List<String> hostnames; + try { + hostnames = curator.framework().getChildren().forPath(hostsRootPath); + } catch (NoNodeException e) { return new HostInfos(); - } else { - List<String> hostnames = uncheck(() -> curator.framework().getChildren().forPath(hostsRootPath)); - Map<HostName, HostInfo> hostInfos = hostnames.stream().collect(Collectors.toMap( - hostname -> new HostName(hostname), - hostname -> { - byte[] bytes = uncheck(() -> curator.framework().getData().forPath(hostsRootPath + "/" + hostname)); - return WireHostInfo.deserialize(bytes); - })); - return new HostInfos(hostInfos); + } catch (Exception e) { + throw new RuntimeException(e); } + + var hostInfos = new HashMap<HostName, HostInfo>(); + for (var hostname : hostnames) { + byte[] bytes; + try { + bytes = curator.framework().getData().forPath(hostsRootPath + "/" + hostname); + } catch (NoNodeException e) { + // OK, node has been removed since getChildren() + continue; + } catch (Exception e) { + throw new RuntimeException(e); + } + + hostInfos.put(new HostName(hostname), WireHostInfo.deserialize(bytes)); + } + + return new HostInfos(hostInfos); } @Override diff --git a/parent/pom.xml b/parent/pom.xml index 454b4677cfa..ce40eb464fc 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -220,6 +220,7 @@ <configuration> <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> <systemPropertyVariables> + <jdisc.logger.level>INFO</jdisc.logger.level> <java.io.tmpdir>${project.build.directory}</java.io.tmpdir> </systemPropertyVariables> <trimStackTrace>false</trimStackTrace> @@ -356,6 +357,12 @@ <goal>javadoc</goal> </goals> </execution> + <execution> + <id>attach-javadocs</id> + <goals> + <goal>jar</goal> + </goals> + </execution> </executions> <configuration> <doclint>${doclint},-missing</doclint> @@ -738,7 +745,7 @@ <dependency> <groupId>xerces</groupId> <artifactId>xercesImpl</artifactId> - <version>2.11.0</version> + <version>2.12.0</version> </dependency> </dependencies> </dependencyManagement> @@ -746,7 +753,7 @@ <properties> <antlr.version>3.5.2</antlr.version> <antlr4.version>4.5</antlr4.version> - <apache.httpclient.version>4.5.11</apache.httpclient.version> + <apache.httpclient.version>4.5.12</apache.httpclient.version> <apache.httpcore.version>4.4.13</apache.httpcore.version> <asm.version>7.0</asm.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> @@ -780,7 +787,8 @@ <protobuf.version>3.7.0</protobuf.version> <surefire.version>2.22.0</surefire.version> <tensorflow.version>1.12.0</tensorflow.version> - <zookeeper.client.version>3.5.6</zookeeper.client.version> + <zookeeper.client.version>3.5.8</zookeeper.client.version> + <zookeeper.server.version>3.5.6</zookeeper.server.version> <doclint>all</doclint> <test.hide>true</test.hide> diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index b45a4131e7c..036a2b87692 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -625,6 +625,7 @@ TEST_F(ConformanceTest, testPutNewDocumentVersion) EXPECT_EQ(Result::ErrorType::NONE, gr.getErrorCode()); EXPECT_EQ(Timestamp(4), gr.getTimestamp()); + EXPECT_FALSE(gr.is_tombstone()); if (!((*doc2)==gr.getDocument())) { std::cerr << "Document returned is not the expected one: \n" @@ -676,6 +677,7 @@ TEST_F(ConformanceTest, testPutOlderDocumentVersion) EXPECT_EQ(Result::ErrorType::NONE, gr.getErrorCode()); EXPECT_EQ(Timestamp(5), gr.getTimestamp()); EXPECT_EQ(*doc1, gr.getDocument()); + EXPECT_FALSE(gr.is_tombstone()); } TEST_F(ConformanceTest, testPutDuplicate) @@ -798,8 +800,9 @@ TEST_F(ConformanceTest, testRemove) context); EXPECT_EQ(Result::ErrorType::NONE, getResult.getErrorCode()); - EXPECT_EQ(Timestamp(0), getResult.getTimestamp()); - EXPECT_TRUE(!getResult.hasDocument()); + EXPECT_EQ(Timestamp(9), getResult.getTimestamp()); + EXPECT_TRUE(getResult.is_tombstone()); + EXPECT_FALSE(getResult.hasDocument()); } TEST_F(ConformanceTest, testRemoveMerge) @@ -939,6 +942,7 @@ TEST_F(ConformanceTest, testUpdate) EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(Timestamp(4), result.getTimestamp()); + EXPECT_FALSE(result.is_tombstone()); EXPECT_EQ(document::IntFieldValue(42), static_cast<document::IntFieldValue&>( *result.getDocument().getValue("headerval"))); @@ -953,8 +957,9 @@ TEST_F(ConformanceTest, testUpdate) context); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); - EXPECT_EQ(Timestamp(0), result.getTimestamp()); - EXPECT_TRUE(!result.hasDocument()); + EXPECT_EQ(Timestamp(5), result.getTimestamp()); + EXPECT_FALSE(result.hasDocument()); + EXPECT_TRUE(result.is_tombstone()); } { @@ -968,8 +973,9 @@ TEST_F(ConformanceTest, testUpdate) { GetResult result = spi->get(bucket, document::AllFields(), doc1->getId(), context); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); - EXPECT_EQ(Timestamp(0), result.getTimestamp()); - EXPECT_TRUE(!result.hasDocument()); + EXPECT_EQ(Timestamp(5), result.getTimestamp()); + EXPECT_FALSE(result.hasDocument()); + EXPECT_TRUE(result.is_tombstone()); } update->setCreateIfNonExistent(true); @@ -985,6 +991,7 @@ TEST_F(ConformanceTest, testUpdate) GetResult result = spi->get(bucket, document::AllFields(), doc1->getId(), context); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(Timestamp(7), result.getTimestamp()); + EXPECT_FALSE(result.is_tombstone()); EXPECT_EQ(document::IntFieldValue(42), reinterpret_cast<document::IntFieldValue&>( *result.getDocument().getValue("headerval"))); @@ -1008,6 +1015,7 @@ TEST_F(ConformanceTest, testGet) EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(Timestamp(0), result.getTimestamp()); + EXPECT_FALSE(result.is_tombstone()); } spi->put(bucket, Timestamp(3), doc1, context); @@ -1017,6 +1025,7 @@ TEST_F(ConformanceTest, testGet) doc1->getId(), context); EXPECT_EQ(*doc1, result.getDocument()); EXPECT_EQ(Timestamp(3), result.getTimestamp()); + EXPECT_FALSE(result.is_tombstone()); } spi->remove(bucket, Timestamp(4), doc1->getId(), context); @@ -1026,7 +1035,8 @@ TEST_F(ConformanceTest, testGet) doc1->getId(), context); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); - EXPECT_EQ(Timestamp(0), result.getTimestamp()); + EXPECT_EQ(Timestamp(4), result.getTimestamp()); + EXPECT_TRUE(result.is_tombstone()); } } diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 5720a0ba662..92fea5ff6e4 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -533,7 +533,10 @@ DummyPersistence::get(const Bucket& b, const document::FieldSet& fieldSet, const if (!bc.get()) { } else { DocEntry::SP entry((*bc)->getEntry(did)); - if (entry.get() == 0 || entry->isRemove()) { + if (!entry) { + return GetResult(); + } else if (entry->isRemove()) { + return GetResult::make_for_tombstone(entry->getTimestamp()); } else { Document::UP doc(entry->getDocument()->clone()); if (fieldSet.getType() != document::FieldSet::ALL) { diff --git a/persistence/src/vespa/persistence/spi/result.cpp b/persistence/src/vespa/persistence/spi/result.cpp index f5131f08b1f..8de6245a8d2 100644 --- a/persistence/src/vespa/persistence/spi/result.cpp +++ b/persistence/src/vespa/persistence/spi/result.cpp @@ -30,8 +30,17 @@ std::ostream & operator << (std::ostream & os, const Result::ErrorType &errorCod GetResult::GetResult(Document::UP doc, Timestamp timestamp) : Result(), _timestamp(timestamp), - _doc(std::move(doc)) -{ } + _doc(std::move(doc)), + _is_tombstone(false) +{ +} + +GetResult::GetResult(Timestamp removed_at_ts) + : _timestamp(removed_at_ts), + _doc(), + _is_tombstone(true) +{ +} GetResult::~GetResult() = default; BucketIdListResult::~BucketIdListResult() = default; diff --git a/persistence/src/vespa/persistence/spi/result.h b/persistence/src/vespa/persistence/spi/result.h index fe8e74706bb..714d71e37ee 100644 --- a/persistence/src/vespa/persistence/spi/result.h +++ b/persistence/src/vespa/persistence/spi/result.h @@ -156,13 +156,20 @@ public: */ GetResult(ErrorType error, const vespalib::string& errorMessage) : Result(error, errorMessage), - _timestamp(0) { } + _timestamp(0), + _is_tombstone(false) + { + } /** * Constructor to use when we didn't find the document in question. */ GetResult() - : _timestamp(0) { } + : _timestamp(0), + _doc(), + _is_tombstone(false) + { + } /** * Constructor to use when we found the document asked for. @@ -172,12 +179,22 @@ public: */ GetResult(DocumentUP doc, Timestamp timestamp); - ~GetResult(); + static GetResult make_for_tombstone(Timestamp removed_at_ts) { + return GetResult(removed_at_ts); + } + + ~GetResult() override; - Timestamp getTimestamp() const { return _timestamp; } + [[nodiscard]] Timestamp getTimestamp() const { + return _timestamp; + } - bool hasDocument() const { - return _doc.get() != NULL; + [[nodiscard]] bool hasDocument() const { + return (_doc.get() != nullptr); + } + + [[nodiscard]] bool is_tombstone() const noexcept { + return _is_tombstone; } const Document& getDocument() const { @@ -193,8 +210,12 @@ public: } private: + // Explicitly creates a tombstone (remove entry) GetResult with no document. + explicit GetResult(Timestamp removed_at_ts); + Timestamp _timestamp; DocumentSP _doc; + bool _is_tombstone; }; class BucketIdListResult : public Result { @@ -31,6 +31,8 @@ <module>bundle-plugin-test</module> <module>chain</module> <module>client</module> + <module>cloud-tenant-base</module> + <module>cloud-tenant-cd</module> <module>clustercontroller-apps</module> <module>clustercontroller-apputil</module> <module>clustercontroller-core</module> @@ -80,6 +82,9 @@ <module>filedistribution</module> <module>flags</module> <module>fsa</module> + <module>hosted-api</module> + <module>hosted-tenant-base</module> + <module>hosted-zone-api</module> <module>http-utils</module> <module>indexinglanguage</module> <module>jaxrs_client_utils</module> @@ -123,7 +128,7 @@ <module>storage</module> <module>tenant-auth</module> <module>tenant-base</module> - <module>tenant-cd</module> + <module>tenant-cd-api</module> <module>testutil</module> <module>vdslib</module> <module>vespaclient-core</module> @@ -144,7 +149,6 @@ <module>zkfacade</module> <module>zookeeper-command-line-client</module> <module>zookeeper-server</module> - <module>hosted-api</module> </modules> </project> diff --git a/processing/src/main/java/com/yahoo/processing/request/CompoundName.java b/processing/src/main/java/com/yahoo/processing/request/CompoundName.java index 2185aac7adc..976fb3e2796 100644 --- a/processing/src/main/java/com/yahoo/processing/request/CompoundName.java +++ b/processing/src/main/java/com/yahoo/processing/request/CompoundName.java @@ -20,6 +20,12 @@ import static com.yahoo.text.Lowercase.toLowerCase; */ public final class CompoundName { + /** + * The string name of this compound. + */ + private final String name; + private final String lowerCasedName; + private final ImmutableList<String> compounds; /** A hashcode which is always derived from the compounds (NEVER the string) */ @@ -37,7 +43,7 @@ public final class CompoundName { * @throws NullPointerException if name is null */ public CompoundName(String name) { - this(parse(name)); + this(name, parse(name)); } /** Constructs this from an array of name components which are assumed not to contain dots */ @@ -47,6 +53,21 @@ public final class CompoundName { /** Constructs this from a list of compounds. */ public CompoundName(List<String> compounds) { + this(toCompoundString(compounds), compounds); + } + + /** + * Constructs this from a name with already parsed compounds. + * Private to avoid creating names with inconsistencies. + * + * @param name the string representation of the compounds + * @param compounds the compounds of this name + */ + private CompoundName(String name, List<String> compounds) { + if (name == null) throw new NullPointerException("Name can not be null"); + + this.name = name; + this.lowerCasedName = toLowerCase(name); if (compounds.size()==1 && compounds.get(0).isEmpty()) this.compounds = ImmutableList.of(); else @@ -90,7 +111,7 @@ public final class CompoundName { if (isEmpty()) return new CompoundName(name); List<String> newCompounds = new ArrayList<>(compounds); newCompounds.addAll(parse(name)); - return new CompoundName(newCompounds); + return new CompoundName(concat(this.name, name), newCompounds); } /** @@ -103,7 +124,11 @@ public final class CompoundName { if (isEmpty()) return name; List<String> newCompounds = new ArrayList<>(compounds); newCompounds.addAll(name.compounds); - return new CompoundName(newCompounds); + return new CompoundName(concat(this.name, name.name), newCompounds); + } + + private String concat(String name1, String name2) { + return name1 + "." + name2; } /** @@ -217,11 +242,14 @@ public final class CompoundName { public boolean hasPrefix(CompoundName prefix) { if (prefix.size() > this.size()) return false; - for (int i = 0; i < prefix.size(); i++) { - if ( ! this.compounds.get(i).equals(prefix.get(i))) - return false; - } - return true; + int prefixLength = prefix.name.length(); + if (prefixLength == 0) + return true; + + if (name.length() > prefixLength && name.charAt(prefixLength) != '.') + return false; + + return name.startsWith(prefix.name); } /** @@ -239,22 +267,24 @@ public final class CompoundName { if (o == this) return true; if ( ! (o instanceof CompoundName)) return false; CompoundName other = (CompoundName)o; - return this.compounds.equals(other.compounds); + return this.name.equals(other.name); } /** * Returns the string representation of this - all the name components in order separated by dots. */ @Override - public String toString() { + public String toString() { return name; } + + public String getLowerCasedName() { + return lowerCasedName; + } + + private static String toCompoundString(List<String> compounds) { StringBuilder b = new StringBuilder(); for (String compound : compounds) b.append(compound).append("."); return b.length()==0 ? "" : b.substring(0, b.length()-1); } - public String getLowerCasedName() { - return toString().toLowerCase(); - } - } diff --git a/python/vespa/.gitignore b/python/vespa/.gitignore new file mode 100644 index 00000000000..9ca09886342 --- /dev/null +++ b/python/vespa/.gitignore @@ -0,0 +1,141 @@ +*.bak +.gitattributes +.last_checked +.gitconfig +*.bak +*.log +*~ +~* +_tmp* +tmp* +tags + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +.vscode +*.swp + +# osx generated files +.DS_Store +.DS_Store? +.Trashes +ehthumbs.db +Thumbs.db +.idea + +# pytest +.pytest_cache + +# tools/trust-doc-nbs +docs_src/.last_checked + +# symlinks to fastai +docs_src/fastai +tools/fastai + +# link checker +checklink/cookies.txt + +# .gitconfig is now autogenerated +.gitconfig + diff --git a/python/vespa/CONTRIBUTING.md b/python/vespa/CONTRIBUTING.md new file mode 100644 index 00000000000..38fbde54884 --- /dev/null +++ b/python/vespa/CONTRIBUTING.md @@ -0,0 +1,33 @@ +# How to contribute + +## How to get started + +Before anything else, please install the git hooks that run automatic scripts during each commit and merge to strip the notebooks of superfluous metadata (and avoid merge conflicts). After cloning the repository, run the following command inside it: +``` +nbdev_install_git_hooks +``` + +## Did you find a bug? + +* Ensure the bug was not already reported by searching on GitHub under Issues. +* If you're unable to find an open issue addressing the problem, open a new one. Be sure to include a title and clear description, as much relevant information as possible, and a code sample or an executable test case demonstrating the expected behavior that is not occurring. +* Be sure to add the complete error messages. + +#### Did you write a patch that fixes a bug? + +* Open a new GitHub pull request with the patch. +* Ensure that your PR includes a test that fails without your patch, and pass with it. +* Ensure the PR description clearly describes the problem and solution. Include the relevant issue number if applicable. + +## PR submission guidelines + +* Keep each PR focused. While it's more convenient, do not combine several unrelated fixes together. Create as many branches as needing to keep each PR focused. +* Do not mix style changes/fixes with "functional" changes. It's very difficult to review such PRs and it most likely get rejected. +* Do not add/remove vertical whitespace. Preserve the original style of the file you edit as much as you can. +* Do not turn an already submitted PR into your development playground. If after you submitted PR, you discovered that more work is needed - close the PR, do the required work and then submit a new PR. Otherwise each of your commits requires attention from maintainers of the project. +* If, however, you submitted a PR and received a request for changes, you should proceed with commits inside that PR, so that the maintainer can see the incremental fixes and won't need to review the whole PR again. In the exception case where you realize it'll take many many commits to complete the requests, then it's probably best to close the PR, do the work and then submit it again. Use common sense where you'd choose one way over another. + +## Do you want to contribute to the documentation? + +* Docs are automatically created from the notebooks in the nbs folder. + diff --git a/python/vespa/MANIFEST.in b/python/vespa/MANIFEST.in new file mode 100644 index 00000000000..5c0e7ced193 --- /dev/null +++ b/python/vespa/MANIFEST.in @@ -0,0 +1,5 @@ +include settings.ini +include LICENSE +include CONTRIBUTING.md +include README.md +recursive-exclude * __pycache__ diff --git a/python/vespa/Pipfile b/python/vespa/Pipfile new file mode 100644 index 00000000000..b723d0199f8 --- /dev/null +++ b/python/vespa/Pipfile @@ -0,0 +1,11 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] + +[requires] +python_version = "3.7" diff --git a/python/vespa/README.md b/python/vespa/README.md new file mode 100644 index 00000000000..00d8cc2e769 --- /dev/null +++ b/python/vespa/README.md @@ -0,0 +1,118 @@ +# Vespa library for data analysis +> Provide data analysis support for Vespa applications + + +## Install + +`pip install pyvespa` + +## Connect to a Vespa app + +> Connect to a running Vespa application + +``` +from vespa.application import Vespa + +app = Vespa(url = "https://api.cord19.vespa.ai") +``` + +## Define a Query model + +> Easily define matching and ranking criteria + +``` +from vespa.query import Query, Union, WeakAnd, ANN, RankProfile +from random import random + +match_phase = Union( + WeakAnd(hits = 10), + ANN( + doc_vector="title_embedding", + query_vector="title_vector", + embedding_model=lambda x: [random() for x in range(768)], + hits = 10, + label="title" + ) +) + +rank_profile = RankProfile(name="bm25", list_features=True) + +query_model = Query(match_phase=match_phase, rank_profile=rank_profile) +``` + +## Query the vespa app + +> Send queries via the query API. See the [query page](/vespa/query) for more examples. + +``` +query_result = app.query( + query="Is remdesivir an effective treatment for COVID-19?", + query_model=query_model +) +``` + +``` +query_result.number_documents_retrieved +``` + +## Labelled data + +> How to structure labelled data + +``` +labelled_data = [ + { + "query_id": 0, + "query": "Intrauterine virus infections and congenital heart disease", + "relevant_docs": [{"id": 0, "score": 1}, {"id": 3, "score": 1}] + }, + { + "query_id": 1, + "query": "Clinical and immunologic studies in identical twins discordant for systemic lupus erythematosus", + "relevant_docs": [{"id": 1, "score": 1}, {"id": 5, "score": 1}] + } +] +``` + +Non-relevant documents are assigned `"score": 0` by default. Relevant documents will be assigned `"score": 1` by default if the field is missing from the labelled data. The defaults for both relevant and non-relevant documents can be modified on the appropriate methods. + +## Collect training data + +> Collect training data to analyse and/or improve ranking functions. See the [collect training data page](/vespa/collect_training_data) for more examples. + +``` +training_data_batch = app.collect_training_data( + labelled_data = labelled_data, + id_field = "id", + query_model = query_model, + number_additional_docs = 2 +) +training_data_batch +``` + +## Evaluating a query model + +> Define metrics and evaluate query models. See the [evaluation page](/vespa/evaluation) for more examples. + +We will define the following evaluation metrics: +* % of documents retrieved per query +* recall @ 10 per query +* MRR @ 10 per query + +``` +from vespa.evaluation import MatchRatio, Recall, ReciprocalRank + +eval_metrics = [MatchRatio(), Recall(at=10), ReciprocalRank(at=10)] +``` + +Evaluate: + +``` +evaluation = app.evaluate( + labelled_data = labelled_data, + eval_metrics = eval_metrics, + query_model = query_model, + id_field = "id", +) +evaluation +``` diff --git a/python/vespa/docs/.gitignore b/python/vespa/docs/.gitignore new file mode 100644 index 00000000000..57510a2be4e --- /dev/null +++ b/python/vespa/docs/.gitignore @@ -0,0 +1 @@ +_site/ diff --git a/python/vespa/docs/Gemfile b/python/vespa/docs/Gemfile new file mode 100644 index 00000000000..f09053c16a3 --- /dev/null +++ b/python/vespa/docs/Gemfile @@ -0,0 +1,7 @@ +source "https://rubygems.org" + +gem 'github-pages', group: :jekyll_plugins + + +# Added at 2019-11-25 10:11:40 -0800 by jhoward: +gem "jekyll", "~> 3.7" diff --git a/python/vespa/docs/Gemfile.lock b/python/vespa/docs/Gemfile.lock new file mode 100644 index 00000000000..4b79042ad18 --- /dev/null +++ b/python/vespa/docs/Gemfile.lock @@ -0,0 +1,250 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (4.2.11.1) + i18n (~> 0.7) + minitest (~> 5.1) + thread_safe (~> 0.3, >= 0.3.4) + tzinfo (~> 1.1) + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) + coffee-script (2.4.1) + coffee-script-source + execjs + coffee-script-source (1.11.1) + colorator (1.1.0) + commonmarker (0.17.13) + ruby-enum (~> 0.5) + concurrent-ruby (1.1.5) + dnsruby (1.61.3) + addressable (~> 2.5) + em-websocket (0.5.1) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0.6.0) + ethon (0.12.0) + ffi (>= 1.3.0) + eventmachine (1.2.7) + execjs (2.7.0) + faraday (0.17.0) + multipart-post (>= 1.2, < 3) + ffi (1.11.3) + forwardable-extended (2.6.0) + gemoji (3.0.1) + github-pages (202) + activesupport (= 4.2.11.1) + github-pages-health-check (= 1.16.1) + jekyll (= 3.8.5) + jekyll-avatar (= 0.6.0) + jekyll-coffeescript (= 1.1.1) + jekyll-commonmark-ghpages (= 0.1.6) + jekyll-default-layout (= 0.1.4) + jekyll-feed (= 0.11.0) + jekyll-gist (= 1.5.0) + jekyll-github-metadata (= 2.12.1) + jekyll-mentions (= 1.4.1) + jekyll-optional-front-matter (= 0.3.0) + jekyll-paginate (= 1.1.0) + jekyll-readme-index (= 0.2.0) + jekyll-redirect-from (= 0.14.0) + jekyll-relative-links (= 0.6.0) + jekyll-remote-theme (= 0.4.0) + jekyll-sass-converter (= 1.5.2) + jekyll-seo-tag (= 2.5.0) + jekyll-sitemap (= 1.2.0) + jekyll-swiss (= 0.4.0) + jekyll-theme-architect (= 0.1.1) + jekyll-theme-cayman (= 0.1.1) + jekyll-theme-dinky (= 0.1.1) + jekyll-theme-hacker (= 0.1.1) + jekyll-theme-leap-day (= 0.1.1) + jekyll-theme-merlot (= 0.1.1) + jekyll-theme-midnight (= 0.1.1) + jekyll-theme-minimal (= 0.1.1) + jekyll-theme-modernist (= 0.1.1) + jekyll-theme-primer (= 0.5.3) + jekyll-theme-slate (= 0.1.1) + jekyll-theme-tactile (= 0.1.1) + jekyll-theme-time-machine (= 0.1.1) + jekyll-titles-from-headings (= 0.5.1) + jemoji (= 0.10.2) + kramdown (= 1.17.0) + liquid (= 4.0.0) + listen (= 3.1.5) + mercenary (~> 0.3) + minima (= 2.5.0) + nokogiri (>= 1.10.4, < 2.0) + rouge (= 3.11.0) + terminal-table (~> 1.4) + github-pages-health-check (1.16.1) + addressable (~> 2.3) + dnsruby (~> 1.60) + octokit (~> 4.0) + public_suffix (~> 3.0) + typhoeus (~> 1.3) + html-pipeline (2.12.2) + activesupport (>= 2) + nokogiri (>= 1.4) + http_parser.rb (0.6.0) + i18n (0.9.5) + concurrent-ruby (~> 1.0) + jekyll (3.8.5) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (~> 0.7) + jekyll-sass-converter (~> 1.0) + jekyll-watch (~> 2.0) + kramdown (~> 1.14) + liquid (~> 4.0) + mercenary (~> 0.3.3) + pathutil (~> 0.9) + rouge (>= 1.7, < 4) + safe_yaml (~> 1.0) + jekyll-avatar (0.6.0) + jekyll (~> 3.0) + jekyll-coffeescript (1.1.1) + coffee-script (~> 2.2) + coffee-script-source (~> 1.11.1) + jekyll-commonmark (1.3.1) + commonmarker (~> 0.14) + jekyll (>= 3.7, < 5.0) + jekyll-commonmark-ghpages (0.1.6) + commonmarker (~> 0.17.6) + jekyll-commonmark (~> 1.2) + rouge (>= 2.0, < 4.0) + jekyll-default-layout (0.1.4) + jekyll (~> 3.0) + jekyll-feed (0.11.0) + jekyll (~> 3.3) + jekyll-gist (1.5.0) + octokit (~> 4.2) + jekyll-github-metadata (2.12.1) + jekyll (~> 3.4) + octokit (~> 4.0, != 4.4.0) + jekyll-mentions (1.4.1) + html-pipeline (~> 2.3) + jekyll (~> 3.0) + jekyll-optional-front-matter (0.3.0) + jekyll (~> 3.0) + jekyll-paginate (1.1.0) + jekyll-readme-index (0.2.0) + jekyll (~> 3.0) + jekyll-redirect-from (0.14.0) + jekyll (~> 3.3) + jekyll-relative-links (0.6.0) + jekyll (~> 3.3) + jekyll-remote-theme (0.4.0) + addressable (~> 2.0) + jekyll (~> 3.5) + rubyzip (>= 1.2.1, < 3.0) + jekyll-sass-converter (1.5.2) + sass (~> 3.4) + jekyll-seo-tag (2.5.0) + jekyll (~> 3.3) + jekyll-sitemap (1.2.0) + jekyll (~> 3.3) + jekyll-swiss (0.4.0) + jekyll-theme-architect (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-cayman (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-dinky (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-hacker (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-leap-day (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-merlot (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-midnight (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-minimal (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-modernist (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-primer (0.5.3) + jekyll (~> 3.5) + jekyll-github-metadata (~> 2.9) + jekyll-seo-tag (~> 2.0) + jekyll-theme-slate (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-tactile (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-time-machine (0.1.1) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-titles-from-headings (0.5.1) + jekyll (~> 3.3) + jekyll-watch (2.2.1) + listen (~> 3.0) + jemoji (0.10.2) + gemoji (~> 3.0) + html-pipeline (~> 2.2) + jekyll (~> 3.0) + kramdown (1.17.0) + liquid (4.0.0) + listen (3.1.5) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + ruby_dep (~> 1.2) + mercenary (0.3.6) + mini_portile2 (2.4.0) + minima (2.5.0) + jekyll (~> 3.5) + jekyll-feed (~> 0.9) + jekyll-seo-tag (~> 2.1) + minitest (5.13.0) + multipart-post (2.1.1) + nokogiri (1.10.8) + mini_portile2 (~> 2.4.0) + octokit (4.14.0) + sawyer (~> 0.8.0, >= 0.5.3) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (3.1.1) + rb-fsevent (0.10.3) + rb-inotify (0.10.0) + ffi (~> 1.0) + rouge (3.11.0) + ruby-enum (0.7.2) + i18n + ruby_dep (1.5.0) + rubyzip (2.0.0) + safe_yaml (1.0.5) + sass (3.7.4) + sass-listen (~> 4.0.0) + sass-listen (4.0.0) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + sawyer (0.8.2) + addressable (>= 2.3.5) + faraday (> 0.8, < 2.0) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) + thread_safe (0.3.6) + typhoeus (1.3.1) + ethon (>= 0.9.0) + tzinfo (1.2.5) + thread_safe (~> 0.1) + unicode-display_width (1.6.0) + +PLATFORMS + ruby + +DEPENDENCIES + github-pages + jekyll (~> 3.7) + +BUNDLED WITH + 2.0.2 diff --git a/python/vespa/docs/_config.yml b/python/vespa/docs/_config.yml new file mode 100644 index 00000000000..1704c669c00 --- /dev/null +++ b/python/vespa/docs/_config.yml @@ -0,0 +1,64 @@ +repository: vespa-engine/vespa +output: web +topnav_title: vespa +site_title: vespa +company_name: Verizon Media +description: Vespa python API +# Set to false to disable KaTeX math +use_math: true +# Add Google analytics id if you have one and want to use it here +google_analytics: +# See http://nbdev.fast.ai/search for help with adding Search +google_search: + +host: 127.0.0.1 +# the preview server used. Leave as is. +port: 4000 +# the port where the preview is rendered. + +exclude: + - .idea/ + - .gitignore + - vendor + +exclude: [vendor] + +highlighter: rouge +markdown: kramdown +kramdown: + input: GFM + auto_ids: true + hard_wrap: false + syntax_highlighter: rouge + +collections: + tooltips: + output: false + +defaults: + - + scope: + path: "" + type: "pages" + values: + layout: "page" + comments: true + search: true + sidebar: home_sidebar + topnav: topnav + - + scope: + path: "" + type: "tooltips" + values: + layout: "page" + comments: true + search: true + tooltip: true + +sidebars: +- home_sidebar +permalink: pretty + +theme: jekyll-theme-cayman +baseurl: /pyvespa/
\ No newline at end of file diff --git a/python/vespa/docs/_data/alerts.yml b/python/vespa/docs/_data/alerts.yml new file mode 100644 index 00000000000..157e1622b00 --- /dev/null +++ b/python/vespa/docs/_data/alerts.yml @@ -0,0 +1,15 @@ +tip: '<div class="alert alert-success" role="alert"><i class="fa fa-check-square-o"></i> <b>Tip: </b>' +note: '<div class="alert alert-info" role="alert"><i class="fa fa-info-circle"></i> <b>Note: </b>' +important: '<div class="alert alert-warning" role="alert"><i class="fa fa-warning"></i> <b>Important: </b>' +warning: '<div class="alert alert-danger" role="alert"><i class="fa fa-exclamation-circle"></i> <b>Warning: </b>' +end: '</div>' + +callout_danger: '<div class="bs-callout bs-callout-danger">' +callout_default: '<div class="bs-callout bs-callout-default">' +callout_primary: '<div class="bs-callout bs-callout-primary">' +callout_success: '<div class="bs-callout bs-callout-success">' +callout_info: '<div class="bs-callout bs-callout-info">' +callout_warning: '<div class="bs-callout bs-callout-warning">' + +hr_faded: '<hr class="faded"/>' +hr_shaded: '<hr class="shaded"/>'
\ No newline at end of file diff --git a/python/vespa/docs/_data/definitions.yml b/python/vespa/docs/_data/definitions.yml new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/python/vespa/docs/_data/definitions.yml @@ -0,0 +1 @@ + diff --git a/python/vespa/docs/_data/glossary.yml b/python/vespa/docs/_data/glossary.yml new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/python/vespa/docs/_data/glossary.yml @@ -0,0 +1 @@ + diff --git a/python/vespa/docs/_data/sidebars/home_sidebar.yml b/python/vespa/docs/_data/sidebars/home_sidebar.yml new file mode 100644 index 00000000000..03a90e47d89 --- /dev/null +++ b/python/vespa/docs/_data/sidebars/home_sidebar.yml @@ -0,0 +1,24 @@ + +################################################# +### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ### +################################################# +# Instead edit ../../sidebar.json +entries: +- folders: + - folderitems: + - output: web,pdf + title: Overview + url: / + - output: web,pdf + title: Vespa - collect training data + url: /collect_training_data + - output: web,pdf + title: Vespa - Evaluate query models + url: /evaluation + - output: web,pdf + title: Query API + url: /query + output: web + title: pyvespa + output: web + title: Sidebar diff --git a/python/vespa/docs/_data/tags.yml b/python/vespa/docs/_data/tags.yml new file mode 100644 index 00000000000..7853ec711dc --- /dev/null +++ b/python/vespa/docs/_data/tags.yml @@ -0,0 +1,3 @@ +allowed-tags: + - getting_started + - navigation diff --git a/python/vespa/docs/_data/terms.yml b/python/vespa/docs/_data/terms.yml new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/python/vespa/docs/_data/terms.yml diff --git a/python/vespa/docs/_data/topnav.yml b/python/vespa/docs/_data/topnav.yml new file mode 100644 index 00000000000..b554be4daf5 --- /dev/null +++ b/python/vespa/docs/_data/topnav.yml @@ -0,0 +1,10 @@ +topnav: +- title: Topnav + items: + - title: github + external_url: https://github.com/vespa-engine/vespa/tree/master/ + +#Topnav dropdowns +topnav_dropdowns: +- title: Topnav dropdowns + folders:
\ No newline at end of file diff --git a/python/vespa/docs/_includes/archive.html b/python/vespa/docs/_includes/archive.html new file mode 100644 index 00000000000..275850c9cb3 --- /dev/null +++ b/python/vespa/docs/_includes/archive.html @@ -0,0 +1,15 @@ +--- +layout: default +type: archive +--- + +<div class="post-header"> + <h1 class="post-title-main">{{ page.title }}</h1> +</div> +<div class="post-content"> + +{{ content }} +</div> + + + diff --git a/python/vespa/docs/_includes/callout.html b/python/vespa/docs/_includes/callout.html new file mode 100644 index 00000000000..d492b1830d3 --- /dev/null +++ b/python/vespa/docs/_includes/callout.html @@ -0,0 +1 @@ +<div markdown="span" class="bs-callout bs-callout-{{include.type}}">{{include.content}}</div> diff --git a/python/vespa/docs/_includes/footer.html b/python/vespa/docs/_includes/footer.html new file mode 100755 index 00000000000..178b47cbf4a --- /dev/null +++ b/python/vespa/docs/_includes/footer.html @@ -0,0 +1,9 @@ +<footer> + <div class="row"> + <div class="col-lg-12 footer"> + <p><img src="{{ "/images/company_logo.png" | prepend: site.baseurl }}" alt="Company logo"/></p> + ©{{ site.time | date: "%Y" }} {{site.company_name}}. All rights reserved. <br /> +{% if page.last_updated %}<span>Page last updated:</span> {{page.last_updated}}<br/>{% endif %} Site last generated: {{ site.time | date: "%b %-d, %Y" }} <br /> + </div> + </div> +</footer> diff --git a/python/vespa/docs/_includes/google_analytics.html b/python/vespa/docs/_includes/google_analytics.html new file mode 100644 index 00000000000..56b2ee88c5d --- /dev/null +++ b/python/vespa/docs/_includes/google_analytics.html @@ -0,0 +1,6 @@ +<!-- the google_analytics_id gets auto inserted from the config file --> + +{% if site.google_analytics %} + +<script>(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)})(window,document,'script','//www.google-analytics.com/analytics.js','ga');ga('create','{{site.google_analytics}}','auto');ga('require','displayfeatures');ga('send','pageview');</script> +{% endif %}
\ No newline at end of file diff --git a/python/vespa/docs/_includes/head.html b/python/vespa/docs/_includes/head.html new file mode 100644 index 00000000000..02eb796a54a --- /dev/null +++ b/python/vespa/docs/_includes/head.html @@ -0,0 +1,82 @@ +<meta charset="utf-8"> +<meta http-equiv="X-UA-Compatible" content="IE=edge"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<meta name="description" content="{% if page.summary %}{{ page.summary | strip_html | strip_newlines | truncate: 160 }}{% endif %}"> +<meta name="keywords" content="{{page.tags}}{% if page.tags %}, {% endif %} {{page.keywords}}"> +<title>{{ page.title }} | {{ site.site_title }}</title> +<link rel="stylesheet" href="{{ "/css/syntax.css" | prepend: site.baseurl }}"> + +<link rel="stylesheet" type="text/css" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css"> +<!--<link rel="stylesheet" type="text/css" href="/css/bootstrap.min.css">--> +<link rel="stylesheet" href="{{ "/css/modern-business.css" | prepend: site.baseurl }}"> +<!-- Latest compiled and minified CSS --> +<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous"> +<link rel="stylesheet" href="{{ "/css/customstyles.css" | prepend: site.baseurl }}"> +<link rel="stylesheet" href="{{ "/css/boxshadowproperties.css" | prepend: site.baseurl }}"> +<!-- most color styles are extracted out to here --> +<link rel="stylesheet" href="{{ "/css/theme-blue.css" | prepend: site.baseurl }}"> + +<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.4/jquery.min.js"></script> + +<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-cookie/1.4.1/jquery.cookie.min.js"></script> +<script src="{{ "/js/jquery.navgoco.min.js" | prepend: site.baseurl }}"></script> + +{% if site.use_math %} +<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css" integrity="sha384-zB1R0rpPzHqg7Kpt0Aljp8JPLqbXI3bhnPWROx27a9N0Ll6ZP/+DiW/UqRcLbRjq" crossorigin="anonymous"> +<script defer src="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.js" integrity="sha384-y23I5Q6l+B6vatafAwxRu/0oK/79VlbSz7Q9aiSZUvyWYIYsd+qj+o24G5ZU2zJz" crossorigin="anonymous"></script> +<script defer src="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/contrib/auto-render.min.js" integrity="sha384-kWPLUVMOks5AQFrykwIup5lo0m3iMkkHrD0uJ4H5cjeGihAutqP0yW0J6dpFiVkI" crossorigin="anonymous"></script> +<script> +document.addEventListener("DOMContentLoaded", function() { + renderMathInElement( document.body, { + delimiters: [ + {left: "$$", right: "$$", display: true}, + {left: "[%", right: "%]", display: true}, + {left: "$", right: "$", display: false} + ]} + ); +}); +</script> +{% endif %} + +<!-- Latest compiled and minified JavaScript --> +<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script> +<!-- Anchor.js --> +<script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/2.0.0/anchor.min.js"></script> +<script src="{{ "/js/toc.js" | prepend: site.baseurl }}"></script> +<script src="{{ "/js/customscripts.js" | prepend: site.baseurl }}"></script> + +<link rel="shortcut icon" href="{{ "/images/favicon.ico?" | prepend: site.baseurl }}"> + +<!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries --> +<!-- WARNING: Respond.js doesn't work if you view the page via file:// --> +<!--[if lt IE 9]> +<script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script> +<script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script> +<![endif]--> + +<link rel="alternate" type="application/rss+xml" title="{{ site.title }}" href="{{ "/feed.xml" | prepend: site.baseurl | prepend: site.url }}"> + +<!-- Twitter cards --> +{% if site.twitter_username %} +<meta name="twitter:site" content="@{{ site.twitter_username }}"> +<meta name="twitter:creator" content="@{{ page.author }}"> +<meta name="twitter:title" content="{{ page.title }}"> +{% endif %} + +{% if page.summary %} +<meta name="twitter:description" content="{{ page.summary }}"> +{% else %} +<meta name="twitter:description" content="{{ site.description }}"> +{% endif %} + +{% if page.image %} +<meta name="twitter:card" content="summary_large_image"> +<meta name="twitter:image" content="{{ site.url }}{{ page.image }}"> +{% else %} +<!-- <meta name="twitter:card" content="summary"> --> +<!-- <meta name="twitter:image" content="{{ site.title_image }}"> --> +{% endif %} +<!-- end of Twitter cards --> + + + diff --git a/python/vespa/docs/_includes/head_print.html b/python/vespa/docs/_includes/head_print.html new file mode 100644 index 00000000000..12b861e6245 --- /dev/null +++ b/python/vespa/docs/_includes/head_print.html @@ -0,0 +1,28 @@ +<meta charset="utf-8"> +<meta http-equiv="X-UA-Compatible" content="IE=edge"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<meta name="description" content="{% if page.summary %}{{ page.summary | strip_html | strip_newlines | truncate: 160 }}{% endif %}"> +<meta name="keywords" content="{{page.tags}}{% if page.tags %}, {% endif %} {{page.keywords}}"> +<title>{% if page.homepage == true %} {{site.homepage_title}} {% elsif page.title %}{{ page.title }}{% endif %} | {{ site.site_title }}</title> + + +<link rel="stylesheet" href="{{ "/css/syntax.css" | prepend: site.baseurl | prepend: site.url }}"> +<link rel="stylesheet" href="{{ "/css/font-awesome.min.css" | prepend: site.baseurl | prepend: site.url }}"> +<link rel="stylesheet" href="{{ "/css/bootstrap.min.css" | prepend: site.baseurl | prepend: site.url }}"> +<link rel="stylesheet" href="{{ "/css/modern-business.css" | prepend: site.baseurl | prepend: site.url }}"> +<link rel="stylesheet" href="{{ "/css/customstyles.css" | prepend: site.baseurl | prepend: site.url }}"> +<link rel="stylesheet" href="{{ "/css/theme-blue.css" | prepend: site.baseurl | prepend: site.url }}"> +<link rel="stylesheet" href="{{ "/css/syntax.css" | prepend: site.baseurl | prepend: site.url }}"> +<link rel="stylesheet" href="{{ "/css/printstyles.css" | prepend: site.baseurl }}"> + +<script> + Prince.addScriptFunc("datestamp", function() { + return "PDF last generated: {{ site.time | date: '%B %d, %Y' }}"; + }); +</script> + +<script> + Prince.addScriptFunc("guideName", function() { + return "{{site.print_title}} User Guide"; + }); +</script> diff --git a/python/vespa/docs/_includes/image.html b/python/vespa/docs/_includes/image.html new file mode 100644 index 00000000000..fb5ab05308f --- /dev/null +++ b/python/vespa/docs/_includes/image.html @@ -0,0 +1 @@ +<figure>{% if {{include.url}} %}<a class="no_icon" target="_blank" href="{{include.url}}">{% endif %}<img class="docimage" src="{{include.file}}" alt="{{include.alt}}" {% if {{include.max-width}} %}style="max-width: {{include.max-width}}px"{% endif %} />{% if {{include.url}} %}</a>{% endif %}{% if {{include.caption}} %}<figcaption>{{include.caption}}</figcaption>{% endif %}</figure> diff --git a/python/vespa/docs/_includes/important.html b/python/vespa/docs/_includes/important.html new file mode 100644 index 00000000000..af8824b9f4a --- /dev/null +++ b/python/vespa/docs/_includes/important.html @@ -0,0 +1 @@ +<div markdown="span" class="alert alert-warning" role="alert"><i class="fa fa-warning"></i> <b>Important:</b> {{include.content}}</div>
\ No newline at end of file diff --git a/python/vespa/docs/_includes/initialize_shuffle.html b/python/vespa/docs/_includes/initialize_shuffle.html new file mode 100644 index 00000000000..9a0f048dd11 --- /dev/null +++ b/python/vespa/docs/_includes/initialize_shuffle.html @@ -0,0 +1,130 @@ +<script type="text/javascript"> +$(document).ready(function() { + $('#toc').toc({ minimumHeaders: 0, listType: 'ul', showSpeed: 0, headers: 'h2,h3,h4' }); +}); + +</script> +<!-- shuffle --> +<script> +var shuffleme = (function( $ ) { + 'use strict'; + + var $grid = $('#grid'), + $filterOptions = $('.filter-options'), + $sizer = $grid.find('.shuffle_sizer'), + + init = function() { + + // None of these need to be executed synchronously + setTimeout(function() { + listen(); + setupFilters(); + }, 100); + + // instantiate the plugin + $grid.shuffle({ + itemSelector: '[class*="col-"]', + sizer: $sizer + }); + }, + + // Set up button clicks + setupFilters = function() { + var $btns = $filterOptions.children(); + $btns.on('click', function() { + var $this = $(this), + isActive = $this.hasClass( 'active' ), + group = isActive ? 'all' : $this.data('group'); + + // Hide current label, show current label in title + if ( !isActive ) { + $('.filter-options .active').removeClass('active'); + } + + $this.toggleClass('active'); + + // Filter elements + $grid.shuffle( 'shuffle', group ); + }); + + $btns = null; + }, + + // Re layout shuffle when images load. This is only needed + // below 768 pixels because the .picture-item height is auto and therefore + // the height of the picture-item is dependent on the image + // I recommend using imagesloaded to determine when an image is loaded + // but that doesn't support IE7 + listen = function() { + var debouncedLayout = $.throttle( 300, function() { + $grid.shuffle('update'); + }); + + // Get all images inside shuffle + $grid.find('img').each(function() { + var proxyImage; + + // Image already loaded + if ( this.complete && this.naturalWidth !== undefined ) { + return; + } + + // If none of the checks above matched, simulate loading on detached element. + proxyImage = new Image(); + $( proxyImage ).on('load', function() { + $(this).off('load'); + debouncedLayout(); + }); + + proxyImage.src = this.src; + }); + + // Because this method doesn't seem to be perfect. + setTimeout(function() { + debouncedLayout(); + }, 500); + }; + + return { + init: init + }; +}( jQuery )); + + + +$(document).ready(function() { + shuffleme.init(); +}); + + </script> + + <!-- new attempt--> + + <script> + $(document).ready(function() { + + /* initialize shuffle plugin */ + var $grid = $('#grid'); + + $grid.shuffle({ + itemSelector: '.item' // the selector for the items in the grid + }); + +});</script> + +<script> +$('#filter a').click(function (e) { + e.preventDefault(); + + // set active class + $('#filter a').removeClass('active'); + $(this).addClass('active'); + + // get group name from clicked item + var groupName = $(this).attr('data-group'); + + // reshuffle grid + $grid.shuffle('shuffle', groupName ); +});</script> + + diff --git a/python/vespa/docs/_includes/inline_image.html b/python/vespa/docs/_includes/inline_image.html new file mode 100644 index 00000000000..1e7fd187cb7 --- /dev/null +++ b/python/vespa/docs/_includes/inline_image.html @@ -0,0 +1 @@ +<img class="inline" src="images/{{include.file}}" alt="{{include.alt}}" /> diff --git a/python/vespa/docs/_includes/links.html b/python/vespa/docs/_includes/links.html new file mode 100644 index 00000000000..4f99e9422ef --- /dev/null +++ b/python/vespa/docs/_includes/links.html @@ -0,0 +1,44 @@ +{% comment %}Get links from each sidebar, as listed in the _config.yml file under sidebars{% endcomment %} + +{% for sidebar in site.sidebars %} +{% for entry in site.data.sidebars[sidebar].entries %} +{% for folder in entry.folders %} +{% for folderitem in folder.folderitems %} +{% if folderitem.url contains "html#" %} +[{{folderitem.url | remove: "/" }}]: {{folderitem.url | remove: "/"}} +{% else %} +[{{folderitem.url | remove: "/" | remove: ".html"}}]: {{folderitem.url | remove: "/"}} +{% endif %} +{% for subfolders in folderitem.subfolders %} +{% for subfolderitem in subfolders.subfolderitems %} +[{{subfolderitem.url | remove: "/" | remove: ".html"}}]: {{subfolderitem.url | remove: "/"}} +{% endfor %} +{% endfor %} +{% endfor %} +{% endfor %} +{% endfor %} +{% endfor %} + + +{% comment %} Get links from topnav {% endcomment %} + +{% for entry in site.data.topnav.topnav %} +{% for item in entry.items %} +{% if item.external_url == null %} +[{{item.url | remove: "/" | remove: ".html"}}]: {{item.url | remove: "/"}} +{% endif %} +{% endfor %} +{% endfor %} + +{% comment %}Get links from topnav dropdowns {% endcomment %} + +{% for entry in site.data.topnav.topnav_dropdowns %} +{% for folder in entry.folders %} +{% for folderitem in folder.folderitems %} +{% if folderitem.external_url == null %} +[{{folderitem.url | remove: "/" | remove: ".html"}}]: {{folderitem.url | remove: "/"}} +{% endif %} +{% endfor %} +{% endfor %} +{% endfor %} + diff --git a/python/vespa/docs/_includes/note.html b/python/vespa/docs/_includes/note.html new file mode 100644 index 00000000000..2c1cfe967c1 --- /dev/null +++ b/python/vespa/docs/_includes/note.html @@ -0,0 +1 @@ +<div markdown="span" class="alert alert-info" role="alert"><i class="fa fa-info-circle"></i> <b>Note:</b> {{include.content}}</div> diff --git a/python/vespa/docs/_includes/search_google_custom.html b/python/vespa/docs/_includes/search_google_custom.html new file mode 100644 index 00000000000..27ca1c44c2a --- /dev/null +++ b/python/vespa/docs/_includes/search_google_custom.html @@ -0,0 +1,16 @@ +<script> + (function() { + var cx = '{{site.google_search}}'; + var gcse = document.createElement('script'); + gcse.type = 'text/javascript'; + gcse.async = true; + gcse.src = 'https://cse.google.com/cse.js?cx=' + cx; + var s = document.getElementsByTagName('script')[0]; + s.parentNode.insertBefore(gcse, s); + })(); +</script> + +<div id="gcs-search-container"> + <gcse:search></gcse:search> +</div> + diff --git a/python/vespa/docs/_includes/search_simple_jekyll.html b/python/vespa/docs/_includes/search_simple_jekyll.html new file mode 100644 index 00000000000..eee4d3cefbe --- /dev/null +++ b/python/vespa/docs/_includes/search_simple_jekyll.html @@ -0,0 +1,16 @@ +<div id="search-demo-container"> + <input type="text" id="search-input" placeholder="{{site.data.strings.search_placeholder_text}}"> + <ul id="results-container"></ul> +</div> +<script src="{{ "js/jekyll-search.js"}}" type="text/javascript"></script> +<script type="text/javascript"> + SimpleJekyllSearch.init({ + searchInput: document.getElementById('search-input'), + resultsContainer: document.getElementById('results-container'), + dataSource: '{{ "search.json" }}', + searchResultTemplate: '<li><a href="{url}" title="{{page.title | escape }}">{title}</a></li>', +noResultsText: '{{site.data.strings.search_no_results_text}}', + limit: 10, + fuzzy: true, +}) +</script> diff --git a/python/vespa/docs/_includes/sidebar.html b/python/vespa/docs/_includes/sidebar.html new file mode 100644 index 00000000000..b402b9b59c0 --- /dev/null +++ b/python/vespa/docs/_includes/sidebar.html @@ -0,0 +1,59 @@ +{% assign sidebar = site.data.sidebars[page.sidebar].entries %} +{% assign pageurl = page.url | remove: ".html" %} + +<ul id="mysidebar" class="nav"> + <li class="sidebarTitle">{{sidebar[0].product}} {{sidebar[0].version}}</li> + {% for entry in sidebar %} + {% for folder in entry.folders %} + {% if folder.output contains "web" %} + <li> + <a href="#">{{ folder.title }}</a> + <ul> + {% for folderitem in folder.folderitems %} + {% if folderitem.output contains "web" %} + {% if folderitem.external_url %} + <li><a href="{{folderitem.external_url}}" target="_blank">{{folderitem.title}}</a></li> + {% elsif pageurl == folderitem.url %} + <li class="active"><a href="{{folderitem.url | prepend: site.baseurl}}">{{folderitem.title}}</a></li> + {% elsif folderitem.type == "empty" %} + <li><a href="{{folderitem.url | prepend: site.baseurl}}">{{folderitem.title}}</a></li> + + {% else %} + <li><a href="{{folderitem.url | prepend: site.baseurl}}">{{folderitem.title}}</a></li> + {% endif %} + {% for subfolders in folderitem.subfolders %} + {% if subfolders.output contains "web" %} + <li class="subfolders"> + <a href="#">{{ subfolders.title }}</a> + <ul> + {% for subfolderitem in subfolders.subfolderitems %} + {% if subfolderitem.output contains "web" %} + {% if subfolderitem.external_url %} + <li><a href="{{subfolderitem.external_url}}" target="_blank">{{subfolderitem.title}}</a></li> + {% elsif pageurl == subfolderitem.url %} + <li class="active"><a href="{{subfolderitem.url | prepend: site.baseurl}}">{{subfolderitem.title}}</a></li> + {% else %} + <li><a href="{{subfolderitem.url | prepend: site.baseurl}}">{{subfolderitem.title}}</a></li> + {% endif %} + {% endif %} + {% endfor %} + </ul> + </li> + {% endif %} + {% endfor %} + {% endif %} + {% endfor %} + </ul> + </li> + {% endif %} + {% endfor %} + {% endfor %} + <!-- if you aren't using the accordion, uncomment this block: + <p class="external"> + <a href="#" id="collapseAll">Collapse All</a> | <a href="#" id="expandAll">Expand All</a> + </p> + --> +</ul> + +<!-- this highlights the active parent class in the navgoco sidebar. this is critical so that the parent expands when you're viewing a page. This must appear below the sidebar code above. Otherwise, if placed inside customscripts.js, the script runs before the sidebar code runs and the class never gets inserted.--> +<script>$("li.active").parents('li').toggleClass("active");</script> diff --git a/python/vespa/docs/_includes/tip.html b/python/vespa/docs/_includes/tip.html new file mode 100644 index 00000000000..faf48afd861 --- /dev/null +++ b/python/vespa/docs/_includes/tip.html @@ -0,0 +1 @@ +<div markdown="span" class="alert alert-success" role="alert"><i class="fa fa-check-square-o"></i> <b>Tip:</b> {{include.content}}</div>
\ No newline at end of file diff --git a/python/vespa/docs/_includes/toc.html b/python/vespa/docs/_includes/toc.html new file mode 100644 index 00000000000..067141a6ac7 --- /dev/null +++ b/python/vespa/docs/_includes/toc.html @@ -0,0 +1,21 @@ + +<!-- this handles the automatic toc. use ## for subheads to auto-generate the on-page minitoc. if you use html tags, you must supply an ID for the heading element in order for it to appear in the minitoc. --> +<script> +$( document ).ready(function() { + // Handler for .ready() called. + +$('#toc').toc({ minimumHeaders: 0, listType: 'ul', showSpeed: 0, headers: 'h2,h3,h4' }); + +/* this offset helps account for the space taken up by the floating toolbar. */ +$('#toc').on('click', 'a', function() { + var target = $(this.getAttribute('href')) + , scroll_target = target.offset().top + + $(window).scrollTop(scroll_target - 10); + return false +}) + +}); +</script> + +<div id="toc"></div> diff --git a/python/vespa/docs/_includes/topnav.html b/python/vespa/docs/_includes/topnav.html new file mode 100644 index 00000000000..4d93732e6ef --- /dev/null +++ b/python/vespa/docs/_includes/topnav.html @@ -0,0 +1,62 @@ +<!-- Navigation --> +<nav class="navbar navbar-inverse navbar-static-top"> + <div class="container topnavlinks"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="fa fa-home fa-lg navbar-brand" href="index.html"> <span class="projectTitle"> {{site.topnav_title}}</span></a> + </div> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right"> + <!-- toggle sidebar button --> + <li><a id="tg-sb-link" href="#"><i id="tg-sb-icon" class="fa fa-toggle-on"></i> Nav</a></li> + <!-- entries without drop-downs appear here --> + +{% assign topnav = site.data[page.topnav] %} +{% assign topnav_dropdowns = site.data[page.topnav].topnav_dropdowns %} + + {% for entry in topnav.topnav %} + {% for item in entry.items %} + {% if item.external_url %} + <li><a href="{{item.external_url}}" target="_blank">{{item.title}}</a></li> + {% elsif page.url contains item.url %} + <li class="active"><a href="{{item.url | remove: "/"}}">{{item.title}}</a></li> + {% else %} + <li><a href="{{item.url | remove: "/"}}">{{item.title}}</a></li> + {% endif %} + {% endfor %} + {% endfor %} + <!-- entries with drop-downs appear here --> + <!-- conditional logic to control which topnav appears for the audience defined in the configuration file.--> + {% for entry in topnav_dropdowns %} + {% for folder in entry.folders %} + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">{{ folder.title }}<b class="caret"></b></a> + <ul class="dropdown-menu"> + {% for folderitem in folder.folderitems %} + {% if folderitem.external_url %} + <li><a href="{{folderitem.external_url}}" target="_blank">{{folderitem.title}}</a></li> + {% elsif page.url contains folderitem.url %} + <li class="dropdownActive"><a href="{{folderitem.url | remove: "/"}}">{{folderitem.title}}</a></li> + {% else %} + <li><a href="{{folderitem.url | remove: "/"}}">{{folderitem.title}}</a></li> + {% endif %} + {% endfor %} + </ul> + </li> + {% endfor %} + {% endfor %} + {% if site.google_search %} + <li> + {% include search_google_custom.html %} + </li> + {% endif %} + </ul> + </div> + </div> + <!-- /.container --> +</nav> diff --git a/python/vespa/docs/_includes/warning.html b/python/vespa/docs/_includes/warning.html new file mode 100644 index 00000000000..e08268c9335 --- /dev/null +++ b/python/vespa/docs/_includes/warning.html @@ -0,0 +1 @@ +<div markdown="span" class="alert alert-danger" role="alert"><i class="fa fa-exclamation-circle"></i> <b>Warning:</b> {{include.content}}</div>
\ No newline at end of file diff --git a/python/vespa/docs/_layouts/default.html b/python/vespa/docs/_layouts/default.html new file mode 100644 index 00000000000..c940af55369 --- /dev/null +++ b/python/vespa/docs/_layouts/default.html @@ -0,0 +1,110 @@ +<!DOCTYPE html> +<html> +<head> + {% include head.html %} + <script> + $(document).ready(function() { + // Initialize navgoco with default options + $("#mysidebar").navgoco({ + caretHtml: '', + accordion: true, + openClass: 'active', // open + save: false, // leave false or nav highlighting doesn't work right + cookie: { + name: 'navgoco', + expires: false, + path: '/' + }, + slide: { + duration: 400, + easing: 'swing' + } + }); + + $("#collapseAll").click(function(e) { + e.preventDefault(); + $("#mysidebar").navgoco('toggle', false); + }); + + $("#expandAll").click(function(e) { + e.preventDefault(); + $("#mysidebar").navgoco('toggle', true); + }); + + // activate menu items where href is matching to current page + $("#mysidebar a[href='" + location.pathname.match(/\/([^\/]*)$/)[1] + "']") + .parents('li').addClass('active') + .parents('ul').css('display', 'block'); + }); + + </script> + <script> + $(function () { + $('[data-toggle="tooltip"]').tooltip() + }) + </script> + <script> + $(document).ready(function() { + $("#tg-sb-link").click(function() { + $("#tg-sb-sidebar").toggle(); + $("#tg-sb-content").toggleClass('col-md-9'); + $("#tg-sb-content").toggleClass('col-md-12'); + $("#tg-sb-icon").toggleClass('fa-toggle-on'); + $("#tg-sb-icon").toggleClass('fa-toggle-off'); + }); + }); + </script> + {% if page.datatable == true %} + <!-- Include the standard DataTables bits --> + <link rel="stylesheet" type="text/css" href="//cdn.datatables.net/1.10.13/css/jquery.dataTables.css"> + <script type="text/javascript" charset="utf8" src="//cdn.datatables.net/1.10.13/js/jquery.dataTables.js"></script> + <!-- First, this walks through the tables that occur between ...-begin + and ...-end and add the "datatable" class to them. + Then it invokes DataTable's standard initializer + Credit here: http://www.beardedhacker.com/blog/2015/08/28/add-class-attribute-to-markdown-table/ + --> + <script> + $(document).ready(function(){ + $('div.datatable-begin').nextUntil('div.datatable-end', 'table').addClass('display'); + $('table.display').DataTable( { + paging: true, + stateSave: true, + searching: true + }); + }); + </script> + {% endif %} + +</head> +<body> +{% include topnav.html %} +<!-- Page Content --> +<div class="container"> + <div id="main"> + <!-- Content Row --> + <div class="row"> + {% assign content_col_size = "col-md-12" %} + {% unless page.hide_sidebar %} + <!-- Sidebar Column --> + <div class="col-md-3" id="tg-sb-sidebar"> + {% include sidebar.html %} + </div> + {% assign content_col_size = "col-md-9" %} + {% endunless %} + + <!-- Content Column --> + <div class="{{content_col_size}}" id="tg-sb-content"> + {{content}} + </div> + <!-- /.row --> +</div> +<!-- /.container --> +</div> +<!-- /#main --> + </div> + +</body> +{% if site.google_analytics %} +{% include google_analytics.html %} +{% endif %} +</html> diff --git a/python/vespa/docs/_layouts/default_print.html b/python/vespa/docs/_layouts/default_print.html new file mode 100644 index 00000000000..4bf619b4735 --- /dev/null +++ b/python/vespa/docs/_layouts/default_print.html @@ -0,0 +1,25 @@ +<!DOCTYPE html> +<html lang="en"> +<html> +<head> + {% include head_print.html %} + + +</head> + +<body class="{% if page.type == "title"%}title{% elsif page.type == "frontmatter" %}frontmatter{% elsif page.type == "first_page" %}first_page{% endif %} print"> + +<!-- Page Content --> +<div class="container"> + <!-- Content Column --> + <div class="col-md-9"> + + {{content}} + </div> + +</div> <!-- /.container --> + +</body> + +</html> + diff --git a/python/vespa/docs/_layouts/none.html b/python/vespa/docs/_layouts/none.html new file mode 100644 index 00000000000..60887a9201d --- /dev/null +++ b/python/vespa/docs/_layouts/none.html @@ -0,0 +1,3 @@ +--- +--- +{{content}}
\ No newline at end of file diff --git a/python/vespa/docs/_layouts/page.html b/python/vespa/docs/_layouts/page.html new file mode 100644 index 00000000000..6962addaa17 --- /dev/null +++ b/python/vespa/docs/_layouts/page.html @@ -0,0 +1,67 @@ +--- +layout: default +--- + +<div class="post-header"> + <a id="{{page.title}}"></a> + <h1 class="post-title-main">{{ page.title }}</h1> +</div> + +{% if page.simple_map == true %} + +<script> + $(document).ready ( function(){ + $('.box{{page.box_number}}').addClass('active'); + }); +</script> + +{% include custom/{{page.map_name}}.html %} + +{% elsif page.complex_map == true %} + +<script> + $(document).ready ( function(){ + $('.modalButton{{page.box_number}}').addClass('active'); + }); +</script> + +{% include custom/{{page.map_name}}.html %} + +{% endif %} + +<div class="post-content"> + + {% if page.summary %} + <div class="summary">{{page.summary}}</div> + {% endif %} + + {% unless page.toc == false %} + {% include toc.html %} + {% endunless %} + + + {% if site.github_editme_path %} + + <a target="_blank" href="https://github.com/{{site.github_editme_path}}{{page.path}}" class="btn btn-default githubEditButton" role="button"><i class="fa fa-github fa-lg"></i> Edit me</a> + + {% endif %} + + {{content}} + + <div class="tags"> + {% if page.tags != null %} + <b>Tags: </b> + {% assign projectTags = site.data.tags.allowed-tags %} + {% for tag in page.tags %} + {% if projectTags contains tag %} + <a href="{{ "tag_" | append: tag | append: ".html" }}" class="btn btn-default navbar-btn cursorNorm" role="button">{{page.tagName}}{{tag}}</a> + {% endif %} + {% endfor %} + {% endif %} + </div> + +</div> + +{{site.data.alerts.hr_shaded}} + +{% include footer.html %} diff --git a/python/vespa/docs/_layouts/page_print.html b/python/vespa/docs/_layouts/page_print.html new file mode 100644 index 00000000000..9e04604a9cd --- /dev/null +++ b/python/vespa/docs/_layouts/page_print.html @@ -0,0 +1,15 @@ +--- +layout: default_print +comments: true +--- +<div class="post-header"> + <h1 class="post-title-main" id="{{page.permalink | replace: '/', '' }}">{{ page.title }}</h1> +</div> + +<div class="post-content"> + + {% if page.summary %} + <div class="summary">{{page.summary}}</div> + {% endif %} + {{ content }} +</div> diff --git a/python/vespa/docs/collect_training_data.html b/python/vespa/docs/collect_training_data.html new file mode 100644 index 00000000000..609b7182482 --- /dev/null +++ b/python/vespa/docs/collect_training_data.html @@ -0,0 +1,869 @@ +--- + +title: Vespa - collect training data + +keywords: fastai +sidebar: home_sidebar + +summary: "Collect training data to analyse and/or improve ranking functions" +description: "Collect training data to analyse and/or improve ranking functions" +--- +<!-- + +################################################# +### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ### +################################################# +# file to edit: notebooks/collect_training_data.ipynb +# command to build the docs after a change: nbdev_build_docs + +--> + +<div class="container" id="notebook-container"> + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Example-setup">Example setup<a class="anchor-link" href="#Example-setup"> </a></h2> +</div> +</div> +</div> +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>Connect to the application and define a query model.</p> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.application</span> <span class="k">import</span> <span class="n">Vespa</span> +<span class="kn">from</span> <span class="nn">vespa.query</span> <span class="k">import</span> <span class="n">Query</span><span class="p">,</span> <span class="n">RankProfile</span><span class="p">,</span> <span class="n">OR</span> + +<span class="n">app</span> <span class="o">=</span> <span class="n">Vespa</span><span class="p">(</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"https://api.cord19.vespa.ai"</span><span class="p">)</span> +<span class="n">query_model</span> <span class="o">=</span> <span class="n">Query</span><span class="p">(</span> + <span class="n">match_phase</span> <span class="o">=</span> <span class="n">OR</span><span class="p">(),</span> + <span class="n">rank_profile</span> <span class="o">=</span> <span class="n">RankProfile</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"bm25"</span><span class="p">,</span> <span class="n">list_features</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>Define some labelled data.</p> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">labelled_data</span> <span class="o">=</span> <span class="p">[</span> + <span class="p">{</span> + <span class="s2">"query_id"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> + <span class="s2">"query"</span><span class="p">:</span> <span class="s2">"Intrauterine virus infections and congenital heart disease"</span><span class="p">,</span> + <span class="s2">"relevant_docs"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span> <span class="p">{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">}]</span> + <span class="p">},</span> + <span class="p">{</span> + <span class="s2">"query_id"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> + <span class="s2">"query"</span><span class="p">:</span> <span class="s2">"Clinical and immunologic studies in identical twins discordant for systemic lupus erythematosus"</span><span class="p">,</span> + <span class="s2">"relevant_docs"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span> <span class="p">{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">}]</span> + <span class="p">}</span> +<span class="p">]</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Collect-training-data-in-batch">Collect training data in batch<a class="anchor-link" href="#Collect-training-data-in-batch"> </a></h2> +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">training_data_batch</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">collect_training_data</span><span class="p">(</span> + <span class="n">labelled_data</span> <span class="o">=</span> <span class="n">labelled_data</span><span class="p">,</span> + <span class="n">id_field</span> <span class="o">=</span> <span class="s2">"id"</span><span class="p">,</span> + <span class="n">query_model</span> <span class="o">=</span> <span class="n">query_model</span><span class="p">,</span> + <span class="n">number_additional_docs</span> <span class="o">=</span> <span class="mi">2</span> +<span class="p">)</span> +<span class="n">training_data_batch</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + +<div class="output_html rendered_html output_subarea output_execute_result"> +<div> +<style scoped> + .dataframe tbody tr th:only-of-type { + vertical-align: middle; + } + + .dataframe tbody tr th { + vertical-align: top; + } + + .dataframe thead th { + text-align: right; + } +</style> +<table border="1" class="dataframe"> + <thead> + <tr style="text-align: right;"> + <th></th> + <th>attributeMatch(authors.first)</th> + <th>attributeMatch(authors.first).averageWeight</th> + <th>attributeMatch(authors.first).completeness</th> + <th>attributeMatch(authors.first).fieldCompleteness</th> + <th>attributeMatch(authors.first).importance</th> + <th>attributeMatch(authors.first).matches</th> + <th>attributeMatch(authors.first).maxWeight</th> + <th>attributeMatch(authors.first).normalizedWeight</th> + <th>attributeMatch(authors.first).normalizedWeightedWeight</th> + <th>attributeMatch(authors.first).queryCompleteness</th> + <th>...</th> + <th>textSimilarity(results).queryCoverage</th> + <th>textSimilarity(results).score</th> + <th>textSimilarity(title).fieldCoverage</th> + <th>textSimilarity(title).order</th> + <th>textSimilarity(title).proximity</th> + <th>textSimilarity(title).queryCoverage</th> + <th>textSimilarity(title).score</th> + <th>document_id</th> + <th>query_id</th> + <th>relevant</th> + </tr> + </thead> + <tbody> + <tr> + <th>0</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.000000</td> + <td>0.000000</td> + <td>0</td> + <td>0</td> + <td>1</td> + </tr> + <tr> + <th>1</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>1.000000</td> + <td>1.0</td> + <td>1.000000</td> + <td>1.000000</td> + <td>1.000000</td> + <td>56212</td> + <td>0</td> + <td>0</td> + </tr> + <tr> + <th>2</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.187500</td> + <td>0.5</td> + <td>0.617188</td> + <td>0.428571</td> + <td>0.457087</td> + <td>34026</td> + <td>0</td> + <td>0</td> + </tr> + <tr> + <th>3</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.000000</td> + <td>0.000000</td> + <td>3</td> + <td>0</td> + <td>1</td> + </tr> + <tr> + <th>4</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>1.000000</td> + <td>1.0</td> + <td>1.000000</td> + <td>1.000000</td> + <td>1.000000</td> + <td>56212</td> + <td>0</td> + <td>0</td> + </tr> + <tr> + <th>5</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.187500</td> + <td>0.5</td> + <td>0.617188</td> + <td>0.428571</td> + <td>0.457087</td> + <td>34026</td> + <td>0</td> + <td>0</td> + </tr> + <tr> + <th>6</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.071429</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.083333</td> + <td>0.039286</td> + <td>1</td> + <td>1</td> + <td>1</td> + </tr> + <tr> + <th>7</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>1.000000</td> + <td>1.0</td> + <td>1.000000</td> + <td>1.000000</td> + <td>1.000000</td> + <td>29774</td> + <td>1</td> + <td>0</td> + </tr> + <tr> + <th>8</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.500000</td> + <td>1.0</td> + <td>1.000000</td> + <td>0.333333</td> + <td>0.700000</td> + <td>22787</td> + <td>1</td> + <td>0</td> + </tr> + <tr> + <th>9</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.058824</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.083333</td> + <td>0.036765</td> + <td>5</td> + <td>1</td> + <td>1</td> + </tr> + <tr> + <th>10</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>1.000000</td> + <td>1.0</td> + <td>1.000000</td> + <td>1.000000</td> + <td>1.000000</td> + <td>29774</td> + <td>1</td> + <td>0</td> + </tr> + <tr> + <th>11</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.500000</td> + <td>1.0</td> + <td>1.000000</td> + <td>0.333333</td> + <td>0.700000</td> + <td>22787</td> + <td>1</td> + <td>0</td> + </tr> + </tbody> +</table> +<p>12 rows × 984 columns</p> +</div> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Collect-training-data-point">Collect training data point<a class="anchor-link" href="#Collect-training-data-point"> </a></h2><blockquote><p>You can have finer control with the <code>collect_training_data_point</code> method.</p> +</blockquote> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">pandas</span> <span class="k">import</span> <span class="n">concat</span><span class="p">,</span> <span class="n">DataFrame</span> + + +<span class="n">training_data</span> <span class="o">=</span> <span class="p">[]</span> +<span class="k">for</span> <span class="n">query_data</span> <span class="ow">in</span> <span class="n">labelled_data</span><span class="p">:</span> + <span class="k">for</span> <span class="n">doc_data</span> <span class="ow">in</span> <span class="n">query_data</span><span class="p">[</span><span class="s2">"relevant_docs"</span><span class="p">]:</span> + <span class="n">training_data_point</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">collect_training_data_point</span><span class="p">(</span> + <span class="n">query</span> <span class="o">=</span> <span class="n">query_data</span><span class="p">[</span><span class="s2">"query"</span><span class="p">],</span> + <span class="n">query_id</span> <span class="o">=</span> <span class="n">query_data</span><span class="p">[</span><span class="s2">"query_id"</span><span class="p">],</span> + <span class="n">relevant_id</span> <span class="o">=</span> <span class="n">doc_data</span><span class="p">[</span><span class="s2">"id"</span><span class="p">],</span> + <span class="n">id_field</span> <span class="o">=</span> <span class="s2">"id"</span><span class="p">,</span> + <span class="n">query_model</span> <span class="o">=</span> <span class="n">query_model</span><span class="p">,</span> + <span class="n">number_additional_docs</span> <span class="o">=</span> <span class="mi">2</span> + <span class="p">)</span> + <span class="n">training_data</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">training_data_point</span><span class="p">)</span> +<span class="n">training_data</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">from_records</span><span class="p">(</span><span class="n">training_data</span><span class="p">)</span> +<span class="n">training_data</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + +<div class="output_html rendered_html output_subarea output_execute_result"> +<div> +<style scoped> + .dataframe tbody tr th:only-of-type { + vertical-align: middle; + } + + .dataframe tbody tr th { + vertical-align: top; + } + + .dataframe thead th { + text-align: right; + } +</style> +<table border="1" class="dataframe"> + <thead> + <tr style="text-align: right;"> + <th></th> + <th>attributeMatch(authors.first)</th> + <th>attributeMatch(authors.first).averageWeight</th> + <th>attributeMatch(authors.first).completeness</th> + <th>attributeMatch(authors.first).fieldCompleteness</th> + <th>attributeMatch(authors.first).importance</th> + <th>attributeMatch(authors.first).matches</th> + <th>attributeMatch(authors.first).maxWeight</th> + <th>attributeMatch(authors.first).normalizedWeight</th> + <th>attributeMatch(authors.first).normalizedWeightedWeight</th> + <th>attributeMatch(authors.first).queryCompleteness</th> + <th>...</th> + <th>textSimilarity(results).queryCoverage</th> + <th>textSimilarity(results).score</th> + <th>textSimilarity(title).fieldCoverage</th> + <th>textSimilarity(title).order</th> + <th>textSimilarity(title).proximity</th> + <th>textSimilarity(title).queryCoverage</th> + <th>textSimilarity(title).score</th> + <th>document_id</th> + <th>query_id</th> + <th>relevant</th> + </tr> + </thead> + <tbody> + <tr> + <th>0</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.000000</td> + <td>0.000000</td> + <td>0</td> + <td>0</td> + <td>1</td> + </tr> + <tr> + <th>1</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>1.000000</td> + <td>1.0</td> + <td>1.000000</td> + <td>1.000000</td> + <td>1.000000</td> + <td>56212</td> + <td>0</td> + <td>0</td> + </tr> + <tr> + <th>2</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.187500</td> + <td>0.5</td> + <td>0.617188</td> + <td>0.428571</td> + <td>0.457087</td> + <td>34026</td> + <td>0</td> + <td>0</td> + </tr> + <tr> + <th>3</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.000000</td> + <td>0.000000</td> + <td>3</td> + <td>0</td> + <td>1</td> + </tr> + <tr> + <th>4</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>1.000000</td> + <td>1.0</td> + <td>1.000000</td> + <td>1.000000</td> + <td>1.000000</td> + <td>56212</td> + <td>0</td> + <td>0</td> + </tr> + <tr> + <th>5</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.187500</td> + <td>0.5</td> + <td>0.617188</td> + <td>0.428571</td> + <td>0.457087</td> + <td>34026</td> + <td>0</td> + <td>0</td> + </tr> + <tr> + <th>6</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.071429</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.083333</td> + <td>0.039286</td> + <td>1</td> + <td>1</td> + <td>1</td> + </tr> + <tr> + <th>7</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>1.000000</td> + <td>1.0</td> + <td>1.000000</td> + <td>1.000000</td> + <td>1.000000</td> + <td>29774</td> + <td>1</td> + <td>0</td> + </tr> + <tr> + <th>8</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.500000</td> + <td>1.0</td> + <td>1.000000</td> + <td>0.333333</td> + <td>0.700000</td> + <td>22787</td> + <td>1</td> + <td>0</td> + </tr> + <tr> + <th>9</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.058824</td> + <td>0.0</td> + <td>0.000000</td> + <td>0.083333</td> + <td>0.036765</td> + <td>5</td> + <td>1</td> + <td>1</td> + </tr> + <tr> + <th>10</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>1.000000</td> + <td>1.0</td> + <td>1.000000</td> + <td>1.000000</td> + <td>1.000000</td> + <td>29774</td> + <td>1</td> + <td>0</td> + </tr> + <tr> + <th>11</th> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>0.0</td> + <td>...</td> + <td>0.0</td> + <td>0.0</td> + <td>0.500000</td> + <td>1.0</td> + <td>1.000000</td> + <td>0.333333</td> + <td>0.700000</td> + <td>22787</td> + <td>1</td> + <td>0</td> + </tr> + </tbody> +</table> +<p>12 rows × 984 columns</p> +</div> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +</div> + + diff --git a/python/vespa/docs/core.html b/python/vespa/docs/core.html new file mode 100644 index 00000000000..4fd94bd89c2 --- /dev/null +++ b/python/vespa/docs/core.html @@ -0,0 +1,32 @@ +--- + +title: module name here + +keywords: fastai +sidebar: home_sidebar + +summary: "API details." +description: "API details." +--- +<!-- + +################################################# +### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ### +################################################# +# file to edit: notebooks/00_core.ipynb +# command to build the docs after a change: nbdev_build_docs + +--> + +<div class="container" id="notebook-container"> + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> + +</div> + {% endraw %} + +</div> + + diff --git a/python/vespa/docs/css/bootstrap.min.css b/python/vespa/docs/css/bootstrap.min.css new file mode 100755 index 00000000000..783f60d1231 --- /dev/null +++ b/python/vespa/docs/css/bootstrap.min.css @@ -0,0 +1,7535 @@ +/*! + * Bootstrap v3.3.2 (http://getbootstrap.com) + * Copyright 2011-2016 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + */ + + +/*! normalize.css v3.0.2 | MIT License | git.io/normalize */ + +html { + font-family: sans-serif; + -webkit-text-size-adjust: 100%; + -ms-text-size-adjust: 100% +} + +body { + margin: 0 +} + +article, +aside, +details, +figcaption, +figure, +footer, +header, +hgroup, +main, +menu, +nav, +section, +summary { + display: block +} + +audio, +canvas, +progress, +video { + display: inline-block; + vertical-align: baseline +} + +audio:not([controls]) { + display: none; + height: 0 +} + +[hidden], +template { + display: none +} + +a { + background-color: transparent +} + +a:active, +a:hover { + outline: 0 +} + +abbr[title] { + border-bottom: 1px dotted +} + +b, +strong { + font-weight: 700 +} + +dfn { + font-style: italic +} + +h1 { + margin: .67em 0; + font-size: 2em +} + +mark { + color: #000; + background: #ff0 +} + +small { + font-size: 80% +} + +sub, +sup { + position: relative; + font-size: 75%; + line-height: 0; + vertical-align: baseline +} + +sup { + top: -.5em +} + +sub { + bottom: -.25em +} + +img { + border: 0 +} + +svg:not(:root) { + overflow: hidden +} + +figure { + margin: 1em 40px +} + +hr { + height: 0; + -webkit-box-sizing: content-box; + -moz-box-sizing: content-box; + box-sizing: content-box +} + +pre { + overflow: auto +} + +code, +kbd, +pre, +samp { + font-family: monospace, monospace; + font-size: 1em +} + +button, +input, +optgroup, +select, +textarea { + margin: 0; + font: inherit; + color: inherit +} + +button { + overflow: visible +} + +button, +select { + text-transform: none +} + +button, +html input[type=button], +input[type=reset], +input[type=submit] { + -webkit-appearance: button; + cursor: pointer +} + +button[disabled], +html input[disabled] { + cursor: default +} + +button::-moz-focus-inner, +input::-moz-focus-inner { + padding: 0; + border: 0 +} + +input { + line-height: normal +} + +input[type=checkbox], +input[type=radio] { + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + padding: 0 +} + +input[type=number]::-webkit-inner-spin-button, +input[type=number]::-webkit-outer-spin-button { + height: auto +} + +input[type=search] { + -webkit-box-sizing: content-box; + -moz-box-sizing: content-box; + box-sizing: content-box; + -webkit-appearance: textfield +} + +input[type=search]::-webkit-search-cancel-button, +input[type=search]::-webkit-search-decoration { + -webkit-appearance: none +} + +fieldset { + padding: .35em .625em .75em; + margin: 0 2px; + border: 1px solid silver +} + +legend { + padding: 0; + border: 0 +} + +textarea { + overflow: auto +} + +optgroup { + font-weight: 700 +} + +table { + border-spacing: 0; + border-collapse: collapse +} + +td, +th { + padding: 0 +} + + +/*! Source: https://github.com/h5bp/html5-boilerplate/blob/master/src/css/main.css */ + +@media print { + *, + :after, + :before { + /*color:#000!important;*/ + /*background:0 0!important*/ + ; + } + a, + a:visited { + text-decoration: underline + } + a[href]:after { + content: " (" attr(href) ")" + } + abbr[title]:after { + content: " (" attr(title) ")" + } + a[href^="javascript:"]:after, + a[href^="#"]:after { + content: "" + } + blockquote, + pre { + border: 1px solid #999; + page-break-inside: avoid + } + thead { + display: table-header-group + } + img, + tr { + page-break-inside: avoid + } + img { + max-width: 100%!important + } + h2, + h3, + p { + orphans: 3; + widows: 3 + } + h2, + h3 { + page-break-after: avoid + } + select { + background: #fff!important + } + .navbar { + display: none + } + .btn>.caret, + .dropup>.btn>.caret { + border-top-color: #000!important + } + .label { + border: 1px solid #000 + } + .table { + border-collapse: collapse!important + } + .table td, + .table th { + background-color: #fff!important + } + .table-bordered td, + .table-bordered th { + border: 1px solid #ddd!important + } +} + +@font-face { + font-family: 'Glyphicons Halflings'; + src: url(../fonts/glyphicons-halflings-regular.eot); + src: url(../fonts/glyphicons-halflings-regular.eot?#iefix) format('embedded-opentype'), url(../fonts/glyphicons-halflings-regular.woff2) format('woff2'), url(../fonts/glyphicons-halflings-regular.woff) format('woff'), url(../fonts/glyphicons-halflings-regular.ttf) format('truetype'), url(../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular) format('svg') +} + +.glyphicon { + position: relative; + top: 1px; + display: inline-block; + font-family: 'Glyphicons Halflings'; + font-style: normal; + font-weight: 400; + line-height: 1; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale +} + +.glyphicon-asterisk:before { + content: "\2a" +} + +.glyphicon-plus:before { + content: "\2b" +} + +.glyphicon-eur:before, +.glyphicon-euro:before { + content: "\20ac" +} + +.glyphicon-minus:before { + content: "\2212" +} + +.glyphicon-cloud:before { + content: "\2601" +} + +.glyphicon-envelope:before { + content: "\2709" +} + +.glyphicon-pencil:before { + content: "\270f" +} + +.glyphicon-glass:before { + content: "\e001" +} + +.glyphicon-music:before { + content: "\e002" +} + +.glyphicon-search:before { + content: "\e003" +} + +.glyphicon-heart:before { + content: "\e005" +} + +.glyphicon-star:before { + content: "\e006" +} + +.glyphicon-star-empty:before { + content: "\e007" +} + +.glyphicon-user:before { + content: "\e008" +} + +.glyphicon-film:before { + content: "\e009" +} + +.glyphicon-th-large:before { + content: "\e010" +} + +.glyphicon-th:before { + content: "\e011" +} + +.glyphicon-th-list:before { + content: "\e012" +} + +.glyphicon-ok:before { + content: "\e013" +} + +.glyphicon-remove:before { + content: "\e014" +} + +.glyphicon-zoom-in:before { + content: "\e015" +} + +.glyphicon-zoom-out:before { + content: "\e016" +} + +.glyphicon-off:before { + content: "\e017" +} + +.glyphicon-signal:before { + content: "\e018" +} + +.glyphicon-cog:before { + content: "\e019" +} + +.glyphicon-trash:before { + content: "\e020" +} + +.glyphicon-home:before { + content: "\e021" +} + +.glyphicon-file:before { + content: "\e022" +} + +.glyphicon-time:before { + content: "\e023" +} + +.glyphicon-road:before { + content: "\e024" +} + +.glyphicon-download-alt:before { + content: "\e025" +} + +.glyphicon-download:before { + content: "\e026" +} + +.glyphicon-upload:before { + content: "\e027" +} + +.glyphicon-inbox:before { + content: "\e028" +} + +.glyphicon-play-circle:before { + content: "\e029" +} + +.glyphicon-repeat:before { + content: "\e030" +} + +.glyphicon-refresh:before { + content: "\e031" +} + +.glyphicon-list-alt:before { + content: "\e032" +} + +.glyphicon-lock:before { + content: "\e033" +} + +.glyphicon-flag:before { + content: "\e034" +} + +.glyphicon-headphones:before { + content: "\e035" +} + +.glyphicon-volume-off:before { + content: "\e036" +} + +.glyphicon-volume-down:before { + content: "\e037" +} + +.glyphicon-volume-up:before { + content: "\e038" +} + +.glyphicon-qrcode:before { + content: "\e039" +} + +.glyphicon-barcode:before { + content: "\e040" +} + +.glyphicon-tag:before { + content: "\e041" +} + +.glyphicon-tags:before { + content: "\e042" +} + +.glyphicon-book:before { + content: "\e043" +} + +.glyphicon-bookmark:before { + content: "\e044" +} + +.glyphicon-print:before { + content: "\e045" +} + +.glyphicon-camera:before { + content: "\e046" +} + +.glyphicon-font:before { + content: "\e047" +} + +.glyphicon-bold:before { + content: "\e048" +} + +.glyphicon-italic:before { + content: "\e049" +} + +.glyphicon-text-height:before { + content: "\e050" +} + +.glyphicon-text-width:before { + content: "\e051" +} + +.glyphicon-align-left:before { + content: "\e052" +} + +.glyphicon-align-center:before { + content: "\e053" +} + +.glyphicon-align-right:before { + content: "\e054" +} + +.glyphicon-align-justify:before { + content: "\e055" +} + +.glyphicon-list:before { + content: "\e056" +} + +.glyphicon-indent-left:before { + content: "\e057" +} + +.glyphicon-indent-right:before { + content: "\e058" +} + +.glyphicon-facetime-video:before { + content: "\e059" +} + +.glyphicon-picture:before { + content: "\e060" +} + +.glyphicon-map-marker:before { + content: "\e062" +} + +.glyphicon-adjust:before { + content: "\e063" +} + +.glyphicon-tint:before { + content: "\e064" +} + +.glyphicon-edit:before { + content: "\e065" +} + +.glyphicon-share:before { + content: "\e066" +} + +.glyphicon-check:before { + content: "\e067" +} + +.glyphicon-move:before { + content: "\e068" +} + +.glyphicon-step-backward:before { + content: "\e069" +} + +.glyphicon-fast-backward:before { + content: "\e070" +} + +.glyphicon-backward:before { + content: "\e071" +} + +.glyphicon-play:before { + content: "\e072" +} + +.glyphicon-pause:before { + content: "\e073" +} + +.glyphicon-stop:before { + content: "\e074" +} + +.glyphicon-forward:before { + content: "\e075" +} + +.glyphicon-fast-forward:before { + content: "\e076" +} + +.glyphicon-step-forward:before { + content: "\e077" +} + +.glyphicon-eject:before { + content: "\e078" +} + +.glyphicon-chevron-left:before { + content: "\e079" +} + +.glyphicon-chevron-right:before { + content: "\e080" +} + +.glyphicon-plus-sign:before { + content: "\e081" +} + +.glyphicon-minus-sign:before { + content: "\e082" +} + +.glyphicon-remove-sign:before { + content: "\e083" +} + +.glyphicon-ok-sign:before { + content: "\e084" +} + +.glyphicon-question-sign:before { + content: "\e085" +} + +.glyphicon-info-sign:before { + content: "\e086" +} + +.glyphicon-screenshot:before { + content: "\e087" +} + +.glyphicon-remove-circle:before { + content: "\e088" +} + +.glyphicon-ok-circle:before { + content: "\e089" +} + +.glyphicon-ban-circle:before { + content: "\e090" +} + +.glyphicon-arrow-left:before { + content: "\e091" +} + +.glyphicon-arrow-right:before { + content: "\e092" +} + +.glyphicon-arrow-up:before { + content: "\e093" +} + +.glyphicon-arrow-down:before { + content: "\e094" +} + +.glyphicon-share-alt:before { + content: "\e095" +} + +.glyphicon-resize-full:before { + content: "\e096" +} + +.glyphicon-resize-small:before { + content: "\e097" +} + +.glyphicon-exclamation-sign:before { + content: "\e101" +} + +.glyphicon-gift:before { + content: "\e102" +} + +.glyphicon-leaf:before { + content: "\e103" +} + +.glyphicon-fire:before { + content: "\e104" +} + +.glyphicon-eye-open:before { + content: "\e105" +} + +.glyphicon-eye-close:before { + content: "\e106" +} + +.glyphicon-warning-sign:before { + content: "\e107" +} + +.glyphicon-plane:before { + content: "\e108" +} + +.glyphicon-calendar:before { + content: "\e109" +} + +.glyphicon-random:before { + content: "\e110" +} + +.glyphicon-comment:before { + content: "\e111" +} + +.glyphicon-magnet:before { + content: "\e112" +} + +.glyphicon-chevron-up:before { + content: "\e113" +} + +.glyphicon-chevron-down:before { + content: "\e114" +} + +.glyphicon-retweet:before { + content: "\e115" +} + +.glyphicon-shopping-cart:before { + content: "\e116" +} + +.glyphicon-folder-close:before { + content: "\e117" +} + +.glyphicon-folder-open:before { + content: "\e118" +} + +.glyphicon-resize-vertical:before { + content: "\e119" +} + +.glyphicon-resize-horizontal:before { + content: "\e120" +} + +.glyphicon-hdd:before { + content: "\e121" +} + +.glyphicon-bullhorn:before { + content: "\e122" +} + +.glyphicon-bell:before { + content: "\e123" +} + +.glyphicon-certificate:before { + content: "\e124" +} + +.glyphicon-thumbs-up:before { + content: "\e125" +} + +.glyphicon-thumbs-down:before { + content: "\e126" +} + +.glyphicon-hand-right:before { + content: "\e127" +} + +.glyphicon-hand-left:before { + content: "\e128" +} + +.glyphicon-hand-up:before { + content: "\e129" +} + +.glyphicon-hand-down:before { + content: "\e130" +} + +.glyphicon-circle-arrow-right:before { + content: "\e131" +} + +.glyphicon-circle-arrow-left:before { + content: "\e132" +} + +.glyphicon-circle-arrow-up:before { + content: "\e133" +} + +.glyphicon-circle-arrow-down:before { + content: "\e134" +} + +.glyphicon-globe:before { + content: "\e135" +} + +.glyphicon-wrench:before { + content: "\e136" +} + +.glyphicon-tasks:before { + content: "\e137" +} + +.glyphicon-filter:before { + content: "\e138" +} + +.glyphicon-briefcase:before { + content: "\e139" +} + +.glyphicon-fullscreen:before { + content: "\e140" +} + +.glyphicon-dashboard:before { + content: "\e141" +} + +.glyphicon-paperclip:before { + content: "\e142" +} + +.glyphicon-heart-empty:before { + content: "\e143" +} + +.glyphicon-link:before { + content: "\e144" +} + +.glyphicon-phone:before { + content: "\e145" +} + +.glyphicon-pushpin:before { + content: "\e146" +} + +.glyphicon-usd:before { + content: "\e148" +} + +.glyphicon-gbp:before { + content: "\e149" +} + +.glyphicon-sort:before { + content: "\e150" +} + +.glyphicon-sort-by-alphabet:before { + content: "\e151" +} + +.glyphicon-sort-by-alphabet-alt:before { + content: "\e152" +} + +.glyphicon-sort-by-order:before { + content: "\e153" +} + +.glyphicon-sort-by-order-alt:before { + content: "\e154" +} + +.glyphicon-sort-by-attributes:before { + content: "\e155" +} + +.glyphicon-sort-by-attributes-alt:before { + content: "\e156" +} + +.glyphicon-unchecked:before { + content: "\e157" +} + +.glyphicon-expand:before { + content: "\e158" +} + +.glyphicon-collapse-down:before { + content: "\e159" +} + +.glyphicon-collapse-up:before { + content: "\e160" +} + +.glyphicon-log-in:before { + content: "\e161" +} + +.glyphicon-flash:before { + content: "\e162" +} + +.glyphicon-log-out:before { + content: "\e163" +} + +.glyphicon-new-window:before { + content: "\e164" +} + +.glyphicon-record:before { + content: "\e165" +} + +.glyphicon-save:before { + content: "\e166" +} + +.glyphicon-open:before { + content: "\e167" +} + +.glyphicon-saved:before { + content: "\e168" +} + +.glyphicon-import:before { + content: "\e169" +} + +.glyphicon-export:before { + content: "\e170" +} + +.glyphicon-send:before { + content: "\e171" +} + +.glyphicon-floppy-disk:before { + content: "\e172" +} + +.glyphicon-floppy-saved:before { + content: "\e173" +} + +.glyphicon-floppy-remove:before { + content: "\e174" +} + +.glyphicon-floppy-save:before { + content: "\e175" +} + +.glyphicon-floppy-open:before { + content: "\e176" +} + +.glyphicon-credit-card:before { + content: "\e177" +} + +.glyphicon-transfer:before { + content: "\e178" +} + +.glyphicon-cutlery:before { + content: "\e179" +} + +.glyphicon-header:before { + content: "\e180" +} + +.glyphicon-compressed:before { + content: "\e181" +} + +.glyphicon-earphone:before { + content: "\e182" +} + +.glyphicon-phone-alt:before { + content: "\e183" +} + +.glyphicon-tower:before { + content: "\e184" +} + +.glyphicon-stats:before { + content: "\e185" +} + +.glyphicon-sd-video:before { + content: "\e186" +} + +.glyphicon-hd-video:before { + content: "\e187" +} + +.glyphicon-subtitles:before { + content: "\e188" +} + +.glyphicon-sound-stereo:before { + content: "\e189" +} + +.glyphicon-sound-dolby:before { + content: "\e190" +} + +.glyphicon-sound-5-1:before { + content: "\e191" +} + +.glyphicon-sound-6-1:before { + content: "\e192" +} + +.glyphicon-sound-7-1:before { + content: "\e193" +} + +.glyphicon-copyright-mark:before { + content: "\e194" +} + +.glyphicon-registration-mark:before { + content: "\e195" +} + +.glyphicon-cloud-download:before { + content: "\e197" +} + +.glyphicon-cloud-upload:before { + content: "\e198" +} + +.glyphicon-tree-conifer:before { + content: "\e199" +} + +.glyphicon-tree-deciduous:before { + content: "\e200" +} + +.glyphicon-cd:before { + content: "\e201" +} + +.glyphicon-save-file:before { + content: "\e202" +} + +.glyphicon-open-file:before { + content: "\e203" +} + +.glyphicon-level-up:before { + content: "\e204" +} + +.glyphicon-copy:before { + content: "\e205" +} + +.glyphicon-paste:before { + content: "\e206" +} + +.glyphicon-alert:before { + content: "\e209" +} + +.glyphicon-equalizer:before { + content: "\e210" +} + +.glyphicon-king:before { + content: "\e211" +} + +.glyphicon-queen:before { + content: "\e212" +} + +.glyphicon-pawn:before { + content: "\e213" +} + +.glyphicon-bishop:before { + content: "\e214" +} + +.glyphicon-knight:before { + content: "\e215" +} + +.glyphicon-baby-formula:before { + content: "\e216" +} + +.glyphicon-tent:before { + content: "\26fa" +} + +.glyphicon-blackboard:before { + content: "\e218" +} + +.glyphicon-bed:before { + content: "\e219" +} + +.glyphicon-apple:before { + content: "\f8ff" +} + +.glyphicon-erase:before { + content: "\e221" +} + +.glyphicon-hourglass:before { + content: "\231b" +} + +.glyphicon-lamp:before { + content: "\e223" +} + +.glyphicon-duplicate:before { + content: "\e224" +} + +.glyphicon-piggy-bank:before { + content: "\e225" +} + +.glyphicon-scissors:before { + content: "\e226" +} + +.glyphicon-bitcoin:before { + content: "\e227" +} + +.glyphicon-yen:before { + content: "\00a5" +} + +.glyphicon-ruble:before { + content: "\20bd" +} + +.glyphicon-scale:before { + content: "\e230" +} + +.glyphicon-ice-lolly:before { + content: "\e231" +} + +.glyphicon-ice-lolly-tasted:before { + content: "\e232" +} + +.glyphicon-education:before { + content: "\e233" +} + +.glyphicon-option-horizontal:before { + content: "\e234" +} + +.glyphicon-option-vertical:before { + content: "\e235" +} + +.glyphicon-menu-hamburger:before { + content: "\e236" +} + +.glyphicon-modal-window:before { + content: "\e237" +} + +.glyphicon-oil:before { + content: "\e238" +} + +.glyphicon-grain:before { + content: "\e239" +} + +.glyphicon-sunglasses:before { + content: "\e240" +} + +.glyphicon-text-size:before { + content: "\e241" +} + +.glyphicon-text-color:before { + content: "\e242" +} + +.glyphicon-text-background:before { + content: "\e243" +} + +.glyphicon-object-align-top:before { + content: "\e244" +} + +.glyphicon-object-align-bottom:before { + content: "\e245" +} + +.glyphicon-object-align-horizontal:before { + content: "\e246" +} + +.glyphicon-object-align-left:before { + content: "\e247" +} + +.glyphicon-object-align-vertical:before { + content: "\e248" +} + +.glyphicon-object-align-right:before { + content: "\e249" +} + +.glyphicon-triangle-right:before { + content: "\e250" +} + +.glyphicon-triangle-left:before { + content: "\e251" +} + +.glyphicon-triangle-bottom:before { + content: "\e252" +} + +.glyphicon-triangle-top:before { + content: "\e253" +} + +.glyphicon-console:before { + content: "\e254" +} + +.glyphicon-superscript:before { + content: "\e255" +} + +.glyphicon-subscript:before { + content: "\e256" +} + +.glyphicon-menu-left:before { + content: "\e257" +} + +.glyphicon-menu-right:before { + content: "\e258" +} + +.glyphicon-menu-down:before { + content: "\e259" +} + +.glyphicon-menu-up:before { + content: "\e260" +} + +* { + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box +} + +:after, +:before { + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box +} + +html { + font-size: 10px; + -webkit-tap-highlight-color: rgba(0, 0, 0, 0) +} + +body { + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + line-height: 1.42857143; + color: #333; + background-color: #fff +} + +button, +input, +select, +textarea { + font-family: inherit; + font-size: inherit; + line-height: inherit +} + +a { + color: #337ab7; + text-decoration: none +} + +a:focus, +a:hover { + color: #23527c; + text-decoration: underline +} + +a:focus { + outline: thin dotted; + outline: 5px auto -webkit-focus-ring-color; + outline-offset: -2px +} + +figure { + margin: 0 +} + +img { + vertical-align: middle +} + +.carousel-inner>.item>a>img, +.carousel-inner>.item>img, +.img-responsive, +.thumbnail a>img, +.thumbnail>img { + display: block; + max-width: 100%; + height: auto +} + +.img-rounded { + border-radius: 6px +} + +.img-thumbnail { + display: inline-block; + max-width: 100%; + height: auto; + padding: 4px; + line-height: 1.42857143; + background-color: #fff; + border: 1px solid #ddd; + border-radius: 4px; + -webkit-transition: all .2s ease-in-out; + -o-transition: all .2s ease-in-out; + transition: all .2s ease-in-out +} + +.img-circle { + border-radius: 50% +} + +hr { + margin-top: 20px; + margin-bottom: 20px; + border: 0; + border-top: 1px solid #eee +} + +.sr-only { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + border: 0 +} + +.sr-only-focusable:active, +.sr-only-focusable:focus { + position: static; + width: auto; + height: auto; + margin: 0; + overflow: visible; + clip: auto +} + +.h1, +.h2, +.h3, +.h4, +.h5, +.h6, +h1, +h2, +h3, +h4, +h5, +h6 { + font-family: inherit; + font-weight: 500; + line-height: 1.1; + color: inherit +} + +.h1 .small, +.h1 small, +.h2 .small, +.h2 small, +.h3 .small, +.h3 small, +.h4 .small, +.h4 small, +.h5 .small, +.h5 small, +.h6 .small, +.h6 small, +h1 .small, +h1 small, +h2 .small, +h2 small, +h3 .small, +h3 small, +h4 .small, +h4 small, +h5 .small, +h5 small, +h6 .small, +h6 small { + font-weight: 400; + line-height: 1; + color: #777 +} + +.h1, +.h2, +.h3, +h1, +h2, +h3 { + margin-top: 20px; + margin-bottom: 10px +} + +.h1 .small, +.h1 small, +.h2 .small, +.h2 small, +.h3 .small, +.h3 small, +h1 .small, +h1 small, +h2 .small, +h2 small, +h3 .small, +h3 small { + font-size: 65% +} + +.h4, +.h5, +.h6, +h4, +h5, +h6 { + margin-top: 10px; + margin-bottom: 10px +} + +.h4 .small, +.h4 small, +.h5 .small, +.h5 small, +.h6 .small, +.h6 small, +h4 .small, +h4 small, +h5 .small, +h5 small, +h6 .small, +h6 small { + font-size: 75% +} + +.h1, +h1 { + font-size: 36px +} + +.h2, +h2 { + font-size: 30px +} + +.h3, +h3 { + font-size: 24px +} + +.h4, +h4 { + font-size: 18px +} + +.h5, +h5 { + font-size: 14px +} + +.h6, +h6 { + font-size: 12px +} + +p { + margin: 0 0 10px +} + +.lead { + margin-bottom: 20px; + font-size: 16px; + font-weight: 300; + line-height: 1.4 +} + +@media (min-width:768px) { + .lead { + font-size: 21px + } +} + +.small, +small { + font-size: 85% +} + +.mark, +mark { + padding: .2em; + background-color: #fcf8e3 +} + +.text-left { + text-align: left +} + +.text-right { + text-align: right +} + +.text-center { + text-align: center +} + +.text-justify { + text-align: justify +} + +.text-nowrap { + white-space: nowrap +} + +.text-lowercase { + text-transform: lowercase +} + +.text-uppercase { + text-transform: uppercase +} + +.text-capitalize { + text-transform: capitalize +} + +.text-muted { + color: #777 +} + +.text-primary { + color: #337ab7 +} + +a.text-primary:hover { + color: #286090 +} + +.text-success { + color: #3c763d +} + +a.text-success:hover { + color: #2b542c +} + +.text-info { + color: #31708f +} + +a.text-info:hover { + color: #245269 +} + +.text-warning { + color: #8a6d3b +} + +a.text-warning:hover { + color: #66512c +} + +.text-danger { + color: #a94442 +} + +a.text-danger:hover { + color: #843534 +} + +.bg-primary { + color: #fff; + background-color: #337ab7 +} + +a.bg-primary:hover { + background-color: #286090 +} + +.bg-success { + background-color: #dff0d8 +} + +a.bg-success:hover { + background-color: #c1e2b3 +} + +.bg-info { + background-color: #d9edf7 +} + +a.bg-info:hover { + background-color: #afd9ee +} + +.bg-warning { + background-color: #fcf8e3 +} + +a.bg-warning:hover { + background-color: #f7ecb5 +} + +.bg-danger { + background-color: #f2dede +} + +a.bg-danger:hover { + background-color: #e4b9b9 +} + +.page-header { + padding-bottom: 9px; + margin: 40px 0 20px; + border-bottom: 1px solid #eee +} + +ol, +ul { + margin-top: 0; + margin-bottom: 10px +} + +ol ol, +ol ul, +ul ol, +ul ul { + margin-bottom: 0 +} + +.list-unstyled { + padding-left: 0; + list-style: none +} + +.list-inline { + padding-left: 0; + margin-left: -5px; + list-style: none +} + +.list-inline>li { + display: inline-block; + padding-right: 5px; + padding-left: 5px +} + +dl { + margin-top: 0; + margin-bottom: 20px +} + +dd, +dt { + line-height: 1.42857143 +} + +dt { + font-weight: 700 +} + +dd { + margin-left: 0 +} + +@media (min-width:768px) { + .dl-horizontal dt { + float: left; + width: 160px; + overflow: hidden; + clear: left; + text-align: right; + text-overflow: ellipsis; + white-space: nowrap + } + .dl-horizontal dd { + margin-left: 180px + } +} + +abbr[data-original-title], +abbr[title] { + cursor: help; + border-bottom: 1px dotted #777 +} + +.initialism { + font-size: 90%; + text-transform: uppercase +} + +blockquote { + padding: 10px 20px; + margin: 0 0 20px; + font-size: 17.5px; + border-left: 5px solid #eee +} + +blockquote ol:last-child, +blockquote p:last-child, +blockquote ul:last-child { + margin-bottom: 0 +} + +blockquote .small, +blockquote footer, +blockquote small { + display: block; + font-size: 80%; + line-height: 1.42857143; + color: #777 +} + +blockquote .small:before, +blockquote footer:before, +blockquote small:before { + content: '\2014 \00A0' +} + +.blockquote-reverse, +blockquote.pull-right { + padding-right: 15px; + padding-left: 0; + text-align: right; + border-right: 5px solid #eee; + border-left: 0 +} + +.blockquote-reverse .small:before, +.blockquote-reverse footer:before, +.blockquote-reverse small:before, +blockquote.pull-right .small:before, +blockquote.pull-right footer:before, +blockquote.pull-right small:before { + content: '' +} + +.blockquote-reverse .small:after, +.blockquote-reverse footer:after, +.blockquote-reverse small:after, +blockquote.pull-right .small:after, +blockquote.pull-right footer:after, +blockquote.pull-right small:after { + content: '\00A0 \2014' +} + +address { + margin-bottom: 20px; + font-style: normal; + line-height: 1.42857143 +} + +code, +kbd, +pre, +samp, +ouput_html{ + font-family: Menlo, Monaco, Consolas, "Courier New", monospace +} + +code, +ouput_html{ + padding: 2px 4px; + font-size: 90%; + color: #c7254e; + background-color: #f9f2f4; + border-radius: 4px +} + +kbd { + padding: 2px 4px; + font-size: 90%; + color: #fff; + background-color: #333; + border-radius: 3px; +} + +kbd kbd { + padding: 0; + font-size: 100%; + font-weight: 700; +} + +pre, +output_html{ + display: block; + padding: 9.5px; + margin: 0 0 10px; + font-size: 13px; + line-height: 1.42857143; + color: #333; + word-break: break-all; + word-wrap: break-word; + background-color: #f5f5f5; + border: 1px solid #ccc; + border-radius: 4px +} + + +pre, +code, +output_html{ + padding: 0; + font-size: inherit; + color: inherit; + white-space: pre-wrap; + background-color: transparent; + border-radius: 0 +} + +.pre-scrollable { + max-height: 340px; + overflow-y: scroll +} + +.container { + padding-right: 15px; + padding-left: 15px; + margin-right: auto; + margin-left: auto +} + +@media (min-width:768px) { + .container { + width: 750px + } +} + +@media (min-width:992px) { + .container { + width: 970px + } +} + +@media (min-width:1200px) { + .container { + width: 1170px + } +} + +.container-fluid { + padding-right: 15px; + padding-left: 15px; + margin-right: auto; + margin-left: auto +} + +.row { + margin-right: -15px; + margin-left: -15px +} + +.col-lg-1, +.col-lg-10, +.col-lg-11, +.col-lg-12, +.col-lg-2, +.col-lg-3, +.col-lg-4, +.col-lg-5, +.col-lg-6, +.col-lg-7, +.col-lg-8, +.col-lg-9, +.col-md-1, +.col-md-10, +.col-md-11, +.col-md-12, +.col-md-2, +.col-md-3, +.col-md-4, +.col-md-5, +.col-md-6, +.col-md-7, +.col-md-8, +.col-md-9, +.col-sm-1, +.col-sm-10, +.col-sm-11, +.col-sm-12, +.col-sm-2, +.col-sm-3, +.col-sm-4, +.col-sm-5, +.col-sm-6, +.col-sm-7, +.col-sm-8, +.col-sm-9, +.col-xs-1, +.col-xs-10, +.col-xs-11, +.col-xs-12, +.col-xs-2, +.col-xs-3, +.col-xs-4, +.col-xs-5, +.col-xs-6, +.col-xs-7, +.col-xs-8, +.col-xs-9 { + position: relative; + min-height: 1px; + padding-right: 15px; + padding-left: 15px +} + +.col-xs-1, +.col-xs-10, +.col-xs-11, +.col-xs-12, +.col-xs-2, +.col-xs-3, +.col-xs-4, +.col-xs-5, +.col-xs-6, +.col-xs-7, +.col-xs-8, +.col-xs-9 { + float: left +} + +.col-xs-12 { + width: 100% +} + +.col-xs-11 { + width: 91.66666667% +} + +.col-xs-10 { + width: 83.33333333% +} + +.col-xs-9 { + width: 75% +} + +.col-xs-8 { + width: 66.66666667% +} + +.col-xs-7 { + width: 58.33333333% +} + +.col-xs-6 { + width: 50% +} + +.col-xs-5 { + width: 41.66666667% +} + +.col-xs-4 { + width: 33.33333333% +} + +.col-xs-3 { + width: 25% +} + +.col-xs-2 { + width: 16.66666667% +} + +.col-xs-1 { + width: 8.33333333% +} + +.col-xs-pull-12 { + right: 100% +} + +.col-xs-pull-11 { + right: 91.66666667% +} + +.col-xs-pull-10 { + right: 83.33333333% +} + +.col-xs-pull-9 { + right: 75% +} + +.col-xs-pull-8 { + right: 66.66666667% +} + +.col-xs-pull-7 { + right: 58.33333333% +} + +.col-xs-pull-6 { + right: 50% +} + +.col-xs-pull-5 { + right: 41.66666667% +} + +.col-xs-pull-4 { + right: 33.33333333% +} + +.col-xs-pull-3 { + right: 25% +} + +.col-xs-pull-2 { + right: 16.66666667% +} + +.col-xs-pull-1 { + right: 8.33333333% +} + +.col-xs-pull-0 { + right: auto +} + +.col-xs-push-12 { + left: 100% +} + +.col-xs-push-11 { + left: 91.66666667% +} + +.col-xs-push-10 { + left: 83.33333333% +} + +.col-xs-push-9 { + left: 75% +} + +.col-xs-push-8 { + left: 66.66666667% +} + +.col-xs-push-7 { + left: 58.33333333% +} + +.col-xs-push-6 { + left: 50% +} + +.col-xs-push-5 { + left: 41.66666667% +} + +.col-xs-push-4 { + left: 33.33333333% +} + +.col-xs-push-3 { + left: 25% +} + +.col-xs-push-2 { + left: 16.66666667% +} + +.col-xs-push-1 { + left: 8.33333333% +} + +.col-xs-push-0 { + left: auto +} + +.col-xs-offset-12 { + margin-left: 100% +} + +.col-xs-offset-11 { + margin-left: 91.66666667% +} + +.col-xs-offset-10 { + margin-left: 83.33333333% +} + +.col-xs-offset-9 { + margin-left: 75% +} + +.col-xs-offset-8 { + margin-left: 66.66666667% +} + +.col-xs-offset-7 { + margin-left: 58.33333333% +} + +.col-xs-offset-6 { + margin-left: 50% +} + +.col-xs-offset-5 { + margin-left: 41.66666667% +} + +.col-xs-offset-4 { + margin-left: 33.33333333% +} + +.col-xs-offset-3 { + margin-left: 25% +} + +.col-xs-offset-2 { + margin-left: 16.66666667% +} + +.col-xs-offset-1 { + margin-left: 8.33333333% +} + +.col-xs-offset-0 { + margin-left: 0 +} + +@media (min-width:768px) { + .col-sm-1, + .col-sm-10, + .col-sm-11, + .col-sm-12, + .col-sm-2, + .col-sm-3, + .col-sm-4, + .col-sm-5, + .col-sm-6, + .col-sm-7, + .col-sm-8, + .col-sm-9 { + float: left + } + .col-sm-12 { + width: 100% + } + .col-sm-11 { + width: 91.66666667% + } + .col-sm-10 { + width: 83.33333333% + } + .col-sm-9 { + width: 75% + } + .col-sm-8 { + width: 66.66666667% + } + .col-sm-7 { + width: 58.33333333% + } + .col-sm-6 { + width: 50% + } + .col-sm-5 { + width: 41.66666667% + } + .col-sm-4 { + width: 33.33333333% + } + .col-sm-3 { + width: 25% + } + .col-sm-2 { + width: 16.66666667% + } + .col-sm-1 { + width: 8.33333333% + } + .col-sm-pull-12 { + right: 100% + } + .col-sm-pull-11 { + right: 91.66666667% + } + .col-sm-pull-10 { + right: 83.33333333% + } + .col-sm-pull-9 { + right: 75% + } + .col-sm-pull-8 { + right: 66.66666667% + } + .col-sm-pull-7 { + right: 58.33333333% + } + .col-sm-pull-6 { + right: 50% + } + .col-sm-pull-5 { + right: 41.66666667% + } + .col-sm-pull-4 { + right: 33.33333333% + } + .col-sm-pull-3 { + right: 25% + } + .col-sm-pull-2 { + right: 16.66666667% + } + .col-sm-pull-1 { + right: 8.33333333% + } + .col-sm-pull-0 { + right: auto + } + .col-sm-push-12 { + left: 100% + } + .col-sm-push-11 { + left: 91.66666667% + } + .col-sm-push-10 { + left: 83.33333333% + } + .col-sm-push-9 { + left: 75% + } + .col-sm-push-8 { + left: 66.66666667% + } + .col-sm-push-7 { + left: 58.33333333% + } + .col-sm-push-6 { + left: 50% + } + .col-sm-push-5 { + left: 41.66666667% + } + .col-sm-push-4 { + left: 33.33333333% + } + .col-sm-push-3 { + left: 25% + } + .col-sm-push-2 { + left: 16.66666667% + } + .col-sm-push-1 { + left: 8.33333333% + } + .col-sm-push-0 { + left: auto + } + .col-sm-offset-12 { + margin-left: 100% + } + .col-sm-offset-11 { + margin-left: 91.66666667% + } + .col-sm-offset-10 { + margin-left: 83.33333333% + } + .col-sm-offset-9 { + margin-left: 75% + } + .col-sm-offset-8 { + margin-left: 66.66666667% + } + .col-sm-offset-7 { + margin-left: 58.33333333% + } + .col-sm-offset-6 { + margin-left: 50% + } + .col-sm-offset-5 { + margin-left: 41.66666667% + } + .col-sm-offset-4 { + margin-left: 33.33333333% + } + .col-sm-offset-3 { + margin-left: 25% + } + .col-sm-offset-2 { + margin-left: 16.66666667% + } + .col-sm-offset-1 { + margin-left: 8.33333333% + } + .col-sm-offset-0 { + margin-left: 0 + } +} + +@media (min-width:992px) { + .col-md-1, + .col-md-10, + .col-md-11, + .col-md-12, + .col-md-2, + .col-md-3, + .col-md-4, + .col-md-5, + .col-md-6, + .col-md-7, + .col-md-8, + .col-md-9 { + float: left + } + .col-md-12 { + width: 100% + } + .col-md-11 { + width: 91.66666667% + } + .col-md-10 { + width: 83.33333333% + } + .col-md-9 { + width: 75% + } + .col-md-8 { + width: 66.66666667% + } + .col-md-7 { + width: 58.33333333% + } + .col-md-6 { + width: 50% + } + .col-md-5 { + width: 41.66666667% + } + .col-md-4 { + width: 33.33333333% + } + .col-md-3 { + width: 25% + } + .col-md-2 { + width: 16.66666667% + } + .col-md-1 { + width: 8.33333333% + } + .col-md-pull-12 { + right: 100% + } + .col-md-pull-11 { + right: 91.66666667% + } + .col-md-pull-10 { + right: 83.33333333% + } + .col-md-pull-9 { + right: 75% + } + .col-md-pull-8 { + right: 66.66666667% + } + .col-md-pull-7 { + right: 58.33333333% + } + .col-md-pull-6 { + right: 50% + } + .col-md-pull-5 { + right: 41.66666667% + } + .col-md-pull-4 { + right: 33.33333333% + } + .col-md-pull-3 { + right: 25% + } + .col-md-pull-2 { + right: 16.66666667% + } + .col-md-pull-1 { + right: 8.33333333% + } + .col-md-pull-0 { + right: auto + } + .col-md-push-12 { + left: 100% + } + .col-md-push-11 { + left: 91.66666667% + } + .col-md-push-10 { + left: 83.33333333% + } + .col-md-push-9 { + left: 75% + } + .col-md-push-8 { + left: 66.66666667% + } + .col-md-push-7 { + left: 58.33333333% + } + .col-md-push-6 { + left: 50% + } + .col-md-push-5 { + left: 41.66666667% + } + .col-md-push-4 { + left: 33.33333333% + } + .col-md-push-3 { + left: 25% + } + .col-md-push-2 { + left: 16.66666667% + } + .col-md-push-1 { + left: 8.33333333% + } + .col-md-push-0 { + left: auto + } + .col-md-offset-12 { + margin-left: 100% + } + .col-md-offset-11 { + margin-left: 91.66666667% + } + .col-md-offset-10 { + margin-left: 83.33333333% + } + .col-md-offset-9 { + margin-left: 75% + } + .col-md-offset-8 { + margin-left: 66.66666667% + } + .col-md-offset-7 { + margin-left: 58.33333333% + } + .col-md-offset-6 { + margin-left: 50% + } + .col-md-offset-5 { + margin-left: 41.66666667% + } + .col-md-offset-4 { + margin-left: 33.33333333% + } + .col-md-offset-3 { + margin-left: 25% + } + .col-md-offset-2 { + margin-left: 16.66666667% + } + .col-md-offset-1 { + margin-left: 8.33333333% + } + .col-md-offset-0 { + margin-left: 0 + } +} + +@media (min-width:1200px) { + .col-lg-1, + .col-lg-10, + .col-lg-11, + .col-lg-12, + .col-lg-2, + .col-lg-3, + .col-lg-4, + .col-lg-5, + .col-lg-6, + .col-lg-7, + .col-lg-8, + .col-lg-9 { + float: left + } + .col-lg-12 { + width: 100% + } + .col-lg-11 { + width: 91.66666667% + } + .col-lg-10 { + width: 83.33333333% + } + .col-lg-9 { + width: 75% + } + .col-lg-8 { + width: 66.66666667% + } + .col-lg-7 { + width: 58.33333333% + } + .col-lg-6 { + width: 50% + } + .col-lg-5 { + width: 41.66666667% + } + .col-lg-4 { + width: 33.33333333% + } + .col-lg-3 { + width: 25% + } + .col-lg-2 { + width: 16.66666667% + } + .col-lg-1 { + width: 8.33333333% + } + .col-lg-pull-12 { + right: 100% + } + .col-lg-pull-11 { + right: 91.66666667% + } + .col-lg-pull-10 { + right: 83.33333333% + } + .col-lg-pull-9 { + right: 75% + } + .col-lg-pull-8 { + right: 66.66666667% + } + .col-lg-pull-7 { + right: 58.33333333% + } + .col-lg-pull-6 { + right: 50% + } + .col-lg-pull-5 { + right: 41.66666667% + } + .col-lg-pull-4 { + right: 33.33333333% + } + .col-lg-pull-3 { + right: 25% + } + .col-lg-pull-2 { + right: 16.66666667% + } + .col-lg-pull-1 { + right: 8.33333333% + } + .col-lg-pull-0 { + right: auto + } + .col-lg-push-12 { + left: 100% + } + .col-lg-push-11 { + left: 91.66666667% + } + .col-lg-push-10 { + left: 83.33333333% + } + .col-lg-push-9 { + left: 75% + } + .col-lg-push-8 { + left: 66.66666667% + } + .col-lg-push-7 { + left: 58.33333333% + } + .col-lg-push-6 { + left: 50% + } + .col-lg-push-5 { + left: 41.66666667% + } + .col-lg-push-4 { + left: 33.33333333% + } + .col-lg-push-3 { + left: 25% + } + .col-lg-push-2 { + left: 16.66666667% + } + .col-lg-push-1 { + left: 8.33333333% + } + .col-lg-push-0 { + left: auto + } + .col-lg-offset-12 { + margin-left: 100% + } + .col-lg-offset-11 { + margin-left: 91.66666667% + } + .col-lg-offset-10 { + margin-left: 83.33333333% + } + .col-lg-offset-9 { + margin-left: 75% + } + .col-lg-offset-8 { + margin-left: 66.66666667% + } + .col-lg-offset-7 { + margin-left: 58.33333333% + } + .col-lg-offset-6 { + margin-left: 50% + } + .col-lg-offset-5 { + margin-left: 41.66666667% + } + .col-lg-offset-4 { + margin-left: 33.33333333% + } + .col-lg-offset-3 { + margin-left: 25% + } + .col-lg-offset-2 { + margin-left: 16.66666667% + } + .col-lg-offset-1 { + margin-left: 8.33333333% + } + .col-lg-offset-0 { + margin-left: 0 + } +} + +table { + background-color: transparent +} + +caption { + padding-top: 8px; + padding-bottom: 8px; + color: #777; + text-align: left +} + +th { + text-align: left +} + +.table { + width: 100%; + max-width: 100%; + margin-bottom: 20px +} + +.table>tbody>tr>td, +.table>tbody>tr>th, +.table>tfoot>tr>td, +.table>tfoot>tr>th, +.table>thead>tr>td, +.table>thead>tr>th { + padding: 8px; + line-height: 1.42857143; + vertical-align: top; + border-top: 1px solid #ddd +} + +.table>thead>tr>th { + vertical-align: bottom; + border-bottom: 2px solid #ddd +} + +.table>caption+thead>tr:first-child>td, +.table>caption+thead>tr:first-child>th, +.table>colgroup+thead>tr:first-child>td, +.table>colgroup+thead>tr:first-child>th, +.table>thead:first-child>tr:first-child>td, +.table>thead:first-child>tr:first-child>th { + border-top: 0 +} + +.table>tbody+tbody { + border-top: 2px solid #ddd +} + +.table .table { + background-color: #fff +} + +.table-condensed>tbody>tr>td, +.table-condensed>tbody>tr>th, +.table-condensed>tfoot>tr>td, +.table-condensed>tfoot>tr>th, +.table-condensed>thead>tr>td, +.table-condensed>thead>tr>th { + padding: 5px +} + +.table-bordered { + border: 1px solid #ddd +} + +.table-bordered>tbody>tr>td, +.table-bordered>tbody>tr>th, +.table-bordered>tfoot>tr>td, +.table-bordered>tfoot>tr>th, +.table-bordered>thead>tr>td, +.table-bordered>thead>tr>th { + border: 1px solid #ddd +} + +.table-bordered>thead>tr>td, +.table-bordered>thead>tr>th { + border-bottom-width: 2px +} + +.table-striped>tbody>tr:nth-of-type(odd) { + background-color: #f9f9f9 +} + +.table-hover>tbody>tr:hover { + background-color: #f5f5f5 +} + +table col[class*=col-] { + position: static; + display: table-column; + float: none +} + +table td[class*=col-], +table th[class*=col-] { + position: static; + display: table-cell; + float: none +} + +.table>tbody>tr.active>td, +.table>tbody>tr.active>th, +.table>tbody>tr>td.active, +.table>tbody>tr>th.active, +.table>tfoot>tr.active>td, +.table>tfoot>tr.active>th, +.table>tfoot>tr>td.active, +.table>tfoot>tr>th.active, +.table>thead>tr.active>td, +.table>thead>tr.active>th, +.table>thead>tr>td.active, +.table>thead>tr>th.active { + background-color: #f5f5f5 +} + +.table-hover>tbody>tr.active:hover>td, +.table-hover>tbody>tr.active:hover>th, +.table-hover>tbody>tr:hover>.active, +.table-hover>tbody>tr>td.active:hover, +.table-hover>tbody>tr>th.active:hover { + background-color: #e8e8e8 +} + +.table>tbody>tr.success>td, +.table>tbody>tr.success>th, +.table>tbody>tr>td.success, +.table>tbody>tr>th.success, +.table>tfoot>tr.success>td, +.table>tfoot>tr.success>th, +.table>tfoot>tr>td.success, +.table>tfoot>tr>th.success, +.table>thead>tr.success>td, +.table>thead>tr.success>th, +.table>thead>tr>td.success, +.table>thead>tr>th.success { + background-color: #dff0d8 +} + +.table-hover>tbody>tr.success:hover>td, +.table-hover>tbody>tr.success:hover>th, +.table-hover>tbody>tr:hover>.success, +.table-hover>tbody>tr>td.success:hover, +.table-hover>tbody>tr>th.success:hover { + background-color: #d0e9c6 +} + +.table>tbody>tr.info>td, +.table>tbody>tr.info>th, +.table>tbody>tr>td.info, +.table>tbody>tr>th.info, +.table>tfoot>tr.info>td, +.table>tfoot>tr.info>th, +.table>tfoot>tr>td.info, +.table>tfoot>tr>th.info, +.table>thead>tr.info>td, +.table>thead>tr.info>th, +.table>thead>tr>td.info, +.table>thead>tr>th.info { + background-color: #d9edf7 +} + +.table-hover>tbody>tr.info:hover>td, +.table-hover>tbody>tr.info:hover>th, +.table-hover>tbody>tr:hover>.info, +.table-hover>tbody>tr>td.info:hover, +.table-hover>tbody>tr>th.info:hover { + background-color: #c4e3f3 +} + +.table>tbody>tr.warning>td, +.table>tbody>tr.warning>th, +.table>tbody>tr>td.warning, +.table>tbody>tr>th.warning, +.table>tfoot>tr.warning>td, +.table>tfoot>tr.warning>th, +.table>tfoot>tr>td.warning, +.table>tfoot>tr>th.warning, +.table>thead>tr.warning>td, +.table>thead>tr.warning>th, +.table>thead>tr>td.warning, +.table>thead>tr>th.warning { + background-color: #fcf8e3 +} + +.table-hover>tbody>tr.warning:hover>td, +.table-hover>tbody>tr.warning:hover>th, +.table-hover>tbody>tr:hover>.warning, +.table-hover>tbody>tr>td.warning:hover, +.table-hover>tbody>tr>th.warning:hover { + background-color: #faf2cc +} + +.table>tbody>tr.danger>td, +.table>tbody>tr.danger>th, +.table>tbody>tr>td.danger, +.table>tbody>tr>th.danger, +.table>tfoot>tr.danger>td, +.table>tfoot>tr.danger>th, +.table>tfoot>tr>td.danger, +.table>tfoot>tr>th.danger, +.table>thead>tr.danger>td, +.table>thead>tr.danger>th, +.table>thead>tr>td.danger, +.table>thead>tr>th.danger { + background-color: #f2dede +} + +.table-hover>tbody>tr.danger:hover>td, +.table-hover>tbody>tr.danger:hover>th, +.table-hover>tbody>tr:hover>.danger, +.table-hover>tbody>tr>td.danger:hover, +.table-hover>tbody>tr>th.danger:hover { + background-color: #ebcccc +} + +.table-responsive { + min-height: .01%; + overflow-x: auto +} + +@media screen and (max-width:767px) { + .table-responsive { + width: 100%; + margin-bottom: 15px; + overflow-y: hidden; + -ms-overflow-style: -ms-autohiding-scrollbar; + border: 1px solid #ddd + } + .table-responsive>.table { + margin-bottom: 0 + } + .table-responsive>.table>tbody>tr>td, + .table-responsive>.table>tbody>tr>th, + .table-responsive>.table>tfoot>tr>td, + .table-responsive>.table>tfoot>tr>th, + .table-responsive>.table>thead>tr>td, + .table-responsive>.table>thead>tr>th { + white-space: nowrap + } + .table-responsive>.table-bordered { + border: 0 + } + .table-responsive>.table-bordered>tbody>tr>td:first-child, + .table-responsive>.table-bordered>tbody>tr>th:first-child, + .table-responsive>.table-bordered>tfoot>tr>td:first-child, + .table-responsive>.table-bordered>tfoot>tr>th:first-child, + .table-responsive>.table-bordered>thead>tr>td:first-child, + .table-responsive>.table-bordered>thead>tr>th:first-child { + border-left: 0 + } + .table-responsive>.table-bordered>tbody>tr>td:last-child, + .table-responsive>.table-bordered>tbody>tr>th:last-child, + .table-responsive>.table-bordered>tfoot>tr>td:last-child, + .table-responsive>.table-bordered>tfoot>tr>th:last-child, + .table-responsive>.table-bordered>thead>tr>td:last-child, + .table-responsive>.table-bordered>thead>tr>th:last-child { + border-right: 0 + } + .table-responsive>.table-bordered>tbody>tr:last-child>td, + .table-responsive>.table-bordered>tbody>tr:last-child>th, + .table-responsive>.table-bordered>tfoot>tr:last-child>td, + .table-responsive>.table-bordered>tfoot>tr:last-child>th { + border-bottom: 0 + } +} + +fieldset { + min-width: 0; + padding: 0; + margin: 0; + border: 0 +} + +legend { + display: block; + width: 100%; + padding: 0; + margin-bottom: 20px; + font-size: 21px; + line-height: inherit; + color: #333; + border: 0; + border-bottom: 1px solid #e5e5e5 +} + +label { + display: inline-block; + max-width: 100%; + margin-bottom: 5px; + font-weight: 700 +} + +input[type=search] { + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box +} + +input[type=checkbox], +input[type=radio] { + margin: 4px 0 0; + margin-top: 1px \9; + line-height: normal +} + +input[type=file] { + display: block +} + +input[type=range] { + display: block; + width: 100% +} + +select[multiple], +select[size] { + height: auto +} + +input[type=file]:focus, +input[type=checkbox]:focus, +input[type=radio]:focus { + outline: thin dotted; + outline: 5px auto -webkit-focus-ring-color; + outline-offset: -2px +} + +output { + display: block; + padding-top: 7px; + font-size: 14px; + line-height: 1.42857143; + color: #555 +} + +.form-control { + display: block; + width: 100%; + height: 34px; + padding: 6px 12px; + font-size: 14px; + line-height: 1.42857143; + color: #555; + background-color: #fff; + background-image: none; + border: 1px solid #ccc; + border-radius: 4px; +} + +.form-control:focus { + border-color: #66afe9; + outline: 0; +} + +.form-control::-moz-placeholder { + color: #999; + opacity: 1 +} + +.form-control:-ms-input-placeholder { + color: #999 +} + +.form-control::-webkit-input-placeholder { + color: #999 +} + +.form-control[disabled], +.form-control[readonly], +fieldset[disabled] .form-control { + cursor: not-allowed; + background-color: #eee; + opacity: 1 +} + +textarea.form-control { + height: auto +} + +input[type=search] { + -webkit-appearance: none +} + +@media screen and (-webkit-min-device-pixel-ratio:0) { + input[type=date], + input[type=time], + input[type=datetime-local], + input[type=month] { + line-height: 34px + } + .input-group-sm input[type=date], + .input-group-sm input[type=time], + .input-group-sm input[type=datetime-local], + .input-group-sm input[type=month], + input[type=date].input-sm, + input[type=time].input-sm, + input[type=datetime-local].input-sm, + input[type=month].input-sm { + line-height: 30px + } + .input-group-lg input[type=date], + .input-group-lg input[type=time], + .input-group-lg input[type=datetime-local], + .input-group-lg input[type=month], + input[type=date].input-lg, + input[type=time].input-lg, + input[type=datetime-local].input-lg, + input[type=month].input-lg { + line-height: 46px + } +} + +.form-group { + margin-bottom: 15px +} + +.checkbox, +.radio { + position: relative; + display: block; + margin-top: 10px; + margin-bottom: 10px +} + +.checkbox label, +.radio label { + min-height: 20px; + padding-left: 20px; + margin-bottom: 0; + font-weight: 400; + cursor: pointer +} + +.checkbox input[type=checkbox], +.checkbox-inline input[type=checkbox], +.radio input[type=radio], +.radio-inline input[type=radio] { + position: absolute; + margin-top: 4px \9; + margin-left: -20px +} + +.checkbox+.checkbox, +.radio+.radio { + margin-top: -5px +} + +.checkbox-inline, +.radio-inline { + display: inline-block; + padding-left: 20px; + margin-bottom: 0; + font-weight: 400; + vertical-align: middle; + cursor: pointer +} + +.checkbox-inline+.checkbox-inline, +.radio-inline+.radio-inline { + margin-top: 0; + margin-left: 10px +} + +fieldset[disabled] input[type=checkbox], +fieldset[disabled] input[type=radio], +input[type=checkbox].disabled, +input[type=checkbox][disabled], +input[type=radio].disabled, +input[type=radio][disabled] { + cursor: not-allowed +} + +.checkbox-inline.disabled, +.radio-inline.disabled, +fieldset[disabled] .checkbox-inline, +fieldset[disabled] .radio-inline { + cursor: not-allowed +} + +.checkbox.disabled label, +.radio.disabled label, +fieldset[disabled] .checkbox label, +fieldset[disabled] .radio label { + cursor: not-allowed +} + +.form-control-static { + padding-top: 7px; + padding-bottom: 7px; + margin-bottom: 0 +} + +.form-control-static.input-lg, +.form-control-static.input-sm { + padding-right: 0; + padding-left: 0 +} + +.input-sm { + height: 30px; + padding: 5px 10px; + font-size: 12px; + line-height: 1.5; + border-radius: 3px +} + +select.input-sm { + height: 30px; + line-height: 30px +} + +select[multiple].input-sm, +textarea.input-sm { + height: auto +} + +.form-group-sm .form-control { + height: 30px; + padding: 5px 10px; + font-size: 12px; + line-height: 1.5; + border-radius: 3px +} + +select.form-group-sm .form-control { + height: 30px; + line-height: 30px +} + +select[multiple].form-group-sm .form-control, +textarea.form-group-sm .form-control { + height: auto +} + +.form-group-sm .form-control-static { + height: 30px; + padding: 5px 10px; + font-size: 12px; + line-height: 1.5 +} + +.input-lg { + height: 46px; + padding: 10px 16px; + font-size: 18px; + line-height: 1.3333333; + border-radius: 6px +} + +select.input-lg { + height: 46px; + line-height: 46px +} + +select[multiple].input-lg, +textarea.input-lg { + height: auto +} + +.form-group-lg .form-control { + height: 46px; + padding: 10px 16px; + font-size: 18px; + line-height: 1.3333333; + border-radius: 6px +} + +select.form-group-lg .form-control { + height: 46px; + line-height: 46px +} + +select[multiple].form-group-lg .form-control, +textarea.form-group-lg .form-control { + height: auto +} + +.form-group-lg .form-control-static { + height: 46px; + padding: 10px 16px; + font-size: 18px; + line-height: 1.3333333 +} + +.has-feedback { + position: relative +} + +.has-feedback .form-control { + padding-right: 42.5px +} + +.form-control-feedback { + position: absolute; + top: 0; + right: 0; + z-index: 2; + display: block; + width: 34px; + height: 34px; + line-height: 34px; + text-align: center; +} + +.input-lg+.form-control-feedback { + width: 46px; + height: 46px; + line-height: 46px +} + +.input-sm+.form-control-feedback { + width: 30px; + height: 30px; + line-height: 30px +} + +.has-success .checkbox, +.has-success .checkbox-inline, +.has-success .control-label, +.has-success .help-block, +.has-success .radio, +.has-success .radio-inline, +.has-success.checkbox label, +.has-success.checkbox-inline label, +.has-success.radio label, +.has-success.radio-inline label { + color: #3c763d +} + +.has-success .form-control { + border-color: #3c763d; +} + +.has-success .form-control:focus { + border-color: #2b542c; +} + +.has-success .input-group-addon { + color: #3c763d; + background-color: #dff0d8; + border-color: #3c763d +} + +.has-success .form-control-feedback { + color: #3c763d +} + +.has-warning .checkbox, +.has-warning .checkbox-inline, +.has-warning .control-label, +.has-warning .help-block, +.has-warning .radio, +.has-warning .radio-inline, +.has-warning.checkbox label, +.has-warning.checkbox-inline label, +.has-warning.radio label, +.has-warning.radio-inline label { + color: #8a6d3b +} + +.has-warning .form-control { + border-color: #8a6d3b; +} + +.has-warning .form-control:focus { + border-color: #66512c; +} + +.has-warning .input-group-addon { + color: #8a6d3b; + background-color: #fcf8e3; + border-color: #8a6d3b +} + +.has-warning .form-control-feedback { + color: #8a6d3b +} + +.has-error .checkbox, +.has-error .checkbox-inline, +.has-error .control-label, +.has-error .help-block, +.has-error .radio, +.has-error .radio-inline, +.has-error.checkbox label, +.has-error.checkbox-inline label, +.has-error.radio label, +.has-error.radio-inline label { + color: #a94442 +} + +.has-error .form-control { + border-color: #a94442; +} + +.has-error .form-control:focus { + border-color: #843534; +} + +.has-error .input-group-addon { + color: #a94442; + background-color: #f2dede; + border-color: #a94442 +} + +.has-error .form-control-feedback { + color: #a94442 +} + +.has-feedback label~.form-control-feedback { + top: 25px +} + +.has-feedback label.sr-only~.form-control-feedback { + top: 0 +} + +.help-block { + display: block; + margin-top: 5px; + margin-bottom: 10px; + color: #737373 +} + +@media (min-width:768px) { + .form-inline .form-group { + display: inline-block; + margin-bottom: 0; + vertical-align: middle + } + .form-inline .form-control { + display: inline-block; + width: auto; + vertical-align: middle + } + .form-inline .form-control-static { + display: inline-block + } + .form-inline .input-group { + display: inline-table; + vertical-align: middle + } + .form-inline .input-group .form-control, + .form-inline .input-group .input-group-addon, + .form-inline .input-group .input-group-btn { + width: auto + } + .form-inline .input-group>.form-control { + width: 100% + } + .form-inline .control-label { + margin-bottom: 0; + vertical-align: middle + } + .form-inline .checkbox, + .form-inline .radio { + display: inline-block; + margin-top: 0; + margin-bottom: 0; + vertical-align: middle + } + .form-inline .checkbox label, + .form-inline .radio label { + padding-left: 0 + } + .form-inline .checkbox input[type=checkbox], + .form-inline .radio input[type=radio] { + position: relative; + margin-left: 0 + } + .form-inline .has-feedback .form-control-feedback { + top: 0 + } +} + +.form-horizontal .checkbox, +.form-horizontal .checkbox-inline, +.form-horizontal .radio, +.form-horizontal .radio-inline { + padding-top: 7px; + margin-top: 0; + margin-bottom: 0 +} + +.form-horizontal .checkbox, +.form-horizontal .radio { + min-height: 27px +} + +.form-horizontal .form-group { + margin-right: -15px; + margin-left: -15px +} + +@media (min-width:768px) { + .form-horizontal .control-label { + padding-top: 7px; + margin-bottom: 0; + text-align: right + } +} + +.form-horizontal .has-feedback .form-control-feedback { + right: 15px +} + +@media (min-width:768px) { + .form-horizontal .form-group-lg .control-label { + padding-top: 14.33px + } +} + +@media (min-width:768px) { + .form-horizontal .form-group-sm .control-label { + padding-top: 6px + } +} + +.btn { + display: inline-block; + padding: 6px 12px; + margin-bottom: 0; + font-size: 14px; + font-weight: 400; + line-height: 1.42857143; + text-align: center; + white-space: nowrap; + vertical-align: middle; + -ms-touch-action: manipulation; + touch-action: manipulation; + cursor: pointer; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + background-image: none; + border: 1px solid transparent; + border-radius: 4px +} + +.btn.active.focus, +.btn.active:focus, +.btn.focus, +.btn:active.focus, +.btn:active:focus, +.btn:focus { + outline: thin dotted; + outline: 5px auto -webkit-focus-ring-color; + outline-offset: -2px +} + +.btn.focus, +.btn:focus, +.btn:hover { + color: #333; + text-decoration: none +} + +.btn.active, +.btn:active { + background-image: none; + outline: 0; +} + +.btn.disabled, +.btn[disabled], +fieldset[disabled] .btn { + cursor: not-allowed; + filter: alpha(opacity=65); + opacity: .65 +} + +.btn-default { + color: #333; + background-color: #fff; + border-color: #ccc +} + +.btn-default.active, +.btn-default.focus, +.btn-default:active, +.btn-default:focus, +.btn-default:hover, +.open>.dropdown-toggle.btn-default { + color: #333; + background-color: #e6e6e6; + border-color: #adadad +} + +.btn-default.active, +.btn-default:active, +.open>.dropdown-toggle.btn-default { + background-image: none +} + +.btn-default.disabled, +.btn-default.disabled.active, +.btn-default.disabled.focus, +.btn-default.disabled:active, +.btn-default.disabled:focus, +.btn-default.disabled:hover, +.btn-default[disabled], +.btn-default[disabled].active, +.btn-default[disabled].focus, +.btn-default[disabled]:active, +.btn-default[disabled]:focus, +.btn-default[disabled]:hover, +fieldset[disabled] .btn-default, +fieldset[disabled] .btn-default.active, +fieldset[disabled] .btn-default.focus, +fieldset[disabled] .btn-default:active, +fieldset[disabled] .btn-default:focus, +fieldset[disabled] .btn-default:hover { + background-color: #fff; + border-color: #ccc +} + +.btn-default .badge { + color: #fff; + background-color: #333 +} + +.btn-primary { + color: #fff; + background-color: #337ab7; + border-color: #2e6da4 +} + +.btn-primary.active, +.btn-primary.focus, +.btn-primary:active, +.btn-primary:focus, +.btn-primary:hover, +.open>.dropdown-toggle.btn-primary { + color: #fff; + background-color: #286090; + border-color: #204d74 +} + +.btn-primary.active, +.btn-primary:active, +.open>.dropdown-toggle.btn-primary { + background-image: none +} + +.btn-primary.disabled, +.btn-primary.disabled.active, +.btn-primary.disabled.focus, +.btn-primary.disabled:active, +.btn-primary.disabled:focus, +.btn-primary.disabled:hover, +.btn-primary[disabled], +.btn-primary[disabled].active, +.btn-primary[disabled].focus, +.btn-primary[disabled]:active, +.btn-primary[disabled]:focus, +.btn-primary[disabled]:hover, +fieldset[disabled] .btn-primary, +fieldset[disabled] .btn-primary.active, +fieldset[disabled] .btn-primary.focus, +fieldset[disabled] .btn-primary:active, +fieldset[disabled] .btn-primary:focus, +fieldset[disabled] .btn-primary:hover { + background-color: #337ab7; + border-color: #2e6da4 +} + +.btn-primary .badge { + color: #337ab7; + background-color: #fff +} + +.btn-success { + color: #fff; + background-color: #5cb85c; + border-color: #4cae4c +} + +.btn-success.active, +.btn-success.focus, +.btn-success:active, +.btn-success:focus, +.btn-success:hover, +.open>.dropdown-toggle.btn-success { + color: #fff; + background-color: #449d44; + border-color: #398439 +} + +.btn-success.active, +.btn-success:active, +.open>.dropdown-toggle.btn-success { + background-image: none +} + +.btn-success.disabled, +.btn-success.disabled.active, +.btn-success.disabled.focus, +.btn-success.disabled:active, +.btn-success.disabled:focus, +.btn-success.disabled:hover, +.btn-success[disabled], +.btn-success[disabled].active, +.btn-success[disabled].focus, +.btn-success[disabled]:active, +.btn-success[disabled]:focus, +.btn-success[disabled]:hover, +fieldset[disabled] .btn-success, +fieldset[disabled] .btn-success.active, +fieldset[disabled] .btn-success.focus, +fieldset[disabled] .btn-success:active, +fieldset[disabled] .btn-success:focus, +fieldset[disabled] .btn-success:hover { + background-color: #5cb85c; + border-color: #4cae4c +} + +.btn-success .badge { + color: #5cb85c; + background-color: #fff +} + +.btn-info { + color: #fff; + background-color: #5bc0de; + border-color: #46b8da +} + +.btn-info.active, +.btn-info.focus, +.btn-info:active, +.btn-info:focus, +.btn-info:hover, +.open>.dropdown-toggle.btn-info { + color: #fff; + background-color: #31b0d5; + border-color: #269abc +} + +.btn-info.active, +.btn-info:active, +.open>.dropdown-toggle.btn-info { + background-image: none +} + +.btn-info.disabled, +.btn-info.disabled.active, +.btn-info.disabled.focus, +.btn-info.disabled:active, +.btn-info.disabled:focus, +.btn-info.disabled:hover, +.btn-info[disabled], +.btn-info[disabled].active, +.btn-info[disabled].focus, +.btn-info[disabled]:active, +.btn-info[disabled]:focus, +.btn-info[disabled]:hover, +fieldset[disabled] .btn-info, +fieldset[disabled] .btn-info.active, +fieldset[disabled] .btn-info.focus, +fieldset[disabled] .btn-info:active, +fieldset[disabled] .btn-info:focus, +fieldset[disabled] .btn-info:hover { + background-color: #5bc0de; + border-color: #46b8da +} + +.btn-info .badge { + color: #5bc0de; + background-color: #fff +} + +.btn-warning { + color: #fff; + background-color: #f0ad4e; + border-color: #eea236 +} + +.btn-warning.active, +.btn-warning.focus, +.btn-warning:active, +.btn-warning:focus, +.btn-warning:hover, +.open>.dropdown-toggle.btn-warning { + color: #fff; + background-color: #ec971f; + border-color: #d58512 +} + +.btn-warning.active, +.btn-warning:active, +.open>.dropdown-toggle.btn-warning { + background-image: none +} + +.btn-warning.disabled, +.btn-warning.disabled.active, +.btn-warning.disabled.focus, +.btn-warning.disabled:active, +.btn-warning.disabled:focus, +.btn-warning.disabled:hover, +.btn-warning[disabled], +.btn-warning[disabled].active, +.btn-warning[disabled].focus, +.btn-warning[disabled]:active, +.btn-warning[disabled]:focus, +.btn-warning[disabled]:hover, +fieldset[disabled] .btn-warning, +fieldset[disabled] .btn-warning.active, +fieldset[disabled] .btn-warning.focus, +fieldset[disabled] .btn-warning:active, +fieldset[disabled] .btn-warning:focus, +fieldset[disabled] .btn-warning:hover { + background-color: #f0ad4e; + border-color: #eea236 +} + +.btn-warning .badge { + color: #f0ad4e; + background-color: #fff +} + +.btn-danger { + color: #fff; + background-color: #d9534f; + border-color: #d43f3a +} + +.btn-danger.active, +.btn-danger.focus, +.btn-danger:active, +.btn-danger:focus, +.btn-danger:hover, +.open>.dropdown-toggle.btn-danger { + color: #fff; + background-color: #c9302c; + border-color: #ac2925 +} + +.btn-danger.active, +.btn-danger:active, +.open>.dropdown-toggle.btn-danger { + background-image: none +} + +.btn-danger.disabled, +.btn-danger.disabled.active, +.btn-danger.disabled.focus, +.btn-danger.disabled:active, +.btn-danger.disabled:focus, +.btn-danger.disabled:hover, +.btn-danger[disabled], +.btn-danger[disabled].active, +.btn-danger[disabled].focus, +.btn-danger[disabled]:active, +.btn-danger[disabled]:focus, +.btn-danger[disabled]:hover, +fieldset[disabled] .btn-danger, +fieldset[disabled] .btn-danger.active, +fieldset[disabled] .btn-danger.focus, +fieldset[disabled] .btn-danger:active, +fieldset[disabled] .btn-danger:focus, +fieldset[disabled] .btn-danger:hover { + background-color: #d9534f; + border-color: #d43f3a +} + +.btn-danger .badge { + color: #d9534f; + background-color: #fff +} + +.btn-link { + font-weight: 400; + color: #337ab7; + border-radius: 0 +} + +.btn-link, +.btn-link.active, +.btn-link:active, +.btn-link[disabled], +fieldset[disabled] .btn-link { + background-color: transparent; +} + +.btn-link, +.btn-link:active, +.btn-link:focus, +.btn-link:hover { + border-color: transparent +} + +.btn-link:focus, +.btn-link:hover { + color: #23527c; + text-decoration: underline; + background-color: transparent +} + +.btn-link[disabled]:focus, +.btn-link[disabled]:hover, +fieldset[disabled] .btn-link:focus, +fieldset[disabled] .btn-link:hover { + color: #777; + text-decoration: none +} + +.btn-group-lg>.btn, +.btn-lg { + padding: 10px 16px; + font-size: 18px; + line-height: 1.3333333; + border-radius: 6px +} + +.btn-group-sm>.btn, +.btn-sm { + padding: 5px 10px; + font-size: 12px; + line-height: 1.5; + border-radius: 3px +} + +.btn-group-xs>.btn, +.btn-xs { + padding: 1px 5px; + font-size: 12px; + line-height: 1.5; + border-radius: 3px +} + +.btn-block { + display: block; + width: 100% +} + +.btn-block+.btn-block { + margin-top: 5px +} + +input[type=button].btn-block, +input[type=reset].btn-block, +input[type=submit].btn-block { + width: 100% +} + +.fade { + opacity: 0; + -webkit-transition: opacity .15s linear; + -o-transition: opacity .15s linear; + transition: opacity .15s linear +} + +.fade.in { + opacity: 1 +} + +.collapse { + display: none; + visibility: hidden +} + +.collapse.in { + display: block; + visibility: visible +} + +tr.collapse.in { + display: table-row +} + +tbody.collapse.in { + display: table-row-group +} + +.collapsing { + position: relative; + height: 0; + overflow: hidden; + -webkit-transition-timing-function: ease; + -o-transition-timing-function: ease; + transition-timing-function: ease; + -webkit-transition-duration: .35s; + -o-transition-duration: .35s; + transition-duration: .35s; + -webkit-transition-property: height, visibility; + -o-transition-property: height, visibility; + transition-property: height, visibility +} + +.caret { + display: inline-block; + width: 0; + height: 0; + margin-left: 2px; + vertical-align: middle; + border-top: 4px solid; + border-right: 4px solid transparent; + border-left: 4px solid transparent +} + +.dropdown, +.dropup { + position: relative +} + +.dropdown-toggle:focus { + outline: 0 +} + +.dropdown-menu { + position: absolute; + top: 100%; + left: 0; + z-index: 1000; + display: none; + float: left; + min-width: 160px; + padding: 5px 0; + margin: 2px 0 0; + font-size: 14px; + text-align: left; + list-style: none; + background-color: #fff; + -webkit-background-clip: padding-box; + background-clip: padding-box; + border: 1px solid #ccc; + border: 1px solid rgba(0, 0, 0, .15); + border-radius: 4px; +} + +.dropdown-menu.pull-right { + right: 0; + left: auto +} + +.dropdown-menu .divider { + height: 1px; + margin: 9px 0; + overflow: hidden; + background-color: #e5e5e5 +} + +.dropdown-menu>li>a { + display: block; + padding: 3px 20px; + clear: both; + font-weight: 400; + line-height: 1.42857143; + color: #333; + white-space: nowrap +} + +.dropdown-menu>li>a:focus, +.dropdown-menu>li>a:hover { + color: #262626; + text-decoration: none; + background-color: #f5f5f5 +} + +.dropdown-menu>.active>a, +.dropdown-menu>.active>a:focus, +.dropdown-menu>.active>a:hover { + color: #fff; + text-decoration: none; + background-color: #337ab7; + outline: 0 +} + +.dropdown-menu>.disabled>a, +.dropdown-menu>.disabled>a:focus, +.dropdown-menu>.disabled>a:hover { + color: #777 +} + +.dropdown-menu>.disabled>a:focus, +.dropdown-menu>.disabled>a:hover { + text-decoration: none; + cursor: not-allowed; + background-color: transparent; + background-image: none; + filter: progid: DXImageTransform.Microsoft.gradient(enabled=false) +} + +.open>.dropdown-menu { + display: block +} + +.open>a { + outline: 0 +} + +.dropdown-menu-right { + right: 0; + left: auto +} + +.dropdown-menu-left { + right: auto; + left: 0 +} + +.dropdown-header { + display: block; + padding: 3px 20px; + font-size: 12px; + line-height: 1.42857143; + color: #777; + white-space: nowrap +} + +.dropdown-backdrop { + position: fixed; + top: 0; + right: 0; + bottom: 0; + left: 0; + z-index: 990 +} + +.pull-right>.dropdown-menu { + right: 0; + left: auto +} + +.dropup .caret, +.navbar-fixed-bottom .dropdown .caret { + content: ""; + border-top: 0; + border-bottom: 4px solid +} + +.dropup .dropdown-menu, +.navbar-fixed-bottom .dropdown .dropdown-menu { + top: auto; + bottom: 100%; + margin-bottom: 2px +} + +@media (min-width:768px) { + .navbar-right .dropdown-menu { + right: 0; + left: auto + } + .navbar-right .dropdown-menu-left { + right: auto; + left: 0 + } +} + +.btn-group, +.btn-group-vertical { + position: relative; + display: inline-block; + vertical-align: middle +} + +.btn-group-vertical>.btn, +.btn-group>.btn { + position: relative; + float: left +} + +.btn-group-vertical>.btn.active, +.btn-group-vertical>.btn:active, +.btn-group-vertical>.btn:focus, +.btn-group-vertical>.btn:hover, +.btn-group>.btn.active, +.btn-group>.btn:active, +.btn-group>.btn:focus, +.btn-group>.btn:hover { + z-index: 2 +} + +.btn-group .btn+.btn, +.btn-group .btn+.btn-group, +.btn-group .btn-group+.btn, +.btn-group .btn-group+.btn-group { + margin-left: -1px +} + +.btn-toolbar { + margin-left: -5px +} + +.btn-toolbar .btn-group, +.btn-toolbar .input-group { + float: left +} + +.btn-toolbar>.btn, +.btn-toolbar>.btn-group, +.btn-toolbar>.input-group { + margin-left: 5px +} + +.btn-group>.btn:not(:first-child):not(:last-child):not(.dropdown-toggle) { + border-radius: 0 +} + +.btn-group>.btn:first-child { + margin-left: 0 +} + +.btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle) { + border-top-right-radius: 0; + border-bottom-right-radius: 0 +} + +.btn-group>.btn:last-child:not(:first-child), +.btn-group>.dropdown-toggle:not(:first-child) { + border-top-left-radius: 0; + border-bottom-left-radius: 0 +} + +.btn-group>.btn-group { + float: left +} + +.btn-group>.btn-group:not(:first-child):not(:last-child)>.btn { + border-radius: 0 +} + +.btn-group>.btn-group:first-child:not(:last-child)>.btn:last-child, +.btn-group>.btn-group:first-child:not(:last-child)>.dropdown-toggle { + border-top-right-radius: 0; + border-bottom-right-radius: 0 +} + +.btn-group>.btn-group:last-child:not(:first-child)>.btn:first-child { + border-top-left-radius: 0; + border-bottom-left-radius: 0 +} + +.btn-group .dropdown-toggle:active, +.btn-group.open .dropdown-toggle { + outline: 0 +} + +.btn-group>.btn+.dropdown-toggle { + padding-right: 8px; + padding-left: 8px +} + +.btn-group>.btn-lg+.dropdown-toggle { + padding-right: 12px; + padding-left: 12px +} + +.btn-group.open .dropdown-toggle { +} + +.btn-group.open .dropdown-toggle.btn-link { +} + +.btn .caret { + margin-left: 0 +} + +.btn-lg .caret { + border-width: 5px 5px 0; + border-bottom-width: 0 +} + +.dropup .btn-lg .caret { + border-width: 0 5px 5px +} + +.btn-group-vertical>.btn, +.btn-group-vertical>.btn-group, +.btn-group-vertical>.btn-group>.btn { + display: block; + float: none; + width: 100%; + max-width: 100% +} + +.btn-group-vertical>.btn-group>.btn { + float: none +} + +.btn-group-vertical>.btn+.btn, +.btn-group-vertical>.btn+.btn-group, +.btn-group-vertical>.btn-group+.btn, +.btn-group-vertical>.btn-group+.btn-group { + margin-top: -1px; + margin-left: 0 +} + +.btn-group-vertical>.btn:not(:first-child):not(:last-child) { + border-radius: 0 +} + +.btn-group-vertical>.btn:first-child:not(:last-child) { + border-top-right-radius: 4px; + border-bottom-right-radius: 0; + border-bottom-left-radius: 0 +} + +.btn-group-vertical>.btn:last-child:not(:first-child) { + border-top-left-radius: 0; + border-top-right-radius: 0; + border-bottom-left-radius: 4px +} + +.btn-group-vertical>.btn-group:not(:first-child):not(:last-child)>.btn { + border-radius: 0 +} + +.btn-group-vertical>.btn-group:first-child:not(:last-child)>.btn:last-child, +.btn-group-vertical>.btn-group:first-child:not(:last-child)>.dropdown-toggle { + border-bottom-right-radius: 0; + border-bottom-left-radius: 0 +} + +.btn-group-vertical>.btn-group:last-child:not(:first-child)>.btn:first-child { + border-top-left-radius: 0; + border-top-right-radius: 0 +} + +.btn-group-justified { + display: table; + width: 100%; + table-layout: fixed; + border-collapse: separate +} + +.btn-group-justified>.btn, +.btn-group-justified>.btn-group { + display: table-cell; + float: none; + width: 1% +} + +.btn-group-justified>.btn-group .btn { + width: 100% +} + +.btn-group-justified>.btn-group .dropdown-menu { + left: auto +} + +[data-toggle=buttons]>.btn input[type=checkbox], +[data-toggle=buttons]>.btn input[type=radio], +[data-toggle=buttons]>.btn-group>.btn input[type=checkbox], +[data-toggle=buttons]>.btn-group>.btn input[type=radio] { + position: absolute; + clip: rect(0, 0, 0, 0); +} + +.input-group { + position: relative; + display: table; + border-collapse: separate +} + +.input-group[class*=col-] { + float: none; + padding-right: 0; + padding-left: 0 +} + +.input-group .form-control { + position: relative; + z-index: 2; + float: left; + width: 100%; + margin-bottom: 0 +} + +.input-group-lg>.form-control, +.input-group-lg>.input-group-addon, +.input-group-lg>.input-group-btn>.btn { + height: 46px; + padding: 10px 16px; + font-size: 18px; + line-height: 1.3333333; + border-radius: 6px +} + +select.input-group-lg>.form-control, +select.input-group-lg>.input-group-addon, +select.input-group-lg>.input-group-btn>.btn { + height: 46px; + line-height: 46px +} + +select[multiple].input-group-lg>.form-control, +select[multiple].input-group-lg>.input-group-addon, +select[multiple].input-group-lg>.input-group-btn>.btn, +textarea.input-group-lg>.form-control, +textarea.input-group-lg>.input-group-addon, +textarea.input-group-lg>.input-group-btn>.btn { + height: auto +} + +.input-group-sm>.form-control, +.input-group-sm>.input-group-addon, +.input-group-sm>.input-group-btn>.btn { + height: 30px; + padding: 5px 10px; + font-size: 12px; + line-height: 1.5; + border-radius: 3px +} + +select.input-group-sm>.form-control, +select.input-group-sm>.input-group-addon, +select.input-group-sm>.input-group-btn>.btn { + height: 30px; + line-height: 30px +} + +select[multiple].input-group-sm>.form-control, +select[multiple].input-group-sm>.input-group-addon, +select[multiple].input-group-sm>.input-group-btn>.btn, +textarea.input-group-sm>.form-control, +textarea.input-group-sm>.input-group-addon, +textarea.input-group-sm>.input-group-btn>.btn { + height: auto +} + +.input-group .form-control, +.input-group-addon, +.input-group-btn { + display: table-cell +} + +.input-group .form-control:not(:first-child):not(:last-child), +.input-group-addon:not(:first-child):not(:last-child), +.input-group-btn:not(:first-child):not(:last-child) { + border-radius: 0 +} + +.input-group-addon, +.input-group-btn { + width: 1%; + white-space: nowrap; + vertical-align: middle +} + +.input-group-addon { + padding: 6px 12px; + font-size: 14px; + font-weight: 400; + line-height: 1; + color: #555; + text-align: center; + background-color: #eee; + border: 1px solid #ccc; + border-radius: 4px +} + +.input-group-addon.input-sm { + padding: 5px 10px; + font-size: 12px; + border-radius: 3px +} + +.input-group-addon.input-lg { + padding: 10px 16px; + font-size: 18px; + border-radius: 6px +} + +.input-group-addon input[type=checkbox], +.input-group-addon input[type=radio] { + margin-top: 0 +} + +.input-group .form-control:first-child, +.input-group-addon:first-child, +.input-group-btn:first-child>.btn, +.input-group-btn:first-child>.btn-group>.btn, +.input-group-btn:first-child>.dropdown-toggle, +.input-group-btn:last-child>.btn-group:not(:last-child)>.btn, +.input-group-btn:last-child>.btn:not(:last-child):not(.dropdown-toggle) { + border-top-right-radius: 0; + border-bottom-right-radius: 0 +} + +.input-group-addon:first-child { + border-right: 0 +} + +.input-group .form-control:last-child, +.input-group-addon:last-child, +.input-group-btn:first-child>.btn-group:not(:first-child)>.btn, +.input-group-btn:first-child>.btn:not(:first-child), +.input-group-btn:last-child>.btn, +.input-group-btn:last-child>.btn-group>.btn, +.input-group-btn:last-child>.dropdown-toggle { + border-top-left-radius: 0; + border-bottom-left-radius: 0 +} + +.input-group-addon:last-child { + border-left: 0 +} + +.input-group-btn { + position: relative; + font-size: 0; + white-space: nowrap +} + +.input-group-btn>.btn { + position: relative +} + +.input-group-btn>.btn+.btn { + margin-left: -1px +} + +.input-group-btn>.btn:active, +.input-group-btn>.btn:focus, +.input-group-btn>.btn:hover { + z-index: 2 +} + +.input-group-btn:first-child>.btn, +.input-group-btn:first-child>.btn-group { + margin-right: -1px +} + +.input-group-btn:last-child>.btn, +.input-group-btn:last-child>.btn-group { + margin-left: -1px +} + +.nav { + padding-left: 0; + margin-bottom: 0; + list-style: none +} + +.nav>li { + position: relative; + display: block +} + +.nav>li>a { + position: relative; + display: block; + padding: 10px 15px +} + +.nav>li>a:focus, +.nav>li>a:hover { + text-decoration: none; + background-color: #eee +} + +.nav>li.disabled>a { + color: #777 +} + +.nav>li.disabled>a:focus, +.nav>li.disabled>a:hover { + color: #777; + text-decoration: none; + cursor: not-allowed; + background-color: transparent +} + +.nav .open>a, +.nav .open>a:focus, +.nav .open>a:hover { + background-color: #eee; + border-color: #337ab7 +} + +.nav .nav-divider { + height: 1px; + margin: 9px 0; + overflow: hidden; + background-color: #e5e5e5 +} + +.nav>li>a>img { + max-width: none +} + +.nav-tabs { + border-bottom: 1px solid #ddd +} + +.nav-tabs>li { + float: left; + margin-bottom: -1px +} + +.nav-tabs>li>a { + margin-right: 2px; + line-height: 1.42857143; + border: 1px solid transparent; + border-radius: 4px 4px 0 0 +} + +.nav-tabs>li>a:hover { + border-color: #eee #eee #ddd +} + +.nav-tabs>li.active>a, +.nav-tabs>li.active>a:focus, +.nav-tabs>li.active>a:hover { + color: #555; + cursor: default; + background-color: #fff; + border: 1px solid #ddd; + border-bottom-color: transparent +} + +.nav-tabs.nav-justified { + width: 100%; + border-bottom: 0 +} + +.nav-tabs.nav-justified>li { + float: none +} + +.nav-tabs.nav-justified>li>a { + margin-bottom: 5px; + text-align: center +} + +.nav-tabs.nav-justified>.dropdown .dropdown-menu { + top: auto; + left: auto +} + +@media (min-width:768px) { + .nav-tabs.nav-justified>li { + display: table-cell; + width: 1% + } + .nav-tabs.nav-justified>li>a { + margin-bottom: 0 + } +} + +.nav-tabs.nav-justified>li>a { + margin-right: 0; + border-radius: 4px +} + +.nav-tabs.nav-justified>.active>a, +.nav-tabs.nav-justified>.active>a:focus, +.nav-tabs.nav-justified>.active>a:hover { + border: 1px solid #ddd +} + +@media (min-width:768px) { + .nav-tabs.nav-justified>li>a { + border-bottom: 1px solid #ddd; + border-radius: 4px 4px 0 0 + } + .nav-tabs.nav-justified>.active>a, + .nav-tabs.nav-justified>.active>a:focus, + .nav-tabs.nav-justified>.active>a:hover { + border-bottom-color: #fff + } +} + +.nav-pills>li { + float: left +} + +.nav-pills>li>a { + border-radius: 4px +} + +.nav-pills>li+li { + margin-left: 2px +} + +.nav-pills>li.active>a, +.nav-pills>li.active>a:focus, +.nav-pills>li.active>a:hover { + color: #fff; + background-color: #337ab7 +} + +.nav-stacked>li { + float: none +} + +.nav-stacked>li+li { + margin-top: 2px; + margin-left: 0 +} + +.nav-justified { + width: 100% +} + +.nav-justified>li { + float: none +} + +.nav-justified>li>a { + margin-bottom: 5px; + text-align: center +} + +.nav-justified>.dropdown .dropdown-menu { + top: auto; + left: auto +} + +@media (min-width:768px) { + .nav-justified>li { + display: table-cell; + width: 1% + } + .nav-justified>li>a { + margin-bottom: 0 + } +} + +.nav-tabs-justified { + border-bottom: 0 +} + +.nav-tabs-justified>li>a { + margin-right: 0; + border-radius: 4px +} + +.nav-tabs-justified>.active>a, +.nav-tabs-justified>.active>a:focus, +.nav-tabs-justified>.active>a:hover { + border: 1px solid #ddd +} + +@media (min-width:768px) { + .nav-tabs-justified>li>a { + border-bottom: 1px solid #ddd; + border-radius: 4px 4px 0 0 + } + .nav-tabs-justified>.active>a, + .nav-tabs-justified>.active>a:focus, + .nav-tabs-justified>.active>a:hover { + border-bottom-color: #fff + } +} + +.tab-content>.tab-pane { + display: none; + visibility: hidden +} + +.tab-content>.active { + display: block; + visibility: visible +} + +.nav-tabs .dropdown-menu { + margin-top: -1px; + border-top-left-radius: 0; + border-top-right-radius: 0 +} + +.navbar { + position: relative; + min-height: 50px; + margin-bottom: 20px; + border: 1px solid transparent +} + +@media (min-width:768px) { + .navbar { + border-radius: 4px + } +} + +@media (min-width:768px) { + .navbar-header { + float: left + } +} + +.navbar-collapse { + padding-right: 15px; + padding-left: 15px; + overflow-x: visible; + -webkit-overflow-scrolling: touch; + border-top: 1px solid transparent; +} + +.navbar-collapse.in { + overflow-y: auto +} + +@media (min-width:768px) { + .navbar-collapse { + width: auto; + border-top: 0; + } + .navbar-collapse.collapse { + display: block!important; + height: auto!important; + padding-bottom: 0; + overflow: visible!important; + visibility: visible!important + } + .navbar-collapse.in { + overflow-y: visible + } + .navbar-fixed-bottom .navbar-collapse, + .navbar-fixed-top .navbar-collapse, + .navbar-static-top .navbar-collapse { + padding-right: 0; + padding-left: 0 + } +} + +.navbar-fixed-bottom .navbar-collapse, +.navbar-fixed-top .navbar-collapse { + max-height: 340px +} + +@media (max-device-width:480px) and (orientation:landscape) { + .navbar-fixed-bottom .navbar-collapse, + .navbar-fixed-top .navbar-collapse { + max-height: 200px + } +} + +.container-fluid>.navbar-collapse, +.container-fluid>.navbar-header, +.container>.navbar-collapse, +.container>.navbar-header { + margin-right: -15px; + margin-left: -15px +} + +@media (min-width:768px) { + .container-fluid>.navbar-collapse, + .container-fluid>.navbar-header, + .container>.navbar-collapse, + .container>.navbar-header { + margin-right: 0; + margin-left: 0 + } +} + +.navbar-static-top { + z-index: 1000; + border-width: 0 0 1px +} + +@media (min-width:768px) { + .navbar-static-top { + border-radius: 0 + } +} + +.navbar-fixed-bottom, +.navbar-fixed-top { + position: fixed; + right: 0; + left: 0; + z-index: 1030 +} + +@media (min-width:768px) { + .navbar-fixed-bottom, + .navbar-fixed-top { + border-radius: 0 + } +} + +.navbar-fixed-top { + top: 0; + border-width: 0 0 1px +} + +.navbar-fixed-bottom { + bottom: 0; + margin-bottom: 0; + border-width: 1px 0 0 +} + +.navbar-brand { + float: left; + height: 50px; + padding: 15px 15px; + font-size: 18px; + line-height: 20px +} + +.navbar-brand:focus, +.navbar-brand:hover { + text-decoration: none +} + +.navbar-brand>img { + display: block +} + +@media (min-width:768px) { + .navbar>.container .navbar-brand, + .navbar>.container-fluid .navbar-brand { + margin-left: -15px + } +} + +.navbar-toggle { + position: relative; + float: right; + padding: 9px 10px; + margin-top: 8px; + margin-right: 15px; + margin-bottom: 8px; + background-color: transparent; + background-image: none; + border: 1px solid transparent; + border-radius: 4px +} + +.navbar-toggle:focus { + outline: 0 +} + +.navbar-toggle .icon-bar { + display: block; + width: 22px; + height: 2px; + border-radius: 1px +} + +.navbar-toggle .icon-bar+.icon-bar { + margin-top: 4px +} + +@media (min-width:768px) { + .navbar-toggle { + display: none + } +} + +.navbar-nav { + margin: 7.5px -15px +} + +.navbar-nav>li>a { + padding-top: 10px; + padding-bottom: 10px; + line-height: 20px +} + +@media (max-width:767px) { + .navbar-nav .open .dropdown-menu { + position: static; + float: none; + width: auto; + margin-top: 0; + background-color: transparent; + border: 0; + } + .navbar-nav .open .dropdown-menu .dropdown-header, + .navbar-nav .open .dropdown-menu>li>a { + padding: 5px 15px 5px 25px + } + .navbar-nav .open .dropdown-menu>li>a { + line-height: 20px + } + .navbar-nav .open .dropdown-menu>li>a:focus, + .navbar-nav .open .dropdown-menu>li>a:hover { + background-image: none + } +} + +@media (min-width:768px) { + .navbar-nav { + float: left; + margin: 0 + } + .navbar-nav>li { + float: left + } + .navbar-nav>li>a { + padding-top: 15px; + padding-bottom: 15px + } +} + +.navbar-form { + padding: 10px 15px; + margin-top: 8px; + margin-right: -15px; + margin-bottom: 8px; + margin-left: -15px; + border-top: 1px solid transparent; + border-bottom: 1px solid transparent; +} + +@media (min-width:768px) { + .navbar-form .form-group { + display: inline-block; + margin-bottom: 0; + vertical-align: middle + } + .navbar-form .form-control { + display: inline-block; + width: auto; + vertical-align: middle + } + .navbar-form .form-control-static { + display: inline-block + } + .navbar-form .input-group { + display: inline-table; + vertical-align: middle + } + .navbar-form .input-group .form-control, + .navbar-form .input-group .input-group-addon, + .navbar-form .input-group .input-group-btn { + width: auto + } + .navbar-form .input-group>.form-control { + width: 100% + } + .navbar-form .control-label { + margin-bottom: 0; + vertical-align: middle + } + .navbar-form .checkbox, + .navbar-form .radio { + display: inline-block; + margin-top: 0; + margin-bottom: 0; + vertical-align: middle + } + .navbar-form .checkbox label, + .navbar-form .radio label { + padding-left: 0 + } + .navbar-form .checkbox input[type=checkbox], + .navbar-form .radio input[type=radio] { + position: relative; + margin-left: 0 + } + .navbar-form .has-feedback .form-control-feedback { + top: 0 + } +} + +@media (max-width:767px) { + .navbar-form .form-group { + margin-bottom: 5px + } + .navbar-form .form-group:last-child { + margin-bottom: 0 + } +} + +@media (min-width:768px) { + .navbar-form { + width: auto; + padding-top: 0; + padding-bottom: 0; + margin-right: 0; + margin-left: 0; + border: 0; + } +} + +.navbar-nav>li>.dropdown-menu { + margin-top: 0; + border-top-left-radius: 0; + border-top-right-radius: 0 +} + +.navbar-fixed-bottom .navbar-nav>li>.dropdown-menu { + margin-bottom: 0; + border-top-left-radius: 4px; + border-top-right-radius: 4px; + border-bottom-right-radius: 0; + border-bottom-left-radius: 0 +} + +.navbar-btn { + margin-top: 8px; + margin-bottom: 8px +} + +.navbar-btn.btn-sm { + margin-top: 10px; + margin-bottom: 10px +} + +.navbar-btn.btn-xs { + margin-top: 14px; + margin-bottom: 14px +} + +.navbar-text { + margin-top: 15px; + margin-bottom: 15px +} + +@media (min-width:768px) { + .navbar-text { + float: left; + margin-right: 15px; + margin-left: 15px + } +} + +@media (min-width:768px) { + .navbar-left { + float: left!important + } + .navbar-right { + float: right!important; + margin-right: -15px + } + .navbar-right~.navbar-right { + margin-right: 0 + } +} + +.navbar-default { + background-color: #f8f8f8; + border-color: #e7e7e7 +} + +.navbar-default .navbar-brand { + color: #777 +} + +.navbar-default .navbar-brand:focus, +.navbar-default .navbar-brand:hover { + color: #5e5e5e; + background-color: transparent +} + +.navbar-default .navbar-text { + color: #777 +} + +.navbar-default .navbar-nav>li>a { + color: #777 +} + +.navbar-default .navbar-nav>li>a:focus, +.navbar-default .navbar-nav>li>a:hover { + color: #333; + background-color: transparent +} + +.navbar-default .navbar-nav>.active>a, +.navbar-default .navbar-nav>.active>a:focus, +.navbar-default .navbar-nav>.active>a:hover { + color: #555; + background-color: #e7e7e7 +} + +.navbar-default .navbar-nav>.disabled>a, +.navbar-default .navbar-nav>.disabled>a:focus, +.navbar-default .navbar-nav>.disabled>a:hover { + color: #ccc; + background-color: transparent +} + +.navbar-default .navbar-toggle { + border-color: #ddd +} + +.navbar-default .navbar-toggle:focus, +.navbar-default .navbar-toggle:hover { + background-color: #ddd +} + +.navbar-default .navbar-toggle .icon-bar { + background-color: #888 +} + +.navbar-default .navbar-collapse, +.navbar-default .navbar-form { + border-color: #e7e7e7 +} + +.navbar-default .navbar-nav>.open>a, +.navbar-default .navbar-nav>.open>a:focus, +.navbar-default .navbar-nav>.open>a:hover { + color: #555; + background-color: #e7e7e7 +} + +@media (max-width:767px) { + .navbar-default .navbar-nav .open .dropdown-menu>li>a { + color: #777 + } + .navbar-default .navbar-nav .open .dropdown-menu>li>a:focus, + .navbar-default .navbar-nav .open .dropdown-menu>li>a:hover { + color: #333; + background-color: transparent + } + .navbar-default .navbar-nav .open .dropdown-menu>.active>a, + .navbar-default .navbar-nav .open .dropdown-menu>.active>a:focus, + .navbar-default .navbar-nav .open .dropdown-menu>.active>a:hover { + color: #555; + background-color: #e7e7e7 + } + .navbar-default .navbar-nav .open .dropdown-menu>.disabled>a, + .navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:focus, + .navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:hover { + color: #ccc; + background-color: transparent + } +} + +.navbar-default .navbar-link { + color: #777 +} + +.navbar-default .navbar-link:hover { + color: #333 +} + +.navbar-default .btn-link { + color: #777 +} + +.navbar-default .btn-link:focus, +.navbar-default .btn-link:hover { + color: #333 +} + +.navbar-default .btn-link[disabled]:focus, +.navbar-default .btn-link[disabled]:hover, +fieldset[disabled] .navbar-default .btn-link:focus, +fieldset[disabled] .navbar-default .btn-link:hover { + color: #ccc +} + +.navbar-inverse { + background-color: #222; + border-color: #080808 +} + +.navbar-inverse .navbar-brand { + color: #9d9d9d +} + +.navbar-inverse .navbar-brand:focus, +.navbar-inverse .navbar-brand:hover { + color: #fff; + background-color: transparent +} + +.navbar-inverse .navbar-text { + color: #9d9d9d +} + +.navbar-inverse .navbar-nav>li>a { + color: #9d9d9d +} + +.navbar-inverse .navbar-nav>li>a:focus, +.navbar-inverse .navbar-nav>li>a:hover { + color: #fff; + background-color: transparent +} + +.navbar-inverse .navbar-nav>.active>a, +.navbar-inverse .navbar-nav>.active>a:focus, +.navbar-inverse .navbar-nav>.active>a:hover { + color: #fff; + background-color: #080808 +} + +.navbar-inverse .navbar-nav>.disabled>a, +.navbar-inverse .navbar-nav>.disabled>a:focus, +.navbar-inverse .navbar-nav>.disabled>a:hover { + color: #444; + background-color: transparent +} + +.navbar-inverse .navbar-toggle { + border-color: #333 +} + +.navbar-inverse .navbar-toggle:focus, +.navbar-inverse .navbar-toggle:hover { + background-color: #333 +} + +.navbar-inverse .navbar-toggle .icon-bar { + background-color: #fff +} + +.navbar-inverse .navbar-collapse, +.navbar-inverse .navbar-form { + border-color: #101010 +} + +.navbar-inverse .navbar-nav>.open>a, +.navbar-inverse .navbar-nav>.open>a:focus, +.navbar-inverse .navbar-nav>.open>a:hover { + color: #fff; + background-color: #080808 +} + +@media (max-width:767px) { + .navbar-inverse .navbar-nav .open .dropdown-menu>.dropdown-header { + border-color: #080808 + } + .navbar-inverse .navbar-nav .open .dropdown-menu .divider { + background-color: #080808 + } + .navbar-inverse .navbar-nav .open .dropdown-menu>li>a { + color: #9d9d9d + } + .navbar-inverse .navbar-nav .open .dropdown-menu>li>a:focus, + .navbar-inverse .navbar-nav .open .dropdown-menu>li>a:hover { + color: #fff; + background-color: transparent + } + .navbar-inverse .navbar-nav .open .dropdown-menu>.active>a, + .navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:focus, + .navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:hover { + color: #fff; + background-color: #080808 + } + .navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a, + .navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:focus, + .navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:hover { + color: #444; + background-color: transparent + } +} + +.navbar-inverse .navbar-link { + color: #9d9d9d +} + +.navbar-inverse .navbar-link:hover { + color: #fff +} + +.navbar-inverse .btn-link { + color: #9d9d9d +} + +.navbar-inverse .btn-link:focus, +.navbar-inverse .btn-link:hover { + color: #fff +} + +.navbar-inverse .btn-link[disabled]:focus, +.navbar-inverse .btn-link[disabled]:hover, +fieldset[disabled] .navbar-inverse .btn-link:focus, +fieldset[disabled] .navbar-inverse .btn-link:hover { + color: #444 +} + +.breadcrumb { + padding: 8px 15px; + margin-bottom: 20px; + list-style: none; + background-color: #f5f5f5; + border-radius: 4px +} + +.breadcrumb>li { + display: inline-block +} + +.breadcrumb>li+li:before { + padding: 0 5px; + color: #ccc; + content: "/\00a0" +} + +.breadcrumb>.active { + color: #777 +} + +.pagination { + display: inline-block; + padding-left: 0; + margin: 20px 0; + border-radius: 4px +} + +.pagination>li { + display: inline +} + +.pagination>li>a, +.pagination>li>span { + position: relative; + float: left; + padding: 6px 12px; + margin-left: -1px; + line-height: 1.42857143; + color: #337ab7; + text-decoration: none; + background-color: #fff; + border: 1px solid #ddd +} + +.pagination>li:first-child>a, +.pagination>li:first-child>span { + margin-left: 0; + border-top-left-radius: 4px; + border-bottom-left-radius: 4px +} + +.pagination>li:last-child>a, +.pagination>li:last-child>span { + border-top-right-radius: 4px; + border-bottom-right-radius: 4px +} + +.pagination>li>a:focus, +.pagination>li>a:hover, +.pagination>li>span:focus, +.pagination>li>span:hover { + color: #23527c; + background-color: #eee; + border-color: #ddd +} + +.pagination>.active>a, +.pagination>.active>a:focus, +.pagination>.active>a:hover, +.pagination>.active>span, +.pagination>.active>span:focus, +.pagination>.active>span:hover { + z-index: 2; + color: #fff; + cursor: default; + background-color: #337ab7; + border-color: #337ab7 +} + +.pagination>.disabled>a, +.pagination>.disabled>a:focus, +.pagination>.disabled>a:hover, +.pagination>.disabled>span, +.pagination>.disabled>span:focus, +.pagination>.disabled>span:hover { + color: #777; + cursor: not-allowed; + background-color: #fff; + border-color: #ddd +} + +.pagination-lg>li>a, +.pagination-lg>li>span { + padding: 10px 16px; + font-size: 18px +} + +.pagination-lg>li:first-child>a, +.pagination-lg>li:first-child>span { + border-top-left-radius: 6px; + border-bottom-left-radius: 6px +} + +.pagination-lg>li:last-child>a, +.pagination-lg>li:last-child>span { + border-top-right-radius: 6px; + border-bottom-right-radius: 6px +} + +.pagination-sm>li>a, +.pagination-sm>li>span { + padding: 5px 10px; + font-size: 12px +} + +.pagination-sm>li:first-child>a, +.pagination-sm>li:first-child>span { + border-top-left-radius: 3px; + border-bottom-left-radius: 3px +} + +.pagination-sm>li:last-child>a, +.pagination-sm>li:last-child>span { + border-top-right-radius: 3px; + border-bottom-right-radius: 3px +} + +.pager { + padding-left: 0; + margin: 20px 0; + text-align: center; + list-style: none +} + +.pager li { + display: inline +} + +.pager li>a, +.pager li>span { + display: inline-block; + padding: 5px 14px; + background-color: #fff; + border: 1px solid #ddd; + border-radius: 15px +} + +.pager li>a:focus, +.pager li>a:hover { + text-decoration: none; + background-color: #eee +} + +.pager .next>a, +.pager .next>span { + float: right +} + +.pager .previous>a, +.pager .previous>span { + float: left +} + +.pager .disabled>a, +.pager .disabled>a:focus, +.pager .disabled>a:hover, +.pager .disabled>span { + color: #777; + cursor: not-allowed; + background-color: #fff +} + +.label { + display: inline; + padding: .2em .6em .3em; + font-size: 75%; + font-weight: 700; + line-height: 1; + color: #fff; + text-align: center; + white-space: nowrap; + vertical-align: baseline; + border-radius: .25em +} + +a.label:focus, +a.label:hover { + color: #fff; + text-decoration: none; + cursor: pointer +} + +.label:empty { + display: none +} + +.btn .label { + position: relative; + top: -1px +} + +.label-default { + background-color: #777 +} + +.label-default[href]:focus, +.label-default[href]:hover { + background-color: #5e5e5e +} + +.label-primary { + background-color: #337ab7 +} + +.label-primary[href]:focus, +.label-primary[href]:hover { + background-color: #286090 +} + +.label-success { + background-color: #5cb85c +} + +.label-success[href]:focus, +.label-success[href]:hover { + background-color: #449d44 +} + +.label-info { + background-color: #5bc0de +} + +.label-info[href]:focus, +.label-info[href]:hover { + background-color: #31b0d5 +} + +.label-warning { + background-color: #f0ad4e +} + +.label-warning[href]:focus, +.label-warning[href]:hover { + background-color: #ec971f +} + +.label-danger { + background-color: #d9534f +} + +.label-danger[href]:focus, +.label-danger[href]:hover { + background-color: #c9302c +} + +.badge { + display: inline-block; + min-width: 10px; + padding: 3px 7px; + font-size: 12px; + font-weight: 700; + line-height: 1; + color: #fff; + text-align: center; + white-space: nowrap; + vertical-align: baseline; + background-color: #777; + border-radius: 10px +} + +.badge:empty { + display: none +} + +.btn .badge { + position: relative; + top: -1px +} + +.btn-xs .badge { + top: 0; + padding: 1px 5px +} + +a.badge:focus, +a.badge:hover { + color: #fff; + text-decoration: none; + cursor: pointer +} + +.list-group-item.active>.badge, +.nav-pills>.active>a>.badge { + color: #337ab7; + background-color: #fff +} + +.list-group-item>.badge { + float: right +} + +.list-group-item>.badge+.badge { + margin-right: 5px +} + +.nav-pills>li>a>.badge { + margin-left: 3px +} + +.jumbotron { + padding: 30px 15px; + margin-bottom: 30px; + color: inherit; + background-color: #eee +} + +.jumbotron .h1, +.jumbotron h1 { + color: inherit +} + +.jumbotron p { + margin-bottom: 15px; + font-size: 21px; + font-weight: 200 +} + +.jumbotron>hr { + border-top-color: #d5d5d5 +} + +.container .jumbotron, +.container-fluid .jumbotron { + border-radius: 6px +} + +.jumbotron .container { + max-width: 100% +} + +@media screen and (min-width:768px) { + .jumbotron { + padding: 48px 0 + } + .container .jumbotron, + .container-fluid .jumbotron { + padding-right: 60px; + padding-left: 60px + } + .jumbotron .h1, + .jumbotron h1 { + font-size: 63px + } +} + +.thumbnail { + display: block; + padding: 4px; + margin-bottom: 20px; + line-height: 1.42857143; + background-color: #fff; + border: 1px solid #ddd; + border-radius: 4px; + -webkit-transition: border .2s ease-in-out; + -o-transition: border .2s ease-in-out; + transition: border .2s ease-in-out +} + +.thumbnail a>img, +.thumbnail>img { + margin-right: auto; + margin-left: auto +} + +a.thumbnail.active, +a.thumbnail:focus, +a.thumbnail:hover { + border-color: #337ab7 +} + +.thumbnail .caption { + padding: 9px; + color: #333 +} + +.alert { + padding: 15px; + margin-bottom: 20px; + border: 1px solid transparent; + border-radius: 4px +} + +.alert h4 { + margin-top: 0; + color: inherit +} + +.alert .alert-link { + font-weight: 700 +} + +.alert>p, +.alert>ul { + margin-bottom: 0 +} + +.alert>p+p { + margin-top: 5px +} + +.alert-dismissable, +.alert-dismissible { + padding-right: 35px +} + +.alert-dismissable .close, +.alert-dismissible .close { + position: relative; + top: -2px; + right: -21px; + color: inherit +} + +.alert-success { + color: #3c763d; + background-color: #dff0d8; + border-color: #d6e9c6 +} + +.alert-success hr { + border-top-color: #c9e2b3 +} + +.alert-success .alert-link { + color: #2b542c +} + +.alert-info { + color: #31708f; + background-color: #d9edf7; + border-color: #bce8f1 +} + +.alert-info hr { + border-top-color: #a6e1ec +} + +.alert-info .alert-link { + color: #245269 +} + +.alert-warning { + color: #8a6d3b; + background-color: #fcf8e3; + border-color: #faebcc +} + +.alert-warning hr { + border-top-color: #f7e1b5 +} + +.alert-warning .alert-link { + color: #66512c +} + +.alert-danger { + color: #a94442; + background-color: #f2dede; + border-color: #ebccd1 +} + +.alert-danger hr { + border-top-color: #e4b9c0 +} + +.alert-danger .alert-link { + color: #843534 +} + +@-webkit-keyframes progress-bar-stripes { + from { + background-position: 40px 0 + } + to { + background-position: 0 0 + } +} + +@-o-keyframes progress-bar-stripes { + from { + background-position: 40px 0 + } + to { + background-position: 0 0 + } +} + +@keyframes progress-bar-stripes { + from { + background-position: 40px 0 + } + to { + background-position: 0 0 + } +} + +.progress { + height: 20px; + margin-bottom: 20px; + overflow: hidden; + background-color: #f5f5f5; + border-radius: 4px; +} + +.progress-bar { + float: left; + width: 0; + height: 100%; + font-size: 12px; + line-height: 20px; + color: #fff; + text-align: center; + background-color: #337ab7; + -webkit-transition: width .6s ease; + -o-transition: width .6s ease; + transition: width .6s ease +} + +.progress-bar-striped, +.progress-striped .progress-bar { + background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent); + background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent); + background-image: linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent); + -webkit-background-size: 40px 40px; + background-size: 40px 40px +} + +.progress-bar.active, +.progress.active .progress-bar { + -webkit-animation: progress-bar-stripes 2s linear infinite; + -o-animation: progress-bar-stripes 2s linear infinite; + animation: progress-bar-stripes 2s linear infinite +} + +.progress-bar-success { + background-color: #5cb85c +} + +.progress-striped .progress-bar-success { + background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent); + background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent); + background-image: linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent) +} + +.progress-bar-info { + background-color: #5bc0de +} + +.progress-striped .progress-bar-info { + background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent); + background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent); + background-image: linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent) +} + +.progress-bar-warning { + background-color: #f0ad4e +} + +.progress-striped .progress-bar-warning { + background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent); + background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent); + background-image: linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent) +} + +.progress-bar-danger { + background-color: #d9534f +} + +.progress-striped .progress-bar-danger { + background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent); + background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent); + background-image: linear-gradient(45deg, rgba(255, 255, 255, .15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, .15) 50%, rgba(255, 255, 255, .15) 75%, transparent 75%, transparent) +} + +.media { + margin-top: 15px +} + +.media:first-child { + margin-top: 0 +} + +.media, +.media-body { + overflow: hidden; + zoom: 1 +} + +.media-body { + width: 10000px +} + +.media-object { + display: block +} + +.media-right, +.media>.pull-right { + padding-left: 10px +} + +.media-left, +.media>.pull-left { + padding-right: 10px +} + +.media-body, +.media-left, +.media-right { + display: table-cell; + vertical-align: top +} + +.media-middle { + vertical-align: middle +} + +.media-bottom { + vertical-align: bottom +} + +.media-heading { + margin-top: 0; + margin-bottom: 5px +} + +.media-list { + padding-left: 0; + list-style: none +} + +.list-group { + padding-left: 0; + margin-bottom: 20px +} + +.list-group-item { + position: relative; + display: block; + padding: 10px 15px; + margin-bottom: -1px; + background-color: #fff; + border: 1px solid #ddd +} + +.list-group-item:first-child { + border-top-left-radius: 4px; + border-top-right-radius: 4px +} + +.list-group-item:last-child { + margin-bottom: 0; + border-bottom-right-radius: 4px; + border-bottom-left-radius: 4px +} + +a.list-group-item { + color: #555 +} + +a.list-group-item .list-group-item-heading { + color: #333 +} + +a.list-group-item:focus, +a.list-group-item:hover { + color: #555; + text-decoration: none; + background-color: #f5f5f5 +} + +.list-group-item.disabled, +.list-group-item.disabled:focus, +.list-group-item.disabled:hover { + color: #777; + cursor: not-allowed; + background-color: #eee +} + +.list-group-item.disabled .list-group-item-heading, +.list-group-item.disabled:focus .list-group-item-heading, +.list-group-item.disabled:hover .list-group-item-heading { + color: inherit +} + +.list-group-item.disabled .list-group-item-text, +.list-group-item.disabled:focus .list-group-item-text, +.list-group-item.disabled:hover .list-group-item-text { + color: #777 +} + +.list-group-item.active, +.list-group-item.active:focus, +.list-group-item.active:hover { + z-index: 2; + color: #fff; + background-color: #337ab7; + border-color: #337ab7 +} + +.list-group-item.active .list-group-item-heading, +.list-group-item.active .list-group-item-heading>.small, +.list-group-item.active .list-group-item-heading>small, +.list-group-item.active:focus .list-group-item-heading, +.list-group-item.active:focus .list-group-item-heading>.small, +.list-group-item.active:focus .list-group-item-heading>small, +.list-group-item.active:hover .list-group-item-heading, +.list-group-item.active:hover .list-group-item-heading>.small, +.list-group-item.active:hover .list-group-item-heading>small { + color: inherit +} + +.list-group-item.active .list-group-item-text, +.list-group-item.active:focus .list-group-item-text, +.list-group-item.active:hover .list-group-item-text { + color: #c7ddef +} + +.list-group-item-success { + color: #3c763d; + background-color: #dff0d8 +} + +a.list-group-item-success { + color: #3c763d +} + +a.list-group-item-success .list-group-item-heading { + color: inherit +} + +a.list-group-item-success:focus, +a.list-group-item-success:hover { + color: #3c763d; + background-color: #d0e9c6 +} + +a.list-group-item-success.active, +a.list-group-item-success.active:focus, +a.list-group-item-success.active:hover { + color: #fff; + background-color: #3c763d; + border-color: #3c763d +} + +.list-group-item-info { + color: #31708f; + background-color: #d9edf7 +} + +a.list-group-item-info { + color: #31708f +} + +a.list-group-item-info .list-group-item-heading { + color: inherit +} + +a.list-group-item-info:focus, +a.list-group-item-info:hover { + color: #31708f; + background-color: #c4e3f3 +} + +a.list-group-item-info.active, +a.list-group-item-info.active:focus, +a.list-group-item-info.active:hover { + color: #fff; + background-color: #31708f; + border-color: #31708f +} + +.list-group-item-warning { + color: #8a6d3b; + background-color: #fcf8e3 +} + +a.list-group-item-warning { + color: #8a6d3b +} + +a.list-group-item-warning .list-group-item-heading { + color: inherit +} + +a.list-group-item-warning:focus, +a.list-group-item-warning:hover { + color: #8a6d3b; + background-color: #faf2cc +} + +a.list-group-item-warning.active, +a.list-group-item-warning.active:focus, +a.list-group-item-warning.active:hover { + color: #fff; + background-color: #8a6d3b; + border-color: #8a6d3b +} + +.list-group-item-danger { + color: #a94442; + background-color: #f2dede +} + +a.list-group-item-danger { + color: #a94442 +} + +a.list-group-item-danger .list-group-item-heading { + color: inherit +} + +a.list-group-item-danger:focus, +a.list-group-item-danger:hover { + color: #a94442; + background-color: #ebcccc +} + +a.list-group-item-danger.active, +a.list-group-item-danger.active:focus, +a.list-group-item-danger.active:hover { + color: #fff; + background-color: #a94442; + border-color: #a94442 +} + +.list-group-item-heading { + margin-top: 0; + margin-bottom: 5px +} + +.list-group-item-text { + margin-bottom: 0; + line-height: 1.3 +} + +.panel { + margin-bottom: 20px; + background-color: #fff; + border: 1px solid transparent; + border-radius: 4px; +} + +.panel-body { + padding: 15px +} + +.panel-heading { + padding: 10px 15px; + border-bottom: 1px solid transparent; + border-top-left-radius: 3px; + border-top-right-radius: 3px +} + +.panel-heading>.dropdown .dropdown-toggle { + color: inherit +} + +.panel-title { + margin-top: 0; + margin-bottom: 0; + font-size: 16px; + color: inherit +} + +.panel-title>.small, +.panel-title>.small>a, +.panel-title>a, +.panel-title>small, +.panel-title>small>a { + color: inherit +} + +.panel-footer { + padding: 10px 15px; + background-color: #f5f5f5; + border-top: 1px solid #ddd; + border-bottom-right-radius: 3px; + border-bottom-left-radius: 3px +} + +.panel>.list-group, +.panel>.panel-collapse>.list-group { + margin-bottom: 0 +} + +.panel>.list-group .list-group-item, +.panel>.panel-collapse>.list-group .list-group-item { + border-width: 1px 0; + border-radius: 0 +} + +.panel>.list-group:first-child .list-group-item:first-child, +.panel>.panel-collapse>.list-group:first-child .list-group-item:first-child { + border-top: 0; + border-top-left-radius: 3px; + border-top-right-radius: 3px +} + +.panel>.list-group:last-child .list-group-item:last-child, +.panel>.panel-collapse>.list-group:last-child .list-group-item:last-child { + border-bottom: 0; + border-bottom-right-radius: 3px; + border-bottom-left-radius: 3px +} + +.panel-heading+.list-group .list-group-item:first-child { + border-top-width: 0 +} + +.list-group+.panel-footer { + border-top-width: 0 +} + +.panel>.panel-collapse>.table, +.panel>.table, +.panel>.table-responsive>.table { + margin-bottom: 0 +} + +.panel>.panel-collapse>.table caption, +.panel>.table caption, +.panel>.table-responsive>.table caption { + padding-right: 15px; + padding-left: 15px +} + +.panel>.table-responsive:first-child>.table:first-child, +.panel>.table:first-child { + border-top-left-radius: 3px; + border-top-right-radius: 3px +} + +.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child, +.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child, +.panel>.table:first-child>tbody:first-child>tr:first-child, +.panel>.table:first-child>thead:first-child>tr:first-child { + border-top-left-radius: 3px; + border-top-right-radius: 3px +} + +.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:first-child, +.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:first-child, +.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:first-child, +.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:first-child, +.panel>.table:first-child>tbody:first-child>tr:first-child td:first-child, +.panel>.table:first-child>tbody:first-child>tr:first-child th:first-child, +.panel>.table:first-child>thead:first-child>tr:first-child td:first-child, +.panel>.table:first-child>thead:first-child>tr:first-child th:first-child { + border-top-left-radius: 3px +} + +.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:last-child, +.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:last-child, +.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:last-child, +.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:last-child, +.panel>.table:first-child>tbody:first-child>tr:first-child td:last-child, +.panel>.table:first-child>tbody:first-child>tr:first-child th:last-child, +.panel>.table:first-child>thead:first-child>tr:first-child td:last-child, +.panel>.table:first-child>thead:first-child>tr:first-child th:last-child { + border-top-right-radius: 3px +} + +.panel>.table-responsive:last-child>.table:last-child, +.panel>.table:last-child { + border-bottom-right-radius: 3px; + border-bottom-left-radius: 3px +} + +.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child, +.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child, +.panel>.table:last-child>tbody:last-child>tr:last-child, +.panel>.table:last-child>tfoot:last-child>tr:last-child { + border-bottom-right-radius: 3px; + border-bottom-left-radius: 3px +} + +.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:first-child, +.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:first-child, +.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:first-child, +.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:first-child, +.panel>.table:last-child>tbody:last-child>tr:last-child td:first-child, +.panel>.table:last-child>tbody:last-child>tr:last-child th:first-child, +.panel>.table:last-child>tfoot:last-child>tr:last-child td:first-child, +.panel>.table:last-child>tfoot:last-child>tr:last-child th:first-child { + border-bottom-left-radius: 3px +} + +.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:last-child, +.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:last-child, +.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:last-child, +.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:last-child, +.panel>.table:last-child>tbody:last-child>tr:last-child td:last-child, +.panel>.table:last-child>tbody:last-child>tr:last-child th:last-child, +.panel>.table:last-child>tfoot:last-child>tr:last-child td:last-child, +.panel>.table:last-child>tfoot:last-child>tr:last-child th:last-child { + border-bottom-right-radius: 3px +} + +.panel>.panel-body+.table, +.panel>.panel-body+.table-responsive, +.panel>.table+.panel-body, +.panel>.table-responsive+.panel-body { + border-top: 1px solid #ddd +} + +.panel>.table>tbody:first-child>tr:first-child td, +.panel>.table>tbody:first-child>tr:first-child th { + border-top: 0 +} + +.panel>.table-bordered, +.panel>.table-responsive>.table-bordered { + border: 0 +} + +.panel>.table-bordered>tbody>tr>td:first-child, +.panel>.table-bordered>tbody>tr>th:first-child, +.panel>.table-bordered>tfoot>tr>td:first-child, +.panel>.table-bordered>tfoot>tr>th:first-child, +.panel>.table-bordered>thead>tr>td:first-child, +.panel>.table-bordered>thead>tr>th:first-child, +.panel>.table-responsive>.table-bordered>tbody>tr>td:first-child, +.panel>.table-responsive>.table-bordered>tbody>tr>th:first-child, +.panel>.table-responsive>.table-bordered>tfoot>tr>td:first-child, +.panel>.table-responsive>.table-bordered>tfoot>tr>th:first-child, +.panel>.table-responsive>.table-bordered>thead>tr>td:first-child, +.panel>.table-responsive>.table-bordered>thead>tr>th:first-child { + border-left: 0 +} + +.panel>.table-bordered>tbody>tr>td:last-child, +.panel>.table-bordered>tbody>tr>th:last-child, +.panel>.table-bordered>tfoot>tr>td:last-child, +.panel>.table-bordered>tfoot>tr>th:last-child, +.panel>.table-bordered>thead>tr>td:last-child, +.panel>.table-bordered>thead>tr>th:last-child, +.panel>.table-responsive>.table-bordered>tbody>tr>td:last-child, +.panel>.table-responsive>.table-bordered>tbody>tr>th:last-child, +.panel>.table-responsive>.table-bordered>tfoot>tr>td:last-child, +.panel>.table-responsive>.table-bordered>tfoot>tr>th:last-child, +.panel>.table-responsive>.table-bordered>thead>tr>td:last-child, +.panel>.table-responsive>.table-bordered>thead>tr>th:last-child { + border-right: 0 +} + +.panel>.table-bordered>tbody>tr:first-child>td, +.panel>.table-bordered>tbody>tr:first-child>th, +.panel>.table-bordered>thead>tr:first-child>td, +.panel>.table-bordered>thead>tr:first-child>th, +.panel>.table-responsive>.table-bordered>tbody>tr:first-child>td, +.panel>.table-responsive>.table-bordered>tbody>tr:first-child>th, +.panel>.table-responsive>.table-bordered>thead>tr:first-child>td, +.panel>.table-responsive>.table-bordered>thead>tr:first-child>th { + border-bottom: 0 +} + +.panel>.table-bordered>tbody>tr:last-child>td, +.panel>.table-bordered>tbody>tr:last-child>th, +.panel>.table-bordered>tfoot>tr:last-child>td, +.panel>.table-bordered>tfoot>tr:last-child>th, +.panel>.table-responsive>.table-bordered>tbody>tr:last-child>td, +.panel>.table-responsive>.table-bordered>tbody>tr:last-child>th, +.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>td, +.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>th { + border-bottom: 0 +} + +.panel>.table-responsive { + margin-bottom: 0; + border: 0 +} + +.panel-group { + margin-bottom: 20px +} + +.panel-group .panel { + margin-bottom: 0; + border-radius: 4px +} + +.panel-group .panel+.panel { + margin-top: 5px +} + +.panel-group .panel-heading { + border-bottom: 0 +} + +.panel-group .panel-heading+.panel-collapse>.list-group, +.panel-group .panel-heading+.panel-collapse>.panel-body { + border-top: 1px solid #ddd +} + +.panel-group .panel-footer { + border-top: 0 +} + +.panel-group .panel-footer+.panel-collapse .panel-body { + border-bottom: 1px solid #ddd +} + +.panel-default { + border-color: #ddd +} + +.panel-default>.panel-heading { + color: #333; + background-color: #f5f5f5; + border-color: #ddd +} + +.panel-default>.panel-heading+.panel-collapse>.panel-body { + border-top-color: #ddd +} + +.panel-default>.panel-heading .badge { + color: #f5f5f5; + background-color: #333 +} + +.panel-default>.panel-footer+.panel-collapse>.panel-body { + border-bottom-color: #ddd +} + +.panel-primary { + border-color: #337ab7 +} + +.panel-primary>.panel-heading { + color: #fff; + background-color: #337ab7; + border-color: #337ab7 +} + +.panel-primary>.panel-heading+.panel-collapse>.panel-body { + border-top-color: #337ab7 +} + +.panel-primary>.panel-heading .badge { + color: #337ab7; + background-color: #fff +} + +.panel-primary>.panel-footer+.panel-collapse>.panel-body { + border-bottom-color: #337ab7 +} + +.panel-success { + border-color: #d6e9c6 +} + +.panel-success>.panel-heading { + color: #3c763d; + background-color: #dff0d8; + border-color: #d6e9c6 +} + +.panel-success>.panel-heading+.panel-collapse>.panel-body { + border-top-color: #d6e9c6 +} + +.panel-success>.panel-heading .badge { + color: #dff0d8; + background-color: #3c763d +} + +.panel-success>.panel-footer+.panel-collapse>.panel-body { + border-bottom-color: #d6e9c6 +} + +.panel-info { + border-color: #bce8f1 +} + +.panel-info>.panel-heading { + color: #31708f; + background-color: #d9edf7; + border-color: #bce8f1 +} + +.panel-info>.panel-heading+.panel-collapse>.panel-body { + border-top-color: #bce8f1 +} + +.panel-info>.panel-heading .badge { + color: #d9edf7; + background-color: #31708f +} + +.panel-info>.panel-footer+.panel-collapse>.panel-body { + border-bottom-color: #bce8f1 +} + +.panel-warning { + border-color: #faebcc +} + +.panel-warning>.panel-heading { + color: #8a6d3b; + background-color: #fcf8e3; + border-color: #faebcc +} + +.panel-warning>.panel-heading+.panel-collapse>.panel-body { + border-top-color: #faebcc +} + +.panel-warning>.panel-heading .badge { + color: #fcf8e3; + background-color: #8a6d3b +} + +.panel-warning>.panel-footer+.panel-collapse>.panel-body { + border-bottom-color: #faebcc +} + +.panel-danger { + border-color: #ebccd1 +} + +.panel-danger>.panel-heading { + color: #a94442; + background-color: #f2dede; + border-color: #ebccd1 +} + +.panel-danger>.panel-heading+.panel-collapse>.panel-body { + border-top-color: #ebccd1 +} + +.panel-danger>.panel-heading .badge { + color: #f2dede; + background-color: #a94442 +} + +.panel-danger>.panel-footer+.panel-collapse>.panel-body { + border-bottom-color: #ebccd1 +} + +.embed-responsive { + position: relative; + display: block; + height: 0; + padding: 0; + overflow: hidden +} + +.embed-responsive .embed-responsive-item, +.embed-responsive embed, +.embed-responsive iframe, +.embed-responsive object, +.embed-responsive video { + position: absolute; + top: 0; + bottom: 0; + left: 0; + width: 100%; + height: 100%; + border: 0 +} + +.embed-responsive.embed-responsive-16by9 { + padding-bottom: 56.25% +} + +.embed-responsive.embed-responsive-4by3 { + padding-bottom: 75% +} + +.well { + min-height: 20px; + padding: 19px; + margin-bottom: 20px; + background-color: #f5f5f5; + border: 1px solid #e3e3e3; + border-radius: 4px; +} + +.well blockquote { + border-color: #ddd; + border-color: rgba(0, 0, 0, .15) +} + +.well-lg { + padding: 24px; + border-radius: 6px +} + +.well-sm { + padding: 9px; + border-radius: 3px +} + +.close { + float: right; + font-size: 21px; + font-weight: 700; + line-height: 1; + color: #000; + filter: alpha(opacity=20); + opacity: .2 +} + +.close:focus, +.close:hover { + color: #000; + text-decoration: none; + cursor: pointer; + filter: alpha(opacity=50); + opacity: .5 +} + +button.close { + -webkit-appearance: none; + padding: 0; + cursor: pointer; + background: 0 0; + border: 0 +} + +.modal-open { + overflow: hidden +} + +.modal { + position: fixed; + top: 0; + right: 0; + bottom: 0; + left: 0; + z-index: 1040; + display: none; + overflow: hidden; + -webkit-overflow-scrolling: touch; + outline: 0 +} + +.modal.fade .modal-dialog { + -webkit-transition: -webkit-transform .3s ease-out; + -o-transition: -o-transform .3s ease-out; + transition: transform .3s ease-out; + -webkit-transform: translate(0, -25%); + -ms-transform: translate(0, -25%); + -o-transform: translate(0, -25%); + transform: translate(0, -25%) +} + +.modal.in .modal-dialog { + -webkit-transform: translate(0, 0); + -ms-transform: translate(0, 0); + -o-transform: translate(0, 0); + transform: translate(0, 0) +} + +.modal-open .modal { + overflow-x: hidden; + overflow-y: auto +} + +.modal-dialog { + position: relative; + width: auto; + margin: 10px +} + +.modal-content { + position: relative; + background-color: #fff; + -webkit-background-clip: padding-box; + background-clip: padding-box; + border: 1px solid #999; + border: 1px solid rgba(0, 0, 0, .2); + border-radius: 6px; + outline: 0; +} + +.modal-backdrop { + position: absolute; + top: 0; + right: 0; + left: 0; + background-color: #000 +} + +.modal-backdrop.fade { + filter: alpha(opacity=0); + opacity: 0 +} + +.modal-backdrop.in { + filter: alpha(opacity=50); + opacity: .5 +} + +.modal-header { + min-height: 16.43px; + padding: 15px; + border-bottom: 1px solid #e5e5e5 +} + +.modal-header .close { + margin-top: -2px +} + +.modal-title { + margin: 0; + line-height: 1.42857143 +} + +.modal-body { + position: relative; + padding: 15px +} + +.modal-footer { + padding: 15px; + text-align: right; + border-top: 1px solid #e5e5e5 +} + +.modal-footer .btn+.btn { + margin-bottom: 0; + margin-left: 5px +} + +.modal-footer .btn-group .btn+.btn { + margin-left: -1px +} + +.modal-footer .btn-block+.btn-block { + margin-left: 0 +} + +.modal-scrollbar-measure { + position: absolute; + top: -9999px; + width: 50px; + height: 50px; + overflow: scroll +} + +@media (min-width:768px) { + .modal-dialog { + width: 600px; + margin: 30px auto + } + .modal-content { + } + .modal-sm { + width: 300px + } +} + +@media (min-width:992px) { + .modal-lg { + width: 900px + } +} + +.tooltip { + position: absolute; + z-index: 1070; + display: block; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 12px; + font-weight: 400; + line-height: 1.4; + visibility: visible; + filter: alpha(opacity=0); + opacity: 0 +} + +.tooltip.in { + filter: alpha(opacity=90); + opacity: .9 +} + +.tooltip.top { + padding: 5px 0; + margin-top: -3px +} + +.tooltip.right { + padding: 0 5px; + margin-left: 3px +} + +.tooltip.bottom { + padding: 5px 0; + margin-top: 3px +} + +.tooltip.left { + padding: 0 5px; + margin-left: -3px +} + +.tooltip-inner { + max-width: 200px; + padding: 3px 8px; + color: #fff; + text-align: center; + text-decoration: none; + background-color: #000; + border-radius: 4px +} + +.tooltip-arrow { + position: absolute; + width: 0; + height: 0; + border-color: transparent; + border-style: solid +} + +.tooltip.top .tooltip-arrow { + bottom: 0; + left: 50%; + margin-left: -5px; + border-width: 5px 5px 0; + border-top-color: #000 +} + +.tooltip.top-left .tooltip-arrow { + right: 5px; + bottom: 0; + margin-bottom: -5px; + border-width: 5px 5px 0; + border-top-color: #000 +} + +.tooltip.top-right .tooltip-arrow { + bottom: 0; + left: 5px; + margin-bottom: -5px; + border-width: 5px 5px 0; + border-top-color: #000 +} + +.tooltip.right .tooltip-arrow { + top: 50%; + left: 0; + margin-top: -5px; + border-width: 5px 5px 5px 0; + border-right-color: #000 +} + +.tooltip.left .tooltip-arrow { + top: 50%; + right: 0; + margin-top: -5px; + border-width: 5px 0 5px 5px; + border-left-color: #000 +} + +.tooltip.bottom .tooltip-arrow { + top: 0; + left: 50%; + margin-left: -5px; + border-width: 0 5px 5px; + border-bottom-color: #000 +} + +.tooltip.bottom-left .tooltip-arrow { + top: 0; + right: 5px; + margin-top: -5px; + border-width: 0 5px 5px; + border-bottom-color: #000 +} + +.tooltip.bottom-right .tooltip-arrow { + top: 0; + left: 5px; + margin-top: -5px; + border-width: 0 5px 5px; + border-bottom-color: #000 +} + +.popover { + position: absolute; + top: 0; + left: 0; + z-index: 1060; + display: none; + max-width: 276px; + padding: 1px; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 400; + line-height: 1.42857143; + text-align: left; + white-space: normal; + background-color: #fff; + -webkit-background-clip: padding-box; + background-clip: padding-box; + border: 1px solid #ccc; + border: 1px solid rgba(0, 0, 0, .2); + border-radius: 6px; +} + +.popover.top { + margin-top: -10px +} + +.popover.right { + margin-left: 10px +} + +.popover.bottom { + margin-top: 10px +} + +.popover.left { + margin-left: -10px +} + +.popover-title { + padding: 8px 14px; + margin: 0; + font-size: 14px; + background-color: #f7f7f7; + border-bottom: 1px solid #ebebeb; + border-radius: 5px 5px 0 0 +} + +.popover-content { + padding: 9px 14px +} + +.popover>.arrow, +.popover>.arrow:after { + position: absolute; + display: block; + width: 0; + height: 0; + border-color: transparent; + border-style: solid +} + +.popover>.arrow { + border-width: 11px +} + +.popover>.arrow:after { + content: ""; + border-width: 10px +} + +.popover.top>.arrow { + bottom: -11px; + left: 50%; + margin-left: -11px; + border-top-color: #999; + border-top-color: rgba(0, 0, 0, .25); + border-bottom-width: 0 +} + +.popover.top>.arrow:after { + bottom: 1px; + margin-left: -10px; + content: " "; + border-top-color: #fff; + border-bottom-width: 0 +} + +.popover.right>.arrow { + top: 50%; + left: -11px; + margin-top: -11px; + border-right-color: #999; + border-right-color: rgba(0, 0, 0, .25); + border-left-width: 0 +} + +.popover.right>.arrow:after { + bottom: -10px; + left: 1px; + content: " "; + border-right-color: #fff; + border-left-width: 0 +} + +.popover.bottom>.arrow { + top: -11px; + left: 50%; + margin-left: -11px; + border-top-width: 0; + border-bottom-color: #999; + border-bottom-color: rgba(0, 0, 0, .25) +} + +.popover.bottom>.arrow:after { + top: 1px; + margin-left: -10px; + content: " "; + border-top-width: 0; + border-bottom-color: #fff +} + +.popover.left>.arrow { + top: 50%; + right: -11px; + margin-top: -11px; + border-right-width: 0; + border-left-color: #999; + border-left-color: rgba(0, 0, 0, .25) +} + +.popover.left>.arrow:after { + right: 1px; + bottom: -10px; + content: " "; + border-right-width: 0; + border-left-color: #fff +} + + +@media screen and (min-width:768px) { + .carousel-control .glyphicon-chevron-left, + .carousel-control .glyphicon-chevron-right, + .carousel-control .icon-next, + .carousel-control .icon-prev { + width: 30px; + height: 30px; + margin-top: -15px; + font-size: 30px + } + .carousel-control .glyphicon-chevron-left, + .carousel-control .icon-prev { + margin-left: -15px + } + .carousel-control .glyphicon-chevron-right, + .carousel-control .icon-next { + margin-right: -15px + } + .carousel-caption { + right: 20%; + left: 20%; + padding-bottom: 30px + } + .carousel-indicators { + bottom: 20px + } +} + +.btn-group-vertical>.btn-group:after, +.btn-group-vertical>.btn-group:before, +.btn-toolbar:after, +.btn-toolbar:before, +.clearfix:after, +.clearfix:before, +.container-fluid:after, +.container-fluid:before, +.container:after, +.container:before, +.dl-horizontal dd:after, +.dl-horizontal dd:before, +.form-horizontal .form-group:after, +.form-horizontal .form-group:before, +.modal-footer:after, +.modal-footer:before, +.nav:after, +.nav:before, +.navbar-collapse:after, +.navbar-collapse:before, +.navbar-header:after, +.navbar-header:before, +.navbar:after, +.navbar:before, +.pager:after, +.pager:before, +.panel-body:after, +.panel-body:before, +.row:after, +.row:before { + display: table; + content: " " +} + +.btn-group-vertical>.btn-group:after, +.btn-toolbar:after, +.clearfix:after, +.container-fluid:after, +.container:after, +.dl-horizontal dd:after, +.form-horizontal .form-group:after, +.modal-footer:after, +.nav:after, +.navbar-collapse:after, +.navbar-header:after, +.navbar:after, +.pager:after, +.panel-body:after, +.row:after { + clear: both +} + +.center-block { + display: block; + margin-right: auto; + margin-left: auto +} + +.pull-right { + float: right!important +} + +.pull-left { + float: left!important +} + +.hide { + display: none!important +} + +.show { + display: block!important +} + +.invisible { + visibility: hidden +} + +.text-hide { + font: 0/0 a; + color: transparent; + background-color: transparent; + border: 0 +} + +.hidden { + display: none!important; + visibility: hidden!important +} + +.affix { + position: fixed +} + +@-ms-viewport { + width: device-width +} + +.visible-lg, +.visible-md, +.visible-sm, +.visible-xs { + display: none!important +} + +.visible-lg-block, +.visible-lg-inline, +.visible-lg-inline-block, +.visible-md-block, +.visible-md-inline, +.visible-md-inline-block, +.visible-sm-block, +.visible-sm-inline, +.visible-sm-inline-block, +.visible-xs-block, +.visible-xs-inline, +.visible-xs-inline-block { + display: none!important +} + +@media (max-width:767px) { + .visible-xs { + display: block!important + } + table.visible-xs { + display: table + } + tr.visible-xs { + display: table-row!important + } + td.visible-xs, + th.visible-xs { + display: table-cell!important + } +} + +@media (max-width:767px) { + .visible-xs-block { + display: block!important + } +} + +@media (max-width:767px) { + .visible-xs-inline { + display: inline!important + } +} + +@media (max-width:767px) { + .visible-xs-inline-block { + display: inline-block!important + } +} + +@media (min-width:768px) and (max-width:991px) { + .visible-sm { + display: block!important + } + table.visible-sm { + display: table + } + tr.visible-sm { + display: table-row!important + } + td.visible-sm, + th.visible-sm { + display: table-cell!important + } +} + +@media (min-width:768px) and (max-width:991px) { + .visible-sm-block { + display: block!important + } +} + +@media (min-width:768px) and (max-width:991px) { + .visible-sm-inline { + display: inline!important + } +} + +@media (min-width:768px) and (max-width:991px) { + .visible-sm-inline-block { + display: inline-block!important + } +} + +@media (min-width:992px) and (max-width:1199px) { + .visible-md { + display: block!important + } + table.visible-md { + display: table + } + tr.visible-md { + display: table-row!important + } + td.visible-md, + th.visible-md { + display: table-cell!important + } +} + +@media (min-width:992px) and (max-width:1199px) { + .visible-md-block { + display: block!important + } +} + +@media (min-width:992px) and (max-width:1199px) { + .visible-md-inline { + display: inline!important + } +} + +@media (min-width:992px) and (max-width:1199px) { + .visible-md-inline-block { + display: inline-block!important + } +} + +@media (min-width:1200px) { + .visible-lg { + display: block!important + } + table.visible-lg { + display: table + } + tr.visible-lg { + display: table-row!important + } + td.visible-lg, + th.visible-lg { + display: table-cell!important + } +} + +@media (min-width:1200px) { + .visible-lg-block { + display: block!important + } +} + +@media (min-width:1200px) { + .visible-lg-inline { + display: inline!important + } +} + +@media (min-width:1200px) { + .visible-lg-inline-block { + display: inline-block!important + } +} + +@media (max-width:767px) { + .hidden-xs { + display: none!important + } +} + +@media (min-width:768px) and (max-width:991px) { + .hidden-sm { + display: none!important + } +} + +@media (min-width:992px) and (max-width:1199px) { + .hidden-md { + display: none!important + } +} + +@media (min-width:1200px) { + .hidden-lg { + display: none!important + } +} + +.visible-print { + display: none!important +} + +@media print { + .visible-print { + display: block!important + } + table.visible-print { + display: table + } + tr.visible-print { + display: table-row!important + } + td.visible-print, + th.visible-print { + display: table-cell!important + } +} + +.visible-print-block { + display: none!important +} + +@media print { + .visible-print-block { + display: block!important + } +} + +.visible-print-inline { + display: none!important +} + +@media print { + .visible-print-inline { + display: inline!important + } +} + +.visible-print-inline-block { + display: none!important +} + +@media print { + .visible-print-inline-block { + display: inline-block!important + } +} + +@media print { + .hidden-print { + display: none!important + } +} diff --git a/python/vespa/docs/css/boxshadowproperties.css b/python/vespa/docs/css/boxshadowproperties.css new file mode 100644 index 00000000000..0f2e1e6db35 --- /dev/null +++ b/python/vespa/docs/css/boxshadowproperties.css @@ -0,0 +1,24 @@ +/* box-shadow fonts return errors with prince, so extracting here to put in web output only */ + +#search-demo-container ul#results-container { + box-shadow: 2px 3px 2px #dedede; +} + + +hr.shaded { + box-shadow: inset 0 6px 6px -6px rgba(0,0,0,0.5); +} + +.videoThumbs img { + box-shadow: 2px 2px 1px #f0f0f0; +} + +.box { + box-shadow: 2px 2px 4px #dedede; +} + +@media (max-width: 1200px) { + .navbar-collapse { + box-shadow: inset 0 1px 0 rgba(255,255,255,0.1); + } +} diff --git a/python/vespa/docs/css/customstyles.css b/python/vespa/docs/css/customstyles.css new file mode 100644 index 00000000000..e3f1532071c --- /dev/null +++ b/python/vespa/docs/css/customstyles.css @@ -0,0 +1,1306 @@ +.anchor-link { + display: none; +} + +body { + font-size:15px; +} + +.bs-callout { + padding: 20px; + margin: 20px 0; + border: 1px solid #eee; + border-left-width: 5px; + border-radius: 3px; +} +.bs-callout h4 { + margin-top: 0; + margin-bottom: 5px; +} +.bs-callout p:last-child { + margin-bottom: 0; +} +.bs-callout code { + border-radius: 3px; +} +.bs-callout+.bs-callout { + margin-top: -5px; +} +.bs-callout-default { + border-left-color: #777; +} +.bs-callout-default h4 { + color: #777; +} +.bs-callout-primary { + border-left-color: #428bca; +} +.bs-callout-primary h4 { + color: #428bca; +} +.bs-callout-success { + border-left-color: #5cb85c; +} +.bs-callout-success h4 { + color: #5cb85c; +} +.bs-callout-danger { + border-left-color: #d9534f; +} +.bs-callout-danger h4 { + color: #d9534f; +} +.bs-callout-warning { + border-left-color: #f0ad4e; +} +.bs-callout-warning h4 { + color: #f0ad4e; +} +.bs-callout-info { + border-left-color: #5bc0de; +} +.bs-callout-info h4 { + color: #5bc0de; +} + + +.gi-2x{font-size: 2em;} +.gi-3x{font-size: 3em;} +.gi-4x{font-size: 4em;} +.gi-5x{font-size: 5em;} + + +.breadcrumb > .active {color: #777 !important;} + +/* make room for the nav bar */ +h1[id] +/*,h2[id], +h3[id], +h4[id], +h5[id], +h6[id], +dt[id]*/ +{ +padding-top: 60px; +margin-top: -40px +} + +.output_html a{ + font-family: Menlo, Monaco, Consolas, "Courier New", monospace; + margin: 25px 0px; + display: block; + padding: 9.5px; + font-size: 13px; + line-height: 1.42857143; + color: #333; + word-break: break-all; + word-wrap: break-word; + background-color: #F5F5F5; + border: 1px solid #FFA500; + border-radius: 4px; + white-space: pre-wrap; + box-sizing: border-box; + overflow: auto; +} + +.col-md-1, .col-md-10, .col-md-11, .col-md-12, .col-md-2, .col-md-3, .col-md-4, .col-md-5, .col-md-6, .col-md-7, .col-md-8, .col-md-9 { + float: left; +} + +.col-md-9 { + width: 75%; +} + +/* From: https://www.w3schools.com/howto/tryit.asp?filename=tryhow_css_download_button' */ +.cstm_btn a{ + background-color: DodgerBlue; + border: none; + color: white; + padding: 12px 30px; + cursor: pointer; + font-size: 20px; + box-sizing: 15px; + position: absolute; +} + +/* Darker background on mouse-over */ +.cstm_btn a:hover { + background-color: RoyalBlue; +} + +.container #notebook-container{ + width: 100%; + box-sizing: border-box; + } + +.post-content img { + margin: 12px 0px 3px 0px; + width: auto; + height: auto; + max-width: 100%; + max-height: 100%; +} + +.post-content ol li, .post-content ul li { + margin: 10px 0px; +} + +.pageSummary { + font-size:13px; + display:block; + margin-bottom:15px; + padding-left:20px; +} + +.post-summary { + margin-bottom:12px; +} + +.bs-example{ + margin: 20px; +} + +.breadcrumb li { + color: gray; +} + +table { + background-color: transparent; +} +caption { + padding-top: 8px; + padding-bottom: 8px; + color: #777; + text-align: left; +} +th { + text-align: left; +} +table { + max-width: 90%; + margin-bottom: 20px; + border: 1px solid #dedede; +} + +table > thead > tr > th, +table > tbody > tr > th, +table > tfoot > tr > th, +table > thead > tr > td, +table > tbody > tr > td, +table > tfoot > tr > td { + padding: 8px; + line-height: 1.42857143; + vertical-align: top; + border-top: 1px solid #ddd; +} +table > thead > tr > th { + vertical-align: bottom; + border-bottom: 2px solid #ddd; + text-transform: none; + background-color: #777; + color: white; + text-align: left; +} +table > caption + thead > tr:first-child > th, +table > colgroup + thead > tr:first-child > th, +table > thead:first-child > tr:first-child > th, +table > caption + thead > tr:first-child > td, +table > colgroup + thead > tr:first-child > td, +table > thead:first-child > tr:first-child > td { + border-top: 0; +} + +table > tbody > tr:nth-of-type(odd) { + background-color: #f9f9f9; +} + +table col[class*="col-"] { + position: static; + display: table-column; + float: none; +} +table td[class*="col-"], +table th[class*="col-"] { + position: static; + display: table-cell; + float: none; +} + +table tr td { + hyphens: auto; +} + + +p.external a { + text-align:right; + font-size:12px; + color: #0088cc; + display:inline; +} + +#definition-box-container div a.active { + font-weight: bold; +} +p.post-meta {font-size: 80%; color: #777;} + +.entry-date{font-size:14px;font-size:0.875rem;line-height:1.71429;margin-bottom:0;text-transform:uppercase;} + +/* search area */ +#search-demo-container ul#results-container { + list-style: none; + font-size: 12px; + background-color: white; + position: absolute; + top: 40px; /* if you change anything about the nav, you'll prob. need to reset the top and left values here.*/ + left: 20px; + z-index: -1; + width:223px; + border-left: 1px solid #dedede; +} + + +ul#results-container a { + background-color: transparent; +} + +ul#results-container a:hover { + color: black; +} + + +#search-demo-container a:hover { + color: black; +} +#search-input { + padding: .5em; + margin-left:20px; + width:20em; + font-size: 0.8em; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + margin-top:10px; +} +/* end search */ + +.filter-options { + margin-bottom: 20px; +} +.filter-options button { + margin: 3px; +} + +a.source_link { + float:right; + font-size:15px; + font-weight:normal; +} + +div#toc ul { + font-size: 90%; + background-color: whitesmoke; + padding: 5px; + border-radius: 5px; + max-width: 450px; + color: gray; + list-style-type: none; +} + +/* when using relative fonts like 0.9em or 90%, remember to then set the nested items to 1em or 100%, otherwise each level will get smaller and smaller and less readable */ +div#toc ul li { + margin: 8px 0px 8px 22px; + font-size: 100%; + list-style-type: none; +} + +div#toc ul li ul { + font-size: 100%; + padding-left:30px; + padding-top: 0px; + padding-bottom: 0px; + list-style-type: none; +} + +div#toc ul li ul li.hide_content::before { + content: none; +} + +div#toc >ul::before { + content: "Table of Contents"; + font-weight: 500; + color: #555; + text-align:center; + margin-left:auto; + margin-right:auto; + width:70px; + padding-top:150px; + padding-bottom:40px; + padding-left:10px; +} + +li.dropdownActive a { + font-weight: bold; +} + + +.post-content a.fa-rss { + color: orange; +} + + +.navbar-inverse .navbar-nav > li > a { + background-color: transparent; + margin-top:10px; +} + +.post-content .rssfeedLink { + color: #248EC2; +} + +footer { + font-size: smaller; +} + +/* FAQ page */ +#accordion .panel-heading { + font-size: 12px; +} + +a.accordion-toggle, a.accordion-collapsed { + font-size: 14px; + text-decoration: none; +} + +/* navgoco sidebar styles (customized) */ +.nav, .nav ul, .nav li { + list-style: none; +} + +.nav ul { + padding: 0; + /*margin: 0 0 0 18px;*/ + margin:0px; +} + +.nav { + /* padding: 4px;*/ + padding:0px; + margin: 0px; +} + +.nav > li { + margin: 1px 0; +} + +.nav > li li { + margin: 2px 0; +} + +.nav a { + color: #333; + display: block; + outline: none; + /*-webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px;*/ + text-decoration: none; +} + +.nav li > a > span { + float: right; + font-size: 19px; + font-weight: bolder; +} + + +.nav li > a > span:after { + content: '\25be'; +} +.nav li.active > a > span:after { + content: '\25b4'; +} + +.nav a:hover, .nav li.active > a { + background-color: #8D8D8D; + color: #f5f5f5; +} + +.nav > li.active > a { +background-color: #347DBE; +} + +.nav li a { + font-size: 12px; + line-height: 18px; + padding: 2px 10px; + background-color: #f1f1f1; +} + +.nav > li > a { + font-size: 14px; + line-height: 20px; + padding: 4px 10px; +} + +ul#mysidebar { + border-radius:0px; +} + +.nav ul li ul li a { + padding-left:40px; +} + +.nav li.thirdlevel > a { + color: #248EC2; + font-weight:bold; + padding-left:20px; + background-color: whitesmoke !important; +} + + +.nav ul li a { + background-color: #FAFAFA; +} + +.nav li a { + padding-right:10px; +} + +.nav li a:hover { + background-color: #8D8D8D; +} + +.nav ul li a { + border-top:1px solid whitesmoke; + padding-left:10px; +} +/* end sidebar */ + +.navbar-inverse .navbar-nav > .active > a, .navbar-inverse .navbar-nav > .active > a:hover, .navbar-inverse .navbar-nav > .active > a:focus { + border-radius:5px; +} + +.navbar-inverse .navbar-nav>.open>a, .navbar-inverse .navbar-nav>.open>a:focus, .navbar-inverse .navbar-nav>.open>a:hover { + border-radius: 5px; +} + +span.projectTitle { + font-family: Helvetica; + font-weight: bold; +} + +.footer { + text-align: right; +} + +.footerMeta { + background-color: whitesmoke; + padding: 10px; + max-width: 250px; + border-radius: 5px; + margin-top: 50px; + font-style:italic; + font-size:12px; +} + +img.screenshotSmall { + max-width: 300px; +} + + +dl dt p { + margin-left:20px; +} + + +dl dd { + margin-top:10px; + margin-bottom:10px; +} + +dl.dl-horizontal dd { + padding-top: 20px; +} + +figcaption { + + padding-bottom:12px; + padding-top:6px; + max-width: 90%; + margin-bottom:20px; + font-style: italic; + color: gray; + +} + +.testing { + color: orange; +} + +.preference { + color: red; +} + + +table.dataTable thead { + background-color: #444; +} +table td { + hyphens: auto; +} + +section table tr.success { + background-color: #dff0d8 !important; +} + +table tr.info { + background-color: #d9edf7 !important; +} + +section table tr.warning, table tr.testing, table tr.testing > td.sorting_1 { + background-color: #fcf8e3 !important; +} +section table tr.danger, table tr.preference, table tr.preference > td.sorting_1 { + background-color: #f2dede !important; +} + +.orange { + color: orange; +} + +table.profile thead tr th { + background-color: #248ec2; +} + +table.request thead tr th { + background-color: #ED1951; +} + +.audienceLabel { + margin: 10px; + float: right; + border:1px solid #dedede; + padding:7px; +} + +.prefaceAudienceLabel { + color: gray; + text-align: center; + margin:5px; +} +span.myLabel { + padding-left:10px; + padding-right:10px; +} + +button.cursorNorm { + cursor: default; +} + +a.dropdown-toggle, .navbar-inverse .navbar-nav > li > a { + margin-left: 10px; +} + +hr.faded { + border: 0; + height: 1px; + background-image: -webkit-linear-gradient(left, rgba(0,0,0,0), rgba(0,0,0,0.75), rgba(0,0,0,0)); + background-image: -moz-linear-gradient(left, rgba(0,0,0,0), rgba(0,0,0,0.75), rgba(0,0,0,0)); + background-image: -ms-linear-gradient(left, rgba(0,0,0,0), rgba(0,0,0,0.75), rgba(0,0,0,0)); + background-image: -o-linear-gradient(left, rgba(0,0,0,0), rgba(0,0,0,0.75), rgba(0,0,0,0)); +} + +hr.shaded { + height: 12px; + border: 0; + margin-top: 70px; + background: white; + width: 100%; + margin-bottom: 10px; +} + +.fa-6x{font-size:900%;} +.fa-7x{font-size:1100%;} +.fa-8x{font-size:1300%;} +.fa-9x{font-size:1500%;} +.fa-10x{font-size:1700%;} + +i.border { + padding: 10px 20px; + background-color: whitesmoke; +} + +a[data-toggle] { + color: #248EC2; +} + +.summary { + font-size:120%; + color: #808080; + margin:20px 0px 20px 0px; + border-left: 5px solid #ED1951; + padding-left: 10px; + +} + +a.fa.fa-envelope-o.mailto { + font-weight: 600; +} + +h3 {color: #ED1951; font-weight:normal; font-size:130%;} +h4 {color: #000000; font-weight:normal; font-size:120%; font-weight:bold;} + +.alert, .callout { + overflow: hidden; +} + +.nav-tabs > li.active > a, .nav-tabs > li.active > a:hover, .nav-tabs > li.active > a:focus { + background-color: #248ec2; + color: white; +} + +ol li ol li {list-style-type: lower-alpha;} +ol li ul li {list-style-type: disc;} + +li img {clear:both; } + +div#toc ul li ul li { + list-style-type: none; + margin: 5px 0px 0px 0px; +} + +.tab-content { + padding: 15px; + background-color: #FAFAFA; +} + +span.tagTitle {font-weight: 500;} + +li.activeSeries { + font-weight: bold; +} + +.seriesContext .dropdown-menu li.active { + font-weight: bold; + margin-left: 43px; + font-size:18px; +} + +.alert-warning { + color: #444; +} + +div.alert code, h2 code { + background-color: transparent !important; +} +/* without this, the links in these notes aren't visible.*/ +.alert a { + text-decoration: underline; +} + +div.tags {padding: 10px 5px;} + +.tabLabel { + font-weight: normal; +} + +hr { + background: #999; + margin: 30px 0px; + width: 90%; + margin-left: auto; + margin-right: auto; +} + +button.cursorNorm { + cursor: pointer; +} + +h2 { + font-size:24px; + line-height:29px; + border-top: 3px solid blue; + padding-top: 5px; +} + +span.otherProgrammingLanguages { + font-style: normal; +} + +a[data-toggle="tooltip"] { + color: #649345; + font-style: italic; + cursor: default; +} + +.seriesNext, .seriesContext { + margin-top: 15px; + margin-bottom: 15px; +} + +.seriescontext ol li { + list-style-type: upper-roman; +} + +ol.series li { + list-style-type: decimal; + margin-left: 40px; + padding-left: 0px; +} + +.siteTagline { + font-size: 200%; + font-weight: bold; + color: silver; + font-family: monospace; + text-align: center; + line-height: 10px; + margin: 20px 0px; + display: block; +} + +.versionTagline { + text-align: center; + margin-bottom: 20px; + font-family: courier; + color: silver; + color: #444; + display:block; +} + +/* not sure if using this ...*/ +.navbar-inverse .navbar-collapse, .navbar-inverse .navbar-form { + border-color: #248ec2 !important; +} + +#mysidebar .nav ul { + background-color: #FAFAFA; +} +.nav ul.series li { + list-style: decimal; + font-size:12px; +} + +.nav ul.series li a:hover { + background-color: gray; +} +.nav ul.series { + padding-left: 30px; +} + +.nav ul.series { + background-color: #FAFAFA; +} + +/* +a.dropdown-toggle.otherProgLangs { + color: #f7e68f !important; +} +*/ + +span.muted {color: #666;} + +table code {background-color: transparent;} + +.highlight .err { + color: #a61717; + background-color: transparent !important; +} + +table p { + margin-top: 12px; + margin-bottom: 12px; +} + +pre, table code { + white-space: pre-wrap; /* css-3 */ + white-space: -moz-pre-wrap; /* Mozilla, since 1999 */ + white-space: -pre-wrap; /* Opera 4-6 */ + white-space: -o-pre-wrap; /* Opera 7 */ + word-wrap: break-word; /* Internet Explorer 5.5+ */ +} + +pre { + margin: 25px 0px; +} + +#json-box-container pre { + margin: 0px; +} + +.video-js { + margin: 30px 0px; +} + +video { + display: block; + margin: 30px 0px; + border: 1px solid #c0c0c0; +} + + +p.required, p.dataType {display: block; color: #c0c0c0; font-size: 80%; margin-left:4px;} + +dd {margin-left:20px;} + +.post-content img.inline { + margin:0px; + margin-bottom:6px; +} +.panel-heading { + font-weight: bold; +} + +.note code, .alert code, .warning code, div#toc code, h2 code, h3 code, h4 code { + color: inherit; + padding: 0px; +} + +.alert { + margin-bottom:10px; + margin-top:10px; +} + +a.accordion-toggle { + font-style: normal; +} + +span.red { + color: red; + font-family: Monaco, Menlo, Consolas, "Courier New", monospace; +} + +h3.codeExplanation { + font-size:18px; + font-style:normal; + color: black; + line-height: 24px; +} + +span.soft { + color: #c0c0c0; +} + +.githubEditButton { + margin-bottom:7px; +} + +.endpoint { + padding: 15px; + background-color: #f0f0f0; + font-family: courier; + font-size: 110%; + margin: 20px 0px; + color: #444; +} + +.parameter { + font-family: courier; + color: red !important; +} + +.formBoundary { + border: 1px solid gray; + padding: 15px; + margin: 15px 0px; + background-color: whitesmoke; +} + +@media (max-width: 767px) { + .navbar-inverse .navbar-nav .open .dropdown-menu > li > a { + color: #444; + } +} + +@media (max-width: 990px) { + #mysidebar { + position: relative; + } + .col-md-9 { + width: 95%; + } +} + + +@media (min-width: 1000px) { + + ul#mysidebar { + width: 225px; + } +} + +@media (max-width: 900px) { + + ul#mysidebar { + max-width: 100%; + } +} + +.col-md-9 img { + max-width: 100%; + max-height: 100%; +} + + +.post-content img { + margin: 12px 0px 3px 0px; + width: auto; + height: auto; + max-width: 100%; + max-height: 100%; +} +.col-md-9 img { + max-width: 100%; + max-height: 100%; +} + + +.post-content img { + margin: 12px 0px 3px 0px; + width: auto; + height: auto; + max-width: 100%; + max-height: 100%; +} + +.videoThumbs img { + float: left; + margin:15px 15px 15px 0px; + border: 1px solid #dedede; +} + + +@media only screen and (min-width: 900px), only screen and (min-device-width: 900px) { + .col-md-9 img { + max-width: 990px; + max-height: 700px; + } +} + +*:hover > .anchorjs-link { + transition: color .25s linear; + text-decoration: none; +} + +.kbCaption { + color: white; + background-color: #444; + padding:10px; +} + +/* Strip the outbound icon when this class is present */ +a[href].noCrossRef::after, +a.no_icon:after + { + content:"" !important; + padding-left: 0; +} + +.btn-default { + margin-bottom: 10px; +} + +/* algolia search */ + +.search { + text-align: left; +} +.search input { + font-size: 20px; + width: 300px; +} +.results { + margin: auto; + text-align: left; +} +.results ul { + list-style-type: none; + padding: 0; +} + +/* algolia */ + +div.results { + position: absolute; + background-color: white; + width: 100%; +} + +.post-meta { + font-size: 14px; + color: #828282; +} + +.post-link { + font-size: 22px; +} + +.post-list p { + margin: 10px 0px; +} + +time { + margin-right: 10px; +} + +p.post-meta time { + margin-right: 0px; +} + +span.label.label-default { + background-color: gray; +} + +span.label.label-primary { + background-color: #f0ad4e; +} +.col-lg-12 .nav li a {background-color: white} + + +.nav li.active > a.subfoldersTitle { + background-color: whitesmoke; + font-weight: bold; + color: black; + } + +code { + color: #1d1f5b; + background-color: #f3f3f3; +} + +a code { + color: #0082C2; +} + +table th code { + color: white; +} + +ol li ul li ol li { + list-style: decimal; +} + +ol li ul li ol li ul li{ + list-style: disc; +} + +.post-content table th { + vertical-align: top; +} + +table thead th code.highlighter-rouge { + background-color: transparent; +} + + +.box { + padding: 10px; + border: 1px solid #888; + width: 100px; + height: 80px; + background-color: #f5f5f5; + font-family: Arial; + font-size: 12px; + hyphens: auto; + float: left; + font-size: 12px; +} + +.box:hover { + background-color: #f0f0f0; +} + +#userMap { + overflow-x: auto; + overflow-y: auto; + padding: 20px; + min-width: 770px; +} + +#userMap .active { + background-color: #d6f5d6; + border:1px solid #555; + font-weight: bold; +} + +h2.userMapTitle { + font-family: Arial; +} + +#userMap a:hover { + text-decoration: none; + } + +div.arrow { + max-width: 50px; + margin-left: 15px; + margin-right: 15px; + font-size: 20px; +} + +div.content { + max-width: 110px +} + +#userMap div.arrow, #userMap div.content { + float: left; +} + +.clearfix { + clear: both; +} + + +#userMap div.arrow { + position: relative; + top: 30px; +} + +.box1 { + margin-left:0px; +} + +button.btn.btn-default.btn-lg.modalButton1 { + margin-left: -20px; +} + +div.box.box1 { + margin-left: -20px; +} + +#userMap .btn-lg { + width: 100px; + height: 80px; + +} + +#userMap .complexArrow { + font-size: 22px; + margin: 0px 10px; +} + + +#userMap .btn-lg .active { + background-color: #d6f5d6; +} + +#userMap .btn-lg { + white-space: pre-wrap; /* css-3 */ + white-space: -moz-pre-wrap; /* Mozilla, since 1999 */ + white-space: -pre-wrap; /* Opera 4-6 */ + white-space: -o-pre-wrap; /* Opera 7 */ + word-wrap: break-word; /* Internet Explorer 5.5+ */ + font-size: 14px; + } + +/* + * Let's target IE to respect aspect ratios and sizes for img tags containing SVG files + * + * [1] IE9 + * [2] IE10+ + */ +/* 1 */ +.ie9 img[src$=".svg"] { + width: 100%; +} +/* 2 */ +@media screen and (-ms-high-contrast: active), (-ms-high-contrast: none) { + img[src$=".svg"] { + width: 100%; + } +} + +h4.panel-title { + padding-top: 0px; + margin-top: 0px; +} + +/*set navbar breakpoint so that it converts to hamburger earlier */ + +@media (max-width: 1200px) { + .navbar-header { + float: none; + } + .navbar-left,.navbar-right { + float: none !important; + } + .navbar-toggle { + display: block; + } + .navbar-collapse { + border-top: 1px solid transparent; + } + .navbar-fixed-top { + top: 0; + border-width: 0 0 1px; + } + .navbar-collapse.collapse { + display: none!important; + } + .navbar-nav { + float: none!important; + margin-top: 7.5px; + } + .navbar-nav>li { + float: none; + } + .navbar-nav>li>a { + padding-top: 10px; + padding-bottom: 10px; + } + .collapse.in{ + display:block !important; + } +} + +.input_area pre { + padding-bottom: 3; + margin-top: 10px; + background-color: #f5f1e0; + border: 0px; +} +.output_area pre { + padding-top: 3; + margin-top: -25px; + margin-bottom: 10px; + border: 0px; + padding-left: 2em; +} +.pytest_card { + padding: 10px 1em 4px 1em; + background-color: #eef2ff; + border-radius: 5px; + margin-bottom: 20px; +} +.pytest_card .close { + color: #000 +} +blockquote { + font-family: Menlo,Monaco,Consolas,"Courier New",monospace; +} +blockquote code { + background-color: white; +} +h4 code { + background-color: white; +} + +/* Google Custom Search styling */ + +#gcs-search-container { + width: 340px !important; /* ToDo: define width without using absolute px value */ + display: inline-block !important; +} + +.gsc-search-box.gsc-search-box-tools table { + margin-bottom: 0px !important; +} +.gsc-input, .gsc-search-button { + padding: 0; +} +.gsc-control-cse { + padding: 6px !important; +} +input.gsc-input { + font-size: 12px !important; +} diff --git a/python/vespa/docs/css/font-awesome.min.css b/python/vespa/docs/css/font-awesome.min.css new file mode 100644 index 00000000000..0e0645d2ace --- /dev/null +++ b/python/vespa/docs/css/font-awesome.min.css @@ -0,0 +1,4 @@ +/*! + * Font Awesome 4.7.0 by @davegandy - http://fontawesome.io - @fontawesome + * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) + */@font-face{font-family:'FontAwesome';src:url('fonts/fontawesome-webfont.eot?v=4.7.0');src:url('fonts/fontawesome-webfont.eot?#iefix&v=4.7.0') format('embedded-opentype'),url('fonts/fontawesome-webfont.woff2?v=4.7.0') format('woff2'),url('fonts/fontawesome-webfont.woff?v=4.7.0') format('woff'),url('fonts/fontawesome-webfont.ttf?v=4.7.0') format('truetype'),url('fonts/fontawesome-webfont.svg?v=4.7.0#fontawesomeregular') format('svg');font-weight:normal;font-style:normal}.fa{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571429em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14285714em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14285714em;width:2.14285714em;top:.14285714em;text-align:center}.fa-li.fa-lg{left:-1.85714286em}.fa-border{padding:.2em .25em .15em;border:solid .08em #eee;border-radius:.1em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left{margin-right:.3em}.fa.fa-pull-right{margin-left:.3em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left{margin-right:.3em}.fa.pull-right{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s infinite linear;animation:fa-spin 2s infinite linear}.fa-pulse{-webkit-animation:fa-spin 1s infinite steps(8);animation:fa-spin 1s infinite steps(8)}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scale(-1, 1);-ms-transform:scale(-1, 1);transform:scale(-1, 1)}.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";-webkit-transform:scale(1, -1);-ms-transform:scale(1, -1);transform:scale(1, -1)}:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270,:root .fa-flip-horizontal,:root .fa-flip-vertical{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:"\f000"}.fa-music:before{content:"\f001"}.fa-search:before{content:"\f002"}.fa-envelope-o:before{content:"\f003"}.fa-heart:before{content:"\f004"}.fa-star:before{content:"\f005"}.fa-star-o:before{content:"\f006"}.fa-user:before{content:"\f007"}.fa-film:before{content:"\f008"}.fa-th-large:before{content:"\f009"}.fa-th:before{content:"\f00a"}.fa-th-list:before{content:"\f00b"}.fa-check:before{content:"\f00c"}.fa-remove:before,.fa-close:before,.fa-times:before{content:"\f00d"}.fa-search-plus:before{content:"\f00e"}.fa-search-minus:before{content:"\f010"}.fa-power-off:before{content:"\f011"}.fa-signal:before{content:"\f012"}.fa-gear:before,.fa-cog:before{content:"\f013"}.fa-trash-o:before{content:"\f014"}.fa-home:before{content:"\f015"}.fa-file-o:before{content:"\f016"}.fa-clock-o:before{content:"\f017"}.fa-road:before{content:"\f018"}.fa-download:before{content:"\f019"}.fa-arrow-circle-o-down:before{content:"\f01a"}.fa-arrow-circle-o-up:before{content:"\f01b"}.fa-inbox:before{content:"\f01c"}.fa-play-circle-o:before{content:"\f01d"}.fa-rotate-right:before,.fa-repeat:before{content:"\f01e"}.fa-refresh:before{content:"\f021"}.fa-list-alt:before{content:"\f022"}.fa-lock:before{content:"\f023"}.fa-flag:before{content:"\f024"}.fa-headphones:before{content:"\f025"}.fa-volume-off:before{content:"\f026"}.fa-volume-down:before{content:"\f027"}.fa-volume-up:before{content:"\f028"}.fa-qrcode:before{content:"\f029"}.fa-barcode:before{content:"\f02a"}.fa-tag:before{content:"\f02b"}.fa-tags:before{content:"\f02c"}.fa-book:before{content:"\f02d"}.fa-bookmark:before{content:"\f02e"}.fa-print:before{content:"\f02f"}.fa-camera:before{content:"\f030"}.fa-font:before{content:"\f031"}.fa-bold:before{content:"\f032"}.fa-italic:before{content:"\f033"}.fa-text-height:before{content:"\f034"}.fa-text-width:before{content:"\f035"}.fa-align-left:before{content:"\f036"}.fa-align-center:before{content:"\f037"}.fa-align-right:before{content:"\f038"}.fa-align-justify:before{content:"\f039"}.fa-list:before{content:"\f03a"}.fa-dedent:before,.fa-outdent:before{content:"\f03b"}.fa-indent:before{content:"\f03c"}.fa-video-camera:before{content:"\f03d"}.fa-photo:before,.fa-image:before,.fa-picture-o:before{content:"\f03e"}.fa-pencil:before{content:"\f040"}.fa-map-marker:before{content:"\f041"}.fa-adjust:before{content:"\f042"}.fa-tint:before{content:"\f043"}.fa-edit:before,.fa-pencil-square-o:before{content:"\f044"}.fa-share-square-o:before{content:"\f045"}.fa-check-square-o:before{content:"\f046"}.fa-arrows:before{content:"\f047"}.fa-step-backward:before{content:"\f048"}.fa-fast-backward:before{content:"\f049"}.fa-backward:before{content:"\f04a"}.fa-play:before{content:"\f04b"}.fa-pause:before{content:"\f04c"}.fa-stop:before{content:"\f04d"}.fa-forward:before{content:"\f04e"}.fa-fast-forward:before{content:"\f050"}.fa-step-forward:before{content:"\f051"}.fa-eject:before{content:"\f052"}.fa-chevron-left:before{content:"\f053"}.fa-chevron-right:before{content:"\f054"}.fa-plus-circle:before{content:"\f055"}.fa-minus-circle:before{content:"\f056"}.fa-times-circle:before{content:"\f057"}.fa-check-circle:before{content:"\f058"}.fa-question-circle:before{content:"\f059"}.fa-info-circle:before{content:"\f05a"}.fa-crosshairs:before{content:"\f05b"}.fa-times-circle-o:before{content:"\f05c"}.fa-check-circle-o:before{content:"\f05d"}.fa-ban:before{content:"\f05e"}.fa-arrow-left:before{content:"\f060"}.fa-arrow-right:before{content:"\f061"}.fa-arrow-up:before{content:"\f062"}.fa-arrow-down:before{content:"\f063"}.fa-mail-forward:before,.fa-share:before{content:"\f064"}.fa-expand:before{content:"\f065"}.fa-compress:before{content:"\f066"}.fa-plus:before{content:"\f067"}.fa-minus:before{content:"\f068"}.fa-asterisk:before{content:"\f069"}.fa-exclamation-circle:before{content:"\f06a"}.fa-gift:before{content:"\f06b"}.fa-leaf:before{content:"\f06c"}.fa-fire:before{content:"\f06d"}.fa-eye:before{content:"\f06e"}.fa-eye-slash:before{content:"\f070"}.fa-warning:before,.fa-exclamation-triangle:before{content:"\f071"}.fa-plane:before{content:"\f072"}.fa-calendar:before{content:"\f073"}.fa-random:before{content:"\f074"}.fa-comment:before{content:"\f075"}.fa-magnet:before{content:"\f076"}.fa-chevron-up:before{content:"\f077"}.fa-chevron-down:before{content:"\f078"}.fa-retweet:before{content:"\f079"}.fa-shopping-cart:before{content:"\f07a"}.fa-folder:before{content:"\f07b"}.fa-folder-open:before{content:"\f07c"}.fa-arrows-v:before{content:"\f07d"}.fa-arrows-h:before{content:"\f07e"}.fa-bar-chart-o:before,.fa-bar-chart:before{content:"\f080"}.fa-twitter-square:before{content:"\f081"}.fa-facebook-square:before{content:"\f082"}.fa-camera-retro:before{content:"\f083"}.fa-key:before{content:"\f084"}.fa-gears:before,.fa-cogs:before{content:"\f085"}.fa-comments:before{content:"\f086"}.fa-thumbs-o-up:before{content:"\f087"}.fa-thumbs-o-down:before{content:"\f088"}.fa-star-half:before{content:"\f089"}.fa-heart-o:before{content:"\f08a"}.fa-sign-out:before{content:"\f08b"}.fa-linkedin-square:before{content:"\f08c"}.fa-thumb-tack:before{content:"\f08d"}.fa-external-link:before{content:"\f08e"}.fa-sign-in:before{content:"\f090"}.fa-trophy:before{content:"\f091"}.fa-github-square:before{content:"\f092"}.fa-upload:before{content:"\f093"}.fa-lemon-o:before{content:"\f094"}.fa-phone:before{content:"\f095"}.fa-square-o:before{content:"\f096"}.fa-bookmark-o:before{content:"\f097"}.fa-phone-square:before{content:"\f098"}.fa-twitter:before{content:"\f099"}.fa-facebook-f:before,.fa-facebook:before{content:"\f09a"}.fa-github:before{content:"\f09b"}.fa-unlock:before{content:"\f09c"}.fa-credit-card:before{content:"\f09d"}.fa-feed:before,.fa-rss:before{content:"\f09e"}.fa-hdd-o:before{content:"\f0a0"}.fa-bullhorn:before{content:"\f0a1"}.fa-bell:before{content:"\f0f3"}.fa-certificate:before{content:"\f0a3"}.fa-hand-o-right:before{content:"\f0a4"}.fa-hand-o-left:before{content:"\f0a5"}.fa-hand-o-up:before{content:"\f0a6"}.fa-hand-o-down:before{content:"\f0a7"}.fa-arrow-circle-left:before{content:"\f0a8"}.fa-arrow-circle-right:before{content:"\f0a9"}.fa-arrow-circle-up:before{content:"\f0aa"}.fa-arrow-circle-down:before{content:"\f0ab"}.fa-globe:before{content:"\f0ac"}.fa-wrench:before{content:"\f0ad"}.fa-tasks:before{content:"\f0ae"}.fa-filter:before{content:"\f0b0"}.fa-briefcase:before{content:"\f0b1"}.fa-arrows-alt:before{content:"\f0b2"}.fa-group:before,.fa-users:before{content:"\f0c0"}.fa-chain:before,.fa-link:before{content:"\f0c1"}.fa-cloud:before{content:"\f0c2"}.fa-flask:before{content:"\f0c3"}.fa-cut:before,.fa-scissors:before{content:"\f0c4"}.fa-copy:before,.fa-files-o:before{content:"\f0c5"}.fa-paperclip:before{content:"\f0c6"}.fa-save:before,.fa-floppy-o:before{content:"\f0c7"}.fa-square:before{content:"\f0c8"}.fa-navicon:before,.fa-reorder:before,.fa-bars:before{content:"\f0c9"}.fa-list-ul:before{content:"\f0ca"}.fa-list-ol:before{content:"\f0cb"}.fa-strikethrough:before{content:"\f0cc"}.fa-underline:before{content:"\f0cd"}.fa-table:before{content:"\f0ce"}.fa-magic:before{content:"\f0d0"}.fa-truck:before{content:"\f0d1"}.fa-pinterest:before{content:"\f0d2"}.fa-pinterest-square:before{content:"\f0d3"}.fa-google-plus-square:before{content:"\f0d4"}.fa-google-plus:before{content:"\f0d5"}.fa-money:before{content:"\f0d6"}.fa-caret-down:before{content:"\f0d7"}.fa-caret-up:before{content:"\f0d8"}.fa-caret-left:before{content:"\f0d9"}.fa-caret-right:before{content:"\f0da"}.fa-columns:before{content:"\f0db"}.fa-unsorted:before,.fa-sort:before{content:"\f0dc"}.fa-sort-down:before,.fa-sort-desc:before{content:"\f0dd"}.fa-sort-up:before,.fa-sort-asc:before{content:"\f0de"}.fa-envelope:before{content:"\f0e0"}.fa-linkedin:before{content:"\f0e1"}.fa-rotate-left:before,.fa-undo:before{content:"\f0e2"}.fa-legal:before,.fa-gavel:before{content:"\f0e3"}.fa-dashboard:before,.fa-tachometer:before{content:"\f0e4"}.fa-comment-o:before{content:"\f0e5"}.fa-comments-o:before{content:"\f0e6"}.fa-flash:before,.fa-bolt:before{content:"\f0e7"}.fa-sitemap:before{content:"\f0e8"}.fa-umbrella:before{content:"\f0e9"}.fa-paste:before,.fa-clipboard:before{content:"\f0ea"}.fa-lightbulb-o:before{content:"\f0eb"}.fa-exchange:before{content:"\f0ec"}.fa-cloud-download:before{content:"\f0ed"}.fa-cloud-upload:before{content:"\f0ee"}.fa-user-md:before{content:"\f0f0"}.fa-stethoscope:before{content:"\f0f1"}.fa-suitcase:before{content:"\f0f2"}.fa-bell-o:before{content:"\f0a2"}.fa-coffee:before{content:"\f0f4"}.fa-cutlery:before{content:"\f0f5"}.fa-file-text-o:before{content:"\f0f6"}.fa-building-o:before{content:"\f0f7"}.fa-hospital-o:before{content:"\f0f8"}.fa-ambulance:before{content:"\f0f9"}.fa-medkit:before{content:"\f0fa"}.fa-fighter-jet:before{content:"\f0fb"}.fa-beer:before{content:"\f0fc"}.fa-h-square:before{content:"\f0fd"}.fa-plus-square:before{content:"\f0fe"}.fa-angle-double-left:before{content:"\f100"}.fa-angle-double-right:before{content:"\f101"}.fa-angle-double-up:before{content:"\f102"}.fa-angle-double-down:before{content:"\f103"}.fa-angle-left:before{content:"\f104"}.fa-angle-right:before{content:"\f105"}.fa-angle-up:before{content:"\f106"}.fa-angle-down:before{content:"\f107"}.fa-desktop:before{content:"\f108"}.fa-laptop:before{content:"\f109"}.fa-tablet:before{content:"\f10a"}.fa-mobile-phone:before,.fa-mobile:before{content:"\f10b"}.fa-circle-o:before{content:"\f10c"}.fa-quote-left:before{content:"\f10d"}.fa-quote-right:before{content:"\f10e"}.fa-spinner:before{content:"\f110"}.fa-circle:before{content:"\f111"}.fa-mail-reply:before,.fa-reply:before{content:"\f112"}.fa-github-alt:before{content:"\f113"}.fa-folder-o:before{content:"\f114"}.fa-folder-open-o:before{content:"\f115"}.fa-smile-o:before{content:"\f118"}.fa-frown-o:before{content:"\f119"}.fa-meh-o:before{content:"\f11a"}.fa-gamepad:before{content:"\f11b"}.fa-keyboard-o:before{content:"\f11c"}.fa-flag-o:before{content:"\f11d"}.fa-flag-checkered:before{content:"\f11e"}.fa-terminal:before{content:"\f120"}.fa-code:before{content:"\f121"}.fa-mail-reply-all:before,.fa-reply-all:before{content:"\f122"}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:"\f123"}.fa-location-arrow:before{content:"\f124"}.fa-crop:before{content:"\f125"}.fa-code-fork:before{content:"\f126"}.fa-unlink:before,.fa-chain-broken:before{content:"\f127"}.fa-question:before{content:"\f128"}.fa-info:before{content:"\f129"}.fa-exclamation:before{content:"\f12a"}.fa-superscript:before{content:"\f12b"}.fa-subscript:before{content:"\f12c"}.fa-eraser:before{content:"\f12d"}.fa-puzzle-piece:before{content:"\f12e"}.fa-microphone:before{content:"\f130"}.fa-microphone-slash:before{content:"\f131"}.fa-shield:before{content:"\f132"}.fa-calendar-o:before{content:"\f133"}.fa-fire-extinguisher:before{content:"\f134"}.fa-rocket:before{content:"\f135"}.fa-maxcdn:before{content:"\f136"}.fa-chevron-circle-left:before{content:"\f137"}.fa-chevron-circle-right:before{content:"\f138"}.fa-chevron-circle-up:before{content:"\f139"}.fa-chevron-circle-down:before{content:"\f13a"}.fa-html5:before{content:"\f13b"}.fa-css3:before{content:"\f13c"}.fa-anchor:before{content:"\f13d"}.fa-unlock-alt:before{content:"\f13e"}.fa-bullseye:before{content:"\f140"}.fa-ellipsis-h:before{content:"\f141"}.fa-ellipsis-v:before{content:"\f142"}.fa-rss-square:before{content:"\f143"}.fa-play-circle:before{content:"\f144"}.fa-ticket:before{content:"\f145"}.fa-minus-square:before{content:"\f146"}.fa-minus-square-o:before{content:"\f147"}.fa-level-up:before{content:"\f148"}.fa-level-down:before{content:"\f149"}.fa-check-square:before{content:"\f14a"}.fa-pencil-square:before{content:"\f14b"}.fa-external-link-square:before{content:"\f14c"}.fa-share-square:before{content:"\f14d"}.fa-compass:before{content:"\f14e"}.fa-toggle-down:before,.fa-caret-square-o-down:before{content:"\f150"}.fa-toggle-up:before,.fa-caret-square-o-up:before{content:"\f151"}.fa-toggle-right:before,.fa-caret-square-o-right:before{content:"\f152"}.fa-euro:before,.fa-eur:before{content:"\f153"}.fa-gbp:before{content:"\f154"}.fa-dollar:before,.fa-usd:before{content:"\f155"}.fa-rupee:before,.fa-inr:before{content:"\f156"}.fa-cny:before,.fa-rmb:before,.fa-yen:before,.fa-jpy:before{content:"\f157"}.fa-ruble:before,.fa-rouble:before,.fa-rub:before{content:"\f158"}.fa-won:before,.fa-krw:before{content:"\f159"}.fa-bitcoin:before,.fa-btc:before{content:"\f15a"}.fa-file:before{content:"\f15b"}.fa-file-text:before{content:"\f15c"}.fa-sort-alpha-asc:before{content:"\f15d"}.fa-sort-alpha-desc:before{content:"\f15e"}.fa-sort-amount-asc:before{content:"\f160"}.fa-sort-amount-desc:before{content:"\f161"}.fa-sort-numeric-asc:before{content:"\f162"}.fa-sort-numeric-desc:before{content:"\f163"}.fa-thumbs-up:before{content:"\f164"}.fa-thumbs-down:before{content:"\f165"}.fa-youtube-square:before{content:"\f166"}.fa-youtube:before{content:"\f167"}.fa-xing:before{content:"\f168"}.fa-xing-square:before{content:"\f169"}.fa-youtube-play:before{content:"\f16a"}.fa-dropbox:before{content:"\f16b"}.fa-stack-overflow:before{content:"\f16c"}.fa-instagram:before{content:"\f16d"}.fa-flickr:before{content:"\f16e"}.fa-adn:before{content:"\f170"}.fa-bitbucket:before{content:"\f171"}.fa-bitbucket-square:before{content:"\f172"}.fa-tumblr:before{content:"\f173"}.fa-tumblr-square:before{content:"\f174"}.fa-long-arrow-down:before{content:"\f175"}.fa-long-arrow-up:before{content:"\f176"}.fa-long-arrow-left:before{content:"\f177"}.fa-long-arrow-right:before{content:"\f178"}.fa-apple:before{content:"\f179"}.fa-windows:before{content:"\f17a"}.fa-android:before{content:"\f17b"}.fa-linux:before{content:"\f17c"}.fa-dribbble:before{content:"\f17d"}.fa-skype:before{content:"\f17e"}.fa-foursquare:before{content:"\f180"}.fa-trello:before{content:"\f181"}.fa-female:before{content:"\f182"}.fa-male:before{content:"\f183"}.fa-gittip:before,.fa-gratipay:before{content:"\f184"}.fa-sun-o:before{content:"\f185"}.fa-moon-o:before{content:"\f186"}.fa-archive:before{content:"\f187"}.fa-bug:before{content:"\f188"}.fa-vk:before{content:"\f189"}.fa-weibo:before{content:"\f18a"}.fa-renren:before{content:"\f18b"}.fa-pagelines:before{content:"\f18c"}.fa-stack-exchange:before{content:"\f18d"}.fa-arrow-circle-o-right:before{content:"\f18e"}.fa-arrow-circle-o-left:before{content:"\f190"}.fa-toggle-left:before,.fa-caret-square-o-left:before{content:"\f191"}.fa-dot-circle-o:before{content:"\f192"}.fa-wheelchair:before{content:"\f193"}.fa-vimeo-square:before{content:"\f194"}.fa-turkish-lira:before,.fa-try:before{content:"\f195"}.fa-plus-square-o:before{content:"\f196"}.fa-space-shuttle:before{content:"\f197"}.fa-slack:before{content:"\f198"}.fa-envelope-square:before{content:"\f199"}.fa-wordpress:before{content:"\f19a"}.fa-openid:before{content:"\f19b"}.fa-institution:before,.fa-bank:before,.fa-university:before{content:"\f19c"}.fa-mortar-board:before,.fa-graduation-cap:before{content:"\f19d"}.fa-yahoo:before{content:"\f19e"}.fa-google:before{content:"\f1a0"}.fa-reddit:before{content:"\f1a1"}.fa-reddit-square:before{content:"\f1a2"}.fa-stumbleupon-circle:before{content:"\f1a3"}.fa-stumbleupon:before{content:"\f1a4"}.fa-delicious:before{content:"\f1a5"}.fa-digg:before{content:"\f1a6"}.fa-pied-piper-pp:before{content:"\f1a7"}.fa-pied-piper-alt:before{content:"\f1a8"}.fa-drupal:before{content:"\f1a9"}.fa-joomla:before{content:"\f1aa"}.fa-language:before{content:"\f1ab"}.fa-fax:before{content:"\f1ac"}.fa-building:before{content:"\f1ad"}.fa-child:before{content:"\f1ae"}.fa-paw:before{content:"\f1b0"}.fa-spoon:before{content:"\f1b1"}.fa-cube:before{content:"\f1b2"}.fa-cubes:before{content:"\f1b3"}.fa-behance:before{content:"\f1b4"}.fa-behance-square:before{content:"\f1b5"}.fa-steam:before{content:"\f1b6"}.fa-steam-square:before{content:"\f1b7"}.fa-recycle:before{content:"\f1b8"}.fa-automobile:before,.fa-car:before{content:"\f1b9"}.fa-cab:before,.fa-taxi:before{content:"\f1ba"}.fa-tree:before{content:"\f1bb"}.fa-spotify:before{content:"\f1bc"}.fa-deviantart:before{content:"\f1bd"}.fa-soundcloud:before{content:"\f1be"}.fa-database:before{content:"\f1c0"}.fa-file-pdf-o:before{content:"\f1c1"}.fa-file-word-o:before{content:"\f1c2"}.fa-file-excel-o:before{content:"\f1c3"}.fa-file-powerpoint-o:before{content:"\f1c4"}.fa-file-photo-o:before,.fa-file-picture-o:before,.fa-file-image-o:before{content:"\f1c5"}.fa-file-zip-o:before,.fa-file-archive-o:before{content:"\f1c6"}.fa-file-sound-o:before,.fa-file-audio-o:before{content:"\f1c7"}.fa-file-movie-o:before,.fa-file-video-o:before{content:"\f1c8"}.fa-file-code-o:before{content:"\f1c9"}.fa-vine:before{content:"\f1ca"}.fa-codepen:before{content:"\f1cb"}.fa-jsfiddle:before{content:"\f1cc"}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-saver:before,.fa-support:before,.fa-life-ring:before{content:"\f1cd"}.fa-circle-o-notch:before{content:"\f1ce"}.fa-ra:before,.fa-resistance:before,.fa-rebel:before{content:"\f1d0"}.fa-ge:before,.fa-empire:before{content:"\f1d1"}.fa-git-square:before{content:"\f1d2"}.fa-git:before{content:"\f1d3"}.fa-y-combinator-square:before,.fa-yc-square:before,.fa-hacker-news:before{content:"\f1d4"}.fa-tencent-weibo:before{content:"\f1d5"}.fa-qq:before{content:"\f1d6"}.fa-wechat:before,.fa-weixin:before{content:"\f1d7"}.fa-send:before,.fa-paper-plane:before{content:"\f1d8"}.fa-send-o:before,.fa-paper-plane-o:before{content:"\f1d9"}.fa-history:before{content:"\f1da"}.fa-circle-thin:before{content:"\f1db"}.fa-header:before{content:"\f1dc"}.fa-paragraph:before{content:"\f1dd"}.fa-sliders:before{content:"\f1de"}.fa-share-alt:before{content:"\f1e0"}.fa-share-alt-square:before{content:"\f1e1"}.fa-bomb:before{content:"\f1e2"}.fa-soccer-ball-o:before,.fa-futbol-o:before{content:"\f1e3"}.fa-tty:before{content:"\f1e4"}.fa-binoculars:before{content:"\f1e5"}.fa-plug:before{content:"\f1e6"}.fa-slideshare:before{content:"\f1e7"}.fa-twitch:before{content:"\f1e8"}.fa-yelp:before{content:"\f1e9"}.fa-newspaper-o:before{content:"\f1ea"}.fa-wifi:before{content:"\f1eb"}.fa-calculator:before{content:"\f1ec"}.fa-paypal:before{content:"\f1ed"}.fa-google-wallet:before{content:"\f1ee"}.fa-cc-visa:before{content:"\f1f0"}.fa-cc-mastercard:before{content:"\f1f1"}.fa-cc-discover:before{content:"\f1f2"}.fa-cc-amex:before{content:"\f1f3"}.fa-cc-paypal:before{content:"\f1f4"}.fa-cc-stripe:before{content:"\f1f5"}.fa-bell-slash:before{content:"\f1f6"}.fa-bell-slash-o:before{content:"\f1f7"}.fa-trash:before{content:"\f1f8"}.fa-copyright:before{content:"\f1f9"}.fa-at:before{content:"\f1fa"}.fa-eyedropper:before{content:"\f1fb"}.fa-paint-brush:before{content:"\f1fc"}.fa-birthday-cake:before{content:"\f1fd"}.fa-area-chart:before{content:"\f1fe"}.fa-pie-chart:before{content:"\f200"}.fa-line-chart:before{content:"\f201"}.fa-lastfm:before{content:"\f202"}.fa-lastfm-square:before{content:"\f203"}.fa-toggle-off:before{content:"\f204"}.fa-toggle-on:before{content:"\f205"}.fa-bicycle:before{content:"\f206"}.fa-bus:before{content:"\f207"}.fa-ioxhost:before{content:"\f208"}.fa-angellist:before{content:"\f209"}.fa-cc:before{content:"\f20a"}.fa-shekel:before,.fa-sheqel:before,.fa-ils:before{content:"\f20b"}.fa-meanpath:before{content:"\f20c"}.fa-buysellads:before{content:"\f20d"}.fa-connectdevelop:before{content:"\f20e"}.fa-dashcube:before{content:"\f210"}.fa-forumbee:before{content:"\f211"}.fa-leanpub:before{content:"\f212"}.fa-sellsy:before{content:"\f213"}.fa-shirtsinbulk:before{content:"\f214"}.fa-simplybuilt:before{content:"\f215"}.fa-skyatlas:before{content:"\f216"}.fa-cart-plus:before{content:"\f217"}.fa-cart-arrow-down:before{content:"\f218"}.fa-diamond:before{content:"\f219"}.fa-ship:before{content:"\f21a"}.fa-user-secret:before{content:"\f21b"}.fa-motorcycle:before{content:"\f21c"}.fa-street-view:before{content:"\f21d"}.fa-heartbeat:before{content:"\f21e"}.fa-venus:before{content:"\f221"}.fa-mars:before{content:"\f222"}.fa-mercury:before{content:"\f223"}.fa-intersex:before,.fa-transgender:before{content:"\f224"}.fa-transgender-alt:before{content:"\f225"}.fa-venus-double:before{content:"\f226"}.fa-mars-double:before{content:"\f227"}.fa-venus-mars:before{content:"\f228"}.fa-mars-stroke:before{content:"\f229"}.fa-mars-stroke-v:before{content:"\f22a"}.fa-mars-stroke-h:before{content:"\f22b"}.fa-neuter:before{content:"\f22c"}.fa-genderless:before{content:"\f22d"}.fa-facebook-official:before{content:"\f230"}.fa-pinterest-p:before{content:"\f231"}.fa-whatsapp:before{content:"\f232"}.fa-server:before{content:"\f233"}.fa-user-plus:before{content:"\f234"}.fa-user-times:before{content:"\f235"}.fa-hotel:before,.fa-bed:before{content:"\f236"}.fa-viacoin:before{content:"\f237"}.fa-train:before{content:"\f238"}.fa-subway:before{content:"\f239"}.fa-medium:before{content:"\f23a"}.fa-yc:before,.fa-y-combinator:before{content:"\f23b"}.fa-optin-monster:before{content:"\f23c"}.fa-opencart:before{content:"\f23d"}.fa-expeditedssl:before{content:"\f23e"}.fa-battery-4:before,.fa-battery:before,.fa-battery-full:before{content:"\f240"}.fa-battery-3:before,.fa-battery-three-quarters:before{content:"\f241"}.fa-battery-2:before,.fa-battery-half:before{content:"\f242"}.fa-battery-1:before,.fa-battery-quarter:before{content:"\f243"}.fa-battery-0:before,.fa-battery-empty:before{content:"\f244"}.fa-mouse-pointer:before{content:"\f245"}.fa-i-cursor:before{content:"\f246"}.fa-object-group:before{content:"\f247"}.fa-object-ungroup:before{content:"\f248"}.fa-sticky-note:before{content:"\f249"}.fa-sticky-note-o:before{content:"\f24a"}.fa-cc-jcb:before{content:"\f24b"}.fa-cc-diners-club:before{content:"\f24c"}.fa-clone:before{content:"\f24d"}.fa-balance-scale:before{content:"\f24e"}.fa-hourglass-o:before{content:"\f250"}.fa-hourglass-1:before,.fa-hourglass-start:before{content:"\f251"}.fa-hourglass-2:before,.fa-hourglass-half:before{content:"\f252"}.fa-hourglass-3:before,.fa-hourglass-end:before{content:"\f253"}.fa-hourglass:before{content:"\f254"}.fa-hand-grab-o:before,.fa-hand-rock-o:before{content:"\f255"}.fa-hand-stop-o:before,.fa-hand-paper-o:before{content:"\f256"}.fa-hand-scissors-o:before{content:"\f257"}.fa-hand-lizard-o:before{content:"\f258"}.fa-hand-spock-o:before{content:"\f259"}.fa-hand-pointer-o:before{content:"\f25a"}.fa-hand-peace-o:before{content:"\f25b"}.fa-trademark:before{content:"\f25c"}.fa-registered:before{content:"\f25d"}.fa-creative-commons:before{content:"\f25e"}.fa-gg:before{content:"\f260"}.fa-gg-circle:before{content:"\f261"}.fa-tripadvisor:before{content:"\f262"}.fa-odnoklassniki:before{content:"\f263"}.fa-odnoklassniki-square:before{content:"\f264"}.fa-get-pocket:before{content:"\f265"}.fa-wikipedia-w:before{content:"\f266"}.fa-safari:before{content:"\f267"}.fa-chrome:before{content:"\f268"}.fa-firefox:before{content:"\f269"}.fa-opera:before{content:"\f26a"}.fa-internet-explorer:before{content:"\f26b"}.fa-tv:before,.fa-television:before{content:"\f26c"}.fa-contao:before{content:"\f26d"}.fa-500px:before{content:"\f26e"}.fa-amazon:before{content:"\f270"}.fa-calendar-plus-o:before{content:"\f271"}.fa-calendar-minus-o:before{content:"\f272"}.fa-calendar-times-o:before{content:"\f273"}.fa-calendar-check-o:before{content:"\f274"}.fa-industry:before{content:"\f275"}.fa-map-pin:before{content:"\f276"}.fa-map-signs:before{content:"\f277"}.fa-map-o:before{content:"\f278"}.fa-map:before{content:"\f279"}.fa-commenting:before{content:"\f27a"}.fa-commenting-o:before{content:"\f27b"}.fa-houzz:before{content:"\f27c"}.fa-vimeo:before{content:"\f27d"}.fa-black-tie:before{content:"\f27e"}.fa-fonticons:before{content:"\f280"}.fa-reddit-alien:before{content:"\f281"}.fa-edge:before{content:"\f282"}.fa-credit-card-alt:before{content:"\f283"}.fa-codiepie:before{content:"\f284"}.fa-modx:before{content:"\f285"}.fa-fort-awesome:before{content:"\f286"}.fa-usb:before{content:"\f287"}.fa-product-hunt:before{content:"\f288"}.fa-mixcloud:before{content:"\f289"}.fa-scribd:before{content:"\f28a"}.fa-pause-circle:before{content:"\f28b"}.fa-pause-circle-o:before{content:"\f28c"}.fa-stop-circle:before{content:"\f28d"}.fa-stop-circle-o:before{content:"\f28e"}.fa-shopping-bag:before{content:"\f290"}.fa-shopping-basket:before{content:"\f291"}.fa-hashtag:before{content:"\f292"}.fa-bluetooth:before{content:"\f293"}.fa-bluetooth-b:before{content:"\f294"}.fa-percent:before{content:"\f295"}.fa-gitlab:before{content:"\f296"}.fa-wpbeginner:before{content:"\f297"}.fa-wpforms:before{content:"\f298"}.fa-envira:before{content:"\f299"}.fa-universal-access:before{content:"\f29a"}.fa-wheelchair-alt:before{content:"\f29b"}.fa-question-circle-o:before{content:"\f29c"}.fa-blind:before{content:"\f29d"}.fa-audio-description:before{content:"\f29e"}.fa-volume-control-phone:before{content:"\f2a0"}.fa-braille:before{content:"\f2a1"}.fa-assistive-listening-systems:before{content:"\f2a2"}.fa-asl-interpreting:before,.fa-american-sign-language-interpreting:before{content:"\f2a3"}.fa-deafness:before,.fa-hard-of-hearing:before,.fa-deaf:before{content:"\f2a4"}.fa-glide:before{content:"\f2a5"}.fa-glide-g:before{content:"\f2a6"}.fa-signing:before,.fa-sign-language:before{content:"\f2a7"}.fa-low-vision:before{content:"\f2a8"}.fa-viadeo:before{content:"\f2a9"}.fa-viadeo-square:before{content:"\f2aa"}.fa-snapchat:before{content:"\f2ab"}.fa-snapchat-ghost:before{content:"\f2ac"}.fa-snapchat-square:before{content:"\f2ad"}.fa-pied-piper:before{content:"\f2ae"}.fa-first-order:before{content:"\f2b0"}.fa-yoast:before{content:"\f2b1"}.fa-themeisle:before{content:"\f2b2"}.fa-google-plus-circle:before,.fa-google-plus-official:before{content:"\f2b3"}.fa-fa:before,.fa-font-awesome:before{content:"\f2b4"}.fa-handshake-o:before{content:"\f2b5"}.fa-envelope-open:before{content:"\f2b6"}.fa-envelope-open-o:before{content:"\f2b7"}.fa-linode:before{content:"\f2b8"}.fa-address-book:before{content:"\f2b9"}.fa-address-book-o:before{content:"\f2ba"}.fa-vcard:before,.fa-address-card:before{content:"\f2bb"}.fa-vcard-o:before,.fa-address-card-o:before{content:"\f2bc"}.fa-user-circle:before{content:"\f2bd"}.fa-user-circle-o:before{content:"\f2be"}.fa-user-o:before{content:"\f2c0"}.fa-id-badge:before{content:"\f2c1"}.fa-drivers-license:before,.fa-id-card:before{content:"\f2c2"}.fa-drivers-license-o:before,.fa-id-card-o:before{content:"\f2c3"}.fa-quora:before{content:"\f2c4"}.fa-free-code-camp:before{content:"\f2c5"}.fa-telegram:before{content:"\f2c6"}.fa-thermometer-4:before,.fa-thermometer:before,.fa-thermometer-full:before{content:"\f2c7"}.fa-thermometer-3:before,.fa-thermometer-three-quarters:before{content:"\f2c8"}.fa-thermometer-2:before,.fa-thermometer-half:before{content:"\f2c9"}.fa-thermometer-1:before,.fa-thermometer-quarter:before{content:"\f2ca"}.fa-thermometer-0:before,.fa-thermometer-empty:before{content:"\f2cb"}.fa-shower:before{content:"\f2cc"}.fa-bathtub:before,.fa-s15:before,.fa-bath:before{content:"\f2cd"}.fa-podcast:before{content:"\f2ce"}.fa-window-maximize:before{content:"\f2d0"}.fa-window-minimize:before{content:"\f2d1"}.fa-window-restore:before{content:"\f2d2"}.fa-times-rectangle:before,.fa-window-close:before{content:"\f2d3"}.fa-times-rectangle-o:before,.fa-window-close-o:before{content:"\f2d4"}.fa-bandcamp:before{content:"\f2d5"}.fa-grav:before{content:"\f2d6"}.fa-etsy:before{content:"\f2d7"}.fa-imdb:before{content:"\f2d8"}.fa-ravelry:before{content:"\f2d9"}.fa-eercast:before{content:"\f2da"}.fa-microchip:before{content:"\f2db"}.fa-snowflake-o:before{content:"\f2dc"}.fa-superpowers:before{content:"\f2dd"}.fa-wpexplorer:before{content:"\f2de"}.fa-meetup:before{content:"\f2e0"}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto} diff --git a/python/vespa/docs/css/fonts/FontAwesome.otf b/python/vespa/docs/css/fonts/FontAwesome.otf Binary files differnew file mode 100644 index 00000000000..401ec0f36e4 --- /dev/null +++ b/python/vespa/docs/css/fonts/FontAwesome.otf diff --git a/python/vespa/docs/css/fonts/fontawesome-webfont.eot b/python/vespa/docs/css/fonts/fontawesome-webfont.eot Binary files differnew file mode 100644 index 00000000000..e9f60ca953f --- /dev/null +++ b/python/vespa/docs/css/fonts/fontawesome-webfont.eot diff --git a/python/vespa/docs/css/fonts/fontawesome-webfont.svg b/python/vespa/docs/css/fonts/fontawesome-webfont.svg new file mode 100644 index 00000000000..855c845e538 --- /dev/null +++ b/python/vespa/docs/css/fonts/fontawesome-webfont.svg @@ -0,0 +1,2671 @@ +<?xml version="1.0" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd" > +<svg> +<metadata> +Created by FontForge 20120731 at Mon Oct 24 17:37:40 2016 + By ,,, +Copyright Dave Gandy 2016. All rights reserved. +</metadata> +<defs> +<font id="FontAwesome" horiz-adv-x="1536" > + <font-face + font-family="FontAwesome" + font-weight="400" + font-stretch="normal" + units-per-em="1792" + panose-1="0 0 0 0 0 0 0 0 0 0" + ascent="1536" + descent="-256" + bbox="-1.02083 -256.962 2304.6 1537.02" + underline-thickness="0" + underline-position="0" + unicode-range="U+0020-F500" + /> +<missing-glyph horiz-adv-x="896" +d="M224 112h448v1312h-448v-1312zM112 0v1536h672v-1536h-672z" /> + <glyph glyph-name=".notdef" horiz-adv-x="896" +d="M224 112h448v1312h-448v-1312zM112 0v1536h672v-1536h-672z" /> + <glyph glyph-name=".null" horiz-adv-x="0" + /> + <glyph glyph-name="nonmarkingreturn" horiz-adv-x="597" + /> + <glyph glyph-name="space" unicode=" " horiz-adv-x="448" + /> + <glyph glyph-name="dieresis" unicode="¨" horiz-adv-x="1792" + /> + <glyph glyph-name="copyright" unicode="©" horiz-adv-x="1792" + /> + <glyph glyph-name="registered" unicode="®" horiz-adv-x="1792" + /> + <glyph glyph-name="acute" unicode="´" horiz-adv-x="1792" + /> + <glyph glyph-name="AE" unicode="Æ" horiz-adv-x="1792" + /> + <glyph glyph-name="Oslash" unicode="Ø" horiz-adv-x="1792" + /> + <glyph glyph-name="trademark" unicode="™" horiz-adv-x="1792" + /> + <glyph glyph-name="infinity" unicode="∞" horiz-adv-x="1792" + /> + <glyph glyph-name="notequal" unicode="≠" horiz-adv-x="1792" + /> + <glyph glyph-name="glass" unicode="" horiz-adv-x="1792" +d="M1699 1350q0 -35 -43 -78l-632 -632v-768h320q26 0 45 -19t19 -45t-19 -45t-45 -19h-896q-26 0 -45 19t-19 45t19 45t45 19h320v768l-632 632q-43 43 -43 78q0 23 18 36.5t38 17.5t43 4h1408q23 0 43 -4t38 -17.5t18 -36.5z" /> + <glyph glyph-name="music" unicode="" +d="M1536 1312v-1120q0 -50 -34 -89t-86 -60.5t-103.5 -32t-96.5 -10.5t-96.5 10.5t-103.5 32t-86 60.5t-34 89t34 89t86 60.5t103.5 32t96.5 10.5q105 0 192 -39v537l-768 -237v-709q0 -50 -34 -89t-86 -60.5t-103.5 -32t-96.5 -10.5t-96.5 10.5t-103.5 32t-86 60.5t-34 89 +t34 89t86 60.5t103.5 32t96.5 10.5q105 0 192 -39v967q0 31 19 56.5t49 35.5l832 256q12 4 28 4q40 0 68 -28t28 -68z" /> + <glyph glyph-name="search" unicode="" horiz-adv-x="1664" +d="M1152 704q0 185 -131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5t316.5 131.5t131.5 316.5zM1664 -128q0 -52 -38 -90t-90 -38q-54 0 -90 38l-343 342q-179 -124 -399 -124q-143 0 -273.5 55.5t-225 150t-150 225t-55.5 273.5 +t55.5 273.5t150 225t225 150t273.5 55.5t273.5 -55.5t225 -150t150 -225t55.5 -273.5q0 -220 -124 -399l343 -343q37 -37 37 -90z" /> + <glyph glyph-name="envelope" unicode="" horiz-adv-x="1792" +d="M1664 32v768q-32 -36 -69 -66q-268 -206 -426 -338q-51 -43 -83 -67t-86.5 -48.5t-102.5 -24.5h-1h-1q-48 0 -102.5 24.5t-86.5 48.5t-83 67q-158 132 -426 338q-37 30 -69 66v-768q0 -13 9.5 -22.5t22.5 -9.5h1472q13 0 22.5 9.5t9.5 22.5zM1664 1083v11v13.5t-0.5 13 +t-3 12.5t-5.5 9t-9 7.5t-14 2.5h-1472q-13 0 -22.5 -9.5t-9.5 -22.5q0 -168 147 -284q193 -152 401 -317q6 -5 35 -29.5t46 -37.5t44.5 -31.5t50.5 -27.5t43 -9h1h1q20 0 43 9t50.5 27.5t44.5 31.5t46 37.5t35 29.5q208 165 401 317q54 43 100.5 115.5t46.5 131.5z +M1792 1120v-1088q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h1472q66 0 113 -47t47 -113z" /> + <glyph glyph-name="heart" unicode="" horiz-adv-x="1792" +d="M896 -128q-26 0 -44 18l-624 602q-10 8 -27.5 26t-55.5 65.5t-68 97.5t-53.5 121t-23.5 138q0 220 127 344t351 124q62 0 126.5 -21.5t120 -58t95.5 -68.5t76 -68q36 36 76 68t95.5 68.5t120 58t126.5 21.5q224 0 351 -124t127 -344q0 -221 -229 -450l-623 -600 +q-18 -18 -44 -18z" /> + <glyph glyph-name="star" unicode="" horiz-adv-x="1664" +d="M1664 889q0 -22 -26 -48l-363 -354l86 -500q1 -7 1 -20q0 -21 -10.5 -35.5t-30.5 -14.5q-19 0 -40 12l-449 236l-449 -236q-22 -12 -40 -12q-21 0 -31.5 14.5t-10.5 35.5q0 6 2 20l86 500l-364 354q-25 27 -25 48q0 37 56 46l502 73l225 455q19 41 49 41t49 -41l225 -455 +l502 -73q56 -9 56 -46z" /> + <glyph glyph-name="star_empty" unicode="" horiz-adv-x="1664" +d="M1137 532l306 297l-422 62l-189 382l-189 -382l-422 -62l306 -297l-73 -421l378 199l377 -199zM1664 889q0 -22 -26 -48l-363 -354l86 -500q1 -7 1 -20q0 -50 -41 -50q-19 0 -40 12l-449 236l-449 -236q-22 -12 -40 -12q-21 0 -31.5 14.5t-10.5 35.5q0 6 2 20l86 500 +l-364 354q-25 27 -25 48q0 37 56 46l502 73l225 455q19 41 49 41t49 -41l225 -455l502 -73q56 -9 56 -46z" /> + <glyph glyph-name="user" unicode="" horiz-adv-x="1280" +d="M1280 137q0 -109 -62.5 -187t-150.5 -78h-854q-88 0 -150.5 78t-62.5 187q0 85 8.5 160.5t31.5 152t58.5 131t94 89t134.5 34.5q131 -128 313 -128t313 128q76 0 134.5 -34.5t94 -89t58.5 -131t31.5 -152t8.5 -160.5zM1024 1024q0 -159 -112.5 -271.5t-271.5 -112.5 +t-271.5 112.5t-112.5 271.5t112.5 271.5t271.5 112.5t271.5 -112.5t112.5 -271.5z" /> + <glyph glyph-name="film" unicode="" horiz-adv-x="1920" +d="M384 -64v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM384 320v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM384 704v128q0 26 -19 45t-45 19h-128 +q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1408 -64v512q0 26 -19 45t-45 19h-768q-26 0 -45 -19t-19 -45v-512q0 -26 19 -45t45 -19h768q26 0 45 19t19 45zM384 1088v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45 +t45 -19h128q26 0 45 19t19 45zM1792 -64v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1408 704v512q0 26 -19 45t-45 19h-768q-26 0 -45 -19t-19 -45v-512q0 -26 19 -45t45 -19h768q26 0 45 19t19 45zM1792 320v128 +q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1792 704v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1792 1088v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19 +t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1920 1248v-1344q0 -66 -47 -113t-113 -47h-1600q-66 0 -113 47t-47 113v1344q0 66 47 113t113 47h1600q66 0 113 -47t47 -113z" /> + <glyph glyph-name="th_large" unicode="" horiz-adv-x="1664" +d="M768 512v-384q0 -52 -38 -90t-90 -38h-512q-52 0 -90 38t-38 90v384q0 52 38 90t90 38h512q52 0 90 -38t38 -90zM768 1280v-384q0 -52 -38 -90t-90 -38h-512q-52 0 -90 38t-38 90v384q0 52 38 90t90 38h512q52 0 90 -38t38 -90zM1664 512v-384q0 -52 -38 -90t-90 -38 +h-512q-52 0 -90 38t-38 90v384q0 52 38 90t90 38h512q52 0 90 -38t38 -90zM1664 1280v-384q0 -52 -38 -90t-90 -38h-512q-52 0 -90 38t-38 90v384q0 52 38 90t90 38h512q52 0 90 -38t38 -90z" /> + <glyph glyph-name="th" unicode="" horiz-adv-x="1792" +d="M512 288v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM512 800v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1152 288v-192q0 -40 -28 -68t-68 -28h-320 +q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM512 1312v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1152 800v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28 +h320q40 0 68 -28t28 -68zM1792 288v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1152 1312v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1792 800v-192 +q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1792 1312v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68z" /> + <glyph glyph-name="th_list" unicode="" horiz-adv-x="1792" +d="M512 288v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM512 800v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1792 288v-192q0 -40 -28 -68t-68 -28h-960 +q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h960q40 0 68 -28t28 -68zM512 1312v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1792 800v-192q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v192q0 40 28 68t68 28 +h960q40 0 68 -28t28 -68zM1792 1312v-192q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h960q40 0 68 -28t28 -68z" /> + <glyph glyph-name="ok" unicode="" horiz-adv-x="1792" +d="M1671 970q0 -40 -28 -68l-724 -724l-136 -136q-28 -28 -68 -28t-68 28l-136 136l-362 362q-28 28 -28 68t28 68l136 136q28 28 68 28t68 -28l294 -295l656 657q28 28 68 28t68 -28l136 -136q28 -28 28 -68z" /> + <glyph glyph-name="remove" unicode="" horiz-adv-x="1408" +d="M1298 214q0 -40 -28 -68l-136 -136q-28 -28 -68 -28t-68 28l-294 294l-294 -294q-28 -28 -68 -28t-68 28l-136 136q-28 28 -28 68t28 68l294 294l-294 294q-28 28 -28 68t28 68l136 136q28 28 68 28t68 -28l294 -294l294 294q28 28 68 28t68 -28l136 -136q28 -28 28 -68 +t-28 -68l-294 -294l294 -294q28 -28 28 -68z" /> + <glyph glyph-name="zoom_in" unicode="" horiz-adv-x="1664" +d="M1024 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-224v-224q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v224h-224q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h224v224q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5v-224h224 +q13 0 22.5 -9.5t9.5 -22.5zM1152 704q0 185 -131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5t316.5 131.5t131.5 316.5zM1664 -128q0 -53 -37.5 -90.5t-90.5 -37.5q-54 0 -90 38l-343 342q-179 -124 -399 -124q-143 0 -273.5 55.5 +t-225 150t-150 225t-55.5 273.5t55.5 273.5t150 225t225 150t273.5 55.5t273.5 -55.5t225 -150t150 -225t55.5 -273.5q0 -220 -124 -399l343 -343q37 -37 37 -90z" /> + <glyph glyph-name="zoom_out" unicode="" horiz-adv-x="1664" +d="M1024 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-576q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h576q13 0 22.5 -9.5t9.5 -22.5zM1152 704q0 185 -131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5t316.5 131.5t131.5 316.5z +M1664 -128q0 -53 -37.5 -90.5t-90.5 -37.5q-54 0 -90 38l-343 342q-179 -124 -399 -124q-143 0 -273.5 55.5t-225 150t-150 225t-55.5 273.5t55.5 273.5t150 225t225 150t273.5 55.5t273.5 -55.5t225 -150t150 -225t55.5 -273.5q0 -220 -124 -399l343 -343q37 -37 37 -90z +" /> + <glyph glyph-name="off" unicode="" +d="M1536 640q0 -156 -61 -298t-164 -245t-245 -164t-298 -61t-298 61t-245 164t-164 245t-61 298q0 182 80.5 343t226.5 270q43 32 95.5 25t83.5 -50q32 -42 24.5 -94.5t-49.5 -84.5q-98 -74 -151.5 -181t-53.5 -228q0 -104 40.5 -198.5t109.5 -163.5t163.5 -109.5 +t198.5 -40.5t198.5 40.5t163.5 109.5t109.5 163.5t40.5 198.5q0 121 -53.5 228t-151.5 181q-42 32 -49.5 84.5t24.5 94.5q31 43 84 50t95 -25q146 -109 226.5 -270t80.5 -343zM896 1408v-640q0 -52 -38 -90t-90 -38t-90 38t-38 90v640q0 52 38 90t90 38t90 -38t38 -90z" /> + <glyph glyph-name="signal" unicode="" horiz-adv-x="1792" +d="M256 96v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM640 224v-320q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v320q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1024 480v-576q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23 +v576q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1408 864v-960q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v960q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1792 1376v-1472q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v1472q0 14 9 23t23 9h192q14 0 23 -9t9 -23z" /> + <glyph glyph-name="cog" unicode="" +d="M1024 640q0 106 -75 181t-181 75t-181 -75t-75 -181t75 -181t181 -75t181 75t75 181zM1536 749v-222q0 -12 -8 -23t-20 -13l-185 -28q-19 -54 -39 -91q35 -50 107 -138q10 -12 10 -25t-9 -23q-27 -37 -99 -108t-94 -71q-12 0 -26 9l-138 108q-44 -23 -91 -38 +q-16 -136 -29 -186q-7 -28 -36 -28h-222q-14 0 -24.5 8.5t-11.5 21.5l-28 184q-49 16 -90 37l-141 -107q-10 -9 -25 -9q-14 0 -25 11q-126 114 -165 168q-7 10 -7 23q0 12 8 23q15 21 51 66.5t54 70.5q-27 50 -41 99l-183 27q-13 2 -21 12.5t-8 23.5v222q0 12 8 23t19 13 +l186 28q14 46 39 92q-40 57 -107 138q-10 12 -10 24q0 10 9 23q26 36 98.5 107.5t94.5 71.5q13 0 26 -10l138 -107q44 23 91 38q16 136 29 186q7 28 36 28h222q14 0 24.5 -8.5t11.5 -21.5l28 -184q49 -16 90 -37l142 107q9 9 24 9q13 0 25 -10q129 -119 165 -170q7 -8 7 -22 +q0 -12 -8 -23q-15 -21 -51 -66.5t-54 -70.5q26 -50 41 -98l183 -28q13 -2 21 -12.5t8 -23.5z" /> + <glyph glyph-name="trash" unicode="" horiz-adv-x="1408" +d="M512 800v-576q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v576q0 14 9 23t23 9h64q14 0 23 -9t9 -23zM768 800v-576q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v576q0 14 9 23t23 9h64q14 0 23 -9t9 -23zM1024 800v-576q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v576 +q0 14 9 23t23 9h64q14 0 23 -9t9 -23zM1152 76v948h-896v-948q0 -22 7 -40.5t14.5 -27t10.5 -8.5h832q3 0 10.5 8.5t14.5 27t7 40.5zM480 1152h448l-48 117q-7 9 -17 11h-317q-10 -2 -17 -11zM1408 1120v-64q0 -14 -9 -23t-23 -9h-96v-948q0 -83 -47 -143.5t-113 -60.5h-832 +q-66 0 -113 58.5t-47 141.5v952h-96q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h309l70 167q15 37 54 63t79 26h320q40 0 79 -26t54 -63l70 -167h309q14 0 23 -9t9 -23z" /> + <glyph glyph-name="home" unicode="" horiz-adv-x="1664" +d="M1408 544v-480q0 -26 -19 -45t-45 -19h-384v384h-256v-384h-384q-26 0 -45 19t-19 45v480q0 1 0.5 3t0.5 3l575 474l575 -474q1 -2 1 -6zM1631 613l-62 -74q-8 -9 -21 -11h-3q-13 0 -21 7l-692 577l-692 -577q-12 -8 -24 -7q-13 2 -21 11l-62 74q-8 10 -7 23.5t11 21.5 +l719 599q32 26 76 26t76 -26l244 -204v195q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-408l219 -182q10 -8 11 -21.5t-7 -23.5z" /> + <glyph glyph-name="file_alt" unicode="" +d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z +" /> + <glyph glyph-name="time" unicode="" +d="M896 992v-448q0 -14 -9 -23t-23 -9h-320q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h224v352q0 14 9 23t23 9h64q14 0 23 -9t9 -23zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640 +q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="road" unicode="" horiz-adv-x="1920" +d="M1111 540v4l-24 320q-1 13 -11 22.5t-23 9.5h-186q-13 0 -23 -9.5t-11 -22.5l-24 -320v-4q-1 -12 8 -20t21 -8h244q12 0 21 8t8 20zM1870 73q0 -73 -46 -73h-704q13 0 22 9.5t8 22.5l-20 256q-1 13 -11 22.5t-23 9.5h-272q-13 0 -23 -9.5t-11 -22.5l-20 -256 +q-1 -13 8 -22.5t22 -9.5h-704q-46 0 -46 73q0 54 26 116l417 1044q8 19 26 33t38 14h339q-13 0 -23 -9.5t-11 -22.5l-15 -192q-1 -14 8 -23t22 -9h166q13 0 22 9t8 23l-15 192q-1 13 -11 22.5t-23 9.5h339q20 0 38 -14t26 -33l417 -1044q26 -62 26 -116z" /> + <glyph glyph-name="download_alt" unicode="" horiz-adv-x="1664" +d="M1280 192q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1536 192q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1664 416v-320q0 -40 -28 -68t-68 -28h-1472q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h465l135 -136 +q58 -56 136 -56t136 56l136 136h464q40 0 68 -28t28 -68zM1339 985q17 -41 -14 -70l-448 -448q-18 -19 -45 -19t-45 19l-448 448q-31 29 -14 70q17 39 59 39h256v448q0 26 19 45t45 19h256q26 0 45 -19t19 -45v-448h256q42 0 59 -39z" /> + <glyph glyph-name="download" unicode="" +d="M1120 608q0 -12 -10 -24l-319 -319q-11 -9 -23 -9t-23 9l-320 320q-15 16 -7 35q8 20 30 20h192v352q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-352h192q14 0 23 -9t9 -23zM768 1184q-148 0 -273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273 +t-73 273t-198 198t-273 73zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="upload" unicode="" +d="M1118 660q-8 -20 -30 -20h-192v-352q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v352h-192q-14 0 -23 9t-9 23q0 12 10 24l319 319q11 9 23 9t23 -9l320 -320q15 -16 7 -35zM768 1184q-148 0 -273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198 +t73 273t-73 273t-198 198t-273 73zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="inbox" unicode="" +d="M1023 576h316q-1 3 -2.5 8.5t-2.5 7.5l-212 496h-708l-212 -496q-1 -3 -2.5 -8.5t-2.5 -7.5h316l95 -192h320zM1536 546v-482q0 -26 -19 -45t-45 -19h-1408q-26 0 -45 19t-19 45v482q0 62 25 123l238 552q10 25 36.5 42t52.5 17h832q26 0 52.5 -17t36.5 -42l238 -552 +q25 -61 25 -123z" /> + <glyph glyph-name="play_circle" unicode="" +d="M1184 640q0 -37 -32 -55l-544 -320q-15 -9 -32 -9q-16 0 -32 8q-32 19 -32 56v640q0 37 32 56q33 18 64 -1l544 -320q32 -18 32 -55zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640 +q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="repeat" unicode="" +d="M1536 1280v-448q0 -26 -19 -45t-45 -19h-448q-42 0 -59 40q-17 39 14 69l138 138q-148 137 -349 137q-104 0 -198.5 -40.5t-163.5 -109.5t-109.5 -163.5t-40.5 -198.5t40.5 -198.5t109.5 -163.5t163.5 -109.5t198.5 -40.5q119 0 225 52t179 147q7 10 23 12q15 0 25 -9 +l137 -138q9 -8 9.5 -20.5t-7.5 -22.5q-109 -132 -264 -204.5t-327 -72.5q-156 0 -298 61t-245 164t-164 245t-61 298t61 298t164 245t245 164t298 61q147 0 284.5 -55.5t244.5 -156.5l130 129q29 31 70 14q39 -17 39 -59z" /> + <glyph glyph-name="refresh" unicode="" +d="M1511 480q0 -5 -1 -7q-64 -268 -268 -434.5t-478 -166.5q-146 0 -282.5 55t-243.5 157l-129 -129q-19 -19 -45 -19t-45 19t-19 45v448q0 26 19 45t45 19h448q26 0 45 -19t19 -45t-19 -45l-137 -137q71 -66 161 -102t187 -36q134 0 250 65t186 179q11 17 53 117 +q8 23 30 23h192q13 0 22.5 -9.5t9.5 -22.5zM1536 1280v-448q0 -26 -19 -45t-45 -19h-448q-26 0 -45 19t-19 45t19 45l138 138q-148 137 -349 137q-134 0 -250 -65t-186 -179q-11 -17 -53 -117q-8 -23 -30 -23h-199q-13 0 -22.5 9.5t-9.5 22.5v7q65 268 270 434.5t480 166.5 +q146 0 284 -55.5t245 -156.5l130 129q19 19 45 19t45 -19t19 -45z" /> + <glyph glyph-name="list_alt" unicode="" horiz-adv-x="1792" +d="M384 352v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 608v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M384 864v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM1536 352v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-960q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h960q13 0 22.5 -9.5t9.5 -22.5z +M1536 608v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-960q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h960q13 0 22.5 -9.5t9.5 -22.5zM1536 864v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-960q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h960q13 0 22.5 -9.5 +t9.5 -22.5zM1664 160v832q0 13 -9.5 22.5t-22.5 9.5h-1472q-13 0 -22.5 -9.5t-9.5 -22.5v-832q0 -13 9.5 -22.5t22.5 -9.5h1472q13 0 22.5 9.5t9.5 22.5zM1792 1248v-1088q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h1472q66 0 113 -47 +t47 -113z" /> + <glyph glyph-name="lock" unicode="" horiz-adv-x="1152" +d="M320 768h512v192q0 106 -75 181t-181 75t-181 -75t-75 -181v-192zM1152 672v-576q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v576q0 40 28 68t68 28h32v192q0 184 132 316t316 132t316 -132t132 -316v-192h32q40 0 68 -28t28 -68z" /> + <glyph glyph-name="flag" unicode="" horiz-adv-x="1792" +d="M320 1280q0 -72 -64 -110v-1266q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v1266q-64 38 -64 110q0 53 37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1792 1216v-763q0 -25 -12.5 -38.5t-39.5 -27.5q-215 -116 -369 -116q-61 0 -123.5 22t-108.5 48 +t-115.5 48t-142.5 22q-192 0 -464 -146q-17 -9 -33 -9q-26 0 -45 19t-19 45v742q0 32 31 55q21 14 79 43q236 120 421 120q107 0 200 -29t219 -88q38 -19 88 -19q54 0 117.5 21t110 47t88 47t54.5 21q26 0 45 -19t19 -45z" /> + <glyph glyph-name="headphones" unicode="" horiz-adv-x="1664" +d="M1664 650q0 -166 -60 -314l-20 -49l-185 -33q-22 -83 -90.5 -136.5t-156.5 -53.5v-32q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v576q0 14 9 23t23 9h64q14 0 23 -9t9 -23v-32q71 0 130 -35.5t93 -95.5l68 12q29 95 29 193q0 148 -88 279t-236.5 209t-315.5 78 +t-315.5 -78t-236.5 -209t-88 -279q0 -98 29 -193l68 -12q34 60 93 95.5t130 35.5v32q0 14 9 23t23 9h64q14 0 23 -9t9 -23v-576q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v32q-88 0 -156.5 53.5t-90.5 136.5l-185 33l-20 49q-60 148 -60 314q0 151 67 291t179 242.5 +t266 163.5t320 61t320 -61t266 -163.5t179 -242.5t67 -291z" /> + <glyph glyph-name="volume_off" unicode="" horiz-adv-x="768" +d="M768 1184v-1088q0 -26 -19 -45t-45 -19t-45 19l-333 333h-262q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h262l333 333q19 19 45 19t45 -19t19 -45z" /> + <glyph glyph-name="volume_down" unicode="" horiz-adv-x="1152" +d="M768 1184v-1088q0 -26 -19 -45t-45 -19t-45 19l-333 333h-262q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h262l333 333q19 19 45 19t45 -19t19 -45zM1152 640q0 -76 -42.5 -141.5t-112.5 -93.5q-10 -5 -25 -5q-26 0 -45 18.5t-19 45.5q0 21 12 35.5t29 25t34 23t29 36 +t12 56.5t-12 56.5t-29 36t-34 23t-29 25t-12 35.5q0 27 19 45.5t45 18.5q15 0 25 -5q70 -27 112.5 -93t42.5 -142z" /> + <glyph glyph-name="volume_up" unicode="" horiz-adv-x="1664" +d="M768 1184v-1088q0 -26 -19 -45t-45 -19t-45 19l-333 333h-262q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h262l333 333q19 19 45 19t45 -19t19 -45zM1152 640q0 -76 -42.5 -141.5t-112.5 -93.5q-10 -5 -25 -5q-26 0 -45 18.5t-19 45.5q0 21 12 35.5t29 25t34 23t29 36 +t12 56.5t-12 56.5t-29 36t-34 23t-29 25t-12 35.5q0 27 19 45.5t45 18.5q15 0 25 -5q70 -27 112.5 -93t42.5 -142zM1408 640q0 -153 -85 -282.5t-225 -188.5q-13 -5 -25 -5q-27 0 -46 19t-19 45q0 39 39 59q56 29 76 44q74 54 115.5 135.5t41.5 173.5t-41.5 173.5 +t-115.5 135.5q-20 15 -76 44q-39 20 -39 59q0 26 19 45t45 19q13 0 26 -5q140 -59 225 -188.5t85 -282.5zM1664 640q0 -230 -127 -422.5t-338 -283.5q-13 -5 -26 -5q-26 0 -45 19t-19 45q0 36 39 59q7 4 22.5 10.5t22.5 10.5q46 25 82 51q123 91 192 227t69 289t-69 289 +t-192 227q-36 26 -82 51q-7 4 -22.5 10.5t-22.5 10.5q-39 23 -39 59q0 26 19 45t45 19q13 0 26 -5q211 -91 338 -283.5t127 -422.5z" /> + <glyph glyph-name="qrcode" unicode="" horiz-adv-x="1408" +d="M384 384v-128h-128v128h128zM384 1152v-128h-128v128h128zM1152 1152v-128h-128v128h128zM128 129h384v383h-384v-383zM128 896h384v384h-384v-384zM896 896h384v384h-384v-384zM640 640v-640h-640v640h640zM1152 128v-128h-128v128h128zM1408 128v-128h-128v128h128z +M1408 640v-384h-384v128h-128v-384h-128v640h384v-128h128v128h128zM640 1408v-640h-640v640h640zM1408 1408v-640h-640v640h640z" /> + <glyph glyph-name="barcode" unicode="" horiz-adv-x="1792" +d="M63 0h-63v1408h63v-1408zM126 1h-32v1407h32v-1407zM220 1h-31v1407h31v-1407zM377 1h-31v1407h31v-1407zM534 1h-62v1407h62v-1407zM660 1h-31v1407h31v-1407zM723 1h-31v1407h31v-1407zM786 1h-31v1407h31v-1407zM943 1h-63v1407h63v-1407zM1100 1h-63v1407h63v-1407z +M1226 1h-63v1407h63v-1407zM1352 1h-63v1407h63v-1407zM1446 1h-63v1407h63v-1407zM1635 1h-94v1407h94v-1407zM1698 1h-32v1407h32v-1407zM1792 0h-63v1408h63v-1408z" /> + <glyph glyph-name="tag" unicode="" +d="M448 1088q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1515 512q0 -53 -37 -90l-491 -492q-39 -37 -91 -37q-53 0 -90 37l-715 716q-38 37 -64.5 101t-26.5 117v416q0 52 38 90t90 38h416q53 0 117 -26.5t102 -64.5 +l715 -714q37 -39 37 -91z" /> + <glyph glyph-name="tags" unicode="" horiz-adv-x="1920" +d="M448 1088q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1515 512q0 -53 -37 -90l-491 -492q-39 -37 -91 -37q-53 0 -90 37l-715 716q-38 37 -64.5 101t-26.5 117v416q0 52 38 90t90 38h416q53 0 117 -26.5t102 -64.5 +l715 -714q37 -39 37 -91zM1899 512q0 -53 -37 -90l-491 -492q-39 -37 -91 -37q-36 0 -59 14t-53 45l470 470q37 37 37 90q0 52 -37 91l-715 714q-38 38 -102 64.5t-117 26.5h224q53 0 117 -26.5t102 -64.5l715 -714q37 -39 37 -91z" /> + <glyph glyph-name="book" unicode="" horiz-adv-x="1664" +d="M1639 1058q40 -57 18 -129l-275 -906q-19 -64 -76.5 -107.5t-122.5 -43.5h-923q-77 0 -148.5 53.5t-99.5 131.5q-24 67 -2 127q0 4 3 27t4 37q1 8 -3 21.5t-3 19.5q2 11 8 21t16.5 23.5t16.5 23.5q23 38 45 91.5t30 91.5q3 10 0.5 30t-0.5 28q3 11 17 28t17 23 +q21 36 42 92t25 90q1 9 -2.5 32t0.5 28q4 13 22 30.5t22 22.5q19 26 42.5 84.5t27.5 96.5q1 8 -3 25.5t-2 26.5q2 8 9 18t18 23t17 21q8 12 16.5 30.5t15 35t16 36t19.5 32t26.5 23.5t36 11.5t47.5 -5.5l-1 -3q38 9 51 9h761q74 0 114 -56t18 -130l-274 -906 +q-36 -119 -71.5 -153.5t-128.5 -34.5h-869q-27 0 -38 -15q-11 -16 -1 -43q24 -70 144 -70h923q29 0 56 15.5t35 41.5l300 987q7 22 5 57q38 -15 59 -43zM575 1056q-4 -13 2 -22.5t20 -9.5h608q13 0 25.5 9.5t16.5 22.5l21 64q4 13 -2 22.5t-20 9.5h-608q-13 0 -25.5 -9.5 +t-16.5 -22.5zM492 800q-4 -13 2 -22.5t20 -9.5h608q13 0 25.5 9.5t16.5 22.5l21 64q4 13 -2 22.5t-20 9.5h-608q-13 0 -25.5 -9.5t-16.5 -22.5z" /> + <glyph glyph-name="bookmark" unicode="" horiz-adv-x="1280" +d="M1164 1408q23 0 44 -9q33 -13 52.5 -41t19.5 -62v-1289q0 -34 -19.5 -62t-52.5 -41q-19 -8 -44 -8q-48 0 -83 32l-441 424l-441 -424q-36 -33 -83 -33q-23 0 -44 9q-33 13 -52.5 41t-19.5 62v1289q0 34 19.5 62t52.5 41q21 9 44 9h1048z" /> + <glyph glyph-name="print" unicode="" horiz-adv-x="1664" +d="M384 0h896v256h-896v-256zM384 640h896v384h-160q-40 0 -68 28t-28 68v160h-640v-640zM1536 576q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1664 576v-416q0 -13 -9.5 -22.5t-22.5 -9.5h-224v-160q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68 +v160h-224q-13 0 -22.5 9.5t-9.5 22.5v416q0 79 56.5 135.5t135.5 56.5h64v544q0 40 28 68t68 28h672q40 0 88 -20t76 -48l152 -152q28 -28 48 -76t20 -88v-256h64q79 0 135.5 -56.5t56.5 -135.5z" /> + <glyph glyph-name="camera" unicode="" horiz-adv-x="1920" +d="M960 864q119 0 203.5 -84.5t84.5 -203.5t-84.5 -203.5t-203.5 -84.5t-203.5 84.5t-84.5 203.5t84.5 203.5t203.5 84.5zM1664 1280q106 0 181 -75t75 -181v-896q0 -106 -75 -181t-181 -75h-1408q-106 0 -181 75t-75 181v896q0 106 75 181t181 75h224l51 136 +q19 49 69.5 84.5t103.5 35.5h512q53 0 103.5 -35.5t69.5 -84.5l51 -136h224zM960 128q185 0 316.5 131.5t131.5 316.5t-131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5z" /> + <glyph glyph-name="font" unicode="" horiz-adv-x="1664" +d="M725 977l-170 -450q33 0 136.5 -2t160.5 -2q19 0 57 2q-87 253 -184 452zM0 -128l2 79q23 7 56 12.5t57 10.5t49.5 14.5t44.5 29t31 50.5l237 616l280 724h75h53q8 -14 11 -21l205 -480q33 -78 106 -257.5t114 -274.5q15 -34 58 -144.5t72 -168.5q20 -45 35 -57 +q19 -15 88 -29.5t84 -20.5q6 -38 6 -57q0 -5 -0.5 -13.5t-0.5 -12.5q-63 0 -190 8t-191 8q-76 0 -215 -7t-178 -8q0 43 4 78l131 28q1 0 12.5 2.5t15.5 3.5t14.5 4.5t15 6.5t11 8t9 11t2.5 14q0 16 -31 96.5t-72 177.5t-42 100l-450 2q-26 -58 -76.5 -195.5t-50.5 -162.5 +q0 -22 14 -37.5t43.5 -24.5t48.5 -13.5t57 -8.5t41 -4q1 -19 1 -58q0 -9 -2 -27q-58 0 -174.5 10t-174.5 10q-8 0 -26.5 -4t-21.5 -4q-80 -14 -188 -14z" /> + <glyph glyph-name="bold" unicode="" horiz-adv-x="1408" +d="M555 15q74 -32 140 -32q376 0 376 335q0 114 -41 180q-27 44 -61.5 74t-67.5 46.5t-80.5 25t-84 10.5t-94.5 2q-73 0 -101 -10q0 -53 -0.5 -159t-0.5 -158q0 -8 -1 -67.5t-0.5 -96.5t4.5 -83.5t12 -66.5zM541 761q42 -7 109 -7q82 0 143 13t110 44.5t74.5 89.5t25.5 142 +q0 70 -29 122.5t-79 82t-108 43.5t-124 14q-50 0 -130 -13q0 -50 4 -151t4 -152q0 -27 -0.5 -80t-0.5 -79q0 -46 1 -69zM0 -128l2 94q15 4 85 16t106 27q7 12 12.5 27t8.5 33.5t5.5 32.5t3 37.5t0.5 34v35.5v30q0 982 -22 1025q-4 8 -22 14.5t-44.5 11t-49.5 7t-48.5 4.5 +t-30.5 3l-4 83q98 2 340 11.5t373 9.5q23 0 68 -0.5t68 -0.5q70 0 136.5 -13t128.5 -42t108 -71t74 -104.5t28 -137.5q0 -52 -16.5 -95.5t-39 -72t-64.5 -57.5t-73 -45t-84 -40q154 -35 256.5 -134t102.5 -248q0 -100 -35 -179.5t-93.5 -130.5t-138 -85.5t-163.5 -48.5 +t-176 -14q-44 0 -132 3t-132 3q-106 0 -307 -11t-231 -12z" /> + <glyph glyph-name="italic" unicode="" horiz-adv-x="1024" +d="M0 -126l17 85q22 7 61.5 16.5t72 19t59.5 23.5q28 35 41 101q1 7 62 289t114 543.5t52 296.5v25q-24 13 -54.5 18.5t-69.5 8t-58 5.5l19 103q33 -2 120 -6.5t149.5 -7t120.5 -2.5q48 0 98.5 2.5t121 7t98.5 6.5q-5 -39 -19 -89q-30 -10 -101.5 -28.5t-108.5 -33.5 +q-8 -19 -14 -42.5t-9 -40t-7.5 -45.5t-6.5 -42q-27 -148 -87.5 -419.5t-77.5 -355.5q-2 -9 -13 -58t-20 -90t-16 -83.5t-6 -57.5l1 -18q17 -4 185 -31q-3 -44 -16 -99q-11 0 -32.5 -1.5t-32.5 -1.5q-29 0 -87 10t-86 10q-138 2 -206 2q-51 0 -143 -9t-121 -11z" /> + <glyph glyph-name="text_height" unicode="" horiz-adv-x="1792" +d="M1744 128q33 0 42 -18.5t-11 -44.5l-126 -162q-20 -26 -49 -26t-49 26l-126 162q-20 26 -11 44.5t42 18.5h80v1024h-80q-33 0 -42 18.5t11 44.5l126 162q20 26 49 26t49 -26l126 -162q20 -26 11 -44.5t-42 -18.5h-80v-1024h80zM81 1407l54 -27q12 -5 211 -5q44 0 132 2 +t132 2q36 0 107.5 -0.5t107.5 -0.5h293q6 0 21 -0.5t20.5 0t16 3t17.5 9t15 17.5l42 1q4 0 14 -0.5t14 -0.5q2 -112 2 -336q0 -80 -5 -109q-39 -14 -68 -18q-25 44 -54 128q-3 9 -11 48t-14.5 73.5t-7.5 35.5q-6 8 -12 12.5t-15.5 6t-13 2.5t-18 0.5t-16.5 -0.5 +q-17 0 -66.5 0.5t-74.5 0.5t-64 -2t-71 -6q-9 -81 -8 -136q0 -94 2 -388t2 -455q0 -16 -2.5 -71.5t0 -91.5t12.5 -69q40 -21 124 -42.5t120 -37.5q5 -40 5 -50q0 -14 -3 -29l-34 -1q-76 -2 -218 8t-207 10q-50 0 -151 -9t-152 -9q-3 51 -3 52v9q17 27 61.5 43t98.5 29t78 27 +q19 42 19 383q0 101 -3 303t-3 303v117q0 2 0.5 15.5t0.5 25t-1 25.5t-3 24t-5 14q-11 12 -162 12q-33 0 -93 -12t-80 -26q-19 -13 -34 -72.5t-31.5 -111t-42.5 -53.5q-42 26 -56 44v383z" /> + <glyph glyph-name="text_width" unicode="" +d="M81 1407l54 -27q12 -5 211 -5q44 0 132 2t132 2q70 0 246.5 1t304.5 0.5t247 -4.5q33 -1 56 31l42 1q4 0 14 -0.5t14 -0.5q2 -112 2 -336q0 -80 -5 -109q-39 -14 -68 -18q-25 44 -54 128q-3 9 -11 47.5t-15 73.5t-7 36q-10 13 -27 19q-5 2 -66 2q-30 0 -93 1t-103 1 +t-94 -2t-96 -7q-9 -81 -8 -136l1 -152v52q0 -55 1 -154t1.5 -180t0.5 -153q0 -16 -2.5 -71.5t0 -91.5t12.5 -69q40 -21 124 -42.5t120 -37.5q5 -40 5 -50q0 -14 -3 -29l-34 -1q-76 -2 -218 8t-207 10q-50 0 -151 -9t-152 -9q-3 51 -3 52v9q17 27 61.5 43t98.5 29t78 27 +q7 16 11.5 74t6 145.5t1.5 155t-0.5 153.5t-0.5 89q0 7 -2.5 21.5t-2.5 22.5q0 7 0.5 44t1 73t0 76.5t-3 67.5t-6.5 32q-11 12 -162 12q-41 0 -163 -13.5t-138 -24.5q-19 -12 -34 -71.5t-31.5 -111.5t-42.5 -54q-42 26 -56 44v383zM1310 125q12 0 42 -19.5t57.5 -41.5 +t59.5 -49t36 -30q26 -21 26 -49t-26 -49q-4 -3 -36 -30t-59.5 -49t-57.5 -41.5t-42 -19.5q-13 0 -20.5 10.5t-10 28.5t-2.5 33.5t1.5 33t1.5 19.5h-1024q0 -2 1.5 -19.5t1.5 -33t-2.5 -33.5t-10 -28.5t-20.5 -10.5q-12 0 -42 19.5t-57.5 41.5t-59.5 49t-36 30q-26 21 -26 49 +t26 49q4 3 36 30t59.5 49t57.5 41.5t42 19.5q13 0 20.5 -10.5t10 -28.5t2.5 -33.5t-1.5 -33t-1.5 -19.5h1024q0 2 -1.5 19.5t-1.5 33t2.5 33.5t10 28.5t20.5 10.5z" /> + <glyph glyph-name="align_left" unicode="" horiz-adv-x="1792" +d="M1792 192v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1408 576v-128q0 -26 -19 -45t-45 -19h-1280q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1280q26 0 45 -19t19 -45zM1664 960v-128q0 -26 -19 -45 +t-45 -19h-1536q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1536q26 0 45 -19t19 -45zM1280 1344v-128q0 -26 -19 -45t-45 -19h-1152q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1152q26 0 45 -19t19 -45z" /> + <glyph glyph-name="align_center" unicode="" horiz-adv-x="1792" +d="M1792 192v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1408 576v-128q0 -26 -19 -45t-45 -19h-896q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h896q26 0 45 -19t19 -45zM1664 960v-128q0 -26 -19 -45t-45 -19 +h-1408q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1408q26 0 45 -19t19 -45zM1280 1344v-128q0 -26 -19 -45t-45 -19h-640q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h640q26 0 45 -19t19 -45z" /> + <glyph glyph-name="align_right" unicode="" horiz-adv-x="1792" +d="M1792 192v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 576v-128q0 -26 -19 -45t-45 -19h-1280q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1280q26 0 45 -19t19 -45zM1792 960v-128q0 -26 -19 -45 +t-45 -19h-1536q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1536q26 0 45 -19t19 -45zM1792 1344v-128q0 -26 -19 -45t-45 -19h-1152q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1152q26 0 45 -19t19 -45z" /> + <glyph glyph-name="align_justify" unicode="" horiz-adv-x="1792" +d="M1792 192v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 576v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 960v-128q0 -26 -19 -45 +t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 1344v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45z" /> + <glyph glyph-name="list" unicode="" horiz-adv-x="1792" +d="M256 224v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-192q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h192q13 0 22.5 -9.5t9.5 -22.5zM256 608v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-192q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h192q13 0 22.5 -9.5 +t9.5 -22.5zM256 992v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-192q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h192q13 0 22.5 -9.5t9.5 -22.5zM1792 224v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1344q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1344 +q13 0 22.5 -9.5t9.5 -22.5zM256 1376v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-192q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h192q13 0 22.5 -9.5t9.5 -22.5zM1792 608v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1344q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5 +t22.5 9.5h1344q13 0 22.5 -9.5t9.5 -22.5zM1792 992v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1344q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1344q13 0 22.5 -9.5t9.5 -22.5zM1792 1376v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1344q-13 0 -22.5 9.5t-9.5 22.5v192 +q0 13 9.5 22.5t22.5 9.5h1344q13 0 22.5 -9.5t9.5 -22.5z" /> + <glyph glyph-name="indent_left" unicode="" horiz-adv-x="1792" +d="M384 992v-576q0 -13 -9.5 -22.5t-22.5 -9.5q-14 0 -23 9l-288 288q-9 9 -9 23t9 23l288 288q9 9 23 9q13 0 22.5 -9.5t9.5 -22.5zM1792 224v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1728q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1728q13 0 22.5 -9.5 +t9.5 -22.5zM1792 608v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1088q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1088q13 0 22.5 -9.5t9.5 -22.5zM1792 992v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1088q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1088 +q13 0 22.5 -9.5t9.5 -22.5zM1792 1376v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1728q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1728q13 0 22.5 -9.5t9.5 -22.5z" /> + <glyph glyph-name="indent_right" unicode="" horiz-adv-x="1792" +d="M352 704q0 -14 -9 -23l-288 -288q-9 -9 -23 -9q-13 0 -22.5 9.5t-9.5 22.5v576q0 13 9.5 22.5t22.5 9.5q14 0 23 -9l288 -288q9 -9 9 -23zM1792 224v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1728q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1728q13 0 22.5 -9.5 +t9.5 -22.5zM1792 608v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1088q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1088q13 0 22.5 -9.5t9.5 -22.5zM1792 992v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1088q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1088 +q13 0 22.5 -9.5t9.5 -22.5zM1792 1376v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1728q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1728q13 0 22.5 -9.5t9.5 -22.5z" /> + <glyph glyph-name="facetime_video" unicode="" horiz-adv-x="1792" +d="M1792 1184v-1088q0 -42 -39 -59q-13 -5 -25 -5q-27 0 -45 19l-403 403v-166q0 -119 -84.5 -203.5t-203.5 -84.5h-704q-119 0 -203.5 84.5t-84.5 203.5v704q0 119 84.5 203.5t203.5 84.5h704q119 0 203.5 -84.5t84.5 -203.5v-165l403 402q18 19 45 19q12 0 25 -5 +q39 -17 39 -59z" /> + <glyph glyph-name="picture" unicode="" horiz-adv-x="1920" +d="M640 960q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM1664 576v-448h-1408v192l320 320l160 -160l512 512zM1760 1280h-1600q-13 0 -22.5 -9.5t-9.5 -22.5v-1216q0 -13 9.5 -22.5t22.5 -9.5h1600q13 0 22.5 9.5t9.5 22.5v1216 +q0 13 -9.5 22.5t-22.5 9.5zM1920 1248v-1216q0 -66 -47 -113t-113 -47h-1600q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1600q66 0 113 -47t47 -113z" /> + <glyph glyph-name="pencil" unicode="" +d="M363 0l91 91l-235 235l-91 -91v-107h128v-128h107zM886 928q0 22 -22 22q-10 0 -17 -7l-542 -542q-7 -7 -7 -17q0 -22 22 -22q10 0 17 7l542 542q7 7 7 17zM832 1120l416 -416l-832 -832h-416v416zM1515 1024q0 -53 -37 -90l-166 -166l-416 416l166 165q36 38 90 38 +q53 0 91 -38l235 -234q37 -39 37 -91z" /> + <glyph glyph-name="map_marker" unicode="" horiz-adv-x="1024" +d="M768 896q0 106 -75 181t-181 75t-181 -75t-75 -181t75 -181t181 -75t181 75t75 181zM1024 896q0 -109 -33 -179l-364 -774q-16 -33 -47.5 -52t-67.5 -19t-67.5 19t-46.5 52l-365 774q-33 70 -33 179q0 212 150 362t362 150t362 -150t150 -362z" /> + <glyph glyph-name="adjust" unicode="" +d="M768 96v1088q-148 0 -273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="tint" unicode="" horiz-adv-x="1024" +d="M512 384q0 36 -20 69q-1 1 -15.5 22.5t-25.5 38t-25 44t-21 50.5q-4 16 -21 16t-21 -16q-7 -23 -21 -50.5t-25 -44t-25.5 -38t-15.5 -22.5q-20 -33 -20 -69q0 -53 37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1024 512q0 -212 -150 -362t-362 -150t-362 150t-150 362 +q0 145 81 275q6 9 62.5 90.5t101 151t99.5 178t83 201.5q9 30 34 47t51 17t51.5 -17t33.5 -47q28 -93 83 -201.5t99.5 -178t101 -151t62.5 -90.5q81 -127 81 -275z" /> + <glyph glyph-name="edit" unicode="" horiz-adv-x="1792" +d="M888 352l116 116l-152 152l-116 -116v-56h96v-96h56zM1328 1072q-16 16 -33 -1l-350 -350q-17 -17 -1 -33t33 1l350 350q17 17 1 33zM1408 478v-190q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h832 +q63 0 117 -25q15 -7 18 -23q3 -17 -9 -29l-49 -49q-14 -14 -32 -8q-23 6 -45 6h-832q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832q66 0 113 47t47 113v126q0 13 9 22l64 64q15 15 35 7t20 -29zM1312 1216l288 -288l-672 -672h-288v288zM1756 1084l-92 -92 +l-288 288l92 92q28 28 68 28t68 -28l152 -152q28 -28 28 -68t-28 -68z" /> + <glyph glyph-name="share" unicode="" horiz-adv-x="1664" +d="M1408 547v-259q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h255v0q13 0 22.5 -9.5t9.5 -22.5q0 -27 -26 -32q-77 -26 -133 -60q-10 -4 -16 -4h-112q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832 +q66 0 113 47t47 113v214q0 19 18 29q28 13 54 37q16 16 35 8q21 -9 21 -29zM1645 1043l-384 -384q-18 -19 -45 -19q-12 0 -25 5q-39 17 -39 59v192h-160q-323 0 -438 -131q-119 -137 -74 -473q3 -23 -20 -34q-8 -2 -12 -2q-16 0 -26 13q-10 14 -21 31t-39.5 68.5t-49.5 99.5 +t-38.5 114t-17.5 122q0 49 3.5 91t14 90t28 88t47 81.5t68.5 74t94.5 61.5t124.5 48.5t159.5 30.5t196.5 11h160v192q0 42 39 59q13 5 25 5q26 0 45 -19l384 -384q19 -19 19 -45t-19 -45z" /> + <glyph glyph-name="check" unicode="" horiz-adv-x="1664" +d="M1408 606v-318q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h832q63 0 117 -25q15 -7 18 -23q3 -17 -9 -29l-49 -49q-10 -10 -23 -10q-3 0 -9 2q-23 6 -45 6h-832q-66 0 -113 -47t-47 -113v-832 +q0 -66 47 -113t113 -47h832q66 0 113 47t47 113v254q0 13 9 22l64 64q10 10 23 10q6 0 12 -3q20 -8 20 -29zM1639 1095l-814 -814q-24 -24 -57 -24t-57 24l-430 430q-24 24 -24 57t24 57l110 110q24 24 57 24t57 -24l263 -263l647 647q24 24 57 24t57 -24l110 -110 +q24 -24 24 -57t-24 -57z" /> + <glyph glyph-name="move" unicode="" horiz-adv-x="1792" +d="M1792 640q0 -26 -19 -45l-256 -256q-19 -19 -45 -19t-45 19t-19 45v128h-384v-384h128q26 0 45 -19t19 -45t-19 -45l-256 -256q-19 -19 -45 -19t-45 19l-256 256q-19 19 -19 45t19 45t45 19h128v384h-384v-128q0 -26 -19 -45t-45 -19t-45 19l-256 256q-19 19 -19 45 +t19 45l256 256q19 19 45 19t45 -19t19 -45v-128h384v384h-128q-26 0 -45 19t-19 45t19 45l256 256q19 19 45 19t45 -19l256 -256q19 -19 19 -45t-19 -45t-45 -19h-128v-384h384v128q0 26 19 45t45 19t45 -19l256 -256q19 -19 19 -45z" /> + <glyph glyph-name="step_backward" unicode="" horiz-adv-x="1024" +d="M979 1395q19 19 32 13t13 -32v-1472q0 -26 -13 -32t-32 13l-710 710q-9 9 -13 19v-678q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-678q4 10 13 19z" /> + <glyph glyph-name="fast_backward" unicode="" horiz-adv-x="1792" +d="M1747 1395q19 19 32 13t13 -32v-1472q0 -26 -13 -32t-32 13l-710 710q-9 9 -13 19v-710q0 -26 -13 -32t-32 13l-710 710q-9 9 -13 19v-678q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-678q4 10 13 19l710 710 +q19 19 32 13t13 -32v-710q4 10 13 19z" /> + <glyph glyph-name="backward" unicode="" horiz-adv-x="1664" +d="M1619 1395q19 19 32 13t13 -32v-1472q0 -26 -13 -32t-32 13l-710 710q-9 9 -13 19v-710q0 -26 -13 -32t-32 13l-710 710q-19 19 -19 45t19 45l710 710q19 19 32 13t13 -32v-710q4 10 13 19z" /> + <glyph glyph-name="play" unicode="" horiz-adv-x="1408" +d="M1384 609l-1328 -738q-23 -13 -39.5 -3t-16.5 36v1472q0 26 16.5 36t39.5 -3l1328 -738q23 -13 23 -31t-23 -31z" /> + <glyph glyph-name="pause" unicode="" +d="M1536 1344v-1408q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h512q26 0 45 -19t19 -45zM640 1344v-1408q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h512q26 0 45 -19t19 -45z" /> + <glyph glyph-name="stop" unicode="" +d="M1536 1344v-1408q0 -26 -19 -45t-45 -19h-1408q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h1408q26 0 45 -19t19 -45z" /> + <glyph glyph-name="forward" unicode="" horiz-adv-x="1664" +d="M45 -115q-19 -19 -32 -13t-13 32v1472q0 26 13 32t32 -13l710 -710q9 -9 13 -19v710q0 26 13 32t32 -13l710 -710q19 -19 19 -45t-19 -45l-710 -710q-19 -19 -32 -13t-13 32v710q-4 -10 -13 -19z" /> + <glyph glyph-name="fast_forward" unicode="" horiz-adv-x="1792" +d="M45 -115q-19 -19 -32 -13t-13 32v1472q0 26 13 32t32 -13l710 -710q9 -9 13 -19v710q0 26 13 32t32 -13l710 -710q9 -9 13 -19v678q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-1408q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v678q-4 -10 -13 -19l-710 -710 +q-19 -19 -32 -13t-13 32v710q-4 -10 -13 -19z" /> + <glyph glyph-name="step_forward" unicode="" horiz-adv-x="1024" +d="M45 -115q-19 -19 -32 -13t-13 32v1472q0 26 13 32t32 -13l710 -710q9 -9 13 -19v678q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-1408q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v678q-4 -10 -13 -19z" /> + <glyph glyph-name="eject" unicode="" horiz-adv-x="1538" +d="M14 557l710 710q19 19 45 19t45 -19l710 -710q19 -19 13 -32t-32 -13h-1472q-26 0 -32 13t13 32zM1473 0h-1408q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h1408q26 0 45 -19t19 -45v-256q0 -26 -19 -45t-45 -19z" /> + <glyph glyph-name="chevron_left" unicode="" horiz-adv-x="1280" +d="M1171 1235l-531 -531l531 -531q19 -19 19 -45t-19 -45l-166 -166q-19 -19 -45 -19t-45 19l-742 742q-19 19 -19 45t19 45l742 742q19 19 45 19t45 -19l166 -166q19 -19 19 -45t-19 -45z" /> + <glyph glyph-name="chevron_right" unicode="" horiz-adv-x="1280" +d="M1107 659l-742 -742q-19 -19 -45 -19t-45 19l-166 166q-19 19 -19 45t19 45l531 531l-531 531q-19 19 -19 45t19 45l166 166q19 19 45 19t45 -19l742 -742q19 -19 19 -45t-19 -45z" /> + <glyph glyph-name="plus_sign" unicode="" +d="M1216 576v128q0 26 -19 45t-45 19h-256v256q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-256h-256q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h256v-256q0 -26 19 -45t45 -19h128q26 0 45 19t19 45v256h256q26 0 45 19t19 45zM1536 640q0 -209 -103 -385.5 +t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="minus_sign" unicode="" +d="M1216 576v128q0 26 -19 45t-45 19h-768q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h768q26 0 45 19t19 45zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5 +t103 -385.5z" /> + <glyph glyph-name="remove_sign" unicode="" +d="M1149 414q0 26 -19 45l-181 181l181 181q19 19 19 45q0 27 -19 46l-90 90q-19 19 -46 19q-26 0 -45 -19l-181 -181l-181 181q-19 19 -45 19q-27 0 -46 -19l-90 -90q-19 -19 -19 -46q0 -26 19 -45l181 -181l-181 -181q-19 -19 -19 -45q0 -27 19 -46l90 -90q19 -19 46 -19 +q26 0 45 19l181 181l181 -181q19 -19 45 -19q27 0 46 19l90 90q19 19 19 46zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="ok_sign" unicode="" +d="M1284 802q0 28 -18 46l-91 90q-19 19 -45 19t-45 -19l-408 -407l-226 226q-19 19 -45 19t-45 -19l-91 -90q-18 -18 -18 -46q0 -27 18 -45l362 -362q19 -19 45 -19q27 0 46 19l543 543q18 18 18 45zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103 +t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="question_sign" unicode="" +d="M896 160v192q0 14 -9 23t-23 9h-192q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h192q14 0 23 9t9 23zM1152 832q0 88 -55.5 163t-138.5 116t-170 41q-243 0 -371 -213q-15 -24 8 -42l132 -100q7 -6 19 -6q16 0 25 12q53 68 86 92q34 24 86 24q48 0 85.5 -26t37.5 -59 +q0 -38 -20 -61t-68 -45q-63 -28 -115.5 -86.5t-52.5 -125.5v-36q0 -14 9 -23t23 -9h192q14 0 23 9t9 23q0 19 21.5 49.5t54.5 49.5q32 18 49 28.5t46 35t44.5 48t28 60.5t12.5 81zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5 +t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="info_sign" unicode="" +d="M1024 160v160q0 14 -9 23t-23 9h-96v512q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-160q0 -14 9 -23t23 -9h96v-320h-96q-14 0 -23 -9t-9 -23v-160q0 -14 9 -23t23 -9h448q14 0 23 9t9 23zM896 1056v160q0 14 -9 23t-23 9h-192q-14 0 -23 -9t-9 -23v-160q0 -14 9 -23 +t23 -9h192q14 0 23 9t9 23zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="screenshot" unicode="" +d="M1197 512h-109q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h109q-32 108 -112.5 188.5t-188.5 112.5v-109q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v109q-108 -32 -188.5 -112.5t-112.5 -188.5h109q26 0 45 -19t19 -45v-128q0 -26 -19 -45t-45 -19h-109 +q32 -108 112.5 -188.5t188.5 -112.5v109q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-109q108 32 188.5 112.5t112.5 188.5zM1536 704v-128q0 -26 -19 -45t-45 -19h-143q-37 -161 -154.5 -278.5t-278.5 -154.5v-143q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v143 +q-161 37 -278.5 154.5t-154.5 278.5h-143q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h143q37 161 154.5 278.5t278.5 154.5v143q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-143q161 -37 278.5 -154.5t154.5 -278.5h143q26 0 45 -19t19 -45z" /> + <glyph glyph-name="remove_circle" unicode="" +d="M1097 457l-146 -146q-10 -10 -23 -10t-23 10l-137 137l-137 -137q-10 -10 -23 -10t-23 10l-146 146q-10 10 -10 23t10 23l137 137l-137 137q-10 10 -10 23t10 23l146 146q10 10 23 10t23 -10l137 -137l137 137q10 10 23 10t23 -10l146 -146q10 -10 10 -23t-10 -23 +l-137 -137l137 -137q10 -10 10 -23t-10 -23zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5 +t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="ok_circle" unicode="" +d="M1171 723l-422 -422q-19 -19 -45 -19t-45 19l-294 294q-19 19 -19 45t19 45l102 102q19 19 45 19t45 -19l147 -147l275 275q19 19 45 19t45 -19l102 -102q19 -19 19 -45t-19 -45zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273t73 -273t198 -198 +t273 -73t273 73t198 198t73 273zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="ban_circle" unicode="" +d="M1312 643q0 161 -87 295l-754 -753q137 -89 297 -89q111 0 211.5 43.5t173.5 116.5t116 174.5t43 212.5zM313 344l755 754q-135 91 -300 91q-148 0 -273 -73t-198 -199t-73 -274q0 -162 89 -299zM1536 643q0 -157 -61 -300t-163.5 -246t-245 -164t-298.5 -61t-298.5 61 +t-245 164t-163.5 246t-61 300t61 299.5t163.5 245.5t245 164t298.5 61t298.5 -61t245 -164t163.5 -245.5t61 -299.5z" /> + <glyph glyph-name="arrow_left" unicode="" +d="M1536 640v-128q0 -53 -32.5 -90.5t-84.5 -37.5h-704l293 -294q38 -36 38 -90t-38 -90l-75 -76q-37 -37 -90 -37q-52 0 -91 37l-651 652q-37 37 -37 90q0 52 37 91l651 650q38 38 91 38q52 0 90 -38l75 -74q38 -38 38 -91t-38 -91l-293 -293h704q52 0 84.5 -37.5 +t32.5 -90.5z" /> + <glyph glyph-name="arrow_right" unicode="" +d="M1472 576q0 -54 -37 -91l-651 -651q-39 -37 -91 -37q-51 0 -90 37l-75 75q-38 38 -38 91t38 91l293 293h-704q-52 0 -84.5 37.5t-32.5 90.5v128q0 53 32.5 90.5t84.5 37.5h704l-293 294q-38 36 -38 90t38 90l75 75q38 38 90 38q53 0 91 -38l651 -651q37 -35 37 -90z" /> + <glyph glyph-name="arrow_up" unicode="" horiz-adv-x="1664" +d="M1611 565q0 -51 -37 -90l-75 -75q-38 -38 -91 -38q-54 0 -90 38l-294 293v-704q0 -52 -37.5 -84.5t-90.5 -32.5h-128q-53 0 -90.5 32.5t-37.5 84.5v704l-294 -293q-36 -38 -90 -38t-90 38l-75 75q-38 38 -38 90q0 53 38 91l651 651q35 37 90 37q54 0 91 -37l651 -651 +q37 -39 37 -91z" /> + <glyph glyph-name="arrow_down" unicode="" horiz-adv-x="1664" +d="M1611 704q0 -53 -37 -90l-651 -652q-39 -37 -91 -37q-53 0 -90 37l-651 652q-38 36 -38 90q0 53 38 91l74 75q39 37 91 37q53 0 90 -37l294 -294v704q0 52 38 90t90 38h128q52 0 90 -38t38 -90v-704l294 294q37 37 90 37q52 0 91 -37l75 -75q37 -39 37 -91z" /> + <glyph glyph-name="share_alt" unicode="" horiz-adv-x="1792" +d="M1792 896q0 -26 -19 -45l-512 -512q-19 -19 -45 -19t-45 19t-19 45v256h-224q-98 0 -175.5 -6t-154 -21.5t-133 -42.5t-105.5 -69.5t-80 -101t-48.5 -138.5t-17.5 -181q0 -55 5 -123q0 -6 2.5 -23.5t2.5 -26.5q0 -15 -8.5 -25t-23.5 -10q-16 0 -28 17q-7 9 -13 22 +t-13.5 30t-10.5 24q-127 285 -127 451q0 199 53 333q162 403 875 403h224v256q0 26 19 45t45 19t45 -19l512 -512q19 -19 19 -45z" /> + <glyph glyph-name="resize_full" unicode="" +d="M755 480q0 -13 -10 -23l-332 -332l144 -144q19 -19 19 -45t-19 -45t-45 -19h-448q-26 0 -45 19t-19 45v448q0 26 19 45t45 19t45 -19l144 -144l332 332q10 10 23 10t23 -10l114 -114q10 -10 10 -23zM1536 1344v-448q0 -26 -19 -45t-45 -19t-45 19l-144 144l-332 -332 +q-10 -10 -23 -10t-23 10l-114 114q-10 10 -10 23t10 23l332 332l-144 144q-19 19 -19 45t19 45t45 19h448q26 0 45 -19t19 -45z" /> + <glyph glyph-name="resize_small" unicode="" +d="M768 576v-448q0 -26 -19 -45t-45 -19t-45 19l-144 144l-332 -332q-10 -10 -23 -10t-23 10l-114 114q-10 10 -10 23t10 23l332 332l-144 144q-19 19 -19 45t19 45t45 19h448q26 0 45 -19t19 -45zM1523 1248q0 -13 -10 -23l-332 -332l144 -144q19 -19 19 -45t-19 -45 +t-45 -19h-448q-26 0 -45 19t-19 45v448q0 26 19 45t45 19t45 -19l144 -144l332 332q10 10 23 10t23 -10l114 -114q10 -10 10 -23z" /> + <glyph glyph-name="plus" unicode="" horiz-adv-x="1408" +d="M1408 800v-192q0 -40 -28 -68t-68 -28h-416v-416q0 -40 -28 -68t-68 -28h-192q-40 0 -68 28t-28 68v416h-416q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h416v416q0 40 28 68t68 28h192q40 0 68 -28t28 -68v-416h416q40 0 68 -28t28 -68z" /> + <glyph glyph-name="minus" unicode="" horiz-adv-x="1408" +d="M1408 800v-192q0 -40 -28 -68t-68 -28h-1216q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h1216q40 0 68 -28t28 -68z" /> + <glyph glyph-name="asterisk" unicode="" horiz-adv-x="1664" +d="M1482 486q46 -26 59.5 -77.5t-12.5 -97.5l-64 -110q-26 -46 -77.5 -59.5t-97.5 12.5l-266 153v-307q0 -52 -38 -90t-90 -38h-128q-52 0 -90 38t-38 90v307l-266 -153q-46 -26 -97.5 -12.5t-77.5 59.5l-64 110q-26 46 -12.5 97.5t59.5 77.5l266 154l-266 154 +q-46 26 -59.5 77.5t12.5 97.5l64 110q26 46 77.5 59.5t97.5 -12.5l266 -153v307q0 52 38 90t90 38h128q52 0 90 -38t38 -90v-307l266 153q46 26 97.5 12.5t77.5 -59.5l64 -110q26 -46 12.5 -97.5t-59.5 -77.5l-266 -154z" /> + <glyph glyph-name="exclamation_sign" unicode="" +d="M768 1408q209 0 385.5 -103t279.5 -279.5t103 -385.5t-103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103zM896 161v190q0 14 -9 23.5t-22 9.5h-192q-13 0 -23 -10t-10 -23v-190q0 -13 10 -23t23 -10h192 +q13 0 22 9.5t9 23.5zM894 505l18 621q0 12 -10 18q-10 8 -24 8h-220q-14 0 -24 -8q-10 -6 -10 -18l17 -621q0 -10 10 -17.5t24 -7.5h185q14 0 23.5 7.5t10.5 17.5z" /> + <glyph glyph-name="gift" unicode="" +d="M928 180v56v468v192h-320v-192v-468v-56q0 -25 18 -38.5t46 -13.5h192q28 0 46 13.5t18 38.5zM472 1024h195l-126 161q-26 31 -69 31q-40 0 -68 -28t-28 -68t28 -68t68 -28zM1160 1120q0 40 -28 68t-68 28q-43 0 -69 -31l-125 -161h194q40 0 68 28t28 68zM1536 864v-320 +q0 -14 -9 -23t-23 -9h-96v-416q0 -40 -28 -68t-68 -28h-1088q-40 0 -68 28t-28 68v416h-96q-14 0 -23 9t-9 23v320q0 14 9 23t23 9h440q-93 0 -158.5 65.5t-65.5 158.5t65.5 158.5t158.5 65.5q107 0 168 -77l128 -165l128 165q61 77 168 77q93 0 158.5 -65.5t65.5 -158.5 +t-65.5 -158.5t-158.5 -65.5h440q14 0 23 -9t9 -23z" /> + <glyph glyph-name="leaf" unicode="" horiz-adv-x="1792" +d="M1280 832q0 26 -19 45t-45 19q-172 0 -318 -49.5t-259.5 -134t-235.5 -219.5q-19 -21 -19 -45q0 -26 19 -45t45 -19q24 0 45 19q27 24 74 71t67 66q137 124 268.5 176t313.5 52q26 0 45 19t19 45zM1792 1030q0 -95 -20 -193q-46 -224 -184.5 -383t-357.5 -268 +q-214 -108 -438 -108q-148 0 -286 47q-15 5 -88 42t-96 37q-16 0 -39.5 -32t-45 -70t-52.5 -70t-60 -32q-43 0 -63.5 17.5t-45.5 59.5q-2 4 -6 11t-5.5 10t-3 9.5t-1.5 13.5q0 35 31 73.5t68 65.5t68 56t31 48q0 4 -14 38t-16 44q-9 51 -9 104q0 115 43.5 220t119 184.5 +t170.5 139t204 95.5q55 18 145 25.5t179.5 9t178.5 6t163.5 24t113.5 56.5l29.5 29.5t29.5 28t27 20t36.5 16t43.5 4.5q39 0 70.5 -46t47.5 -112t24 -124t8 -96z" /> + <glyph glyph-name="fire" unicode="" horiz-adv-x="1408" +d="M1408 -160v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-1344q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h1344q13 0 22.5 -9.5t9.5 -22.5zM1152 896q0 -78 -24.5 -144t-64 -112.5t-87.5 -88t-96 -77.5t-87.5 -72t-64 -81.5t-24.5 -96.5q0 -96 67 -224l-4 1l1 -1 +q-90 41 -160 83t-138.5 100t-113.5 122.5t-72.5 150.5t-27.5 184q0 78 24.5 144t64 112.5t87.5 88t96 77.5t87.5 72t64 81.5t24.5 96.5q0 94 -66 224l3 -1l-1 1q90 -41 160 -83t138.5 -100t113.5 -122.5t72.5 -150.5t27.5 -184z" /> + <glyph glyph-name="eye_open" unicode="" horiz-adv-x="1792" +d="M1664 576q-152 236 -381 353q61 -104 61 -225q0 -185 -131.5 -316.5t-316.5 -131.5t-316.5 131.5t-131.5 316.5q0 121 61 225q-229 -117 -381 -353q133 -205 333.5 -326.5t434.5 -121.5t434.5 121.5t333.5 326.5zM944 960q0 20 -14 34t-34 14q-125 0 -214.5 -89.5 +t-89.5 -214.5q0 -20 14 -34t34 -14t34 14t14 34q0 86 61 147t147 61q20 0 34 14t14 34zM1792 576q0 -34 -20 -69q-140 -230 -376.5 -368.5t-499.5 -138.5t-499.5 139t-376.5 368q-20 35 -20 69t20 69q140 229 376.5 368t499.5 139t499.5 -139t376.5 -368q20 -35 20 -69z" /> + <glyph glyph-name="eye_close" unicode="" horiz-adv-x="1792" +d="M555 201l78 141q-87 63 -136 159t-49 203q0 121 61 225q-229 -117 -381 -353q167 -258 427 -375zM944 960q0 20 -14 34t-34 14q-125 0 -214.5 -89.5t-89.5 -214.5q0 -20 14 -34t34 -14t34 14t14 34q0 86 61 147t147 61q20 0 34 14t14 34zM1307 1151q0 -7 -1 -9 +q-106 -189 -316 -567t-315 -566l-49 -89q-10 -16 -28 -16q-12 0 -134 70q-16 10 -16 28q0 12 44 87q-143 65 -263.5 173t-208.5 245q-20 31 -20 69t20 69q153 235 380 371t496 136q89 0 180 -17l54 97q10 16 28 16q5 0 18 -6t31 -15.5t33 -18.5t31.5 -18.5t19.5 -11.5 +q16 -10 16 -27zM1344 704q0 -139 -79 -253.5t-209 -164.5l280 502q8 -45 8 -84zM1792 576q0 -35 -20 -69q-39 -64 -109 -145q-150 -172 -347.5 -267t-419.5 -95l74 132q212 18 392.5 137t301.5 307q-115 179 -282 294l63 112q95 -64 182.5 -153t144.5 -184q20 -34 20 -69z +" /> + <glyph glyph-name="warning_sign" unicode="" horiz-adv-x="1792" +d="M1024 161v190q0 14 -9.5 23.5t-22.5 9.5h-192q-13 0 -22.5 -9.5t-9.5 -23.5v-190q0 -14 9.5 -23.5t22.5 -9.5h192q13 0 22.5 9.5t9.5 23.5zM1022 535l18 459q0 12 -10 19q-13 11 -24 11h-220q-11 0 -24 -11q-10 -7 -10 -21l17 -457q0 -10 10 -16.5t24 -6.5h185 +q14 0 23.5 6.5t10.5 16.5zM1008 1469l768 -1408q35 -63 -2 -126q-17 -29 -46.5 -46t-63.5 -17h-1536q-34 0 -63.5 17t-46.5 46q-37 63 -2 126l768 1408q17 31 47 49t65 18t65 -18t47 -49z" /> + <glyph glyph-name="plane" unicode="" horiz-adv-x="1408" +d="M1376 1376q44 -52 12 -148t-108 -172l-161 -161l160 -696q5 -19 -12 -33l-128 -96q-7 -6 -19 -6q-4 0 -7 1q-15 3 -21 16l-279 508l-259 -259l53 -194q5 -17 -8 -31l-96 -96q-9 -9 -23 -9h-2q-15 2 -24 13l-189 252l-252 189q-11 7 -13 23q-1 13 9 25l96 97q9 9 23 9 +q6 0 8 -1l194 -53l259 259l-508 279q-14 8 -17 24q-2 16 9 27l128 128q14 13 30 8l665 -159l160 160q76 76 172 108t148 -12z" /> + <glyph glyph-name="calendar" unicode="" horiz-adv-x="1664" +d="M128 -128h288v288h-288v-288zM480 -128h320v288h-320v-288zM128 224h288v320h-288v-320zM480 224h320v320h-320v-320zM128 608h288v288h-288v-288zM864 -128h320v288h-320v-288zM480 608h320v288h-320v-288zM1248 -128h288v288h-288v-288zM864 224h320v320h-320v-320z +M512 1088v288q0 13 -9.5 22.5t-22.5 9.5h-64q-13 0 -22.5 -9.5t-9.5 -22.5v-288q0 -13 9.5 -22.5t22.5 -9.5h64q13 0 22.5 9.5t9.5 22.5zM1248 224h288v320h-288v-320zM864 608h320v288h-320v-288zM1248 608h288v288h-288v-288zM1280 1088v288q0 13 -9.5 22.5t-22.5 9.5h-64 +q-13 0 -22.5 -9.5t-9.5 -22.5v-288q0 -13 9.5 -22.5t22.5 -9.5h64q13 0 22.5 9.5t9.5 22.5zM1664 1152v-1280q0 -52 -38 -90t-90 -38h-1408q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h128v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h384v96q0 66 47 113t113 47 +h64q66 0 113 -47t47 -113v-96h128q52 0 90 -38t38 -90z" /> + <glyph glyph-name="random" unicode="" horiz-adv-x="1792" +d="M666 1055q-60 -92 -137 -273q-22 45 -37 72.5t-40.5 63.5t-51 56.5t-63 35t-81.5 14.5h-224q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h224q250 0 410 -225zM1792 256q0 -14 -9 -23l-320 -320q-9 -9 -23 -9q-13 0 -22.5 9.5t-9.5 22.5v192q-32 0 -85 -0.5t-81 -1t-73 1 +t-71 5t-64 10.5t-63 18.5t-58 28.5t-59 40t-55 53.5t-56 69.5q59 93 136 273q22 -45 37 -72.5t40.5 -63.5t51 -56.5t63 -35t81.5 -14.5h256v192q0 14 9 23t23 9q12 0 24 -10l319 -319q9 -9 9 -23zM1792 1152q0 -14 -9 -23l-320 -320q-9 -9 -23 -9q-13 0 -22.5 9.5t-9.5 22.5 +v192h-256q-48 0 -87 -15t-69 -45t-51 -61.5t-45 -77.5q-32 -62 -78 -171q-29 -66 -49.5 -111t-54 -105t-64 -100t-74 -83t-90 -68.5t-106.5 -42t-128 -16.5h-224q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h224q48 0 87 15t69 45t51 61.5t45 77.5q32 62 78 171q29 66 49.5 111 +t54 105t64 100t74 83t90 68.5t106.5 42t128 16.5h256v192q0 14 9 23t23 9q12 0 24 -10l319 -319q9 -9 9 -23z" /> + <glyph glyph-name="comment" unicode="" horiz-adv-x="1792" +d="M1792 640q0 -174 -120 -321.5t-326 -233t-450 -85.5q-70 0 -145 8q-198 -175 -460 -242q-49 -14 -114 -22q-17 -2 -30.5 9t-17.5 29v1q-3 4 -0.5 12t2 10t4.5 9.5l6 9t7 8.5t8 9q7 8 31 34.5t34.5 38t31 39.5t32.5 51t27 59t26 76q-157 89 -247.5 220t-90.5 281 +q0 130 71 248.5t191 204.5t286 136.5t348 50.5q244 0 450 -85.5t326 -233t120 -321.5z" /> + <glyph glyph-name="magnet" unicode="" +d="M1536 704v-128q0 -201 -98.5 -362t-274 -251.5t-395.5 -90.5t-395.5 90.5t-274 251.5t-98.5 362v128q0 26 19 45t45 19h384q26 0 45 -19t19 -45v-128q0 -52 23.5 -90t53.5 -57t71 -30t64 -13t44 -2t44 2t64 13t71 30t53.5 57t23.5 90v128q0 26 19 45t45 19h384 +q26 0 45 -19t19 -45zM512 1344v-384q0 -26 -19 -45t-45 -19h-384q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h384q26 0 45 -19t19 -45zM1536 1344v-384q0 -26 -19 -45t-45 -19h-384q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h384q26 0 45 -19t19 -45z" /> + <glyph glyph-name="chevron_up" unicode="" horiz-adv-x="1792" +d="M1683 205l-166 -165q-19 -19 -45 -19t-45 19l-531 531l-531 -531q-19 -19 -45 -19t-45 19l-166 165q-19 19 -19 45.5t19 45.5l742 741q19 19 45 19t45 -19l742 -741q19 -19 19 -45.5t-19 -45.5z" /> + <glyph glyph-name="chevron_down" unicode="" horiz-adv-x="1792" +d="M1683 728l-742 -741q-19 -19 -45 -19t-45 19l-742 741q-19 19 -19 45.5t19 45.5l166 165q19 19 45 19t45 -19l531 -531l531 531q19 19 45 19t45 -19l166 -165q19 -19 19 -45.5t-19 -45.5z" /> + <glyph glyph-name="retweet" unicode="" horiz-adv-x="1920" +d="M1280 32q0 -13 -9.5 -22.5t-22.5 -9.5h-960q-8 0 -13.5 2t-9 7t-5.5 8t-3 11.5t-1 11.5v13v11v160v416h-192q-26 0 -45 19t-19 45q0 24 15 41l320 384q19 22 49 22t49 -22l320 -384q15 -17 15 -41q0 -26 -19 -45t-45 -19h-192v-384h576q16 0 25 -11l160 -192q7 -10 7 -21 +zM1920 448q0 -24 -15 -41l-320 -384q-20 -23 -49 -23t-49 23l-320 384q-15 17 -15 41q0 26 19 45t45 19h192v384h-576q-16 0 -25 12l-160 192q-7 9 -7 20q0 13 9.5 22.5t22.5 9.5h960q8 0 13.5 -2t9 -7t5.5 -8t3 -11.5t1 -11.5v-13v-11v-160v-416h192q26 0 45 -19t19 -45z +" /> + <glyph glyph-name="shopping_cart" unicode="" horiz-adv-x="1664" +d="M640 0q0 -52 -38 -90t-90 -38t-90 38t-38 90t38 90t90 38t90 -38t38 -90zM1536 0q0 -52 -38 -90t-90 -38t-90 38t-38 90t38 90t90 38t90 -38t38 -90zM1664 1088v-512q0 -24 -16.5 -42.5t-40.5 -21.5l-1044 -122q13 -60 13 -70q0 -16 -24 -64h920q26 0 45 -19t19 -45 +t-19 -45t-45 -19h-1024q-26 0 -45 19t-19 45q0 11 8 31.5t16 36t21.5 40t15.5 29.5l-177 823h-204q-26 0 -45 19t-19 45t19 45t45 19h256q16 0 28.5 -6.5t19.5 -15.5t13 -24.5t8 -26t5.5 -29.5t4.5 -26h1201q26 0 45 -19t19 -45z" /> + <glyph glyph-name="folder_close" unicode="" horiz-adv-x="1664" +d="M1664 928v-704q0 -92 -66 -158t-158 -66h-1216q-92 0 -158 66t-66 158v960q0 92 66 158t158 66h320q92 0 158 -66t66 -158v-32h672q92 0 158 -66t66 -158z" /> + <glyph glyph-name="folder_open" unicode="" horiz-adv-x="1920" +d="M1879 584q0 -31 -31 -66l-336 -396q-43 -51 -120.5 -86.5t-143.5 -35.5h-1088q-34 0 -60.5 13t-26.5 43q0 31 31 66l336 396q43 51 120.5 86.5t143.5 35.5h1088q34 0 60.5 -13t26.5 -43zM1536 928v-160h-832q-94 0 -197 -47.5t-164 -119.5l-337 -396l-5 -6q0 4 -0.5 12.5 +t-0.5 12.5v960q0 92 66 158t158 66h320q92 0 158 -66t66 -158v-32h544q92 0 158 -66t66 -158z" /> + <glyph glyph-name="resize_vertical" unicode="" horiz-adv-x="768" +d="M704 1216q0 -26 -19 -45t-45 -19h-128v-1024h128q26 0 45 -19t19 -45t-19 -45l-256 -256q-19 -19 -45 -19t-45 19l-256 256q-19 19 -19 45t19 45t45 19h128v1024h-128q-26 0 -45 19t-19 45t19 45l256 256q19 19 45 19t45 -19l256 -256q19 -19 19 -45z" /> + <glyph glyph-name="resize_horizontal" unicode="" horiz-adv-x="1792" +d="M1792 640q0 -26 -19 -45l-256 -256q-19 -19 -45 -19t-45 19t-19 45v128h-1024v-128q0 -26 -19 -45t-45 -19t-45 19l-256 256q-19 19 -19 45t19 45l256 256q19 19 45 19t45 -19t19 -45v-128h1024v128q0 26 19 45t45 19t45 -19l256 -256q19 -19 19 -45z" /> + <glyph glyph-name="bar_chart" unicode="" horiz-adv-x="2048" +d="M640 640v-512h-256v512h256zM1024 1152v-1024h-256v1024h256zM2048 0v-128h-2048v1536h128v-1408h1920zM1408 896v-768h-256v768h256zM1792 1280v-1152h-256v1152h256z" /> + <glyph glyph-name="twitter_sign" unicode="" +d="M1280 926q-56 -25 -121 -34q68 40 93 117q-65 -38 -134 -51q-61 66 -153 66q-87 0 -148.5 -61.5t-61.5 -148.5q0 -29 5 -48q-129 7 -242 65t-192 155q-29 -50 -29 -106q0 -114 91 -175q-47 1 -100 26v-2q0 -75 50 -133.5t123 -72.5q-29 -8 -51 -8q-13 0 -39 4 +q21 -63 74.5 -104t121.5 -42q-116 -90 -261 -90q-26 0 -50 3q148 -94 322 -94q112 0 210 35.5t168 95t120.5 137t75 162t24.5 168.5q0 18 -1 27q63 45 105 109zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5 +t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="facebook_sign" unicode="" +d="M1248 1408q119 0 203.5 -84.5t84.5 -203.5v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-188v595h199l30 232h-229v148q0 56 23.5 84t91.5 28l122 1v207q-63 9 -178 9q-136 0 -217.5 -80t-81.5 -226v-171h-200v-232h200v-595h-532q-119 0 -203.5 84.5t-84.5 203.5v960 +q0 119 84.5 203.5t203.5 84.5h960z" /> + <glyph glyph-name="camera_retro" unicode="" horiz-adv-x="1792" +d="M928 704q0 14 -9 23t-23 9q-66 0 -113 -47t-47 -113q0 -14 9 -23t23 -9t23 9t9 23q0 40 28 68t68 28q14 0 23 9t9 23zM1152 574q0 -106 -75 -181t-181 -75t-181 75t-75 181t75 181t181 75t181 -75t75 -181zM128 0h1536v128h-1536v-128zM1280 574q0 159 -112.5 271.5 +t-271.5 112.5t-271.5 -112.5t-112.5 -271.5t112.5 -271.5t271.5 -112.5t271.5 112.5t112.5 271.5zM256 1216h384v128h-384v-128zM128 1024h1536v118v138h-828l-64 -128h-644v-128zM1792 1280v-1280q0 -53 -37.5 -90.5t-90.5 -37.5h-1536q-53 0 -90.5 37.5t-37.5 90.5v1280 +q0 53 37.5 90.5t90.5 37.5h1536q53 0 90.5 -37.5t37.5 -90.5z" /> + <glyph glyph-name="key" unicode="" horiz-adv-x="1792" +d="M832 1024q0 80 -56 136t-136 56t-136 -56t-56 -136q0 -42 19 -83q-41 19 -83 19q-80 0 -136 -56t-56 -136t56 -136t136 -56t136 56t56 136q0 42 -19 83q41 -19 83 -19q80 0 136 56t56 136zM1683 320q0 -17 -49 -66t-66 -49q-9 0 -28.5 16t-36.5 33t-38.5 40t-24.5 26 +l-96 -96l220 -220q28 -28 28 -68q0 -42 -39 -81t-81 -39q-40 0 -68 28l-671 671q-176 -131 -365 -131q-163 0 -265.5 102.5t-102.5 265.5q0 160 95 313t248 248t313 95q163 0 265.5 -102.5t102.5 -265.5q0 -189 -131 -365l355 -355l96 96q-3 3 -26 24.5t-40 38.5t-33 36.5 +t-16 28.5q0 17 49 66t66 49q13 0 23 -10q6 -6 46 -44.5t82 -79.5t86.5 -86t73 -78t28.5 -41z" /> + <glyph glyph-name="cogs" unicode="" horiz-adv-x="1920" +d="M896 640q0 106 -75 181t-181 75t-181 -75t-75 -181t75 -181t181 -75t181 75t75 181zM1664 128q0 52 -38 90t-90 38t-90 -38t-38 -90q0 -53 37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1664 1152q0 52 -38 90t-90 38t-90 -38t-38 -90q0 -53 37.5 -90.5t90.5 -37.5 +t90.5 37.5t37.5 90.5zM1280 731v-185q0 -10 -7 -19.5t-16 -10.5l-155 -24q-11 -35 -32 -76q34 -48 90 -115q7 -11 7 -20q0 -12 -7 -19q-23 -30 -82.5 -89.5t-78.5 -59.5q-11 0 -21 7l-115 90q-37 -19 -77 -31q-11 -108 -23 -155q-7 -24 -30 -24h-186q-11 0 -20 7.5t-10 17.5 +l-23 153q-34 10 -75 31l-118 -89q-7 -7 -20 -7q-11 0 -21 8q-144 133 -144 160q0 9 7 19q10 14 41 53t47 61q-23 44 -35 82l-152 24q-10 1 -17 9.5t-7 19.5v185q0 10 7 19.5t16 10.5l155 24q11 35 32 76q-34 48 -90 115q-7 11 -7 20q0 12 7 20q22 30 82 89t79 59q11 0 21 -7 +l115 -90q34 18 77 32q11 108 23 154q7 24 30 24h186q11 0 20 -7.5t10 -17.5l23 -153q34 -10 75 -31l118 89q8 7 20 7q11 0 21 -8q144 -133 144 -160q0 -8 -7 -19q-12 -16 -42 -54t-45 -60q23 -48 34 -82l152 -23q10 -2 17 -10.5t7 -19.5zM1920 198v-140q0 -16 -149 -31 +q-12 -27 -30 -52q51 -113 51 -138q0 -4 -4 -7q-122 -71 -124 -71q-8 0 -46 47t-52 68q-20 -2 -30 -2t-30 2q-14 -21 -52 -68t-46 -47q-2 0 -124 71q-4 3 -4 7q0 25 51 138q-18 25 -30 52q-149 15 -149 31v140q0 16 149 31q13 29 30 52q-51 113 -51 138q0 4 4 7q4 2 35 20 +t59 34t30 16q8 0 46 -46.5t52 -67.5q20 2 30 2t30 -2q51 71 92 112l6 2q4 0 124 -70q4 -3 4 -7q0 -25 -51 -138q17 -23 30 -52q149 -15 149 -31zM1920 1222v-140q0 -16 -149 -31q-12 -27 -30 -52q51 -113 51 -138q0 -4 -4 -7q-122 -71 -124 -71q-8 0 -46 47t-52 68 +q-20 -2 -30 -2t-30 2q-14 -21 -52 -68t-46 -47q-2 0 -124 71q-4 3 -4 7q0 25 51 138q-18 25 -30 52q-149 15 -149 31v140q0 16 149 31q13 29 30 52q-51 113 -51 138q0 4 4 7q4 2 35 20t59 34t30 16q8 0 46 -46.5t52 -67.5q20 2 30 2t30 -2q51 71 92 112l6 2q4 0 124 -70 +q4 -3 4 -7q0 -25 -51 -138q17 -23 30 -52q149 -15 149 -31z" /> + <glyph glyph-name="comments" unicode="" horiz-adv-x="1792" +d="M1408 768q0 -139 -94 -257t-256.5 -186.5t-353.5 -68.5q-86 0 -176 16q-124 -88 -278 -128q-36 -9 -86 -16h-3q-11 0 -20.5 8t-11.5 21q-1 3 -1 6.5t0.5 6.5t2 6l2.5 5t3.5 5.5t4 5t4.5 5t4 4.5q5 6 23 25t26 29.5t22.5 29t25 38.5t20.5 44q-124 72 -195 177t-71 224 +q0 139 94 257t256.5 186.5t353.5 68.5t353.5 -68.5t256.5 -186.5t94 -257zM1792 512q0 -120 -71 -224.5t-195 -176.5q10 -24 20.5 -44t25 -38.5t22.5 -29t26 -29.5t23 -25q1 -1 4 -4.5t4.5 -5t4 -5t3.5 -5.5l2.5 -5t2 -6t0.5 -6.5t-1 -6.5q-3 -14 -13 -22t-22 -7 +q-50 7 -86 16q-154 40 -278 128q-90 -16 -176 -16q-271 0 -472 132q58 -4 88 -4q161 0 309 45t264 129q125 92 192 212t67 254q0 77 -23 152q129 -71 204 -178t75 -230z" /> + <glyph glyph-name="thumbs_up_alt" unicode="" +d="M256 192q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1408 768q0 51 -39 89.5t-89 38.5h-352q0 58 48 159.5t48 160.5q0 98 -32 145t-128 47q-26 -26 -38 -85t-30.5 -125.5t-59.5 -109.5q-22 -23 -77 -91q-4 -5 -23 -30t-31.5 -41t-34.5 -42.5 +t-40 -44t-38.5 -35.5t-40 -27t-35.5 -9h-32v-640h32q13 0 31.5 -3t33 -6.5t38 -11t35 -11.5t35.5 -12.5t29 -10.5q211 -73 342 -73h121q192 0 192 167q0 26 -5 56q30 16 47.5 52.5t17.5 73.5t-18 69q53 50 53 119q0 25 -10 55.5t-25 47.5q32 1 53.5 47t21.5 81zM1536 769 +q0 -89 -49 -163q9 -33 9 -69q0 -77 -38 -144q3 -21 3 -43q0 -101 -60 -178q1 -139 -85 -219.5t-227 -80.5h-36h-93q-96 0 -189.5 22.5t-216.5 65.5q-116 40 -138 40h-288q-53 0 -90.5 37.5t-37.5 90.5v640q0 53 37.5 90.5t90.5 37.5h274q36 24 137 155q58 75 107 128 +q24 25 35.5 85.5t30.5 126.5t62 108q39 37 90 37q84 0 151 -32.5t102 -101.5t35 -186q0 -93 -48 -192h176q104 0 180 -76t76 -179z" /> + <glyph glyph-name="thumbs_down_alt" unicode="" +d="M256 1088q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1408 512q0 35 -21.5 81t-53.5 47q15 17 25 47.5t10 55.5q0 69 -53 119q18 31 18 69q0 37 -17.5 73.5t-47.5 52.5q5 30 5 56q0 85 -49 126t-136 41h-128q-131 0 -342 -73q-5 -2 -29 -10.5 +t-35.5 -12.5t-35 -11.5t-38 -11t-33 -6.5t-31.5 -3h-32v-640h32q16 0 35.5 -9t40 -27t38.5 -35.5t40 -44t34.5 -42.5t31.5 -41t23 -30q55 -68 77 -91q41 -43 59.5 -109.5t30.5 -125.5t38 -85q96 0 128 47t32 145q0 59 -48 160.5t-48 159.5h352q50 0 89 38.5t39 89.5z +M1536 511q0 -103 -76 -179t-180 -76h-176q48 -99 48 -192q0 -118 -35 -186q-35 -69 -102 -101.5t-151 -32.5q-51 0 -90 37q-34 33 -54 82t-25.5 90.5t-17.5 84.5t-31 64q-48 50 -107 127q-101 131 -137 155h-274q-53 0 -90.5 37.5t-37.5 90.5v640q0 53 37.5 90.5t90.5 37.5 +h288q22 0 138 40q128 44 223 66t200 22h112q140 0 226.5 -79t85.5 -216v-5q60 -77 60 -178q0 -22 -3 -43q38 -67 38 -144q0 -36 -9 -69q49 -73 49 -163z" /> + <glyph glyph-name="star_half" unicode="" horiz-adv-x="896" +d="M832 1504v-1339l-449 -236q-22 -12 -40 -12q-21 0 -31.5 14.5t-10.5 35.5q0 6 2 20l86 500l-364 354q-25 27 -25 48q0 37 56 46l502 73l225 455q19 41 49 41z" /> + <glyph glyph-name="heart_empty" unicode="" horiz-adv-x="1792" +d="M1664 940q0 81 -21.5 143t-55 98.5t-81.5 59.5t-94 31t-98 8t-112 -25.5t-110.5 -64t-86.5 -72t-60 -61.5q-18 -22 -49 -22t-49 22q-24 28 -60 61.5t-86.5 72t-110.5 64t-112 25.5t-98 -8t-94 -31t-81.5 -59.5t-55 -98.5t-21.5 -143q0 -168 187 -355l581 -560l580 559 +q188 188 188 356zM1792 940q0 -221 -229 -450l-623 -600q-18 -18 -44 -18t-44 18l-624 602q-10 8 -27.5 26t-55.5 65.5t-68 97.5t-53.5 121t-23.5 138q0 220 127 344t351 124q62 0 126.5 -21.5t120 -58t95.5 -68.5t76 -68q36 36 76 68t95.5 68.5t120 58t126.5 21.5 +q224 0 351 -124t127 -344z" /> + <glyph glyph-name="signout" unicode="" horiz-adv-x="1664" +d="M640 96q0 -4 1 -20t0.5 -26.5t-3 -23.5t-10 -19.5t-20.5 -6.5h-320q-119 0 -203.5 84.5t-84.5 203.5v704q0 119 84.5 203.5t203.5 84.5h320q13 0 22.5 -9.5t9.5 -22.5q0 -4 1 -20t0.5 -26.5t-3 -23.5t-10 -19.5t-20.5 -6.5h-320q-66 0 -113 -47t-47 -113v-704 +q0 -66 47 -113t113 -47h288h11h13t11.5 -1t11.5 -3t8 -5.5t7 -9t2 -13.5zM1568 640q0 -26 -19 -45l-544 -544q-19 -19 -45 -19t-45 19t-19 45v288h-448q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h448v288q0 26 19 45t45 19t45 -19l544 -544q19 -19 19 -45z" /> + <glyph glyph-name="linkedin_sign" unicode="" +d="M237 122h231v694h-231v-694zM483 1030q-1 52 -36 86t-93 34t-94.5 -34t-36.5 -86q0 -51 35.5 -85.5t92.5 -34.5h1q59 0 95 34.5t36 85.5zM1068 122h231v398q0 154 -73 233t-193 79q-136 0 -209 -117h2v101h-231q3 -66 0 -694h231v388q0 38 7 56q15 35 45 59.5t74 24.5 +q116 0 116 -157v-371zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="pushpin" unicode="" horiz-adv-x="1152" +d="M480 672v448q0 14 -9 23t-23 9t-23 -9t-9 -23v-448q0 -14 9 -23t23 -9t23 9t9 23zM1152 320q0 -26 -19 -45t-45 -19h-429l-51 -483q-2 -12 -10.5 -20.5t-20.5 -8.5h-1q-27 0 -32 27l-76 485h-404q-26 0 -45 19t-19 45q0 123 78.5 221.5t177.5 98.5v512q-52 0 -90 38 +t-38 90t38 90t90 38h640q52 0 90 -38t38 -90t-38 -90t-90 -38v-512q99 0 177.5 -98.5t78.5 -221.5z" /> + <glyph glyph-name="external_link" unicode="" horiz-adv-x="1792" +d="M1408 608v-320q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h704q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-704q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832q66 0 113 47t47 113v320 +q0 14 9 23t23 9h64q14 0 23 -9t9 -23zM1792 1472v-512q0 -26 -19 -45t-45 -19t-45 19l-176 176l-652 -652q-10 -10 -23 -10t-23 10l-114 114q-10 10 -10 23t10 23l652 652l-176 176q-19 19 -19 45t19 45t45 19h512q26 0 45 -19t19 -45z" /> + <glyph glyph-name="signin" unicode="" +d="M1184 640q0 -26 -19 -45l-544 -544q-19 -19 -45 -19t-45 19t-19 45v288h-448q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h448v288q0 26 19 45t45 19t45 -19l544 -544q19 -19 19 -45zM1536 992v-704q0 -119 -84.5 -203.5t-203.5 -84.5h-320q-13 0 -22.5 9.5t-9.5 22.5 +q0 4 -1 20t-0.5 26.5t3 23.5t10 19.5t20.5 6.5h320q66 0 113 47t47 113v704q0 66 -47 113t-113 47h-288h-11h-13t-11.5 1t-11.5 3t-8 5.5t-7 9t-2 13.5q0 4 -1 20t-0.5 26.5t3 23.5t10 19.5t20.5 6.5h320q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="trophy" unicode="" horiz-adv-x="1664" +d="M458 653q-74 162 -74 371h-256v-96q0 -78 94.5 -162t235.5 -113zM1536 928v96h-256q0 -209 -74 -371q141 29 235.5 113t94.5 162zM1664 1056v-128q0 -71 -41.5 -143t-112 -130t-173 -97.5t-215.5 -44.5q-42 -54 -95 -95q-38 -34 -52.5 -72.5t-14.5 -89.5q0 -54 30.5 -91 +t97.5 -37q75 0 133.5 -45.5t58.5 -114.5v-64q0 -14 -9 -23t-23 -9h-832q-14 0 -23 9t-9 23v64q0 69 58.5 114.5t133.5 45.5q67 0 97.5 37t30.5 91q0 51 -14.5 89.5t-52.5 72.5q-53 41 -95 95q-113 5 -215.5 44.5t-173 97.5t-112 130t-41.5 143v128q0 40 28 68t68 28h288v96 +q0 66 47 113t113 47h576q66 0 113 -47t47 -113v-96h288q40 0 68 -28t28 -68z" /> + <glyph glyph-name="github_sign" unicode="" +d="M519 336q4 6 -3 13q-9 7 -14 2q-4 -6 3 -13q9 -7 14 -2zM491 377q-5 7 -12 4q-6 -4 0 -12q7 -8 12 -5q6 4 0 13zM450 417q2 4 -5 8q-7 2 -8 -2q-3 -5 4 -8q8 -2 9 2zM471 394q2 1 1.5 4.5t-3.5 5.5q-6 7 -10 3t1 -11q6 -6 11 -2zM557 319q2 7 -9 11q-9 3 -13 -4 +q-2 -7 9 -11q9 -3 13 4zM599 316q0 8 -12 8q-10 0 -10 -8t11 -8t11 8zM638 323q-2 7 -13 5t-9 -9q2 -8 12 -6t10 10zM1280 640q0 212 -150 362t-362 150t-362 -150t-150 -362q0 -167 98 -300.5t252 -185.5q18 -3 26.5 5t8.5 20q0 52 -1 95q-6 -1 -15.5 -2.5t-35.5 -2t-48 4 +t-43.5 20t-29.5 41.5q-23 59 -57 74q-2 1 -4.5 3.5l-8 8t-7 9.5t4 7.5t19.5 3.5q6 0 15 -2t30 -15.5t33 -35.5q16 -28 37.5 -42t43.5 -14t38 3.5t30 9.5q7 47 33 69q-49 6 -86 18.5t-73 39t-55.5 76t-19.5 119.5q0 79 53 137q-24 62 5 136q19 6 54.5 -7.5t60.5 -29.5l26 -16 +q58 17 128 17t128 -17q11 7 28.5 18t55.5 26t57 9q29 -74 5 -136q53 -58 53 -137q0 -57 -14 -100.5t-35.5 -70t-53.5 -44.5t-62.5 -26t-68.5 -12q35 -31 35 -95q0 -40 -0.5 -89t-0.5 -51q0 -12 8.5 -20t26.5 -5q154 52 252 185.5t98 300.5zM1536 1120v-960 +q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="upload_alt" unicode="" horiz-adv-x="1664" +d="M1280 64q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1536 64q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1664 288v-320q0 -40 -28 -68t-68 -28h-1472q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h427q21 -56 70.5 -92 +t110.5 -36h256q61 0 110.5 36t70.5 92h427q40 0 68 -28t28 -68zM1339 936q-17 -40 -59 -40h-256v-448q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v448h-256q-42 0 -59 40q-17 39 14 69l448 448q18 19 45 19t45 -19l448 -448q31 -30 14 -69z" /> + <glyph glyph-name="lemon" unicode="" +d="M1407 710q0 44 -7 113.5t-18 96.5q-12 30 -17 44t-9 36.5t-4 48.5q0 23 5 68.5t5 67.5q0 37 -10 55q-4 1 -13 1q-19 0 -58 -4.5t-59 -4.5q-60 0 -176 24t-175 24q-43 0 -94.5 -11.5t-85 -23.5t-89.5 -34q-137 -54 -202 -103q-96 -73 -159.5 -189.5t-88 -236t-24.5 -248.5 +q0 -40 12.5 -120t12.5 -121q0 -23 -11 -66.5t-11 -65.5t12 -36.5t34 -14.5q24 0 72.5 11t73.5 11q57 0 169.5 -15.5t169.5 -15.5q181 0 284 36q129 45 235.5 152.5t166 245.5t59.5 275zM1535 712q0 -165 -70 -327.5t-196 -288t-281 -180.5q-124 -44 -326 -44 +q-57 0 -170 14.5t-169 14.5q-24 0 -72.5 -14.5t-73.5 -14.5q-73 0 -123.5 55.5t-50.5 128.5q0 24 11 68t11 67q0 40 -12.5 120.5t-12.5 121.5q0 111 18 217.5t54.5 209.5t100.5 194t150 156q78 59 232 120q194 78 316 78q60 0 175.5 -24t173.5 -24q19 0 57 5t58 5 +q81 0 118 -50.5t37 -134.5q0 -23 -5 -68t-5 -68q0 -13 2 -25t3.5 -16.5t7.5 -20.5t8 -20q16 -40 25 -118.5t9 -136.5z" /> + <glyph glyph-name="phone" unicode="" horiz-adv-x="1408" +d="M1408 296q0 -27 -10 -70.5t-21 -68.5q-21 -50 -122 -106q-94 -51 -186 -51q-27 0 -53 3.5t-57.5 12.5t-47 14.5t-55.5 20.5t-49 18q-98 35 -175 83q-127 79 -264 216t-216 264q-48 77 -83 175q-3 9 -18 49t-20.5 55.5t-14.5 47t-12.5 57.5t-3.5 53q0 92 51 186 +q56 101 106 122q25 11 68.5 21t70.5 10q14 0 21 -3q18 -6 53 -76q11 -19 30 -54t35 -63.5t31 -53.5q3 -4 17.5 -25t21.5 -35.5t7 -28.5q0 -20 -28.5 -50t-62 -55t-62 -53t-28.5 -46q0 -9 5 -22.5t8.5 -20.5t14 -24t11.5 -19q76 -137 174 -235t235 -174q2 -1 19 -11.5t24 -14 +t20.5 -8.5t22.5 -5q18 0 46 28.5t53 62t55 62t50 28.5q14 0 28.5 -7t35.5 -21.5t25 -17.5q25 -15 53.5 -31t63.5 -35t54 -30q70 -35 76 -53q3 -7 3 -21z" /> + <glyph glyph-name="check_empty" unicode="" horiz-adv-x="1408" +d="M1120 1280h-832q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832q66 0 113 47t47 113v832q0 66 -47 113t-113 47zM1408 1120v-832q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h832 +q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="bookmark_empty" unicode="" horiz-adv-x="1280" +d="M1152 1280h-1024v-1242l423 406l89 85l89 -85l423 -406v1242zM1164 1408q23 0 44 -9q33 -13 52.5 -41t19.5 -62v-1289q0 -34 -19.5 -62t-52.5 -41q-19 -8 -44 -8q-48 0 -83 32l-441 424l-441 -424q-36 -33 -83 -33q-23 0 -44 9q-33 13 -52.5 41t-19.5 62v1289 +q0 34 19.5 62t52.5 41q21 9 44 9h1048z" /> + <glyph glyph-name="phone_sign" unicode="" +d="M1280 343q0 11 -2 16t-18 16.5t-40.5 25t-47.5 26.5t-45.5 25t-28.5 15q-5 3 -19 13t-25 15t-21 5q-15 0 -36.5 -20.5t-39.5 -45t-38.5 -45t-33.5 -20.5q-7 0 -16.5 3.5t-15.5 6.5t-17 9.5t-14 8.5q-99 55 -170 126.5t-127 170.5q-2 3 -8.5 14t-9.5 17t-6.5 15.5 +t-3.5 16.5q0 13 20.5 33.5t45 38.5t45 39.5t20.5 36.5q0 10 -5 21t-15 25t-13 19q-3 6 -15 28.5t-25 45.5t-26.5 47.5t-25 40.5t-16.5 18t-16 2q-48 0 -101 -22q-46 -21 -80 -94.5t-34 -130.5q0 -16 2.5 -34t5 -30.5t9 -33t10 -29.5t12.5 -33t11 -30q60 -164 216.5 -320.5 +t320.5 -216.5q6 -2 30 -11t33 -12.5t29.5 -10t33 -9t30.5 -5t34 -2.5q57 0 130.5 34t94.5 80q22 53 22 101zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z +" /> + <glyph glyph-name="twitter" unicode="" horiz-adv-x="1664" +d="M1620 1128q-67 -98 -162 -167q1 -14 1 -42q0 -130 -38 -259.5t-115.5 -248.5t-184.5 -210.5t-258 -146t-323 -54.5q-271 0 -496 145q35 -4 78 -4q225 0 401 138q-105 2 -188 64.5t-114 159.5q33 -5 61 -5q43 0 85 11q-112 23 -185.5 111.5t-73.5 205.5v4q68 -38 146 -41 +q-66 44 -105 115t-39 154q0 88 44 163q121 -149 294.5 -238.5t371.5 -99.5q-8 38 -8 74q0 134 94.5 228.5t228.5 94.5q140 0 236 -102q109 21 205 78q-37 -115 -142 -178q93 10 186 50z" /> + <glyph glyph-name="facebook" unicode="" horiz-adv-x="1024" +d="M959 1524v-264h-157q-86 0 -116 -36t-30 -108v-189h293l-39 -296h-254v-759h-306v759h-255v296h255v218q0 186 104 288.5t277 102.5q147 0 228 -12z" /> + <glyph glyph-name="github" unicode="" +d="M768 1408q209 0 385.5 -103t279.5 -279.5t103 -385.5q0 -251 -146.5 -451.5t-378.5 -277.5q-27 -5 -40 7t-13 30q0 3 0.5 76.5t0.5 134.5q0 97 -52 142q57 6 102.5 18t94 39t81 66.5t53 105t20.5 150.5q0 119 -79 206q37 91 -8 204q-28 9 -81 -11t-92 -44l-38 -24 +q-93 26 -192 26t-192 -26q-16 11 -42.5 27t-83.5 38.5t-85 13.5q-45 -113 -8 -204q-79 -87 -79 -206q0 -85 20.5 -150t52.5 -105t80.5 -67t94 -39t102.5 -18q-39 -36 -49 -103q-21 -10 -45 -15t-57 -5t-65.5 21.5t-55.5 62.5q-19 32 -48.5 52t-49.5 24l-20 3q-21 0 -29 -4.5 +t-5 -11.5t9 -14t13 -12l7 -5q22 -10 43.5 -38t31.5 -51l10 -23q13 -38 44 -61.5t67 -30t69.5 -7t55.5 3.5l23 4q0 -38 0.5 -88.5t0.5 -54.5q0 -18 -13 -30t-40 -7q-232 77 -378.5 277.5t-146.5 451.5q0 209 103 385.5t279.5 279.5t385.5 103zM291 305q3 7 -7 12 +q-10 3 -13 -2q-3 -7 7 -12q9 -6 13 2zM322 271q7 5 -2 16q-10 9 -16 3q-7 -5 2 -16q10 -10 16 -3zM352 226q9 7 0 19q-8 13 -17 6q-9 -5 0 -18t17 -7zM394 184q8 8 -4 19q-12 12 -20 3q-9 -8 4 -19q12 -12 20 -3zM451 159q3 11 -13 16q-15 4 -19 -7t13 -15q15 -6 19 6z +M514 154q0 13 -17 11q-16 0 -16 -11q0 -13 17 -11q16 0 16 11zM572 164q-2 11 -18 9q-16 -3 -14 -15t18 -8t14 14z" /> + <glyph glyph-name="unlock" unicode="" horiz-adv-x="1664" +d="M1664 960v-256q0 -26 -19 -45t-45 -19h-64q-26 0 -45 19t-19 45v256q0 106 -75 181t-181 75t-181 -75t-75 -181v-192h96q40 0 68 -28t28 -68v-576q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v576q0 40 28 68t68 28h672v192q0 185 131.5 316.5t316.5 131.5 +t316.5 -131.5t131.5 -316.5z" /> + <glyph glyph-name="credit_card" unicode="" horiz-adv-x="1920" +d="M1760 1408q66 0 113 -47t47 -113v-1216q0 -66 -47 -113t-113 -47h-1600q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1600zM160 1280q-13 0 -22.5 -9.5t-9.5 -22.5v-224h1664v224q0 13 -9.5 22.5t-22.5 9.5h-1600zM1760 0q13 0 22.5 9.5t9.5 22.5v608h-1664v-608 +q0 -13 9.5 -22.5t22.5 -9.5h1600zM256 128v128h256v-128h-256zM640 128v128h384v-128h-384z" /> + <glyph glyph-name="rss" unicode="" horiz-adv-x="1408" +d="M384 192q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM896 69q2 -28 -17 -48q-18 -21 -47 -21h-135q-25 0 -43 16.5t-20 41.5q-22 229 -184.5 391.5t-391.5 184.5q-25 2 -41.5 20t-16.5 43v135q0 29 21 47q17 17 43 17h5q160 -13 306 -80.5 +t259 -181.5q114 -113 181.5 -259t80.5 -306zM1408 67q2 -27 -18 -47q-18 -20 -46 -20h-143q-26 0 -44.5 17.5t-19.5 42.5q-12 215 -101 408.5t-231.5 336t-336 231.5t-408.5 102q-25 1 -42.5 19.5t-17.5 43.5v143q0 28 20 46q18 18 44 18h3q262 -13 501.5 -120t425.5 -294 +q187 -186 294 -425.5t120 -501.5z" /> + <glyph glyph-name="hdd" unicode="" +d="M1040 320q0 -33 -23.5 -56.5t-56.5 -23.5t-56.5 23.5t-23.5 56.5t23.5 56.5t56.5 23.5t56.5 -23.5t23.5 -56.5zM1296 320q0 -33 -23.5 -56.5t-56.5 -23.5t-56.5 23.5t-23.5 56.5t23.5 56.5t56.5 23.5t56.5 -23.5t23.5 -56.5zM1408 160v320q0 13 -9.5 22.5t-22.5 9.5 +h-1216q-13 0 -22.5 -9.5t-9.5 -22.5v-320q0 -13 9.5 -22.5t22.5 -9.5h1216q13 0 22.5 9.5t9.5 22.5zM178 640h1180l-157 482q-4 13 -16 21.5t-26 8.5h-782q-14 0 -26 -8.5t-16 -21.5zM1536 480v-320q0 -66 -47 -113t-113 -47h-1216q-66 0 -113 47t-47 113v320q0 25 16 75 +l197 606q17 53 63 86t101 33h782q55 0 101 -33t63 -86l197 -606q16 -50 16 -75z" /> + <glyph glyph-name="bullhorn" unicode="" horiz-adv-x="1792" +d="M1664 896q53 0 90.5 -37.5t37.5 -90.5t-37.5 -90.5t-90.5 -37.5v-384q0 -52 -38 -90t-90 -38q-417 347 -812 380q-58 -19 -91 -66t-31 -100.5t40 -92.5q-20 -33 -23 -65.5t6 -58t33.5 -55t48 -50t61.5 -50.5q-29 -58 -111.5 -83t-168.5 -11.5t-132 55.5q-7 23 -29.5 87.5 +t-32 94.5t-23 89t-15 101t3.5 98.5t22 110.5h-122q-66 0 -113 47t-47 113v192q0 66 47 113t113 47h480q435 0 896 384q52 0 90 -38t38 -90v-384zM1536 292v954q-394 -302 -768 -343v-270q377 -42 768 -341z" /> + <glyph glyph-name="bell" unicode="" horiz-adv-x="1792" +d="M912 -160q0 16 -16 16q-59 0 -101.5 42.5t-42.5 101.5q0 16 -16 16t-16 -16q0 -73 51.5 -124.5t124.5 -51.5q16 0 16 16zM246 128h1300q-266 300 -266 832q0 51 -24 105t-69 103t-121.5 80.5t-169.5 31.5t-169.5 -31.5t-121.5 -80.5t-69 -103t-24 -105q0 -532 -266 -832z +M1728 128q0 -52 -38 -90t-90 -38h-448q0 -106 -75 -181t-181 -75t-181 75t-75 181h-448q-52 0 -90 38t-38 90q50 42 91 88t85 119.5t74.5 158.5t50 206t19.5 260q0 152 117 282.5t307 158.5q-8 19 -8 39q0 40 28 68t68 28t68 -28t28 -68q0 -20 -8 -39q190 -28 307 -158.5 +t117 -282.5q0 -139 19.5 -260t50 -206t74.5 -158.5t85 -119.5t91 -88z" /> + <glyph glyph-name="certificate" unicode="" +d="M1376 640l138 -135q30 -28 20 -70q-12 -41 -52 -51l-188 -48l53 -186q12 -41 -19 -70q-29 -31 -70 -19l-186 53l-48 -188q-10 -40 -51 -52q-12 -2 -19 -2q-31 0 -51 22l-135 138l-135 -138q-28 -30 -70 -20q-41 11 -51 52l-48 188l-186 -53q-41 -12 -70 19q-31 29 -19 70 +l53 186l-188 48q-40 10 -52 51q-10 42 20 70l138 135l-138 135q-30 28 -20 70q12 41 52 51l188 48l-53 186q-12 41 19 70q29 31 70 19l186 -53l48 188q10 41 51 51q41 12 70 -19l135 -139l135 139q29 30 70 19q41 -10 51 -51l48 -188l186 53q41 12 70 -19q31 -29 19 -70 +l-53 -186l188 -48q40 -10 52 -51q10 -42 -20 -70z" /> + <glyph glyph-name="hand_right" unicode="" horiz-adv-x="1792" +d="M256 192q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1664 768q0 51 -39 89.5t-89 38.5h-576q0 20 15 48.5t33 55t33 68t15 84.5q0 67 -44.5 97.5t-115.5 30.5q-24 0 -90 -139q-24 -44 -37 -65q-40 -64 -112 -145q-71 -81 -101 -106 +q-69 -57 -140 -57h-32v-640h32q72 0 167 -32t193.5 -64t179.5 -32q189 0 189 167q0 26 -5 56q30 16 47.5 52.5t17.5 73.5t-18 69q53 50 53 119q0 25 -10 55.5t-25 47.5h331q52 0 90 38t38 90zM1792 769q0 -105 -75.5 -181t-180.5 -76h-169q-4 -62 -37 -119q3 -21 3 -43 +q0 -101 -60 -178q1 -139 -85 -219.5t-227 -80.5q-133 0 -322 69q-164 59 -223 59h-288q-53 0 -90.5 37.5t-37.5 90.5v640q0 53 37.5 90.5t90.5 37.5h288q10 0 21.5 4.5t23.5 14t22.5 18t24 22.5t20.5 21.5t19 21.5t14 17q65 74 100 129q13 21 33 62t37 72t40.5 63t55 49.5 +t69.5 17.5q125 0 206.5 -67t81.5 -189q0 -68 -22 -128h374q104 0 180 -76t76 -179z" /> + <glyph glyph-name="hand_left" unicode="" horiz-adv-x="1792" +d="M1376 128h32v640h-32q-35 0 -67.5 12t-62.5 37t-50 46t-49 54q-8 9 -12 14q-72 81 -112 145q-14 22 -38 68q-1 3 -10.5 22.5t-18.5 36t-20 35.5t-21.5 30.5t-18.5 11.5q-71 0 -115.5 -30.5t-44.5 -97.5q0 -43 15 -84.5t33 -68t33 -55t15 -48.5h-576q-50 0 -89 -38.5 +t-39 -89.5q0 -52 38 -90t90 -38h331q-15 -17 -25 -47.5t-10 -55.5q0 -69 53 -119q-18 -32 -18 -69t17.5 -73.5t47.5 -52.5q-4 -24 -4 -56q0 -85 48.5 -126t135.5 -41q84 0 183 32t194 64t167 32zM1664 192q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45z +M1792 768v-640q0 -53 -37.5 -90.5t-90.5 -37.5h-288q-59 0 -223 -59q-190 -69 -317 -69q-142 0 -230 77.5t-87 217.5l1 5q-61 76 -61 178q0 22 3 43q-33 57 -37 119h-169q-105 0 -180.5 76t-75.5 181q0 103 76 179t180 76h374q-22 60 -22 128q0 122 81.5 189t206.5 67 +q38 0 69.5 -17.5t55 -49.5t40.5 -63t37 -72t33 -62q35 -55 100 -129q2 -3 14 -17t19 -21.5t20.5 -21.5t24 -22.5t22.5 -18t23.5 -14t21.5 -4.5h288q53 0 90.5 -37.5t37.5 -90.5z" /> + <glyph glyph-name="hand_up" unicode="" +d="M1280 -64q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1408 700q0 189 -167 189q-26 0 -56 -5q-16 30 -52.5 47.5t-73.5 17.5t-69 -18q-50 53 -119 53q-25 0 -55.5 -10t-47.5 -25v331q0 52 -38 90t-90 38q-51 0 -89.5 -39t-38.5 -89v-576 +q-20 0 -48.5 15t-55 33t-68 33t-84.5 15q-67 0 -97.5 -44.5t-30.5 -115.5q0 -24 139 -90q44 -24 65 -37q64 -40 145 -112q81 -71 106 -101q57 -69 57 -140v-32h640v32q0 72 32 167t64 193.5t32 179.5zM1536 705q0 -133 -69 -322q-59 -164 -59 -223v-288q0 -53 -37.5 -90.5 +t-90.5 -37.5h-640q-53 0 -90.5 37.5t-37.5 90.5v288q0 10 -4.5 21.5t-14 23.5t-18 22.5t-22.5 24t-21.5 20.5t-21.5 19t-17 14q-74 65 -129 100q-21 13 -62 33t-72 37t-63 40.5t-49.5 55t-17.5 69.5q0 125 67 206.5t189 81.5q68 0 128 -22v374q0 104 76 180t179 76 +q105 0 181 -75.5t76 -180.5v-169q62 -4 119 -37q21 3 43 3q101 0 178 -60q139 1 219.5 -85t80.5 -227z" /> + <glyph glyph-name="hand_down" unicode="" +d="M1408 576q0 84 -32 183t-64 194t-32 167v32h-640v-32q0 -35 -12 -67.5t-37 -62.5t-46 -50t-54 -49q-9 -8 -14 -12q-81 -72 -145 -112q-22 -14 -68 -38q-3 -1 -22.5 -10.5t-36 -18.5t-35.5 -20t-30.5 -21.5t-11.5 -18.5q0 -71 30.5 -115.5t97.5 -44.5q43 0 84.5 15t68 33 +t55 33t48.5 15v-576q0 -50 38.5 -89t89.5 -39q52 0 90 38t38 90v331q46 -35 103 -35q69 0 119 53q32 -18 69 -18t73.5 17.5t52.5 47.5q24 -4 56 -4q85 0 126 48.5t41 135.5zM1280 1344q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1536 580 +q0 -142 -77.5 -230t-217.5 -87l-5 1q-76 -61 -178 -61q-22 0 -43 3q-54 -30 -119 -37v-169q0 -105 -76 -180.5t-181 -75.5q-103 0 -179 76t-76 180v374q-54 -22 -128 -22q-121 0 -188.5 81.5t-67.5 206.5q0 38 17.5 69.5t49.5 55t63 40.5t72 37t62 33q55 35 129 100 +q3 2 17 14t21.5 19t21.5 20.5t22.5 24t18 22.5t14 23.5t4.5 21.5v288q0 53 37.5 90.5t90.5 37.5h640q53 0 90.5 -37.5t37.5 -90.5v-288q0 -59 59 -223q69 -190 69 -317z" /> + <glyph glyph-name="circle_arrow_left" unicode="" +d="M1280 576v128q0 26 -19 45t-45 19h-502l189 189q19 19 19 45t-19 45l-91 91q-18 18 -45 18t-45 -18l-362 -362l-91 -91q-18 -18 -18 -45t18 -45l91 -91l362 -362q18 -18 45 -18t45 18l91 91q18 18 18 45t-18 45l-189 189h502q26 0 45 19t19 45zM1536 640 +q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="circle_arrow_right" unicode="" +d="M1285 640q0 27 -18 45l-91 91l-362 362q-18 18 -45 18t-45 -18l-91 -91q-18 -18 -18 -45t18 -45l189 -189h-502q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h502l-189 -189q-19 -19 -19 -45t19 -45l91 -91q18 -18 45 -18t45 18l362 362l91 91q18 18 18 45zM1536 640 +q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="circle_arrow_up" unicode="" +d="M1284 641q0 27 -18 45l-362 362l-91 91q-18 18 -45 18t-45 -18l-91 -91l-362 -362q-18 -18 -18 -45t18 -45l91 -91q18 -18 45 -18t45 18l189 189v-502q0 -26 19 -45t45 -19h128q26 0 45 19t19 45v502l189 -189q19 -19 45 -19t45 19l91 91q18 18 18 45zM1536 640 +q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="circle_arrow_down" unicode="" +d="M1284 639q0 27 -18 45l-91 91q-18 18 -45 18t-45 -18l-189 -189v502q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-502l-189 189q-19 19 -45 19t-45 -19l-91 -91q-18 -18 -18 -45t18 -45l362 -362l91 -91q18 -18 45 -18t45 18l91 91l362 362q18 18 18 45zM1536 640 +q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="globe" unicode="" +d="M768 1408q209 0 385.5 -103t279.5 -279.5t103 -385.5t-103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103zM1042 887q-2 -1 -9.5 -9.5t-13.5 -9.5q2 0 4.5 5t5 11t3.5 7q6 7 22 15q14 6 52 12q34 8 51 -11 +q-2 2 9.5 13t14.5 12q3 2 15 4.5t15 7.5l2 22q-12 -1 -17.5 7t-6.5 21q0 -2 -6 -8q0 7 -4.5 8t-11.5 -1t-9 -1q-10 3 -15 7.5t-8 16.5t-4 15q-2 5 -9.5 11t-9.5 10q-1 2 -2.5 5.5t-3 6.5t-4 5.5t-5.5 2.5t-7 -5t-7.5 -10t-4.5 -5q-3 2 -6 1.5t-4.5 -1t-4.5 -3t-5 -3.5 +q-3 -2 -8.5 -3t-8.5 -2q15 5 -1 11q-10 4 -16 3q9 4 7.5 12t-8.5 14h5q-1 4 -8.5 8.5t-17.5 8.5t-13 6q-8 5 -34 9.5t-33 0.5q-5 -6 -4.5 -10.5t4 -14t3.5 -12.5q1 -6 -5.5 -13t-6.5 -12q0 -7 14 -15.5t10 -21.5q-3 -8 -16 -16t-16 -12q-5 -8 -1.5 -18.5t10.5 -16.5 +q2 -2 1.5 -4t-3.5 -4.5t-5.5 -4t-6.5 -3.5l-3 -2q-11 -5 -20.5 6t-13.5 26q-7 25 -16 30q-23 8 -29 -1q-5 13 -41 26q-25 9 -58 4q6 1 0 15q-7 15 -19 12q3 6 4 17.5t1 13.5q3 13 12 23q1 1 7 8.5t9.5 13.5t0.5 6q35 -4 50 11q5 5 11.5 17t10.5 17q9 6 14 5.5t14.5 -5.5 +t14.5 -5q14 -1 15.5 11t-7.5 20q12 -1 3 17q-4 7 -8 9q-12 4 -27 -5q-8 -4 2 -8q-1 1 -9.5 -10.5t-16.5 -17.5t-16 5q-1 1 -5.5 13.5t-9.5 13.5q-8 0 -16 -15q3 8 -11 15t-24 8q19 12 -8 27q-7 4 -20.5 5t-19.5 -4q-5 -7 -5.5 -11.5t5 -8t10.5 -5.5t11.5 -4t8.5 -3 +q14 -10 8 -14q-2 -1 -8.5 -3.5t-11.5 -4.5t-6 -4q-3 -4 0 -14t-2 -14q-5 5 -9 17.5t-7 16.5q7 -9 -25 -6l-10 1q-4 0 -16 -2t-20.5 -1t-13.5 8q-4 8 0 20q1 4 4 2q-4 3 -11 9.5t-10 8.5q-46 -15 -94 -41q6 -1 12 1q5 2 13 6.5t10 5.5q34 14 42 7l5 5q14 -16 20 -25 +q-7 4 -30 1q-20 -6 -22 -12q7 -12 5 -18q-4 3 -11.5 10t-14.5 11t-15 5q-16 0 -22 -1q-146 -80 -235 -222q7 -7 12 -8q4 -1 5 -9t2.5 -11t11.5 3q9 -8 3 -19q1 1 44 -27q19 -17 21 -21q3 -11 -10 -18q-1 2 -9 9t-9 4q-3 -5 0.5 -18.5t10.5 -12.5q-7 0 -9.5 -16t-2.5 -35.5 +t-1 -23.5l2 -1q-3 -12 5.5 -34.5t21.5 -19.5q-13 -3 20 -43q6 -8 8 -9q3 -2 12 -7.5t15 -10t10 -10.5q4 -5 10 -22.5t14 -23.5q-2 -6 9.5 -20t10.5 -23q-1 0 -2.5 -1t-2.5 -1q3 -7 15.5 -14t15.5 -13q1 -3 2 -10t3 -11t8 -2q2 20 -24 62q-15 25 -17 29q-3 5 -5.5 15.5 +t-4.5 14.5q2 0 6 -1.5t8.5 -3.5t7.5 -4t2 -3q-3 -7 2 -17.5t12 -18.5t17 -19t12 -13q6 -6 14 -19.5t0 -13.5q9 0 20 -10.5t17 -19.5q5 -8 8 -26t5 -24q2 -7 8.5 -13.5t12.5 -9.5l16 -8t13 -7q5 -2 18.5 -10.5t21.5 -11.5q10 -4 16 -4t14.5 2.5t13.5 3.5q15 2 29 -15t21 -21 +q36 -19 55 -11q-2 -1 0.5 -7.5t8 -15.5t9 -14.5t5.5 -8.5q5 -6 18 -15t18 -15q6 4 7 9q-3 -8 7 -20t18 -10q14 3 14 32q-31 -15 -49 18q0 1 -2.5 5.5t-4 8.5t-2.5 8.5t0 7.5t5 3q9 0 10 3.5t-2 12.5t-4 13q-1 8 -11 20t-12 15q-5 -9 -16 -8t-16 9q0 -1 -1.5 -5.5t-1.5 -6.5 +q-13 0 -15 1q1 3 2.5 17.5t3.5 22.5q1 4 5.5 12t7.5 14.5t4 12.5t-4.5 9.5t-17.5 2.5q-19 -1 -26 -20q-1 -3 -3 -10.5t-5 -11.5t-9 -7q-7 -3 -24 -2t-24 5q-13 8 -22.5 29t-9.5 37q0 10 2.5 26.5t3 25t-5.5 24.5q3 2 9 9.5t10 10.5q2 1 4.5 1.5t4.5 0t4 1.5t3 6q-1 1 -4 3 +q-3 3 -4 3q7 -3 28.5 1.5t27.5 -1.5q15 -11 22 2q0 1 -2.5 9.5t-0.5 13.5q5 -27 29 -9q3 -3 15.5 -5t17.5 -5q3 -2 7 -5.5t5.5 -4.5t5 0.5t8.5 6.5q10 -14 12 -24q11 -40 19 -44q7 -3 11 -2t4.5 9.5t0 14t-1.5 12.5l-1 8v18l-1 8q-15 3 -18.5 12t1.5 18.5t15 18.5q1 1 8 3.5 +t15.5 6.5t12.5 8q21 19 15 35q7 0 11 9q-1 0 -5 3t-7.5 5t-4.5 2q9 5 2 16q5 3 7.5 11t7.5 10q9 -12 21 -2q8 8 1 16q5 7 20.5 10.5t18.5 9.5q7 -2 8 2t1 12t3 12q4 5 15 9t13 5l17 11q3 4 0 4q18 -2 31 11q10 11 -6 20q3 6 -3 9.5t-15 5.5q3 1 11.5 0.5t10.5 1.5 +q15 10 -7 16q-17 5 -43 -12zM879 10q206 36 351 189q-3 3 -12.5 4.5t-12.5 3.5q-18 7 -24 8q1 7 -2.5 13t-8 9t-12.5 8t-11 7q-2 2 -7 6t-7 5.5t-7.5 4.5t-8.5 2t-10 -1l-3 -1q-3 -1 -5.5 -2.5t-5.5 -3t-4 -3t0 -2.5q-21 17 -36 22q-5 1 -11 5.5t-10.5 7t-10 1.5t-11.5 -7 +q-5 -5 -6 -15t-2 -13q-7 5 0 17.5t2 18.5q-3 6 -10.5 4.5t-12 -4.5t-11.5 -8.5t-9 -6.5t-8.5 -5.5t-8.5 -7.5q-3 -4 -6 -12t-5 -11q-2 4 -11.5 6.5t-9.5 5.5q2 -10 4 -35t5 -38q7 -31 -12 -48q-27 -25 -29 -40q-4 -22 12 -26q0 -7 -8 -20.5t-7 -21.5q0 -6 2 -16z" /> + <glyph glyph-name="wrench" unicode="" horiz-adv-x="1664" +d="M384 64q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1028 484l-682 -682q-37 -37 -90 -37q-52 0 -91 37l-106 108q-38 36 -38 90q0 53 38 91l681 681q39 -98 114.5 -173.5t173.5 -114.5zM1662 919q0 -39 -23 -106q-47 -134 -164.5 -217.5 +t-258.5 -83.5q-185 0 -316.5 131.5t-131.5 316.5t131.5 316.5t316.5 131.5q58 0 121.5 -16.5t107.5 -46.5q16 -11 16 -28t-16 -28l-293 -169v-224l193 -107q5 3 79 48.5t135.5 81t70.5 35.5q15 0 23.5 -10t8.5 -25z" /> + <glyph glyph-name="tasks" unicode="" horiz-adv-x="1792" +d="M1024 128h640v128h-640v-128zM640 640h1024v128h-1024v-128zM1280 1152h384v128h-384v-128zM1792 320v-256q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 832v-256q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19 +t-19 45v256q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 1344v-256q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h1664q26 0 45 -19t19 -45z" /> + <glyph glyph-name="filter" unicode="" horiz-adv-x="1408" +d="M1403 1241q17 -41 -14 -70l-493 -493v-742q0 -42 -39 -59q-13 -5 -25 -5q-27 0 -45 19l-256 256q-19 19 -19 45v486l-493 493q-31 29 -14 70q17 39 59 39h1280q42 0 59 -39z" /> + <glyph glyph-name="briefcase" unicode="" horiz-adv-x="1792" +d="M640 1280h512v128h-512v-128zM1792 640v-480q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v480h672v-160q0 -26 19 -45t45 -19h320q26 0 45 19t19 45v160h672zM1024 640v-128h-256v128h256zM1792 1120v-384h-1792v384q0 66 47 113t113 47h352v160q0 40 28 68 +t68 28h576q40 0 68 -28t28 -68v-160h352q66 0 113 -47t47 -113z" /> + <glyph glyph-name="fullscreen" unicode="" +d="M1283 995l-355 -355l355 -355l144 144q29 31 70 14q39 -17 39 -59v-448q0 -26 -19 -45t-45 -19h-448q-42 0 -59 40q-17 39 14 69l144 144l-355 355l-355 -355l144 -144q31 -30 14 -69q-17 -40 -59 -40h-448q-26 0 -45 19t-19 45v448q0 42 40 59q39 17 69 -14l144 -144 +l355 355l-355 355l-144 -144q-19 -19 -45 -19q-12 0 -24 5q-40 17 -40 59v448q0 26 19 45t45 19h448q42 0 59 -40q17 -39 -14 -69l-144 -144l355 -355l355 355l-144 144q-31 30 -14 69q17 40 59 40h448q26 0 45 -19t19 -45v-448q0 -42 -39 -59q-13 -5 -25 -5q-26 0 -45 19z +" /> + <glyph glyph-name="group" unicode="" horiz-adv-x="1920" +d="M593 640q-162 -5 -265 -128h-134q-82 0 -138 40.5t-56 118.5q0 353 124 353q6 0 43.5 -21t97.5 -42.5t119 -21.5q67 0 133 23q-5 -37 -5 -66q0 -139 81 -256zM1664 3q0 -120 -73 -189.5t-194 -69.5h-874q-121 0 -194 69.5t-73 189.5q0 53 3.5 103.5t14 109t26.5 108.5 +t43 97.5t62 81t85.5 53.5t111.5 20q10 0 43 -21.5t73 -48t107 -48t135 -21.5t135 21.5t107 48t73 48t43 21.5q61 0 111.5 -20t85.5 -53.5t62 -81t43 -97.5t26.5 -108.5t14 -109t3.5 -103.5zM640 1280q0 -106 -75 -181t-181 -75t-181 75t-75 181t75 181t181 75t181 -75 +t75 -181zM1344 896q0 -159 -112.5 -271.5t-271.5 -112.5t-271.5 112.5t-112.5 271.5t112.5 271.5t271.5 112.5t271.5 -112.5t112.5 -271.5zM1920 671q0 -78 -56 -118.5t-138 -40.5h-134q-103 123 -265 128q81 117 81 256q0 29 -5 66q66 -23 133 -23q59 0 119 21.5t97.5 42.5 +t43.5 21q124 0 124 -353zM1792 1280q0 -106 -75 -181t-181 -75t-181 75t-75 181t75 181t181 75t181 -75t75 -181z" /> + <glyph glyph-name="link" unicode="" horiz-adv-x="1664" +d="M1456 320q0 40 -28 68l-208 208q-28 28 -68 28q-42 0 -72 -32q3 -3 19 -18.5t21.5 -21.5t15 -19t13 -25.5t3.5 -27.5q0 -40 -28 -68t-68 -28q-15 0 -27.5 3.5t-25.5 13t-19 15t-21.5 21.5t-18.5 19q-33 -31 -33 -73q0 -40 28 -68l206 -207q27 -27 68 -27q40 0 68 26 +l147 146q28 28 28 67zM753 1025q0 40 -28 68l-206 207q-28 28 -68 28q-39 0 -68 -27l-147 -146q-28 -28 -28 -67q0 -40 28 -68l208 -208q27 -27 68 -27q42 0 72 31q-3 3 -19 18.5t-21.5 21.5t-15 19t-13 25.5t-3.5 27.5q0 40 28 68t68 28q15 0 27.5 -3.5t25.5 -13t19 -15 +t21.5 -21.5t18.5 -19q33 31 33 73zM1648 320q0 -120 -85 -203l-147 -146q-83 -83 -203 -83q-121 0 -204 85l-206 207q-83 83 -83 203q0 123 88 209l-88 88q-86 -88 -208 -88q-120 0 -204 84l-208 208q-84 84 -84 204t85 203l147 146q83 83 203 83q121 0 204 -85l206 -207 +q83 -83 83 -203q0 -123 -88 -209l88 -88q86 88 208 88q120 0 204 -84l208 -208q84 -84 84 -204z" /> + <glyph glyph-name="cloud" unicode="" horiz-adv-x="1920" +d="M1920 384q0 -159 -112.5 -271.5t-271.5 -112.5h-1088q-185 0 -316.5 131.5t-131.5 316.5q0 132 71 241.5t187 163.5q-2 28 -2 43q0 212 150 362t362 150q158 0 286.5 -88t187.5 -230q70 62 166 62q106 0 181 -75t75 -181q0 -75 -41 -138q129 -30 213 -134.5t84 -239.5z +" /> + <glyph glyph-name="beaker" unicode="" horiz-adv-x="1664" +d="M1527 88q56 -89 21.5 -152.5t-140.5 -63.5h-1152q-106 0 -140.5 63.5t21.5 152.5l503 793v399h-64q-26 0 -45 19t-19 45t19 45t45 19h512q26 0 45 -19t19 -45t-19 -45t-45 -19h-64v-399zM748 813l-272 -429h712l-272 429l-20 31v37v399h-128v-399v-37z" /> + <glyph glyph-name="cut" unicode="" horiz-adv-x="1792" +d="M960 640q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19zM1260 576l507 -398q28 -20 25 -56q-5 -35 -35 -51l-128 -64q-13 -7 -29 -7q-17 0 -31 8l-690 387l-110 -66q-8 -4 -12 -5q14 -49 10 -97q-7 -77 -56 -147.5t-132 -123.5q-132 -84 -277 -84 +q-136 0 -222 78q-90 84 -79 207q7 76 56 147t131 124q132 84 278 84q83 0 151 -31q9 13 22 22l122 73l-122 73q-13 9 -22 22q-68 -31 -151 -31q-146 0 -278 84q-82 53 -131 124t-56 147q-5 59 15.5 113t63.5 93q85 79 222 79q145 0 277 -84q83 -52 132 -123t56 -148 +q4 -48 -10 -97q4 -1 12 -5l110 -66l690 387q14 8 31 8q16 0 29 -7l128 -64q30 -16 35 -51q3 -36 -25 -56zM579 836q46 42 21 108t-106 117q-92 59 -192 59q-74 0 -113 -36q-46 -42 -21 -108t106 -117q92 -59 192 -59q74 0 113 36zM494 91q81 51 106 117t-21 108 +q-39 36 -113 36q-100 0 -192 -59q-81 -51 -106 -117t21 -108q39 -36 113 -36q100 0 192 59zM672 704l96 -58v11q0 36 33 56l14 8l-79 47l-26 -26q-3 -3 -10 -11t-12 -12q-2 -2 -4 -3.5t-3 -2.5zM896 480l96 -32l736 576l-128 64l-768 -431v-113l-160 -96l9 -8q2 -2 7 -6 +q4 -4 11 -12t11 -12l26 -26zM1600 64l128 64l-520 408l-177 -138q-2 -3 -13 -7z" /> + <glyph glyph-name="copy" unicode="" horiz-adv-x="1792" +d="M1696 1152q40 0 68 -28t28 -68v-1216q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v288h-544q-40 0 -68 28t-28 68v672q0 40 20 88t48 76l408 408q28 28 76 48t88 20h416q40 0 68 -28t28 -68v-328q68 40 128 40h416zM1152 939l-299 -299h299v299zM512 1323l-299 -299 +h299v299zM708 676l316 316v416h-384v-416q0 -40 -28 -68t-68 -28h-416v-640h512v256q0 40 20 88t48 76zM1664 -128v1152h-384v-416q0 -40 -28 -68t-68 -28h-416v-640h896z" /> + <glyph glyph-name="paper_clip" unicode="" horiz-adv-x="1408" +d="M1404 151q0 -117 -79 -196t-196 -79q-135 0 -235 100l-777 776q-113 115 -113 271q0 159 110 270t269 111q158 0 273 -113l605 -606q10 -10 10 -22q0 -16 -30.5 -46.5t-46.5 -30.5q-13 0 -23 10l-606 607q-79 77 -181 77q-106 0 -179 -75t-73 -181q0 -105 76 -181 +l776 -777q63 -63 145 -63q64 0 106 42t42 106q0 82 -63 145l-581 581q-26 24 -60 24q-29 0 -48 -19t-19 -48q0 -32 25 -59l410 -410q10 -10 10 -22q0 -16 -31 -47t-47 -31q-12 0 -22 10l-410 410q-63 61 -63 149q0 82 57 139t139 57q88 0 149 -63l581 -581q100 -98 100 -235 +z" /> + <glyph glyph-name="save" unicode="" +d="M384 0h768v384h-768v-384zM1280 0h128v896q0 14 -10 38.5t-20 34.5l-281 281q-10 10 -34 20t-39 10v-416q0 -40 -28 -68t-68 -28h-576q-40 0 -68 28t-28 68v416h-128v-1280h128v416q0 40 28 68t68 28h832q40 0 68 -28t28 -68v-416zM896 928v320q0 13 -9.5 22.5t-22.5 9.5 +h-192q-13 0 -22.5 -9.5t-9.5 -22.5v-320q0 -13 9.5 -22.5t22.5 -9.5h192q13 0 22.5 9.5t9.5 22.5zM1536 896v-928q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1344q0 40 28 68t68 28h928q40 0 88 -20t76 -48l280 -280q28 -28 48 -76t20 -88z" /> + <glyph glyph-name="sign_blank" unicode="" +d="M1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="reorder" unicode="" +d="M1536 192v-128q0 -26 -19 -45t-45 -19h-1408q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1408q26 0 45 -19t19 -45zM1536 704v-128q0 -26 -19 -45t-45 -19h-1408q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1408q26 0 45 -19t19 -45zM1536 1216v-128q0 -26 -19 -45 +t-45 -19h-1408q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1408q26 0 45 -19t19 -45z" /> + <glyph glyph-name="ul" unicode="" horiz-adv-x="1792" +d="M384 128q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM384 640q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM1792 224v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1216q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5 +t22.5 9.5h1216q13 0 22.5 -9.5t9.5 -22.5zM384 1152q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM1792 736v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1216q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1216q13 0 22.5 -9.5t9.5 -22.5z +M1792 1248v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1216q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1216q13 0 22.5 -9.5t9.5 -22.5z" /> + <glyph glyph-name="ol" unicode="" horiz-adv-x="1792" +d="M381 -84q0 -80 -54.5 -126t-135.5 -46q-106 0 -172 66l57 88q49 -45 106 -45q29 0 50.5 14.5t21.5 42.5q0 64 -105 56l-26 56q8 10 32.5 43.5t42.5 54t37 38.5v1q-16 0 -48.5 -1t-48.5 -1v-53h-106v152h333v-88l-95 -115q51 -12 81 -49t30 -88zM383 543v-159h-362 +q-6 36 -6 54q0 51 23.5 93t56.5 68t66 47.5t56.5 43.5t23.5 45q0 25 -14.5 38.5t-39.5 13.5q-46 0 -81 -58l-85 59q24 51 71.5 79.5t105.5 28.5q73 0 123 -41.5t50 -112.5q0 -50 -34 -91.5t-75 -64.5t-75.5 -50.5t-35.5 -52.5h127v60h105zM1792 224v-192q0 -13 -9.5 -22.5 +t-22.5 -9.5h-1216q-13 0 -22.5 9.5t-9.5 22.5v192q0 14 9 23t23 9h1216q13 0 22.5 -9.5t9.5 -22.5zM384 1123v-99h-335v99h107q0 41 0.5 121.5t0.5 121.5v12h-2q-8 -17 -50 -54l-71 76l136 127h106v-404h108zM1792 736v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1216 +q-13 0 -22.5 9.5t-9.5 22.5v192q0 14 9 23t23 9h1216q13 0 22.5 -9.5t9.5 -22.5zM1792 1248v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1216q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1216q13 0 22.5 -9.5t9.5 -22.5z" /> + <glyph glyph-name="strikethrough" unicode="" horiz-adv-x="1792" +d="M1760 640q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-1728q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h1728zM483 704q-28 35 -51 80q-48 98 -48 188q0 181 134 309q133 127 393 127q50 0 167 -19q66 -12 177 -48q10 -38 21 -118q14 -123 14 -183q0 -18 -5 -45l-12 -3l-84 6 +l-14 2q-50 149 -103 205q-88 91 -210 91q-114 0 -182 -59q-67 -58 -67 -146q0 -73 66 -140t279 -129q69 -20 173 -66q58 -28 95 -52h-743zM990 448h411q7 -39 7 -92q0 -111 -41 -212q-23 -56 -71 -104q-37 -35 -109 -81q-80 -48 -153 -66q-80 -21 -203 -21q-114 0 -195 23 +l-140 40q-57 16 -72 28q-8 8 -8 22v13q0 108 -2 156q-1 30 0 68l2 37v44l102 2q15 -34 30 -71t22.5 -56t12.5 -27q35 -57 80 -94q43 -36 105 -57q59 -22 132 -22q64 0 139 27q77 26 122 86q47 61 47 129q0 84 -81 157q-34 29 -137 71z" /> + <glyph glyph-name="underline" unicode="" +d="M48 1313q-37 2 -45 4l-3 88q13 1 40 1q60 0 112 -4q132 -7 166 -7q86 0 168 3q116 4 146 5q56 0 86 2l-1 -14l2 -64v-9q-60 -9 -124 -9q-60 0 -79 -25q-13 -14 -13 -132q0 -13 0.5 -32.5t0.5 -25.5l1 -229l14 -280q6 -124 51 -202q35 -59 96 -92q88 -47 177 -47 +q104 0 191 28q56 18 99 51q48 36 65 64q36 56 53 114q21 73 21 229q0 79 -3.5 128t-11 122.5t-13.5 159.5l-4 59q-5 67 -24 88q-34 35 -77 34l-100 -2l-14 3l2 86h84l205 -10q76 -3 196 10l18 -2q6 -38 6 -51q0 -7 -4 -31q-45 -12 -84 -13q-73 -11 -79 -17q-15 -15 -15 -41 +q0 -7 1.5 -27t1.5 -31q8 -19 22 -396q6 -195 -15 -304q-15 -76 -41 -122q-38 -65 -112 -123q-75 -57 -182 -89q-109 -33 -255 -33q-167 0 -284 46q-119 47 -179 122q-61 76 -83 195q-16 80 -16 237v333q0 188 -17 213q-25 36 -147 39zM1536 -96v64q0 14 -9 23t-23 9h-1472 +q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h1472q14 0 23 9t9 23z" /> + <glyph glyph-name="table" unicode="" horiz-adv-x="1664" +d="M512 160v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM512 544v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1024 160v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23 +v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM512 928v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1024 544v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1536 160v192 +q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1024 928v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1536 544v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192 +q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1536 928v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1664 1248v-1088q0 -66 -47 -113t-113 -47h-1344q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h1344q66 0 113 -47t47 -113 +z" /> + <glyph glyph-name="magic" unicode="" horiz-adv-x="1664" +d="M1190 955l293 293l-107 107l-293 -293zM1637 1248q0 -27 -18 -45l-1286 -1286q-18 -18 -45 -18t-45 18l-198 198q-18 18 -18 45t18 45l1286 1286q18 18 45 18t45 -18l198 -198q18 -18 18 -45zM286 1438l98 -30l-98 -30l-30 -98l-30 98l-98 30l98 30l30 98zM636 1276 +l196 -60l-196 -60l-60 -196l-60 196l-196 60l196 60l60 196zM1566 798l98 -30l-98 -30l-30 -98l-30 98l-98 30l98 30l30 98zM926 1438l98 -30l-98 -30l-30 -98l-30 98l-98 30l98 30l30 98z" /> + <glyph glyph-name="truck" unicode="" horiz-adv-x="1792" +d="M640 128q0 52 -38 90t-90 38t-90 -38t-38 -90t38 -90t90 -38t90 38t38 90zM256 640h384v256h-158q-13 0 -22 -9l-195 -195q-9 -9 -9 -22v-30zM1536 128q0 52 -38 90t-90 38t-90 -38t-38 -90t38 -90t90 -38t90 38t38 90zM1792 1216v-1024q0 -15 -4 -26.5t-13.5 -18.5 +t-16.5 -11.5t-23.5 -6t-22.5 -2t-25.5 0t-22.5 0.5q0 -106 -75 -181t-181 -75t-181 75t-75 181h-384q0 -106 -75 -181t-181 -75t-181 75t-75 181h-64q-3 0 -22.5 -0.5t-25.5 0t-22.5 2t-23.5 6t-16.5 11.5t-13.5 18.5t-4 26.5q0 26 19 45t45 19v320q0 8 -0.5 35t0 38 +t2.5 34.5t6.5 37t14 30.5t22.5 30l198 198q19 19 50.5 32t58.5 13h160v192q0 26 19 45t45 19h1024q26 0 45 -19t19 -45z" /> + <glyph glyph-name="pinterest" unicode="" +d="M1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103q-111 0 -218 32q59 93 78 164q9 34 54 211q20 -39 73 -67.5t114 -28.5q121 0 216 68.5t147 188.5t52 270q0 114 -59.5 214t-172.5 163t-255 63q-105 0 -196 -29t-154.5 -77t-109 -110.5t-67 -129.5t-21.5 -134 +q0 -104 40 -183t117 -111q30 -12 38 20q2 7 8 31t8 30q6 23 -11 43q-51 61 -51 151q0 151 104.5 259.5t273.5 108.5q151 0 235.5 -82t84.5 -213q0 -170 -68.5 -289t-175.5 -119q-61 0 -98 43.5t-23 104.5q8 35 26.5 93.5t30 103t11.5 75.5q0 50 -27 83t-77 33 +q-62 0 -105 -57t-43 -142q0 -73 25 -122l-99 -418q-17 -70 -13 -177q-206 91 -333 281t-127 423q0 209 103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="pinterest_sign" unicode="" +d="M1248 1408q119 0 203.5 -84.5t84.5 -203.5v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-725q85 122 108 210q9 34 53 209q21 -39 73.5 -67t112.5 -28q181 0 295.5 147.5t114.5 373.5q0 84 -35 162.5t-96.5 139t-152.5 97t-197 36.5q-104 0 -194.5 -28.5t-153 -76.5 +t-107.5 -109.5t-66.5 -128t-21.5 -132.5q0 -102 39.5 -180t116.5 -110q13 -5 23.5 0t14.5 19q10 44 15 61q6 23 -11 42q-50 62 -50 150q0 150 103.5 256.5t270.5 106.5q149 0 232.5 -81t83.5 -210q0 -168 -67.5 -286t-173.5 -118q-60 0 -97 43.5t-23 103.5q8 34 26.5 92.5 +t29.5 102t11 74.5q0 49 -26.5 81.5t-75.5 32.5q-61 0 -103.5 -56.5t-42.5 -139.5q0 -72 24 -121l-98 -414q-24 -100 -7 -254h-183q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960z" /> + <glyph glyph-name="google_plus_sign" unicode="" +d="M917 631q0 26 -6 64h-362v-132h217q-3 -24 -16.5 -50t-37.5 -53t-66.5 -44.5t-96.5 -17.5q-99 0 -169 71t-70 171t70 171t169 71q92 0 153 -59l104 101q-108 100 -257 100q-160 0 -272 -112.5t-112 -271.5t112 -271.5t272 -112.5q165 0 266.5 105t101.5 270zM1262 585 +h109v110h-109v110h-110v-110h-110v-110h110v-110h110v110zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="google_plus" unicode="" horiz-adv-x="2304" +d="M1437 623q0 -208 -87 -370.5t-248 -254t-369 -91.5q-149 0 -285 58t-234 156t-156 234t-58 285t58 285t156 234t234 156t285 58q286 0 491 -192l-199 -191q-117 113 -292 113q-123 0 -227.5 -62t-165.5 -168.5t-61 -232.5t61 -232.5t165.5 -168.5t227.5 -62 +q83 0 152.5 23t114.5 57.5t78.5 78.5t49 83t21.5 74h-416v252h692q12 -63 12 -122zM2304 745v-210h-209v-209h-210v209h-209v210h209v209h210v-209h209z" /> + <glyph glyph-name="money" unicode="" horiz-adv-x="1920" +d="M768 384h384v96h-128v448h-114l-148 -137l77 -80q42 37 55 57h2v-288h-128v-96zM1280 640q0 -70 -21 -142t-59.5 -134t-101.5 -101t-138 -39t-138 39t-101.5 101t-59.5 134t-21 142t21 142t59.5 134t101.5 101t138 39t138 -39t101.5 -101t59.5 -134t21 -142zM1792 384 +v512q-106 0 -181 75t-75 181h-1152q0 -106 -75 -181t-181 -75v-512q106 0 181 -75t75 -181h1152q0 106 75 181t181 75zM1920 1216v-1152q0 -26 -19 -45t-45 -19h-1792q-26 0 -45 19t-19 45v1152q0 26 19 45t45 19h1792q26 0 45 -19t19 -45z" /> + <glyph glyph-name="caret_down" unicode="" horiz-adv-x="1024" +d="M1024 832q0 -26 -19 -45l-448 -448q-19 -19 -45 -19t-45 19l-448 448q-19 19 -19 45t19 45t45 19h896q26 0 45 -19t19 -45z" /> + <glyph glyph-name="caret_up" unicode="" horiz-adv-x="1024" +d="M1024 320q0 -26 -19 -45t-45 -19h-896q-26 0 -45 19t-19 45t19 45l448 448q19 19 45 19t45 -19l448 -448q19 -19 19 -45z" /> + <glyph glyph-name="caret_left" unicode="" horiz-adv-x="640" +d="M640 1088v-896q0 -26 -19 -45t-45 -19t-45 19l-448 448q-19 19 -19 45t19 45l448 448q19 19 45 19t45 -19t19 -45z" /> + <glyph glyph-name="caret_right" unicode="" horiz-adv-x="640" +d="M576 640q0 -26 -19 -45l-448 -448q-19 -19 -45 -19t-45 19t-19 45v896q0 26 19 45t45 19t45 -19l448 -448q19 -19 19 -45z" /> + <glyph glyph-name="columns" unicode="" horiz-adv-x="1664" +d="M160 0h608v1152h-640v-1120q0 -13 9.5 -22.5t22.5 -9.5zM1536 32v1120h-640v-1152h608q13 0 22.5 9.5t9.5 22.5zM1664 1248v-1216q0 -66 -47 -113t-113 -47h-1344q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1344q66 0 113 -47t47 -113z" /> + <glyph glyph-name="sort" unicode="" horiz-adv-x="1024" +d="M1024 448q0 -26 -19 -45l-448 -448q-19 -19 -45 -19t-45 19l-448 448q-19 19 -19 45t19 45t45 19h896q26 0 45 -19t19 -45zM1024 832q0 -26 -19 -45t-45 -19h-896q-26 0 -45 19t-19 45t19 45l448 448q19 19 45 19t45 -19l448 -448q19 -19 19 -45z" /> + <glyph glyph-name="sort_down" unicode="" horiz-adv-x="1024" +d="M1024 448q0 -26 -19 -45l-448 -448q-19 -19 -45 -19t-45 19l-448 448q-19 19 -19 45t19 45t45 19h896q26 0 45 -19t19 -45z" /> + <glyph glyph-name="sort_up" unicode="" horiz-adv-x="1024" +d="M1024 832q0 -26 -19 -45t-45 -19h-896q-26 0 -45 19t-19 45t19 45l448 448q19 19 45 19t45 -19l448 -448q19 -19 19 -45z" /> + <glyph glyph-name="envelope_alt" unicode="" horiz-adv-x="1792" +d="M1792 826v-794q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v794q44 -49 101 -87q362 -246 497 -345q57 -42 92.5 -65.5t94.5 -48t110 -24.5h1h1q51 0 110 24.5t94.5 48t92.5 65.5q170 123 498 345q57 39 100 87zM1792 1120q0 -79 -49 -151t-122 -123 +q-376 -261 -468 -325q-10 -7 -42.5 -30.5t-54 -38t-52 -32.5t-57.5 -27t-50 -9h-1h-1q-23 0 -50 9t-57.5 27t-52 32.5t-54 38t-42.5 30.5q-91 64 -262 182.5t-205 142.5q-62 42 -117 115.5t-55 136.5q0 78 41.5 130t118.5 52h1472q65 0 112.5 -47t47.5 -113z" /> + <glyph glyph-name="linkedin" unicode="" +d="M349 911v-991h-330v991h330zM370 1217q1 -73 -50.5 -122t-135.5 -49h-2q-82 0 -132 49t-50 122q0 74 51.5 122.5t134.5 48.5t133 -48.5t51 -122.5zM1536 488v-568h-329v530q0 105 -40.5 164.5t-126.5 59.5q-63 0 -105.5 -34.5t-63.5 -85.5q-11 -30 -11 -81v-553h-329 +q2 399 2 647t-1 296l-1 48h329v-144h-2q20 32 41 56t56.5 52t87 43.5t114.5 15.5q171 0 275 -113.5t104 -332.5z" /> + <glyph glyph-name="undo" unicode="" +d="M1536 640q0 -156 -61 -298t-164 -245t-245 -164t-298 -61q-172 0 -327 72.5t-264 204.5q-7 10 -6.5 22.5t8.5 20.5l137 138q10 9 25 9q16 -2 23 -12q73 -95 179 -147t225 -52q104 0 198.5 40.5t163.5 109.5t109.5 163.5t40.5 198.5t-40.5 198.5t-109.5 163.5 +t-163.5 109.5t-198.5 40.5q-98 0 -188 -35.5t-160 -101.5l137 -138q31 -30 14 -69q-17 -40 -59 -40h-448q-26 0 -45 19t-19 45v448q0 42 40 59q39 17 69 -14l130 -129q107 101 244.5 156.5t284.5 55.5q156 0 298 -61t245 -164t164 -245t61 -298z" /> + <glyph glyph-name="legal" unicode="" horiz-adv-x="1792" +d="M1771 0q0 -53 -37 -90l-107 -108q-39 -37 -91 -37q-53 0 -90 37l-363 364q-38 36 -38 90q0 53 43 96l-256 256l-126 -126q-14 -14 -34 -14t-34 14q2 -2 12.5 -12t12.5 -13t10 -11.5t10 -13.5t6 -13.5t5.5 -16.5t1.5 -18q0 -38 -28 -68q-3 -3 -16.5 -18t-19 -20.5 +t-18.5 -16.5t-22 -15.5t-22 -9t-26 -4.5q-40 0 -68 28l-408 408q-28 28 -28 68q0 13 4.5 26t9 22t15.5 22t16.5 18.5t20.5 19t18 16.5q30 28 68 28q10 0 18 -1.5t16.5 -5.5t13.5 -6t13.5 -10t11.5 -10t13 -12.5t12 -12.5q-14 14 -14 34t14 34l348 348q14 14 34 14t34 -14 +q-2 2 -12.5 12t-12.5 13t-10 11.5t-10 13.5t-6 13.5t-5.5 16.5t-1.5 18q0 38 28 68q3 3 16.5 18t19 20.5t18.5 16.5t22 15.5t22 9t26 4.5q40 0 68 -28l408 -408q28 -28 28 -68q0 -13 -4.5 -26t-9 -22t-15.5 -22t-16.5 -18.5t-20.5 -19t-18 -16.5q-30 -28 -68 -28 +q-10 0 -18 1.5t-16.5 5.5t-13.5 6t-13.5 10t-11.5 10t-13 12.5t-12 12.5q14 -14 14 -34t-14 -34l-126 -126l256 -256q43 43 96 43q52 0 91 -37l363 -363q37 -39 37 -91z" /> + <glyph glyph-name="dashboard" unicode="" horiz-adv-x="1792" +d="M384 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM576 832q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1004 351l101 382q6 26 -7.5 48.5t-38.5 29.5 +t-48 -6.5t-30 -39.5l-101 -382q-60 -5 -107 -43.5t-63 -98.5q-20 -77 20 -146t117 -89t146 20t89 117q16 60 -6 117t-72 91zM1664 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1024 1024q0 53 -37.5 90.5 +t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1472 832q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1792 384q0 -261 -141 -483q-19 -29 -54 -29h-1402q-35 0 -54 29 +q-141 221 -141 483q0 182 71 348t191 286t286 191t348 71t348 -71t286 -191t191 -286t71 -348z" /> + <glyph glyph-name="comment_alt" unicode="" horiz-adv-x="1792" +d="M896 1152q-204 0 -381.5 -69.5t-282 -187.5t-104.5 -255q0 -112 71.5 -213.5t201.5 -175.5l87 -50l-27 -96q-24 -91 -70 -172q152 63 275 171l43 38l57 -6q69 -8 130 -8q204 0 381.5 69.5t282 187.5t104.5 255t-104.5 255t-282 187.5t-381.5 69.5zM1792 640 +q0 -174 -120 -321.5t-326 -233t-450 -85.5q-70 0 -145 8q-198 -175 -460 -242q-49 -14 -114 -22h-5q-15 0 -27 10.5t-16 27.5v1q-3 4 -0.5 12t2 10t4.5 9.5l6 9t7 8.5t8 9q7 8 31 34.5t34.5 38t31 39.5t32.5 51t27 59t26 76q-157 89 -247.5 220t-90.5 281q0 174 120 321.5 +t326 233t450 85.5t450 -85.5t326 -233t120 -321.5z" /> + <glyph glyph-name="comments_alt" unicode="" horiz-adv-x="1792" +d="M704 1152q-153 0 -286 -52t-211.5 -141t-78.5 -191q0 -82 53 -158t149 -132l97 -56l-35 -84q34 20 62 39l44 31l53 -10q78 -14 153 -14q153 0 286 52t211.5 141t78.5 191t-78.5 191t-211.5 141t-286 52zM704 1280q191 0 353.5 -68.5t256.5 -186.5t94 -257t-94 -257 +t-256.5 -186.5t-353.5 -68.5q-86 0 -176 16q-124 -88 -278 -128q-36 -9 -86 -16h-3q-11 0 -20.5 8t-11.5 21q-1 3 -1 6.5t0.5 6.5t2 6l2.5 5t3.5 5.5t4 5t4.5 5t4 4.5q5 6 23 25t26 29.5t22.5 29t25 38.5t20.5 44q-124 72 -195 177t-71 224q0 139 94 257t256.5 186.5 +t353.5 68.5zM1526 111q10 -24 20.5 -44t25 -38.5t22.5 -29t26 -29.5t23 -25q1 -1 4 -4.5t4.5 -5t4 -5t3.5 -5.5l2.5 -5t2 -6t0.5 -6.5t-1 -6.5q-3 -14 -13 -22t-22 -7q-50 7 -86 16q-154 40 -278 128q-90 -16 -176 -16q-271 0 -472 132q58 -4 88 -4q161 0 309 45t264 129 +q125 92 192 212t67 254q0 77 -23 152q129 -71 204 -178t75 -230q0 -120 -71 -224.5t-195 -176.5z" /> + <glyph glyph-name="bolt" unicode="" horiz-adv-x="896" +d="M885 970q18 -20 7 -44l-540 -1157q-13 -25 -42 -25q-4 0 -14 2q-17 5 -25.5 19t-4.5 30l197 808l-406 -101q-4 -1 -12 -1q-18 0 -31 11q-18 15 -13 39l201 825q4 14 16 23t28 9h328q19 0 32 -12.5t13 -29.5q0 -8 -5 -18l-171 -463l396 98q8 2 12 2q19 0 34 -15z" /> + <glyph glyph-name="sitemap" unicode="" horiz-adv-x="1792" +d="M1792 288v-320q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h96v192h-512v-192h96q40 0 68 -28t28 -68v-320q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h96v192h-512v-192h96q40 0 68 -28t28 -68v-320 +q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h96v192q0 52 38 90t90 38h512v192h-96q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h320q40 0 68 -28t28 -68v-320q0 -40 -28 -68t-68 -28h-96v-192h512q52 0 90 -38t38 -90v-192h96q40 0 68 -28t28 -68 +z" /> + <glyph glyph-name="umbrella" unicode="" horiz-adv-x="1664" +d="M896 708v-580q0 -104 -76 -180t-180 -76t-180 76t-76 180q0 26 19 45t45 19t45 -19t19 -45q0 -50 39 -89t89 -39t89 39t39 89v580q33 11 64 11t64 -11zM1664 681q0 -13 -9.5 -22.5t-22.5 -9.5q-11 0 -23 10q-49 46 -93 69t-102 23q-68 0 -128 -37t-103 -97 +q-7 -10 -17.5 -28t-14.5 -24q-11 -17 -28 -17q-18 0 -29 17q-4 6 -14.5 24t-17.5 28q-43 60 -102.5 97t-127.5 37t-127.5 -37t-102.5 -97q-7 -10 -17.5 -28t-14.5 -24q-11 -17 -29 -17q-17 0 -28 17q-4 6 -14.5 24t-17.5 28q-43 60 -103 97t-128 37q-58 0 -102 -23t-93 -69 +q-12 -10 -23 -10q-13 0 -22.5 9.5t-9.5 22.5q0 5 1 7q45 183 172.5 319.5t298 204.5t360.5 68q140 0 274.5 -40t246.5 -113.5t194.5 -187t115.5 -251.5q1 -2 1 -7zM896 1408v-98q-42 2 -64 2t-64 -2v98q0 26 19 45t45 19t45 -19t19 -45z" /> + <glyph glyph-name="paste" unicode="" horiz-adv-x="1792" +d="M768 -128h896v640h-416q-40 0 -68 28t-28 68v416h-384v-1152zM1024 1312v64q0 13 -9.5 22.5t-22.5 9.5h-704q-13 0 -22.5 -9.5t-9.5 -22.5v-64q0 -13 9.5 -22.5t22.5 -9.5h704q13 0 22.5 9.5t9.5 22.5zM1280 640h299l-299 299v-299zM1792 512v-672q0 -40 -28 -68t-68 -28 +h-960q-40 0 -68 28t-28 68v160h-544q-40 0 -68 28t-28 68v1344q0 40 28 68t68 28h1088q40 0 68 -28t28 -68v-328q21 -13 36 -28l408 -408q28 -28 48 -76t20 -88z" /> + <glyph glyph-name="light_bulb" unicode="" horiz-adv-x="1024" +d="M736 960q0 -13 -9.5 -22.5t-22.5 -9.5t-22.5 9.5t-9.5 22.5q0 46 -54 71t-106 25q-13 0 -22.5 9.5t-9.5 22.5t9.5 22.5t22.5 9.5q50 0 99.5 -16t87 -54t37.5 -90zM896 960q0 72 -34.5 134t-90 101.5t-123 62t-136.5 22.5t-136.5 -22.5t-123 -62t-90 -101.5t-34.5 -134 +q0 -101 68 -180q10 -11 30.5 -33t30.5 -33q128 -153 141 -298h228q13 145 141 298q10 11 30.5 33t30.5 33q68 79 68 180zM1024 960q0 -155 -103 -268q-45 -49 -74.5 -87t-59.5 -95.5t-34 -107.5q47 -28 47 -82q0 -37 -25 -64q25 -27 25 -64q0 -52 -45 -81q13 -23 13 -47 +q0 -46 -31.5 -71t-77.5 -25q-20 -44 -60 -70t-87 -26t-87 26t-60 70q-46 0 -77.5 25t-31.5 71q0 24 13 47q-45 29 -45 81q0 37 25 64q-25 27 -25 64q0 54 47 82q-4 50 -34 107.5t-59.5 95.5t-74.5 87q-103 113 -103 268q0 99 44.5 184.5t117 142t164 89t186.5 32.5 +t186.5 -32.5t164 -89t117 -142t44.5 -184.5z" /> + <glyph glyph-name="exchange" unicode="" horiz-adv-x="1792" +d="M1792 352v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1376v-192q0 -13 -9.5 -22.5t-22.5 -9.5q-12 0 -24 10l-319 320q-9 9 -9 22q0 14 9 23l320 320q9 9 23 9q13 0 22.5 -9.5t9.5 -22.5v-192h1376q13 0 22.5 -9.5t9.5 -22.5zM1792 896q0 -14 -9 -23l-320 -320q-9 -9 -23 -9 +q-13 0 -22.5 9.5t-9.5 22.5v192h-1376q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1376v192q0 14 9 23t23 9q12 0 24 -10l319 -319q9 -9 9 -23z" /> + <glyph glyph-name="cloud_download" unicode="" horiz-adv-x="1920" +d="M1280 608q0 14 -9 23t-23 9h-224v352q0 13 -9.5 22.5t-22.5 9.5h-192q-13 0 -22.5 -9.5t-9.5 -22.5v-352h-224q-13 0 -22.5 -9.5t-9.5 -22.5q0 -14 9 -23l352 -352q9 -9 23 -9t23 9l351 351q10 12 10 24zM1920 384q0 -159 -112.5 -271.5t-271.5 -112.5h-1088 +q-185 0 -316.5 131.5t-131.5 316.5q0 130 70 240t188 165q-2 30 -2 43q0 212 150 362t362 150q156 0 285.5 -87t188.5 -231q71 62 166 62q106 0 181 -75t75 -181q0 -76 -41 -138q130 -31 213.5 -135.5t83.5 -238.5z" /> + <glyph glyph-name="cloud_upload" unicode="" horiz-adv-x="1920" +d="M1280 672q0 14 -9 23l-352 352q-9 9 -23 9t-23 -9l-351 -351q-10 -12 -10 -24q0 -14 9 -23t23 -9h224v-352q0 -13 9.5 -22.5t22.5 -9.5h192q13 0 22.5 9.5t9.5 22.5v352h224q13 0 22.5 9.5t9.5 22.5zM1920 384q0 -159 -112.5 -271.5t-271.5 -112.5h-1088 +q-185 0 -316.5 131.5t-131.5 316.5q0 130 70 240t188 165q-2 30 -2 43q0 212 150 362t362 150q156 0 285.5 -87t188.5 -231q71 62 166 62q106 0 181 -75t75 -181q0 -76 -41 -138q130 -31 213.5 -135.5t83.5 -238.5z" /> + <glyph glyph-name="user_md" unicode="" horiz-adv-x="1408" +d="M384 192q0 -26 -19 -45t-45 -19t-45 19t-19 45t19 45t45 19t45 -19t19 -45zM1408 131q0 -121 -73 -190t-194 -69h-874q-121 0 -194 69t-73 190q0 68 5.5 131t24 138t47.5 132.5t81 103t120 60.5q-22 -52 -22 -120v-203q-58 -20 -93 -70t-35 -111q0 -80 56 -136t136 -56 +t136 56t56 136q0 61 -35.5 111t-92.5 70v203q0 62 25 93q132 -104 295 -104t295 104q25 -31 25 -93v-64q-106 0 -181 -75t-75 -181v-89q-32 -29 -32 -71q0 -40 28 -68t68 -28t68 28t28 68q0 42 -32 71v89q0 52 38 90t90 38t90 -38t38 -90v-89q-32 -29 -32 -71q0 -40 28 -68 +t68 -28t68 28t28 68q0 42 -32 71v89q0 68 -34.5 127.5t-93.5 93.5q0 10 0.5 42.5t0 48t-2.5 41.5t-7 47t-13 40q68 -15 120 -60.5t81 -103t47.5 -132.5t24 -138t5.5 -131zM1088 1024q0 -159 -112.5 -271.5t-271.5 -112.5t-271.5 112.5t-112.5 271.5t112.5 271.5t271.5 112.5 +t271.5 -112.5t112.5 -271.5z" /> + <glyph glyph-name="stethoscope" unicode="" horiz-adv-x="1408" +d="M1280 832q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1408 832q0 -62 -35.5 -111t-92.5 -70v-395q0 -159 -131.5 -271.5t-316.5 -112.5t-316.5 112.5t-131.5 271.5v132q-164 20 -274 128t-110 252v512q0 26 19 45t45 19q6 0 16 -2q17 30 47 48 +t65 18q53 0 90.5 -37.5t37.5 -90.5t-37.5 -90.5t-90.5 -37.5q-33 0 -64 18v-402q0 -106 94 -181t226 -75t226 75t94 181v402q-31 -18 -64 -18q-53 0 -90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5q35 0 65 -18t47 -48q10 2 16 2q26 0 45 -19t19 -45v-512q0 -144 -110 -252 +t-274 -128v-132q0 -106 94 -181t226 -75t226 75t94 181v395q-57 21 -92.5 70t-35.5 111q0 80 56 136t136 56t136 -56t56 -136z" /> + <glyph glyph-name="suitcase" unicode="" horiz-adv-x="1792" +d="M640 1152h512v128h-512v-128zM288 1152v-1280h-64q-92 0 -158 66t-66 158v832q0 92 66 158t158 66h64zM1408 1152v-1280h-1024v1280h128v160q0 40 28 68t68 28h576q40 0 68 -28t28 -68v-160h128zM1792 928v-832q0 -92 -66 -158t-158 -66h-64v1280h64q92 0 158 -66 +t66 -158z" /> + <glyph glyph-name="bell_alt" unicode="" horiz-adv-x="1792" +d="M912 -160q0 16 -16 16q-59 0 -101.5 42.5t-42.5 101.5q0 16 -16 16t-16 -16q0 -73 51.5 -124.5t124.5 -51.5q16 0 16 16zM1728 128q0 -52 -38 -90t-90 -38h-448q0 -106 -75 -181t-181 -75t-181 75t-75 181h-448q-52 0 -90 38t-38 90q50 42 91 88t85 119.5t74.5 158.5 +t50 206t19.5 260q0 152 117 282.5t307 158.5q-8 19 -8 39q0 40 28 68t68 28t68 -28t28 -68q0 -20 -8 -39q190 -28 307 -158.5t117 -282.5q0 -139 19.5 -260t50 -206t74.5 -158.5t85 -119.5t91 -88z" /> + <glyph glyph-name="coffee" unicode="" horiz-adv-x="1920" +d="M1664 896q0 80 -56 136t-136 56h-64v-384h64q80 0 136 56t56 136zM0 128h1792q0 -106 -75 -181t-181 -75h-1280q-106 0 -181 75t-75 181zM1856 896q0 -159 -112.5 -271.5t-271.5 -112.5h-64v-32q0 -92 -66 -158t-158 -66h-704q-92 0 -158 66t-66 158v736q0 26 19 45 +t45 19h1152q159 0 271.5 -112.5t112.5 -271.5z" /> + <glyph glyph-name="food" unicode="" horiz-adv-x="1408" +d="M640 1472v-640q0 -61 -35.5 -111t-92.5 -70v-779q0 -52 -38 -90t-90 -38h-128q-52 0 -90 38t-38 90v779q-57 20 -92.5 70t-35.5 111v640q0 26 19 45t45 19t45 -19t19 -45v-416q0 -26 19 -45t45 -19t45 19t19 45v416q0 26 19 45t45 19t45 -19t19 -45v-416q0 -26 19 -45 +t45 -19t45 19t19 45v416q0 26 19 45t45 19t45 -19t19 -45zM1408 1472v-1600q0 -52 -38 -90t-90 -38h-128q-52 0 -90 38t-38 90v512h-224q-13 0 -22.5 9.5t-9.5 22.5v800q0 132 94 226t226 94h256q26 0 45 -19t19 -45z" /> + <glyph glyph-name="file_text_alt" unicode="" +d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z +M384 736q0 14 9 23t23 9h704q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-704q-14 0 -23 9t-9 23v64zM1120 512q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-704q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h704zM1120 256q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-704 +q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h704z" /> + <glyph glyph-name="building" unicode="" horiz-adv-x="1408" +d="M384 224v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M640 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M1152 224v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM896 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M640 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 992v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M1152 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM896 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M640 992v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 1248v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M1152 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM896 992v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M640 1248v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM1152 992v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M896 1248v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM1152 1248v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M896 -128h384v1536h-1152v-1536h384v224q0 13 9.5 22.5t22.5 9.5h320q13 0 22.5 -9.5t9.5 -22.5v-224zM1408 1472v-1664q0 -26 -19 -45t-45 -19h-1280q-26 0 -45 19t-19 45v1664q0 26 19 45t45 19h1280q26 0 45 -19t19 -45z" /> + <glyph glyph-name="hospital" unicode="" horiz-adv-x="1408" +d="M384 224v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M640 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M1152 224v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM896 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M640 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM1152 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M896 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM1152 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z +M896 -128h384v1152h-256v-32q0 -40 -28 -68t-68 -28h-448q-40 0 -68 28t-28 68v32h-256v-1152h384v224q0 13 9.5 22.5t22.5 9.5h320q13 0 22.5 -9.5t9.5 -22.5v-224zM896 1056v320q0 13 -9.5 22.5t-22.5 9.5h-64q-13 0 -22.5 -9.5t-9.5 -22.5v-96h-128v96q0 13 -9.5 22.5 +t-22.5 9.5h-64q-13 0 -22.5 -9.5t-9.5 -22.5v-320q0 -13 9.5 -22.5t22.5 -9.5h64q13 0 22.5 9.5t9.5 22.5v96h128v-96q0 -13 9.5 -22.5t22.5 -9.5h64q13 0 22.5 9.5t9.5 22.5zM1408 1088v-1280q0 -26 -19 -45t-45 -19h-1280q-26 0 -45 19t-19 45v1280q0 26 19 45t45 19h320 +v288q0 40 28 68t68 28h448q40 0 68 -28t28 -68v-288h320q26 0 45 -19t19 -45z" /> + <glyph glyph-name="ambulance" unicode="" horiz-adv-x="1920" +d="M640 128q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM256 640h384v256h-158q-14 -2 -22 -9l-195 -195q-7 -12 -9 -22v-30zM1536 128q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5 +t90.5 37.5t37.5 90.5zM1664 800v192q0 14 -9 23t-23 9h-224v224q0 14 -9 23t-23 9h-192q-14 0 -23 -9t-9 -23v-224h-224q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h224v-224q0 -14 9 -23t23 -9h192q14 0 23 9t9 23v224h224q14 0 23 9t9 23zM1920 1344v-1152 +q0 -26 -19 -45t-45 -19h-192q0 -106 -75 -181t-181 -75t-181 75t-75 181h-384q0 -106 -75 -181t-181 -75t-181 75t-75 181h-128q-26 0 -45 19t-19 45t19 45t45 19v416q0 26 13 58t32 51l198 198q19 19 51 32t58 13h160v320q0 26 19 45t45 19h1152q26 0 45 -19t19 -45z" /> + <glyph glyph-name="medkit" unicode="" horiz-adv-x="1792" +d="M1280 416v192q0 14 -9 23t-23 9h-224v224q0 14 -9 23t-23 9h-192q-14 0 -23 -9t-9 -23v-224h-224q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h224v-224q0 -14 9 -23t23 -9h192q14 0 23 9t9 23v224h224q14 0 23 9t9 23zM640 1152h512v128h-512v-128zM256 1152v-1280h-32 +q-92 0 -158 66t-66 158v832q0 92 66 158t158 66h32zM1440 1152v-1280h-1088v1280h160v160q0 40 28 68t68 28h576q40 0 68 -28t28 -68v-160h160zM1792 928v-832q0 -92 -66 -158t-158 -66h-32v1280h32q92 0 158 -66t66 -158z" /> + <glyph glyph-name="fighter_jet" unicode="" horiz-adv-x="1920" +d="M1920 576q-1 -32 -288 -96l-352 -32l-224 -64h-64l-293 -352h69q26 0 45 -4.5t19 -11.5t-19 -11.5t-45 -4.5h-96h-160h-64v32h64v416h-160l-192 -224h-96l-32 32v192h32v32h128v8l-192 24v128l192 24v8h-128v32h-32v192l32 32h96l192 -224h160v416h-64v32h64h160h96 +q26 0 45 -4.5t19 -11.5t-19 -11.5t-45 -4.5h-69l293 -352h64l224 -64l352 -32q128 -28 200 -52t80 -34z" /> + <glyph glyph-name="beer" unicode="" horiz-adv-x="1664" +d="M640 640v384h-256v-256q0 -53 37.5 -90.5t90.5 -37.5h128zM1664 192v-192h-1152v192l128 192h-128q-159 0 -271.5 112.5t-112.5 271.5v320l-64 64l32 128h480l32 128h960l32 -192l-64 -32v-800z" /> + <glyph glyph-name="h_sign" unicode="" +d="M1280 192v896q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-320h-512v320q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-896q0 -26 19 -45t45 -19h128q26 0 45 19t19 45v320h512v-320q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1536 1120v-960 +q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="f0fe" unicode="" +d="M1280 576v128q0 26 -19 45t-45 19h-320v320q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-320h-320q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h320v-320q0 -26 19 -45t45 -19h128q26 0 45 19t19 45v320h320q26 0 45 19t19 45zM1536 1120v-960 +q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="double_angle_left" unicode="" horiz-adv-x="1024" +d="M627 160q0 -13 -10 -23l-50 -50q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l50 -50q10 -10 10 -23t-10 -23l-393 -393l393 -393q10 -10 10 -23zM1011 160q0 -13 -10 -23l-50 -50q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23 +t10 23l466 466q10 10 23 10t23 -10l50 -50q10 -10 10 -23t-10 -23l-393 -393l393 -393q10 -10 10 -23z" /> + <glyph glyph-name="double_angle_right" unicode="" horiz-adv-x="1024" +d="M595 576q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l393 393l-393 393q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l466 -466q10 -10 10 -23zM979 576q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23 +l393 393l-393 393q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l466 -466q10 -10 10 -23z" /> + <glyph glyph-name="double_angle_up" unicode="" horiz-adv-x="1152" +d="M1075 224q0 -13 -10 -23l-50 -50q-10 -10 -23 -10t-23 10l-393 393l-393 -393q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l466 -466q10 -10 10 -23zM1075 608q0 -13 -10 -23l-50 -50q-10 -10 -23 -10t-23 10l-393 393l-393 -393 +q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l466 -466q10 -10 10 -23z" /> + <glyph glyph-name="double_angle_down" unicode="" horiz-adv-x="1152" +d="M1075 672q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l393 -393l393 393q10 10 23 10t23 -10l50 -50q10 -10 10 -23zM1075 1056q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23 +t10 23l50 50q10 10 23 10t23 -10l393 -393l393 393q10 10 23 10t23 -10l50 -50q10 -10 10 -23z" /> + <glyph glyph-name="angle_left" unicode="" horiz-adv-x="640" +d="M627 992q0 -13 -10 -23l-393 -393l393 -393q10 -10 10 -23t-10 -23l-50 -50q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l50 -50q10 -10 10 -23z" /> + <glyph glyph-name="angle_right" unicode="" horiz-adv-x="640" +d="M595 576q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l393 393l-393 393q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l466 -466q10 -10 10 -23z" /> + <glyph glyph-name="angle_up" unicode="" horiz-adv-x="1152" +d="M1075 352q0 -13 -10 -23l-50 -50q-10 -10 -23 -10t-23 10l-393 393l-393 -393q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l466 -466q10 -10 10 -23z" /> + <glyph glyph-name="angle_down" unicode="" horiz-adv-x="1152" +d="M1075 800q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l393 -393l393 393q10 10 23 10t23 -10l50 -50q10 -10 10 -23z" /> + <glyph glyph-name="desktop" unicode="" horiz-adv-x="1920" +d="M1792 544v832q0 13 -9.5 22.5t-22.5 9.5h-1600q-13 0 -22.5 -9.5t-9.5 -22.5v-832q0 -13 9.5 -22.5t22.5 -9.5h1600q13 0 22.5 9.5t9.5 22.5zM1920 1376v-1088q0 -66 -47 -113t-113 -47h-544q0 -37 16 -77.5t32 -71t16 -43.5q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19 +t-19 45q0 14 16 44t32 70t16 78h-544q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h1600q66 0 113 -47t47 -113z" /> + <glyph glyph-name="laptop" unicode="" horiz-adv-x="1920" +d="M416 256q-66 0 -113 47t-47 113v704q0 66 47 113t113 47h1088q66 0 113 -47t47 -113v-704q0 -66 -47 -113t-113 -47h-1088zM384 1120v-704q0 -13 9.5 -22.5t22.5 -9.5h1088q13 0 22.5 9.5t9.5 22.5v704q0 13 -9.5 22.5t-22.5 9.5h-1088q-13 0 -22.5 -9.5t-9.5 -22.5z +M1760 192h160v-96q0 -40 -47 -68t-113 -28h-1600q-66 0 -113 28t-47 68v96h160h1600zM1040 96q16 0 16 16t-16 16h-160q-16 0 -16 -16t16 -16h160z" /> + <glyph glyph-name="tablet" unicode="" horiz-adv-x="1152" +d="M640 128q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1024 288v960q0 13 -9.5 22.5t-22.5 9.5h-832q-13 0 -22.5 -9.5t-9.5 -22.5v-960q0 -13 9.5 -22.5t22.5 -9.5h832q13 0 22.5 9.5t9.5 22.5zM1152 1248v-1088q0 -66 -47 -113t-113 -47h-832 +q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h832q66 0 113 -47t47 -113z" /> + <glyph glyph-name="mobile_phone" unicode="" horiz-adv-x="768" +d="M464 128q0 33 -23.5 56.5t-56.5 23.5t-56.5 -23.5t-23.5 -56.5t23.5 -56.5t56.5 -23.5t56.5 23.5t23.5 56.5zM672 288v704q0 13 -9.5 22.5t-22.5 9.5h-512q-13 0 -22.5 -9.5t-9.5 -22.5v-704q0 -13 9.5 -22.5t22.5 -9.5h512q13 0 22.5 9.5t9.5 22.5zM480 1136 +q0 16 -16 16h-160q-16 0 -16 -16t16 -16h160q16 0 16 16zM768 1152v-1024q0 -52 -38 -90t-90 -38h-512q-52 0 -90 38t-38 90v1024q0 52 38 90t90 38h512q52 0 90 -38t38 -90z" /> + <glyph glyph-name="circle_blank" unicode="" +d="M768 1184q-148 0 -273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273t-73 273t-198 198t-273 73zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103 +t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="quote_left" unicode="" horiz-adv-x="1664" +d="M768 576v-384q0 -80 -56 -136t-136 -56h-384q-80 0 -136 56t-56 136v704q0 104 40.5 198.5t109.5 163.5t163.5 109.5t198.5 40.5h64q26 0 45 -19t19 -45v-128q0 -26 -19 -45t-45 -19h-64q-106 0 -181 -75t-75 -181v-32q0 -40 28 -68t68 -28h224q80 0 136 -56t56 -136z +M1664 576v-384q0 -80 -56 -136t-136 -56h-384q-80 0 -136 56t-56 136v704q0 104 40.5 198.5t109.5 163.5t163.5 109.5t198.5 40.5h64q26 0 45 -19t19 -45v-128q0 -26 -19 -45t-45 -19h-64q-106 0 -181 -75t-75 -181v-32q0 -40 28 -68t68 -28h224q80 0 136 -56t56 -136z" /> + <glyph glyph-name="quote_right" unicode="" horiz-adv-x="1664" +d="M768 1216v-704q0 -104 -40.5 -198.5t-109.5 -163.5t-163.5 -109.5t-198.5 -40.5h-64q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h64q106 0 181 75t75 181v32q0 40 -28 68t-68 28h-224q-80 0 -136 56t-56 136v384q0 80 56 136t136 56h384q80 0 136 -56t56 -136zM1664 1216 +v-704q0 -104 -40.5 -198.5t-109.5 -163.5t-163.5 -109.5t-198.5 -40.5h-64q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h64q106 0 181 75t75 181v32q0 40 -28 68t-68 28h-224q-80 0 -136 56t-56 136v384q0 80 56 136t136 56h384q80 0 136 -56t56 -136z" /> + <glyph glyph-name="spinner" unicode="" horiz-adv-x="1792" +d="M526 142q0 -53 -37.5 -90.5t-90.5 -37.5q-52 0 -90 38t-38 90q0 53 37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1024 -64q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM320 640q0 -53 -37.5 -90.5t-90.5 -37.5 +t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1522 142q0 -52 -38 -90t-90 -38q-53 0 -90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM558 1138q0 -66 -47 -113t-113 -47t-113 47t-47 113t47 113t113 47t113 -47t47 -113z +M1728 640q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1088 1344q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM1618 1138q0 -93 -66 -158.5t-158 -65.5q-93 0 -158.5 65.5t-65.5 158.5 +q0 92 65.5 158t158.5 66q92 0 158 -66t66 -158z" /> + <glyph glyph-name="circle" unicode="" +d="M1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="reply" unicode="" horiz-adv-x="1792" +d="M1792 416q0 -166 -127 -451q-3 -7 -10.5 -24t-13.5 -30t-13 -22q-12 -17 -28 -17q-15 0 -23.5 10t-8.5 25q0 9 2.5 26.5t2.5 23.5q5 68 5 123q0 101 -17.5 181t-48.5 138.5t-80 101t-105.5 69.5t-133 42.5t-154 21.5t-175.5 6h-224v-256q0 -26 -19 -45t-45 -19t-45 19 +l-512 512q-19 19 -19 45t19 45l512 512q19 19 45 19t45 -19t19 -45v-256h224q713 0 875 -403q53 -134 53 -333z" /> + <glyph glyph-name="github_alt" unicode="" horiz-adv-x="1664" +d="M640 320q0 -40 -12.5 -82t-43 -76t-72.5 -34t-72.5 34t-43 76t-12.5 82t12.5 82t43 76t72.5 34t72.5 -34t43 -76t12.5 -82zM1280 320q0 -40 -12.5 -82t-43 -76t-72.5 -34t-72.5 34t-43 76t-12.5 82t12.5 82t43 76t72.5 34t72.5 -34t43 -76t12.5 -82zM1440 320 +q0 120 -69 204t-187 84q-41 0 -195 -21q-71 -11 -157 -11t-157 11q-152 21 -195 21q-118 0 -187 -84t-69 -204q0 -88 32 -153.5t81 -103t122 -60t140 -29.5t149 -7h168q82 0 149 7t140 29.5t122 60t81 103t32 153.5zM1664 496q0 -207 -61 -331q-38 -77 -105.5 -133t-141 -86 +t-170 -47.5t-171.5 -22t-167 -4.5q-78 0 -142 3t-147.5 12.5t-152.5 30t-137 51.5t-121 81t-86 115q-62 123 -62 331q0 237 136 396q-27 82 -27 170q0 116 51 218q108 0 190 -39.5t189 -123.5q147 35 309 35q148 0 280 -32q105 82 187 121t189 39q51 -102 51 -218 +q0 -87 -27 -168q136 -160 136 -398z" /> + <glyph glyph-name="folder_close_alt" unicode="" horiz-adv-x="1664" +d="M1536 224v704q0 40 -28 68t-68 28h-704q-40 0 -68 28t-28 68v64q0 40 -28 68t-68 28h-320q-40 0 -68 -28t-28 -68v-960q0 -40 28 -68t68 -28h1216q40 0 68 28t28 68zM1664 928v-704q0 -92 -66 -158t-158 -66h-1216q-92 0 -158 66t-66 158v960q0 92 66 158t158 66h320 +q92 0 158 -66t66 -158v-32h672q92 0 158 -66t66 -158z" /> + <glyph glyph-name="folder_open_alt" unicode="" horiz-adv-x="1920" +d="M1781 605q0 35 -53 35h-1088q-40 0 -85.5 -21.5t-71.5 -52.5l-294 -363q-18 -24 -18 -40q0 -35 53 -35h1088q40 0 86 22t71 53l294 363q18 22 18 39zM640 768h768v160q0 40 -28 68t-68 28h-576q-40 0 -68 28t-28 68v64q0 40 -28 68t-68 28h-320q-40 0 -68 -28t-28 -68 +v-853l256 315q44 53 116 87.5t140 34.5zM1909 605q0 -62 -46 -120l-295 -363q-43 -53 -116 -87.5t-140 -34.5h-1088q-92 0 -158 66t-66 158v960q0 92 66 158t158 66h320q92 0 158 -66t66 -158v-32h544q92 0 158 -66t66 -158v-160h192q54 0 99 -24.5t67 -70.5q15 -32 15 -68z +" /> + <glyph glyph-name="expand_alt" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="collapse_alt" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="smile" unicode="" +d="M1134 461q-37 -121 -138 -195t-228 -74t-228 74t-138 195q-8 25 4 48.5t38 31.5q25 8 48.5 -4t31.5 -38q25 -80 92.5 -129.5t151.5 -49.5t151.5 49.5t92.5 129.5q8 26 32 38t49 4t37 -31.5t4 -48.5zM640 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5 +t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1152 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1408 640q0 130 -51 248.5t-136.5 204t-204 136.5t-248.5 51t-248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5 +t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="frown" unicode="" +d="M1134 307q8 -25 -4 -48.5t-37 -31.5t-49 4t-32 38q-25 80 -92.5 129.5t-151.5 49.5t-151.5 -49.5t-92.5 -129.5q-8 -26 -31.5 -38t-48.5 -4q-26 8 -38 31.5t-4 48.5q37 121 138 195t228 74t228 -74t138 -195zM640 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5 +t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1152 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1408 640q0 130 -51 248.5t-136.5 204t-204 136.5t-248.5 51t-248.5 -51t-204 -136.5t-136.5 -204 +t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="meh" unicode="" +d="M1152 448q0 -26 -19 -45t-45 -19h-640q-26 0 -45 19t-19 45t19 45t45 19h640q26 0 45 -19t19 -45zM640 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1152 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5 +t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1408 640q0 130 -51 248.5t-136.5 204t-204 136.5t-248.5 51t-248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5zM1536 640 +q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="gamepad" unicode="" horiz-adv-x="1920" +d="M832 448v128q0 14 -9 23t-23 9h-192v192q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-192h-192q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h192v-192q0 -14 9 -23t23 -9h128q14 0 23 9t9 23v192h192q14 0 23 9t9 23zM1408 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5 +t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1664 640q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1920 512q0 -212 -150 -362t-362 -150q-192 0 -338 128h-220q-146 -128 -338 -128q-212 0 -362 150 +t-150 362t150 362t362 150h896q212 0 362 -150t150 -362z" /> + <glyph glyph-name="keyboard" unicode="" horiz-adv-x="1920" +d="M384 368v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM512 624v-96q0 -16 -16 -16h-224q-16 0 -16 16v96q0 16 16 16h224q16 0 16 -16zM384 880v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1408 368v-96q0 -16 -16 -16 +h-864q-16 0 -16 16v96q0 16 16 16h864q16 0 16 -16zM768 624v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM640 880v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1024 624v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16 +h96q16 0 16 -16zM896 880v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1280 624v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1664 368v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1152 880v-96 +q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1408 880v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1664 880v-352q0 -16 -16 -16h-224q-16 0 -16 16v96q0 16 16 16h112v240q0 16 16 16h96q16 0 16 -16zM1792 128v896h-1664v-896 +h1664zM1920 1024v-896q0 -53 -37.5 -90.5t-90.5 -37.5h-1664q-53 0 -90.5 37.5t-37.5 90.5v896q0 53 37.5 90.5t90.5 37.5h1664q53 0 90.5 -37.5t37.5 -90.5z" /> + <glyph glyph-name="flag_alt" unicode="" horiz-adv-x="1792" +d="M1664 491v616q-169 -91 -306 -91q-82 0 -145 32q-100 49 -184 76.5t-178 27.5q-173 0 -403 -127v-599q245 113 433 113q55 0 103.5 -7.5t98 -26t77 -31t82.5 -39.5l28 -14q44 -22 101 -22q120 0 293 92zM320 1280q0 -35 -17.5 -64t-46.5 -46v-1266q0 -14 -9 -23t-23 -9 +h-64q-14 0 -23 9t-9 23v1266q-29 17 -46.5 46t-17.5 64q0 53 37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1792 1216v-763q0 -39 -35 -57q-10 -5 -17 -9q-218 -116 -369 -116q-88 0 -158 35l-28 14q-64 33 -99 48t-91 29t-114 14q-102 0 -235.5 -44t-228.5 -102 +q-15 -9 -33 -9q-16 0 -32 8q-32 19 -32 56v742q0 35 31 55q35 21 78.5 42.5t114 52t152.5 49.5t155 19q112 0 209 -31t209 -86q38 -19 89 -19q122 0 310 112q22 12 31 17q31 16 62 -2q31 -20 31 -55z" /> + <glyph glyph-name="flag_checkered" unicode="" horiz-adv-x="1792" +d="M832 536v192q-181 -16 -384 -117v-185q205 96 384 110zM832 954v197q-172 -8 -384 -126v-189q215 111 384 118zM1664 491v184q-235 -116 -384 -71v224q-20 6 -39 15q-5 3 -33 17t-34.5 17t-31.5 15t-34.5 15.5t-32.5 13t-36 12.5t-35 8.5t-39.5 7.5t-39.5 4t-44 2 +q-23 0 -49 -3v-222h19q102 0 192.5 -29t197.5 -82q19 -9 39 -15v-188q42 -17 91 -17q120 0 293 92zM1664 918v189q-169 -91 -306 -91q-45 0 -78 8v-196q148 -42 384 90zM320 1280q0 -35 -17.5 -64t-46.5 -46v-1266q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v1266 +q-29 17 -46.5 46t-17.5 64q0 53 37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1792 1216v-763q0 -39 -35 -57q-10 -5 -17 -9q-218 -116 -369 -116q-88 0 -158 35l-28 14q-64 33 -99 48t-91 29t-114 14q-102 0 -235.5 -44t-228.5 -102q-15 -9 -33 -9q-16 0 -32 8 +q-32 19 -32 56v742q0 35 31 55q35 21 78.5 42.5t114 52t152.5 49.5t155 19q112 0 209 -31t209 -86q38 -19 89 -19q122 0 310 112q22 12 31 17q31 16 62 -2q31 -20 31 -55z" /> + <glyph glyph-name="terminal" unicode="" horiz-adv-x="1664" +d="M585 553l-466 -466q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l393 393l-393 393q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l466 -466q10 -10 10 -23t-10 -23zM1664 96v-64q0 -14 -9 -23t-23 -9h-960q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h960q14 0 23 -9 +t9 -23z" /> + <glyph glyph-name="code" unicode="" horiz-adv-x="1920" +d="M617 137l-50 -50q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l50 -50q10 -10 10 -23t-10 -23l-393 -393l393 -393q10 -10 10 -23t-10 -23zM1208 1204l-373 -1291q-4 -13 -15.5 -19.5t-23.5 -2.5l-62 17q-13 4 -19.5 15.5t-2.5 24.5 +l373 1291q4 13 15.5 19.5t23.5 2.5l62 -17q13 -4 19.5 -15.5t2.5 -24.5zM1865 553l-466 -466q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l393 393l-393 393q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l466 -466q10 -10 10 -23t-10 -23z" /> + <glyph glyph-name="reply_all" unicode="" horiz-adv-x="1792" +d="M640 454v-70q0 -42 -39 -59q-13 -5 -25 -5q-27 0 -45 19l-512 512q-19 19 -19 45t19 45l512 512q29 31 70 14q39 -17 39 -59v-69l-397 -398q-19 -19 -19 -45t19 -45zM1792 416q0 -58 -17 -133.5t-38.5 -138t-48 -125t-40.5 -90.5l-20 -40q-8 -17 -28 -17q-6 0 -9 1 +q-25 8 -23 34q43 400 -106 565q-64 71 -170.5 110.5t-267.5 52.5v-251q0 -42 -39 -59q-13 -5 -25 -5q-27 0 -45 19l-512 512q-19 19 -19 45t19 45l512 512q29 31 70 14q39 -17 39 -59v-262q411 -28 599 -221q169 -173 169 -509z" /> + <glyph glyph-name="star_half_empty" unicode="" horiz-adv-x="1664" +d="M1186 579l257 250l-356 52l-66 10l-30 60l-159 322v-963l59 -31l318 -168l-60 355l-12 66zM1638 841l-363 -354l86 -500q5 -33 -6 -51.5t-34 -18.5q-17 0 -40 12l-449 236l-449 -236q-23 -12 -40 -12q-23 0 -34 18.5t-6 51.5l86 500l-364 354q-32 32 -23 59.5t54 34.5 +l502 73l225 455q20 41 49 41q28 0 49 -41l225 -455l502 -73q45 -7 54 -34.5t-24 -59.5z" /> + <glyph glyph-name="location_arrow" unicode="" horiz-adv-x="1408" +d="M1401 1187l-640 -1280q-17 -35 -57 -35q-5 0 -15 2q-22 5 -35.5 22.5t-13.5 39.5v576h-576q-22 0 -39.5 13.5t-22.5 35.5t4 42t29 30l1280 640q13 7 29 7q27 0 45 -19q15 -14 18.5 -34.5t-6.5 -39.5z" /> + <glyph glyph-name="crop" unicode="" horiz-adv-x="1664" +d="M557 256h595v595zM512 301l595 595h-595v-595zM1664 224v-192q0 -14 -9 -23t-23 -9h-224v-224q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v224h-864q-14 0 -23 9t-9 23v864h-224q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h224v224q0 14 9 23t23 9h192q14 0 23 -9t9 -23 +v-224h851l246 247q10 9 23 9t23 -9q9 -10 9 -23t-9 -23l-247 -246v-851h224q14 0 23 -9t9 -23z" /> + <glyph glyph-name="code_fork" unicode="" horiz-adv-x="1024" +d="M288 64q0 40 -28 68t-68 28t-68 -28t-28 -68t28 -68t68 -28t68 28t28 68zM288 1216q0 40 -28 68t-68 28t-68 -28t-28 -68t28 -68t68 -28t68 28t28 68zM928 1088q0 40 -28 68t-68 28t-68 -28t-28 -68t28 -68t68 -28t68 28t28 68zM1024 1088q0 -52 -26 -96.5t-70 -69.5 +q-2 -287 -226 -414q-67 -38 -203 -81q-128 -40 -169.5 -71t-41.5 -100v-26q44 -25 70 -69.5t26 -96.5q0 -80 -56 -136t-136 -56t-136 56t-56 136q0 52 26 96.5t70 69.5v820q-44 25 -70 69.5t-26 96.5q0 80 56 136t136 56t136 -56t56 -136q0 -52 -26 -96.5t-70 -69.5v-497 +q54 26 154 57q55 17 87.5 29.5t70.5 31t59 39.5t40.5 51t28 69.5t8.5 91.5q-44 25 -70 69.5t-26 96.5q0 80 56 136t136 56t136 -56t56 -136z" /> + <glyph glyph-name="unlink" unicode="" horiz-adv-x="1664" +d="M439 265l-256 -256q-11 -9 -23 -9t-23 9q-9 10 -9 23t9 23l256 256q10 9 23 9t23 -9q9 -10 9 -23t-9 -23zM608 224v-320q0 -14 -9 -23t-23 -9t-23 9t-9 23v320q0 14 9 23t23 9t23 -9t9 -23zM384 448q0 -14 -9 -23t-23 -9h-320q-14 0 -23 9t-9 23t9 23t23 9h320 +q14 0 23 -9t9 -23zM1648 320q0 -120 -85 -203l-147 -146q-83 -83 -203 -83q-121 0 -204 85l-334 335q-21 21 -42 56l239 18l273 -274q27 -27 68 -27.5t68 26.5l147 146q28 28 28 67q0 40 -28 68l-274 275l18 239q35 -21 56 -42l336 -336q84 -86 84 -204zM1031 1044l-239 -18 +l-273 274q-28 28 -68 28q-39 0 -68 -27l-147 -146q-28 -28 -28 -67q0 -40 28 -68l274 -274l-18 -240q-35 21 -56 42l-336 336q-84 86 -84 204q0 120 85 203l147 146q83 83 203 83q121 0 204 -85l334 -335q21 -21 42 -56zM1664 960q0 -14 -9 -23t-23 -9h-320q-14 0 -23 9 +t-9 23t9 23t23 9h320q14 0 23 -9t9 -23zM1120 1504v-320q0 -14 -9 -23t-23 -9t-23 9t-9 23v320q0 14 9 23t23 9t23 -9t9 -23zM1527 1353l-256 -256q-11 -9 -23 -9t-23 9q-9 10 -9 23t9 23l256 256q10 9 23 9t23 -9q9 -10 9 -23t-9 -23z" /> + <glyph glyph-name="question" unicode="" horiz-adv-x="1024" +d="M704 280v-240q0 -16 -12 -28t-28 -12h-240q-16 0 -28 12t-12 28v240q0 16 12 28t28 12h240q16 0 28 -12t12 -28zM1020 880q0 -54 -15.5 -101t-35 -76.5t-55 -59.5t-57.5 -43.5t-61 -35.5q-41 -23 -68.5 -65t-27.5 -67q0 -17 -12 -32.5t-28 -15.5h-240q-15 0 -25.5 18.5 +t-10.5 37.5v45q0 83 65 156.5t143 108.5q59 27 84 56t25 76q0 42 -46.5 74t-107.5 32q-65 0 -108 -29q-35 -25 -107 -115q-13 -16 -31 -16q-12 0 -25 8l-164 125q-13 10 -15.5 25t5.5 28q160 266 464 266q80 0 161 -31t146 -83t106 -127.5t41 -158.5z" /> + <glyph glyph-name="_279" unicode="" horiz-adv-x="640" +d="M640 192v-128q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h64v384h-64q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h384q26 0 45 -19t19 -45v-576h64q26 0 45 -19t19 -45zM512 1344v-192q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v192 +q0 26 19 45t45 19h256q26 0 45 -19t19 -45z" /> + <glyph glyph-name="exclamation" unicode="" horiz-adv-x="640" +d="M512 288v-224q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v224q0 26 19 45t45 19h256q26 0 45 -19t19 -45zM542 1344l-28 -768q-1 -26 -20.5 -45t-45.5 -19h-256q-26 0 -45.5 19t-20.5 45l-28 768q-1 26 17.5 45t44.5 19h320q26 0 44.5 -19t17.5 -45z" /> + <glyph glyph-name="superscript" unicode="" +d="M897 167v-167h-248l-159 252l-24 42q-8 9 -11 21h-3q-1 -3 -2.5 -6.5t-3.5 -8t-3 -6.5q-10 -20 -25 -44l-155 -250h-258v167h128l197 291l-185 272h-137v168h276l139 -228q2 -4 23 -42q8 -9 11 -21h3q3 9 11 21l25 42l140 228h257v-168h-125l-184 -267l204 -296h109z +M1534 846v-206h-514l-3 27q-4 28 -4 46q0 64 26 117t65 86.5t84 65t84 54.5t65 54t26 64q0 38 -29.5 62.5t-70.5 24.5q-51 0 -97 -39q-14 -11 -36 -38l-105 92q26 37 63 66q83 65 188 65q110 0 178 -59.5t68 -158.5q0 -56 -24.5 -103t-62 -76.5t-81.5 -58.5t-82 -50.5 +t-65.5 -51.5t-30.5 -63h232v80h126z" /> + <glyph glyph-name="subscript" unicode="" +d="M897 167v-167h-248l-159 252l-24 42q-8 9 -11 21h-3q-1 -3 -2.5 -6.5t-3.5 -8t-3 -6.5q-10 -20 -25 -44l-155 -250h-258v167h128l197 291l-185 272h-137v168h276l139 -228q2 -4 23 -42q8 -9 11 -21h3q3 9 11 21l25 42l140 228h257v-168h-125l-184 -267l204 -296h109z +M1536 -50v-206h-514l-4 27q-3 45 -3 46q0 64 26 117t65 86.5t84 65t84 54.5t65 54t26 64q0 38 -29.5 62.5t-70.5 24.5q-51 0 -97 -39q-14 -11 -36 -38l-105 92q26 37 63 66q80 65 188 65q110 0 178 -59.5t68 -158.5q0 -66 -34.5 -118.5t-84 -86t-99.5 -62.5t-87 -63t-41 -73 +h232v80h126z" /> + <glyph glyph-name="_283" unicode="" horiz-adv-x="1920" +d="M896 128l336 384h-768l-336 -384h768zM1909 1205q15 -34 9.5 -71.5t-30.5 -65.5l-896 -1024q-38 -44 -96 -44h-768q-38 0 -69.5 20.5t-47.5 54.5q-15 34 -9.5 71.5t30.5 65.5l896 1024q38 44 96 44h768q38 0 69.5 -20.5t47.5 -54.5z" /> + <glyph glyph-name="puzzle_piece" unicode="" horiz-adv-x="1664" +d="M1664 438q0 -81 -44.5 -135t-123.5 -54q-41 0 -77.5 17.5t-59 38t-56.5 38t-71 17.5q-110 0 -110 -124q0 -39 16 -115t15 -115v-5q-22 0 -33 -1q-34 -3 -97.5 -11.5t-115.5 -13.5t-98 -5q-61 0 -103 26.5t-42 83.5q0 37 17.5 71t38 56.5t38 59t17.5 77.5q0 79 -54 123.5 +t-135 44.5q-84 0 -143 -45.5t-59 -127.5q0 -43 15 -83t33.5 -64.5t33.5 -53t15 -50.5q0 -45 -46 -89q-37 -35 -117 -35q-95 0 -245 24q-9 2 -27.5 4t-27.5 4l-13 2q-1 0 -3 1q-2 0 -2 1v1024q2 -1 17.5 -3.5t34 -5t21.5 -3.5q150 -24 245 -24q80 0 117 35q46 44 46 89 +q0 22 -15 50.5t-33.5 53t-33.5 64.5t-15 83q0 82 59 127.5t144 45.5q80 0 134 -44.5t54 -123.5q0 -41 -17.5 -77.5t-38 -59t-38 -56.5t-17.5 -71q0 -57 42 -83.5t103 -26.5q64 0 180 15t163 17v-2q-1 -2 -3.5 -17.5t-5 -34t-3.5 -21.5q-24 -150 -24 -245q0 -80 35 -117 +q44 -46 89 -46q22 0 50.5 15t53 33.5t64.5 33.5t83 15q82 0 127.5 -59t45.5 -143z" /> + <glyph glyph-name="microphone" unicode="" horiz-adv-x="1152" +d="M1152 832v-128q0 -221 -147.5 -384.5t-364.5 -187.5v-132h256q26 0 45 -19t19 -45t-19 -45t-45 -19h-640q-26 0 -45 19t-19 45t19 45t45 19h256v132q-217 24 -364.5 187.5t-147.5 384.5v128q0 26 19 45t45 19t45 -19t19 -45v-128q0 -185 131.5 -316.5t316.5 -131.5 +t316.5 131.5t131.5 316.5v128q0 26 19 45t45 19t45 -19t19 -45zM896 1216v-512q0 -132 -94 -226t-226 -94t-226 94t-94 226v512q0 132 94 226t226 94t226 -94t94 -226z" /> + <glyph glyph-name="microphone_off" unicode="" horiz-adv-x="1408" +d="M271 591l-101 -101q-42 103 -42 214v128q0 26 19 45t45 19t45 -19t19 -45v-128q0 -53 15 -113zM1385 1193l-361 -361v-128q0 -132 -94 -226t-226 -94q-55 0 -109 19l-96 -96q97 -51 205 -51q185 0 316.5 131.5t131.5 316.5v128q0 26 19 45t45 19t45 -19t19 -45v-128 +q0 -221 -147.5 -384.5t-364.5 -187.5v-132h256q26 0 45 -19t19 -45t-19 -45t-45 -19h-640q-26 0 -45 19t-19 45t19 45t45 19h256v132q-125 13 -235 81l-254 -254q-10 -10 -23 -10t-23 10l-82 82q-10 10 -10 23t10 23l1234 1234q10 10 23 10t23 -10l82 -82q10 -10 10 -23 +t-10 -23zM1005 1325l-621 -621v512q0 132 94 226t226 94q102 0 184.5 -59t116.5 -152z" /> + <glyph glyph-name="shield" unicode="" horiz-adv-x="1280" +d="M1088 576v640h-448v-1137q119 63 213 137q235 184 235 360zM1280 1344v-768q0 -86 -33.5 -170.5t-83 -150t-118 -127.5t-126.5 -103t-121 -77.5t-89.5 -49.5t-42.5 -20q-12 -6 -26 -6t-26 6q-16 7 -42.5 20t-89.5 49.5t-121 77.5t-126.5 103t-118 127.5t-83 150 +t-33.5 170.5v768q0 26 19 45t45 19h1152q26 0 45 -19t19 -45z" /> + <glyph glyph-name="calendar_empty" unicode="" horiz-adv-x="1664" +d="M128 -128h1408v1024h-1408v-1024zM512 1088v288q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-288q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1280 1088v288q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-288q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1664 1152v-1280 +q0 -52 -38 -90t-90 -38h-1408q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h128v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h384v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h128q52 0 90 -38t38 -90z" /> + <glyph glyph-name="fire_extinguisher" unicode="" horiz-adv-x="1408" +d="M512 1344q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1408 1376v-320q0 -16 -12 -25q-8 -7 -20 -7q-4 0 -7 1l-448 96q-11 2 -18 11t-7 20h-256v-102q111 -23 183.5 -111t72.5 -203v-800q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v800 +q0 106 62.5 190.5t161.5 114.5v111h-32q-59 0 -115 -23.5t-91.5 -53t-66 -66.5t-40.5 -53.5t-14 -24.5q-17 -35 -57 -35q-16 0 -29 7q-23 12 -31.5 37t3.5 49q5 10 14.5 26t37.5 53.5t60.5 70t85 67t108.5 52.5q-25 42 -25 86q0 66 47 113t113 47t113 -47t47 -113 +q0 -33 -14 -64h302q0 11 7 20t18 11l448 96q3 1 7 1q12 0 20 -7q12 -9 12 -25z" /> + <glyph glyph-name="rocket" unicode="" horiz-adv-x="1664" +d="M1440 1088q0 40 -28 68t-68 28t-68 -28t-28 -68t28 -68t68 -28t68 28t28 68zM1664 1376q0 -249 -75.5 -430.5t-253.5 -360.5q-81 -80 -195 -176l-20 -379q-2 -16 -16 -26l-384 -224q-7 -4 -16 -4q-12 0 -23 9l-64 64q-13 14 -8 32l85 276l-281 281l-276 -85q-3 -1 -9 -1 +q-14 0 -23 9l-64 64q-17 19 -5 39l224 384q10 14 26 16l379 20q96 114 176 195q188 187 358 258t431 71q14 0 24 -9.5t10 -22.5z" /> + <glyph glyph-name="maxcdn" unicode="" horiz-adv-x="1792" +d="M1745 763l-164 -763h-334l178 832q13 56 -15 88q-27 33 -83 33h-169l-204 -953h-334l204 953h-286l-204 -953h-334l204 953l-153 327h1276q101 0 189.5 -40.5t147.5 -113.5q60 -73 81 -168.5t0 -194.5z" /> + <glyph glyph-name="chevron_sign_left" unicode="" +d="M909 141l102 102q19 19 19 45t-19 45l-307 307l307 307q19 19 19 45t-19 45l-102 102q-19 19 -45 19t-45 -19l-454 -454q-19 -19 -19 -45t19 -45l454 -454q19 -19 45 -19t45 19zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5 +t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="chevron_sign_right" unicode="" +d="M717 141l454 454q19 19 19 45t-19 45l-454 454q-19 19 -45 19t-45 -19l-102 -102q-19 -19 -19 -45t19 -45l307 -307l-307 -307q-19 -19 -19 -45t19 -45l102 -102q19 -19 45 -19t45 19zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5 +t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="chevron_sign_up" unicode="" +d="M1165 397l102 102q19 19 19 45t-19 45l-454 454q-19 19 -45 19t-45 -19l-454 -454q-19 -19 -19 -45t19 -45l102 -102q19 -19 45 -19t45 19l307 307l307 -307q19 -19 45 -19t45 19zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5 +t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="chevron_sign_down" unicode="" +d="M813 237l454 454q19 19 19 45t-19 45l-102 102q-19 19 -45 19t-45 -19l-307 -307l-307 307q-19 19 -45 19t-45 -19l-102 -102q-19 -19 -19 -45t19 -45l454 -454q19 -19 45 -19t45 19zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5 +t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="html5" unicode="" horiz-adv-x="1408" +d="M1130 939l16 175h-884l47 -534h612l-22 -228l-197 -53l-196 53l-13 140h-175l22 -278l362 -100h4v1l359 99l50 544h-644l-15 181h674zM0 1408h1408l-128 -1438l-578 -162l-574 162z" /> + <glyph glyph-name="css3" unicode="" horiz-adv-x="1792" +d="M275 1408h1505l-266 -1333l-804 -267l-698 267l71 356h297l-29 -147l422 -161l486 161l68 339h-1208l58 297h1209l38 191h-1208z" /> + <glyph glyph-name="anchor" unicode="" horiz-adv-x="1792" +d="M960 1280q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1792 352v-352q0 -22 -20 -30q-8 -2 -12 -2q-12 0 -23 9l-93 93q-119 -143 -318.5 -226.5t-429.5 -83.5t-429.5 83.5t-318.5 226.5l-93 -93q-9 -9 -23 -9q-4 0 -12 2q-20 8 -20 30v352 +q0 14 9 23t23 9h352q22 0 30 -20q8 -19 -7 -35l-100 -100q67 -91 189.5 -153.5t271.5 -82.5v647h-192q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h192v163q-58 34 -93 92.5t-35 128.5q0 106 75 181t181 75t181 -75t75 -181q0 -70 -35 -128.5t-93 -92.5v-163h192q26 0 45 -19 +t19 -45v-128q0 -26 -19 -45t-45 -19h-192v-647q149 20 271.5 82.5t189.5 153.5l-100 100q-15 16 -7 35q8 20 30 20h352q14 0 23 -9t9 -23z" /> + <glyph glyph-name="unlock_alt" unicode="" horiz-adv-x="1152" +d="M1056 768q40 0 68 -28t28 -68v-576q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v576q0 40 28 68t68 28h32v320q0 185 131.5 316.5t316.5 131.5t316.5 -131.5t131.5 -316.5q0 -26 -19 -45t-45 -19h-64q-26 0 -45 19t-19 45q0 106 -75 181t-181 75t-181 -75t-75 -181 +v-320h736z" /> + <glyph glyph-name="bullseye" unicode="" +d="M1024 640q0 -106 -75 -181t-181 -75t-181 75t-75 181t75 181t181 75t181 -75t75 -181zM1152 640q0 159 -112.5 271.5t-271.5 112.5t-271.5 -112.5t-112.5 -271.5t112.5 -271.5t271.5 -112.5t271.5 112.5t112.5 271.5zM1280 640q0 -212 -150 -362t-362 -150t-362 150 +t-150 362t150 362t362 150t362 -150t150 -362zM1408 640q0 130 -51 248.5t-136.5 204t-204 136.5t-248.5 51t-248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5zM1536 640 +q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="ellipsis_horizontal" unicode="" horiz-adv-x="1408" +d="M384 800v-192q0 -40 -28 -68t-68 -28h-192q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68zM896 800v-192q0 -40 -28 -68t-68 -28h-192q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68zM1408 800v-192q0 -40 -28 -68t-68 -28h-192 +q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68z" /> + <glyph glyph-name="ellipsis_vertical" unicode="" horiz-adv-x="384" +d="M384 288v-192q0 -40 -28 -68t-68 -28h-192q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68zM384 800v-192q0 -40 -28 -68t-68 -28h-192q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68zM384 1312v-192q0 -40 -28 -68t-68 -28h-192 +q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68z" /> + <glyph glyph-name="_303" unicode="" +d="M512 256q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM863 162q-13 233 -176.5 396.5t-396.5 176.5q-14 1 -24 -9t-10 -23v-128q0 -13 8.5 -22t21.5 -10q154 -11 264 -121t121 -264q1 -13 10 -21.5t22 -8.5h128 +q13 0 23 10t9 24zM1247 161q-5 154 -56 297.5t-139.5 260t-205 205t-260 139.5t-297.5 56q-14 1 -23 -9q-10 -10 -10 -23v-128q0 -13 9 -22t22 -10q204 -7 378 -111.5t278.5 -278.5t111.5 -378q1 -13 10 -22t22 -9h128q13 0 23 10q11 9 9 23zM1536 1120v-960 +q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="play_sign" unicode="" +d="M768 1408q209 0 385.5 -103t279.5 -279.5t103 -385.5t-103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103zM1152 585q32 18 32 55t-32 55l-544 320q-31 19 -64 1q-32 -19 -32 -56v-640q0 -37 32 -56 +q16 -8 32 -8q17 0 32 9z" /> + <glyph glyph-name="ticket" unicode="" horiz-adv-x="1792" +d="M1024 1084l316 -316l-572 -572l-316 316zM813 105l618 618q19 19 19 45t-19 45l-362 362q-18 18 -45 18t-45 -18l-618 -618q-19 -19 -19 -45t19 -45l362 -362q18 -18 45 -18t45 18zM1702 742l-907 -908q-37 -37 -90.5 -37t-90.5 37l-126 126q56 56 56 136t-56 136 +t-136 56t-136 -56l-125 126q-37 37 -37 90.5t37 90.5l907 906q37 37 90.5 37t90.5 -37l125 -125q-56 -56 -56 -136t56 -136t136 -56t136 56l126 -125q37 -37 37 -90.5t-37 -90.5z" /> + <glyph glyph-name="minus_sign_alt" unicode="" +d="M1280 576v128q0 26 -19 45t-45 19h-896q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h896q26 0 45 19t19 45zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5 +t84.5 -203.5z" /> + <glyph glyph-name="check_minus" unicode="" horiz-adv-x="1408" +d="M1152 736v-64q0 -14 -9 -23t-23 -9h-832q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h832q14 0 23 -9t9 -23zM1280 288v832q0 66 -47 113t-113 47h-832q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832q66 0 113 47t47 113zM1408 1120v-832q0 -119 -84.5 -203.5 +t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h832q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="level_up" unicode="" horiz-adv-x="1024" +d="M1018 933q-18 -37 -58 -37h-192v-864q0 -14 -9 -23t-23 -9h-704q-21 0 -29 18q-8 20 4 35l160 192q9 11 25 11h320v640h-192q-40 0 -58 37q-17 37 9 68l320 384q18 22 49 22t49 -22l320 -384q27 -32 9 -68z" /> + <glyph glyph-name="level_down" unicode="" horiz-adv-x="1024" +d="M32 1280h704q13 0 22.5 -9.5t9.5 -23.5v-863h192q40 0 58 -37t-9 -69l-320 -384q-18 -22 -49 -22t-49 22l-320 384q-26 31 -9 69q18 37 58 37h192v640h-320q-14 0 -25 11l-160 192q-13 14 -4 34q9 19 29 19z" /> + <glyph glyph-name="check_sign" unicode="" +d="M685 237l614 614q19 19 19 45t-19 45l-102 102q-19 19 -45 19t-45 -19l-467 -467l-211 211q-19 19 -45 19t-45 -19l-102 -102q-19 -19 -19 -45t19 -45l358 -358q19 -19 45 -19t45 19zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5 +t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="edit_sign" unicode="" +d="M404 428l152 -152l-52 -52h-56v96h-96v56zM818 818q14 -13 -3 -30l-291 -291q-17 -17 -30 -3q-14 13 3 30l291 291q17 17 30 3zM544 128l544 544l-288 288l-544 -544v-288h288zM1152 736l92 92q28 28 28 68t-28 68l-152 152q-28 28 -68 28t-68 -28l-92 -92zM1536 1120 +v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="_312" unicode="" +d="M1280 608v480q0 26 -19 45t-45 19h-480q-42 0 -59 -39q-17 -41 14 -70l144 -144l-534 -534q-19 -19 -19 -45t19 -45l102 -102q19 -19 45 -19t45 19l534 534l144 -144q18 -19 45 -19q12 0 25 5q39 17 39 59zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960 +q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="share_sign" unicode="" +d="M1005 435l352 352q19 19 19 45t-19 45l-352 352q-30 31 -69 14q-40 -17 -40 -59v-160q-119 0 -216 -19.5t-162.5 -51t-114 -79t-76.5 -95.5t-44.5 -109t-21.5 -111.5t-5 -110.5q0 -181 167 -404q11 -12 25 -12q7 0 13 3q22 9 19 33q-44 354 62 473q46 52 130 75.5 +t224 23.5v-160q0 -42 40 -59q12 -5 24 -5q26 0 45 19zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="compass" unicode="" +d="M640 448l256 128l-256 128v-256zM1024 1039v-542l-512 -256v542zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103 +t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="collapse" unicode="" +d="M1145 861q18 -35 -5 -66l-320 -448q-19 -27 -52 -27t-52 27l-320 448q-23 31 -5 66q17 35 57 35h640q40 0 57 -35zM1280 160v960q0 13 -9.5 22.5t-22.5 9.5h-960q-13 0 -22.5 -9.5t-9.5 -22.5v-960q0 -13 9.5 -22.5t22.5 -9.5h960q13 0 22.5 9.5t9.5 22.5zM1536 1120 +v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="collapse_top" unicode="" +d="M1145 419q-17 -35 -57 -35h-640q-40 0 -57 35q-18 35 5 66l320 448q19 27 52 27t52 -27l320 -448q23 -31 5 -66zM1280 160v960q0 13 -9.5 22.5t-22.5 9.5h-960q-13 0 -22.5 -9.5t-9.5 -22.5v-960q0 -13 9.5 -22.5t22.5 -9.5h960q13 0 22.5 9.5t9.5 22.5zM1536 1120v-960 +q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="_317" unicode="" +d="M1088 640q0 -33 -27 -52l-448 -320q-31 -23 -66 -5q-35 17 -35 57v640q0 40 35 57q35 18 66 -5l448 -320q27 -19 27 -52zM1280 160v960q0 14 -9 23t-23 9h-960q-14 0 -23 -9t-9 -23v-960q0 -14 9 -23t23 -9h960q14 0 23 9t9 23zM1536 1120v-960q0 -119 -84.5 -203.5 +t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="eur" unicode="" horiz-adv-x="1024" +d="M976 229l35 -159q3 -12 -3 -22.5t-17 -14.5l-5 -1q-4 -2 -10.5 -3.5t-16 -4.5t-21.5 -5.5t-25.5 -5t-30 -5t-33.5 -4.5t-36.5 -3t-38.5 -1q-234 0 -409 130.5t-238 351.5h-95q-13 0 -22.5 9.5t-9.5 22.5v113q0 13 9.5 22.5t22.5 9.5h66q-2 57 1 105h-67q-14 0 -23 9 +t-9 23v114q0 14 9 23t23 9h98q67 210 243.5 338t400.5 128q102 0 194 -23q11 -3 20 -15q6 -11 3 -24l-43 -159q-3 -13 -14 -19.5t-24 -2.5l-4 1q-4 1 -11.5 2.5l-17.5 3.5t-22.5 3.5t-26 3t-29 2.5t-29.5 1q-126 0 -226 -64t-150 -176h468q16 0 25 -12q10 -12 7 -26 +l-24 -114q-5 -26 -32 -26h-488q-3 -37 0 -105h459q15 0 25 -12q9 -12 6 -27l-24 -112q-2 -11 -11 -18.5t-20 -7.5h-387q48 -117 149.5 -185.5t228.5 -68.5q18 0 36 1.5t33.5 3.5t29.5 4.5t24.5 5t18.5 4.5l12 3l5 2q13 5 26 -2q12 -7 15 -21z" /> + <glyph glyph-name="gbp" unicode="" horiz-adv-x="1024" +d="M1020 399v-367q0 -14 -9 -23t-23 -9h-956q-14 0 -23 9t-9 23v150q0 13 9.5 22.5t22.5 9.5h97v383h-95q-14 0 -23 9.5t-9 22.5v131q0 14 9 23t23 9h95v223q0 171 123.5 282t314.5 111q185 0 335 -125q9 -8 10 -20.5t-7 -22.5l-103 -127q-9 -11 -22 -12q-13 -2 -23 7 +q-5 5 -26 19t-69 32t-93 18q-85 0 -137 -47t-52 -123v-215h305q13 0 22.5 -9t9.5 -23v-131q0 -13 -9.5 -22.5t-22.5 -9.5h-305v-379h414v181q0 13 9 22.5t23 9.5h162q14 0 23 -9.5t9 -22.5z" /> + <glyph glyph-name="usd" unicode="" horiz-adv-x="1024" +d="M978 351q0 -153 -99.5 -263.5t-258.5 -136.5v-175q0 -14 -9 -23t-23 -9h-135q-13 0 -22.5 9.5t-9.5 22.5v175q-66 9 -127.5 31t-101.5 44.5t-74 48t-46.5 37.5t-17.5 18q-17 21 -2 41l103 135q7 10 23 12q15 2 24 -9l2 -2q113 -99 243 -125q37 -8 74 -8q81 0 142.5 43 +t61.5 122q0 28 -15 53t-33.5 42t-58.5 37.5t-66 32t-80 32.5q-39 16 -61.5 25t-61.5 26.5t-62.5 31t-56.5 35.5t-53.5 42.5t-43.5 49t-35.5 58t-21 66.5t-8.5 78q0 138 98 242t255 134v180q0 13 9.5 22.5t22.5 9.5h135q14 0 23 -9t9 -23v-176q57 -6 110.5 -23t87 -33.5 +t63.5 -37.5t39 -29t15 -14q17 -18 5 -38l-81 -146q-8 -15 -23 -16q-14 -3 -27 7q-3 3 -14.5 12t-39 26.5t-58.5 32t-74.5 26t-85.5 11.5q-95 0 -155 -43t-60 -111q0 -26 8.5 -48t29.5 -41.5t39.5 -33t56 -31t60.5 -27t70 -27.5q53 -20 81 -31.5t76 -35t75.5 -42.5t62 -50 +t53 -63.5t31.5 -76.5t13 -94z" /> + <glyph glyph-name="inr" unicode="" horiz-adv-x="898" +d="M898 1066v-102q0 -14 -9 -23t-23 -9h-168q-23 -144 -129 -234t-276 -110q167 -178 459 -536q14 -16 4 -34q-8 -18 -29 -18h-195q-16 0 -25 12q-306 367 -498 571q-9 9 -9 22v127q0 13 9.5 22.5t22.5 9.5h112q132 0 212.5 43t102.5 125h-427q-14 0 -23 9t-9 23v102 +q0 14 9 23t23 9h413q-57 113 -268 113h-145q-13 0 -22.5 9.5t-9.5 22.5v133q0 14 9 23t23 9h832q14 0 23 -9t9 -23v-102q0 -14 -9 -23t-23 -9h-233q47 -61 64 -144h171q14 0 23 -9t9 -23z" /> + <glyph glyph-name="jpy" unicode="" horiz-adv-x="1027" +d="M603 0h-172q-13 0 -22.5 9t-9.5 23v330h-288q-13 0 -22.5 9t-9.5 23v103q0 13 9.5 22.5t22.5 9.5h288v85h-288q-13 0 -22.5 9t-9.5 23v104q0 13 9.5 22.5t22.5 9.5h214l-321 578q-8 16 0 32q10 16 28 16h194q19 0 29 -18l215 -425q19 -38 56 -125q10 24 30.5 68t27.5 61 +l191 420q8 19 29 19h191q17 0 27 -16q9 -14 1 -31l-313 -579h215q13 0 22.5 -9.5t9.5 -22.5v-104q0 -14 -9.5 -23t-22.5 -9h-290v-85h290q13 0 22.5 -9.5t9.5 -22.5v-103q0 -14 -9.5 -23t-22.5 -9h-290v-330q0 -13 -9.5 -22.5t-22.5 -9.5z" /> + <glyph glyph-name="rub" unicode="" horiz-adv-x="1280" +d="M1043 971q0 100 -65 162t-171 62h-320v-448h320q106 0 171 62t65 162zM1280 971q0 -193 -126.5 -315t-326.5 -122h-340v-118h505q14 0 23 -9t9 -23v-128q0 -14 -9 -23t-23 -9h-505v-192q0 -14 -9.5 -23t-22.5 -9h-167q-14 0 -23 9t-9 23v192h-224q-14 0 -23 9t-9 23v128 +q0 14 9 23t23 9h224v118h-224q-14 0 -23 9t-9 23v149q0 13 9 22.5t23 9.5h224v629q0 14 9 23t23 9h539q200 0 326.5 -122t126.5 -315z" /> + <glyph glyph-name="krw" unicode="" horiz-adv-x="1792" +d="M514 341l81 299h-159l75 -300q1 -1 1 -3t1 -3q0 1 0.5 3.5t0.5 3.5zM630 768l35 128h-292l32 -128h225zM822 768h139l-35 128h-70zM1271 340l78 300h-162l81 -299q0 -1 0.5 -3.5t1.5 -3.5q0 1 0.5 3t0.5 3zM1382 768l33 128h-297l34 -128h230zM1792 736v-64q0 -14 -9 -23 +t-23 -9h-213l-164 -616q-7 -24 -31 -24h-159q-24 0 -31 24l-166 616h-209l-167 -616q-7 -24 -31 -24h-159q-11 0 -19.5 7t-10.5 17l-160 616h-208q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h175l-33 128h-142q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h109l-89 344q-5 15 5 28 +q10 12 26 12h137q26 0 31 -24l90 -360h359l97 360q7 24 31 24h126q24 0 31 -24l98 -360h365l93 360q5 24 31 24h137q16 0 26 -12q10 -13 5 -28l-91 -344h111q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-145l-34 -128h179q14 0 23 -9t9 -23z" /> + <glyph glyph-name="btc" unicode="" horiz-adv-x="1280" +d="M1167 896q18 -182 -131 -258q117 -28 175 -103t45 -214q-7 -71 -32.5 -125t-64.5 -89t-97 -58.5t-121.5 -34.5t-145.5 -15v-255h-154v251q-80 0 -122 1v-252h-154v255q-18 0 -54 0.5t-55 0.5h-200l31 183h111q50 0 58 51v402h16q-6 1 -16 1v287q-13 68 -89 68h-111v164 +l212 -1q64 0 97 1v252h154v-247q82 2 122 2v245h154v-252q79 -7 140 -22.5t113 -45t82.5 -78t36.5 -114.5zM952 351q0 36 -15 64t-37 46t-57.5 30.5t-65.5 18.5t-74 9t-69 3t-64.5 -1t-47.5 -1v-338q8 0 37 -0.5t48 -0.5t53 1.5t58.5 4t57 8.5t55.5 14t47.5 21t39.5 30 +t24.5 40t9.5 51zM881 827q0 33 -12.5 58.5t-30.5 42t-48 28t-55 16.5t-61.5 8t-58 2.5t-54 -1t-39.5 -0.5v-307q5 0 34.5 -0.5t46.5 0t50 2t55 5.5t51.5 11t48.5 18.5t37 27t27 38.5t9 51z" /> + <glyph glyph-name="file" unicode="" +d="M1024 1024v472q22 -14 36 -28l408 -408q14 -14 28 -36h-472zM896 992q0 -40 28 -68t68 -28h544v-1056q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h800v-544z" /> + <glyph glyph-name="file_text" unicode="" +d="M1468 1060q14 -14 28 -36h-472v472q22 -14 36 -28zM992 896h544v-1056q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h800v-544q0 -40 28 -68t68 -28zM1152 160v64q0 14 -9 23t-23 9h-704q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h704 +q14 0 23 9t9 23zM1152 416v64q0 14 -9 23t-23 9h-704q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h704q14 0 23 9t9 23zM1152 672v64q0 14 -9 23t-23 9h-704q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h704q14 0 23 9t9 23z" /> + <glyph glyph-name="sort_by_alphabet" unicode="" horiz-adv-x="1664" +d="M1191 1128h177l-72 218l-12 47q-2 16 -2 20h-4l-3 -20q0 -1 -3.5 -18t-7.5 -29zM736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192q14 0 23 -9t9 -23zM1572 -23 +v-233h-584v90l369 529q12 18 21 27l11 9v3q-2 0 -6.5 -0.5t-7.5 -0.5q-12 -3 -30 -3h-232v-115h-120v229h567v-89l-369 -530q-6 -8 -21 -26l-11 -11v-2l14 2q9 2 30 2h248v119h121zM1661 874v-106h-288v106h75l-47 144h-243l-47 -144h75v-106h-287v106h70l230 662h162 +l230 -662h70z" /> + <glyph glyph-name="_329" unicode="" horiz-adv-x="1664" +d="M1191 104h177l-72 218l-12 47q-2 16 -2 20h-4l-3 -20q0 -1 -3.5 -18t-7.5 -29zM736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192q14 0 23 -9t9 -23zM1661 -150 +v-106h-288v106h75l-47 144h-243l-47 -144h75v-106h-287v106h70l230 662h162l230 -662h70zM1572 1001v-233h-584v90l369 529q12 18 21 27l11 9v3q-2 0 -6.5 -0.5t-7.5 -0.5q-12 -3 -30 -3h-232v-115h-120v229h567v-89l-369 -530q-6 -8 -21 -26l-11 -10v-3l14 3q9 1 30 1h248 +v119h121z" /> + <glyph glyph-name="sort_by_attributes" unicode="" horiz-adv-x="1792" +d="M736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192q14 0 23 -9t9 -23zM1792 -32v-192q0 -14 -9 -23t-23 -9h-832q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h832 +q14 0 23 -9t9 -23zM1600 480v-192q0 -14 -9 -23t-23 -9h-640q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h640q14 0 23 -9t9 -23zM1408 992v-192q0 -14 -9 -23t-23 -9h-448q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h448q14 0 23 -9t9 -23zM1216 1504v-192q0 -14 -9 -23t-23 -9h-256 +q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h256q14 0 23 -9t9 -23z" /> + <glyph glyph-name="sort_by_attributes_alt" unicode="" horiz-adv-x="1792" +d="M1216 -32v-192q0 -14 -9 -23t-23 -9h-256q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h256q14 0 23 -9t9 -23zM736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192 +q14 0 23 -9t9 -23zM1408 480v-192q0 -14 -9 -23t-23 -9h-448q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h448q14 0 23 -9t9 -23zM1600 992v-192q0 -14 -9 -23t-23 -9h-640q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h640q14 0 23 -9t9 -23zM1792 1504v-192q0 -14 -9 -23t-23 -9h-832 +q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h832q14 0 23 -9t9 -23z" /> + <glyph glyph-name="sort_by_order" unicode="" +d="M1346 223q0 63 -44 116t-103 53q-52 0 -83 -37t-31 -94t36.5 -95t104.5 -38q50 0 85 27t35 68zM736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192q14 0 23 -9t9 -23 +zM1486 165q0 -62 -13 -121.5t-41 -114t-68 -95.5t-98.5 -65.5t-127.5 -24.5q-62 0 -108 16q-24 8 -42 15l39 113q15 -7 31 -11q37 -13 75 -13q84 0 134.5 58.5t66.5 145.5h-2q-21 -23 -61.5 -37t-84.5 -14q-106 0 -173 71.5t-67 172.5q0 105 72 178t181 73q123 0 205 -94.5 +t82 -252.5zM1456 882v-114h-469v114h167v432q0 7 0.5 19t0.5 17v16h-2l-7 -12q-8 -13 -26 -31l-62 -58l-82 86l192 185h123v-654h165z" /> + <glyph glyph-name="sort_by_order_alt" unicode="" +d="M1346 1247q0 63 -44 116t-103 53q-52 0 -83 -37t-31 -94t36.5 -95t104.5 -38q50 0 85 27t35 68zM736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192q14 0 23 -9 +t9 -23zM1456 -142v-114h-469v114h167v432q0 7 0.5 19t0.5 17v16h-2l-7 -12q-8 -13 -26 -31l-62 -58l-82 86l192 185h123v-654h165zM1486 1189q0 -62 -13 -121.5t-41 -114t-68 -95.5t-98.5 -65.5t-127.5 -24.5q-62 0 -108 16q-24 8 -42 15l39 113q15 -7 31 -11q37 -13 75 -13 +q84 0 134.5 58.5t66.5 145.5h-2q-21 -23 -61.5 -37t-84.5 -14q-106 0 -173 71.5t-67 172.5q0 105 72 178t181 73q123 0 205 -94.5t82 -252.5z" /> + <glyph glyph-name="_334" unicode="" horiz-adv-x="1664" +d="M256 192q0 26 -19 45t-45 19q-27 0 -45.5 -19t-18.5 -45q0 -27 18.5 -45.5t45.5 -18.5q26 0 45 18.5t19 45.5zM416 704v-640q0 -26 -19 -45t-45 -19h-288q-26 0 -45 19t-19 45v640q0 26 19 45t45 19h288q26 0 45 -19t19 -45zM1600 704q0 -86 -55 -149q15 -44 15 -76 +q3 -76 -43 -137q17 -56 0 -117q-15 -57 -54 -94q9 -112 -49 -181q-64 -76 -197 -78h-36h-76h-17q-66 0 -144 15.5t-121.5 29t-120.5 39.5q-123 43 -158 44q-26 1 -45 19.5t-19 44.5v641q0 25 18 43.5t43 20.5q24 2 76 59t101 121q68 87 101 120q18 18 31 48t17.5 48.5 +t13.5 60.5q7 39 12.5 61t19.5 52t34 50q19 19 45 19q46 0 82.5 -10.5t60 -26t40 -40.5t24 -45t12 -50t5 -45t0.5 -39q0 -38 -9.5 -76t-19 -60t-27.5 -56q-3 -6 -10 -18t-11 -22t-8 -24h277q78 0 135 -57t57 -135z" /> + <glyph glyph-name="_335" unicode="" horiz-adv-x="1664" +d="M256 960q0 -26 -19 -45t-45 -19q-27 0 -45.5 19t-18.5 45q0 27 18.5 45.5t45.5 18.5q26 0 45 -18.5t19 -45.5zM416 448v640q0 26 -19 45t-45 19h-288q-26 0 -45 -19t-19 -45v-640q0 -26 19 -45t45 -19h288q26 0 45 19t19 45zM1545 597q55 -61 55 -149q-1 -78 -57.5 -135 +t-134.5 -57h-277q4 -14 8 -24t11 -22t10 -18q18 -37 27 -57t19 -58.5t10 -76.5q0 -24 -0.5 -39t-5 -45t-12 -50t-24 -45t-40 -40.5t-60 -26t-82.5 -10.5q-26 0 -45 19q-20 20 -34 50t-19.5 52t-12.5 61q-9 42 -13.5 60.5t-17.5 48.5t-31 48q-33 33 -101 120q-49 64 -101 121 +t-76 59q-25 2 -43 20.5t-18 43.5v641q0 26 19 44.5t45 19.5q35 1 158 44q77 26 120.5 39.5t121.5 29t144 15.5h17h76h36q133 -2 197 -78q58 -69 49 -181q39 -37 54 -94q17 -61 0 -117q46 -61 43 -137q0 -32 -15 -76z" /> + <glyph glyph-name="youtube_sign" unicode="" +d="M919 233v157q0 50 -29 50q-17 0 -33 -16v-224q16 -16 33 -16q29 0 29 49zM1103 355h66v34q0 51 -33 51t-33 -51v-34zM532 621v-70h-80v-423h-74v423h-78v70h232zM733 495v-367h-67v40q-39 -45 -76 -45q-33 0 -42 28q-6 17 -6 54v290h66v-270q0 -24 1 -26q1 -15 15 -15 +q20 0 42 31v280h67zM985 384v-146q0 -52 -7 -73q-12 -42 -53 -42q-35 0 -68 41v-36h-67v493h67v-161q32 40 68 40q41 0 53 -42q7 -21 7 -74zM1236 255v-9q0 -29 -2 -43q-3 -22 -15 -40q-27 -40 -80 -40q-52 0 -81 38q-21 27 -21 86v129q0 59 20 86q29 38 80 38t78 -38 +q21 -29 21 -86v-76h-133v-65q0 -51 34 -51q24 0 30 26q0 1 0.5 7t0.5 16.5v21.5h68zM785 1079v-156q0 -51 -32 -51t-32 51v156q0 52 32 52t32 -52zM1318 366q0 177 -19 260q-10 44 -43 73.5t-76 34.5q-136 15 -412 15q-275 0 -411 -15q-44 -5 -76.5 -34.5t-42.5 -73.5 +q-20 -87 -20 -260q0 -176 20 -260q10 -43 42.5 -73t75.5 -35q137 -15 412 -15t412 15q43 5 75.5 35t42.5 73q20 84 20 260zM563 1017l90 296h-75l-51 -195l-53 195h-78q7 -23 23 -69l24 -69q35 -103 46 -158v-201h74v201zM852 936v130q0 58 -21 87q-29 38 -78 38 +q-51 0 -78 -38q-21 -29 -21 -87v-130q0 -58 21 -87q27 -38 78 -38q49 0 78 38q21 27 21 87zM1033 816h67v370h-67v-283q-22 -31 -42 -31q-15 0 -16 16q-1 2 -1 26v272h-67v-293q0 -37 6 -55q11 -27 43 -27q36 0 77 45v-40zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5 +h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="youtube" unicode="" +d="M971 292v-211q0 -67 -39 -67q-23 0 -45 22v301q22 22 45 22q39 0 39 -67zM1309 291v-46h-90v46q0 68 45 68t45 -68zM343 509h107v94h-312v-94h105v-569h100v569zM631 -60h89v494h-89v-378q-30 -42 -57 -42q-18 0 -21 21q-1 3 -1 35v364h-89v-391q0 -49 8 -73 +q12 -37 58 -37q48 0 102 61v-54zM1060 88v197q0 73 -9 99q-17 56 -71 56q-50 0 -93 -54v217h-89v-663h89v48q45 -55 93 -55q54 0 71 55q9 27 9 100zM1398 98v13h-91q0 -51 -2 -61q-7 -36 -40 -36q-46 0 -46 69v87h179v103q0 79 -27 116q-39 51 -106 51q-68 0 -107 -51 +q-28 -37 -28 -116v-173q0 -79 29 -116q39 -51 108 -51q72 0 108 53q18 27 21 54q2 9 2 58zM790 1011v210q0 69 -43 69t-43 -69v-210q0 -70 43 -70t43 70zM1509 260q0 -234 -26 -350q-14 -59 -58 -99t-102 -46q-184 -21 -555 -21t-555 21q-58 6 -102.5 46t-57.5 99 +q-26 112 -26 350q0 234 26 350q14 59 58 99t103 47q183 20 554 20t555 -20q58 -7 102.5 -47t57.5 -99q26 -112 26 -350zM511 1536h102l-121 -399v-271h-100v271q-14 74 -61 212q-37 103 -65 187h106l71 -263zM881 1203v-175q0 -81 -28 -118q-38 -51 -106 -51q-67 0 -105 51 +q-28 38 -28 118v175q0 80 28 117q38 51 105 51q68 0 106 -51q28 -37 28 -117zM1216 1365v-499h-91v55q-53 -62 -103 -62q-46 0 -59 37q-8 24 -8 75v394h91v-367q0 -33 1 -35q3 -22 21 -22q27 0 57 43v381h91z" /> + <glyph glyph-name="xing" unicode="" horiz-adv-x="1408" +d="M597 869q-10 -18 -257 -456q-27 -46 -65 -46h-239q-21 0 -31 17t0 36l253 448q1 0 0 1l-161 279q-12 22 -1 37q9 15 32 15h239q40 0 66 -45zM1403 1511q11 -16 0 -37l-528 -934v-1l336 -615q11 -20 1 -37q-10 -15 -32 -15h-239q-42 0 -66 45l-339 622q18 32 531 942 +q25 45 64 45h241q22 0 31 -15z" /> + <glyph glyph-name="xing_sign" unicode="" +d="M685 771q0 1 -126 222q-21 34 -52 34h-184q-18 0 -26 -11q-7 -12 1 -29l125 -216v-1l-196 -346q-9 -14 0 -28q8 -13 24 -13h185q31 0 50 36zM1309 1268q-7 12 -24 12h-187q-30 0 -49 -35l-411 -729q1 -2 262 -481q20 -35 52 -35h184q18 0 25 12q8 13 -1 28l-260 476v1 +l409 723q8 16 0 28zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="youtube_play" unicode="" horiz-adv-x="1792" +d="M711 408l484 250l-484 253v-503zM896 1270q168 0 324.5 -4.5t229.5 -9.5l73 -4q1 0 17 -1.5t23 -3t23.5 -4.5t28.5 -8t28 -13t31 -19.5t29 -26.5q6 -6 15.5 -18.5t29 -58.5t26.5 -101q8 -64 12.5 -136.5t5.5 -113.5v-40v-136q1 -145 -18 -290q-7 -55 -25 -99.5t-32 -61.5 +l-14 -17q-14 -15 -29 -26.5t-31 -19t-28 -12.5t-28.5 -8t-24 -4.5t-23 -3t-16.5 -1.5q-251 -19 -627 -19q-207 2 -359.5 6.5t-200.5 7.5l-49 4l-36 4q-36 5 -54.5 10t-51 21t-56.5 41q-6 6 -15.5 18.5t-29 58.5t-26.5 101q-8 64 -12.5 136.5t-5.5 113.5v40v136 +q-1 145 18 290q7 55 25 99.5t32 61.5l14 17q14 15 29 26.5t31 19.5t28 13t28.5 8t23.5 4.5t23 3t17 1.5q251 18 627 18z" /> + <glyph glyph-name="dropbox" unicode="" horiz-adv-x="1792" +d="M402 829l494 -305l-342 -285l-490 319zM1388 274v-108l-490 -293v-1l-1 1l-1 -1v1l-489 293v108l147 -96l342 284v2l1 -1l1 1v-2l343 -284zM554 1418l342 -285l-494 -304l-338 270zM1390 829l338 -271l-489 -319l-343 285zM1239 1418l489 -319l-338 -270l-494 304z" /> + <glyph glyph-name="stackexchange" unicode="" +d="M1289 -96h-1118v480h-160v-640h1438v640h-160v-480zM347 428l33 157l783 -165l-33 -156zM450 802l67 146l725 -339l-67 -145zM651 1158l102 123l614 -513l-102 -123zM1048 1536l477 -641l-128 -96l-477 641zM330 65v159h800v-159h-800z" /> + <glyph glyph-name="instagram" unicode="" +d="M1024 640q0 106 -75 181t-181 75t-181 -75t-75 -181t75 -181t181 -75t181 75t75 181zM1162 640q0 -164 -115 -279t-279 -115t-279 115t-115 279t115 279t279 115t279 -115t115 -279zM1270 1050q0 -38 -27 -65t-65 -27t-65 27t-27 65t27 65t65 27t65 -27t27 -65zM768 1270 +q-7 0 -76.5 0.5t-105.5 0t-96.5 -3t-103 -10t-71.5 -18.5q-50 -20 -88 -58t-58 -88q-11 -29 -18.5 -71.5t-10 -103t-3 -96.5t0 -105.5t0.5 -76.5t-0.5 -76.5t0 -105.5t3 -96.5t10 -103t18.5 -71.5q20 -50 58 -88t88 -58q29 -11 71.5 -18.5t103 -10t96.5 -3t105.5 0t76.5 0.5 +t76.5 -0.5t105.5 0t96.5 3t103 10t71.5 18.5q50 20 88 58t58 88q11 29 18.5 71.5t10 103t3 96.5t0 105.5t-0.5 76.5t0.5 76.5t0 105.5t-3 96.5t-10 103t-18.5 71.5q-20 50 -58 88t-88 58q-29 11 -71.5 18.5t-103 10t-96.5 3t-105.5 0t-76.5 -0.5zM1536 640q0 -229 -5 -317 +q-10 -208 -124 -322t-322 -124q-88 -5 -317 -5t-317 5q-208 10 -322 124t-124 322q-5 88 -5 317t5 317q10 208 124 322t322 124q88 5 317 5t317 -5q208 -10 322 -124t124 -322q5 -88 5 -317z" /> + <glyph glyph-name="flickr" unicode="" +d="M1248 1408q119 0 203.5 -84.5t84.5 -203.5v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960zM698 640q0 88 -62 150t-150 62t-150 -62t-62 -150t62 -150t150 -62t150 62t62 150zM1262 640q0 88 -62 150 +t-150 62t-150 -62t-62 -150t62 -150t150 -62t150 62t62 150z" /> + <glyph glyph-name="adn" unicode="" +d="M768 914l201 -306h-402zM1133 384h94l-459 691l-459 -691h94l104 160h522zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="f171" unicode="" horiz-adv-x="1408" +d="M815 677q8 -63 -50.5 -101t-111.5 -6q-39 17 -53.5 58t-0.5 82t52 58q36 18 72.5 12t64 -35.5t27.5 -67.5zM926 698q-14 107 -113 164t-197 13q-63 -28 -100.5 -88.5t-34.5 -129.5q4 -91 77.5 -155t165.5 -56q91 8 152 84t50 168zM1165 1240q-20 27 -56 44.5t-58 22 +t-71 12.5q-291 47 -566 -2q-43 -7 -66 -12t-55 -22t-50 -43q30 -28 76 -45.5t73.5 -22t87.5 -11.5q228 -29 448 -1q63 8 89.5 12t72.5 21.5t75 46.5zM1222 205q-8 -26 -15.5 -76.5t-14 -84t-28.5 -70t-58 -56.5q-86 -48 -189.5 -71.5t-202 -22t-201.5 18.5q-46 8 -81.5 18 +t-76.5 27t-73 43.5t-52 61.5q-25 96 -57 292l6 16l18 9q223 -148 506.5 -148t507.5 148q21 -6 24 -23t-5 -45t-8 -37zM1403 1166q-26 -167 -111 -655q-5 -30 -27 -56t-43.5 -40t-54.5 -31q-252 -126 -610 -88q-248 27 -394 139q-15 12 -25.5 26.5t-17 35t-9 34t-6 39.5 +t-5.5 35q-9 50 -26.5 150t-28 161.5t-23.5 147.5t-22 158q3 26 17.5 48.5t31.5 37.5t45 30t46 22.5t48 18.5q125 46 313 64q379 37 676 -50q155 -46 215 -122q16 -20 16.5 -51t-5.5 -54z" /> + <glyph glyph-name="bitbucket_sign" unicode="" +d="M848 666q0 43 -41 66t-77 1q-43 -20 -42.5 -72.5t43.5 -70.5q39 -23 81 4t36 72zM928 682q8 -66 -36 -121t-110 -61t-119 40t-56 113q-2 49 25.5 93t72.5 64q70 31 141.5 -10t81.5 -118zM1100 1073q-20 -21 -53.5 -34t-53 -16t-63.5 -8q-155 -20 -324 0q-44 6 -63 9.5 +t-52.5 16t-54.5 32.5q13 19 36 31t40 15.5t47 8.5q198 35 408 1q33 -5 51 -8.5t43 -16t39 -31.5zM1142 327q0 7 5.5 26.5t3 32t-17.5 16.5q-161 -106 -365 -106t-366 106l-12 -6l-5 -12q26 -154 41 -210q47 -81 204 -108q249 -46 428 53q34 19 49 51.5t22.5 85.5t12.5 71z +M1272 1020q9 53 -8 75q-43 55 -155 88q-216 63 -487 36q-132 -12 -226 -46q-38 -15 -59.5 -25t-47 -34t-29.5 -54q8 -68 19 -138t29 -171t24 -137q1 -5 5 -31t7 -36t12 -27t22 -28q105 -80 284 -100q259 -28 440 63q24 13 39.5 23t31 29t19.5 40q48 267 80 473zM1536 1120 +v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="tumblr" unicode="" horiz-adv-x="1024" +d="M944 207l80 -237q-23 -35 -111 -66t-177 -32q-104 -2 -190.5 26t-142.5 74t-95 106t-55.5 120t-16.5 118v544h-168v215q72 26 129 69.5t91 90t58 102t34 99t15 88.5q1 5 4.5 8.5t7.5 3.5h244v-424h333v-252h-334v-518q0 -30 6.5 -56t22.5 -52.5t49.5 -41.5t81.5 -14 +q78 2 134 29z" /> + <glyph glyph-name="tumblr_sign" unicode="" +d="M1136 75l-62 183q-44 -22 -103 -22q-36 -1 -62 10.5t-38.5 31.5t-17.5 40.5t-5 43.5v398h257v194h-256v326h-188q-8 0 -9 -10q-5 -44 -17.5 -87t-39 -95t-77 -95t-118.5 -68v-165h130v-418q0 -57 21.5 -115t65 -111t121 -85.5t176.5 -30.5q69 1 136.5 25t85.5 50z +M1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="long_arrow_down" unicode="" horiz-adv-x="768" +d="M765 237q8 -19 -5 -35l-350 -384q-10 -10 -23 -10q-14 0 -24 10l-355 384q-13 16 -5 35q9 19 29 19h224v1248q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1248h224q21 0 29 -19z" /> + <glyph glyph-name="long_arrow_up" unicode="" horiz-adv-x="768" +d="M765 1043q-9 -19 -29 -19h-224v-1248q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v1248h-224q-21 0 -29 19t5 35l350 384q10 10 23 10q14 0 24 -10l355 -384q13 -16 5 -35z" /> + <glyph glyph-name="long_arrow_left" unicode="" horiz-adv-x="1792" +d="M1792 736v-192q0 -14 -9 -23t-23 -9h-1248v-224q0 -21 -19 -29t-35 5l-384 350q-10 10 -10 23q0 14 10 24l384 354q16 14 35 6q19 -9 19 -29v-224h1248q14 0 23 -9t9 -23z" /> + <glyph glyph-name="long_arrow_right" unicode="" horiz-adv-x="1792" +d="M1728 643q0 -14 -10 -24l-384 -354q-16 -14 -35 -6q-19 9 -19 29v224h-1248q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h1248v224q0 21 19 29t35 -5l384 -350q10 -10 10 -23z" /> + <glyph glyph-name="apple" unicode="" horiz-adv-x="1408" +d="M1393 321q-39 -125 -123 -250q-129 -196 -257 -196q-49 0 -140 32q-86 32 -151 32q-61 0 -142 -33q-81 -34 -132 -34q-152 0 -301 259q-147 261 -147 503q0 228 113 374q113 144 284 144q72 0 177 -30q104 -30 138 -30q45 0 143 34q102 34 173 34q119 0 213 -65 +q52 -36 104 -100q-79 -67 -114 -118q-65 -94 -65 -207q0 -124 69 -223t158 -126zM1017 1494q0 -61 -29 -136q-30 -75 -93 -138q-54 -54 -108 -72q-37 -11 -104 -17q3 149 78 257q74 107 250 148q1 -3 2.5 -11t2.5 -11q0 -4 0.5 -10t0.5 -10z" /> + <glyph glyph-name="windows" unicode="" horiz-adv-x="1664" +d="M682 530v-651l-682 94v557h682zM682 1273v-659h-682v565zM1664 530v-786l-907 125v661h907zM1664 1408v-794h-907v669z" /> + <glyph glyph-name="android" unicode="" horiz-adv-x="1408" +d="M493 1053q16 0 27.5 11.5t11.5 27.5t-11.5 27.5t-27.5 11.5t-27 -11.5t-11 -27.5t11 -27.5t27 -11.5zM915 1053q16 0 27 11.5t11 27.5t-11 27.5t-27 11.5t-27.5 -11.5t-11.5 -27.5t11.5 -27.5t27.5 -11.5zM103 869q42 0 72 -30t30 -72v-430q0 -43 -29.5 -73t-72.5 -30 +t-73 30t-30 73v430q0 42 30 72t73 30zM1163 850v-666q0 -46 -32 -78t-77 -32h-75v-227q0 -43 -30 -73t-73 -30t-73 30t-30 73v227h-138v-227q0 -43 -30 -73t-73 -30q-42 0 -72 30t-30 73l-1 227h-74q-46 0 -78 32t-32 78v666h918zM931 1255q107 -55 171 -153.5t64 -215.5 +h-925q0 117 64 215.5t172 153.5l-71 131q-7 13 5 20q13 6 20 -6l72 -132q95 42 201 42t201 -42l72 132q7 12 20 6q12 -7 5 -20zM1408 767v-430q0 -43 -30 -73t-73 -30q-42 0 -72 30t-30 73v430q0 43 30 72.5t72 29.5q43 0 73 -29.5t30 -72.5z" /> + <glyph glyph-name="linux" unicode="" +d="M663 1125q-11 -1 -15.5 -10.5t-8.5 -9.5q-5 -1 -5 5q0 12 19 15h10zM750 1111q-4 -1 -11.5 6.5t-17.5 4.5q24 11 32 -2q3 -6 -3 -9zM399 684q-4 1 -6 -3t-4.5 -12.5t-5.5 -13.5t-10 -13q-10 -11 -1 -12q4 -1 12.5 7t12.5 18q1 3 2 7t2 6t1.5 4.5t0.5 4v3t-1 2.5t-3 2z +M1254 325q0 18 -55 42q4 15 7.5 27.5t5 26t3 21.5t0.5 22.5t-1 19.5t-3.5 22t-4 20.5t-5 25t-5.5 26.5q-10 48 -47 103t-72 75q24 -20 57 -83q87 -162 54 -278q-11 -40 -50 -42q-31 -4 -38.5 18.5t-8 83.5t-11.5 107q-9 39 -19.5 69t-19.5 45.5t-15.5 24.5t-13 15t-7.5 7 +q-14 62 -31 103t-29.5 56t-23.5 33t-15 40q-4 21 6 53.5t4.5 49.5t-44.5 25q-15 3 -44.5 18t-35.5 16q-8 1 -11 26t8 51t36 27q37 3 51 -30t4 -58q-11 -19 -2 -26.5t30 -0.5q13 4 13 36v37q-5 30 -13.5 50t-21 30.5t-23.5 15t-27 7.5q-107 -8 -89 -134q0 -15 -1 -15 +q-9 9 -29.5 10.5t-33 -0.5t-15.5 5q1 57 -16 90t-45 34q-27 1 -41.5 -27.5t-16.5 -59.5q-1 -15 3.5 -37t13 -37.5t15.5 -13.5q10 3 16 14q4 9 -7 8q-7 0 -15.5 14.5t-9.5 33.5q-1 22 9 37t34 14q17 0 27 -21t9.5 -39t-1.5 -22q-22 -15 -31 -29q-8 -12 -27.5 -23.5 +t-20.5 -12.5q-13 -14 -15.5 -27t7.5 -18q14 -8 25 -19.5t16 -19t18.5 -13t35.5 -6.5q47 -2 102 15q2 1 23 7t34.5 10.5t29.5 13t21 17.5q9 14 20 8q5 -3 6.5 -8.5t-3 -12t-16.5 -9.5q-20 -6 -56.5 -21.5t-45.5 -19.5q-44 -19 -70 -23q-25 -5 -79 2q-10 2 -9 -2t17 -19 +q25 -23 67 -22q17 1 36 7t36 14t33.5 17.5t30 17t24.5 12t17.5 2.5t8.5 -11q0 -2 -1 -4.5t-4 -5t-6 -4.5t-8.5 -5t-9 -4.5t-10 -5t-9.5 -4.5q-28 -14 -67.5 -44t-66.5 -43t-49 -1q-21 11 -63 73q-22 31 -25 22q-1 -3 -1 -10q0 -25 -15 -56.5t-29.5 -55.5t-21 -58t11.5 -63 +q-23 -6 -62.5 -90t-47.5 -141q-2 -18 -1.5 -69t-5.5 -59q-8 -24 -29 -3q-32 31 -36 94q-2 28 4 56q4 19 -1 18q-2 -1 -4 -5q-36 -65 10 -166q5 -12 25 -28t24 -20q20 -23 104 -90.5t93 -76.5q16 -15 17.5 -38t-14 -43t-45.5 -23q8 -15 29 -44.5t28 -54t7 -70.5q46 24 7 92 +q-4 8 -10.5 16t-9.5 12t-2 6q3 5 13 9.5t20 -2.5q46 -52 166 -36q133 15 177 87q23 38 34 30q12 -6 10 -52q-1 -25 -23 -92q-9 -23 -6 -37.5t24 -15.5q3 19 14.5 77t13.5 90q2 21 -6.5 73.5t-7.5 97t23 70.5q15 18 51 18q1 37 34.5 53t72.5 10.5t60 -22.5zM626 1152 +q3 17 -2.5 30t-11.5 15q-9 2 -9 -7q2 -5 5 -6q10 0 7 -15q-3 -20 8 -20q3 0 3 3zM1045 955q-2 8 -6.5 11.5t-13 5t-14.5 5.5q-5 3 -9.5 8t-7 8t-5.5 6.5t-4 4t-4 -1.5q-14 -16 7 -43.5t39 -31.5q9 -1 14.5 8t3.5 20zM867 1168q0 11 -5 19.5t-11 12.5t-9 3q-6 0 -8 -2t0 -4 +t5 -3q14 -4 18 -31q0 -3 8 2q2 2 2 3zM921 1401q0 2 -2.5 5t-9 7t-9.5 6q-15 15 -24 15q-9 -1 -11.5 -7.5t-1 -13t-0.5 -12.5q-1 -4 -6 -10.5t-6 -9t3 -8.5q4 -3 8 0t11 9t15 9q1 1 9 1t15 2t9 7zM1486 60q20 -12 31 -24.5t12 -24t-2.5 -22.5t-15.5 -22t-23.5 -19.5 +t-30 -18.5t-31.5 -16.5t-32 -15.5t-27 -13q-38 -19 -85.5 -56t-75.5 -64q-17 -16 -68 -19.5t-89 14.5q-18 9 -29.5 23.5t-16.5 25.5t-22 19.5t-47 9.5q-44 1 -130 1q-19 0 -57 -1.5t-58 -2.5q-44 -1 -79.5 -15t-53.5 -30t-43.5 -28.5t-53.5 -11.5q-29 1 -111 31t-146 43 +q-19 4 -51 9.5t-50 9t-39.5 9.5t-33.5 14.5t-17 19.5q-10 23 7 66.5t18 54.5q1 16 -4 40t-10 42.5t-4.5 36.5t10.5 27q14 12 57 14t60 12q30 18 42 35t12 51q21 -73 -32 -106q-32 -20 -83 -15q-34 3 -43 -10q-13 -15 5 -57q2 -6 8 -18t8.5 -18t4.5 -17t1 -22q0 -15 -17 -49 +t-14 -48q3 -17 37 -26q20 -6 84.5 -18.5t99.5 -20.5q24 -6 74 -22t82.5 -23t55.5 -4q43 6 64.5 28t23 48t-7.5 58.5t-19 52t-20 36.5q-121 190 -169 242q-68 74 -113 40q-11 -9 -15 15q-3 16 -2 38q1 29 10 52t24 47t22 42q8 21 26.5 72t29.5 78t30 61t39 54 +q110 143 124 195q-12 112 -16 310q-2 90 24 151.5t106 104.5q39 21 104 21q53 1 106 -13.5t89 -41.5q57 -42 91.5 -121.5t29.5 -147.5q-5 -95 30 -214q34 -113 133 -218q55 -59 99.5 -163t59.5 -191q8 -49 5 -84.5t-12 -55.5t-20 -22q-10 -2 -23.5 -19t-27 -35.5 +t-40.5 -33.5t-61 -14q-18 1 -31.5 5t-22.5 13.5t-13.5 15.5t-11.5 20.5t-9 19.5q-22 37 -41 30t-28 -49t7 -97q20 -70 1 -195q-10 -65 18 -100.5t73 -33t85 35.5q59 49 89.5 66.5t103.5 42.5q53 18 77 36.5t18.5 34.5t-25 28.5t-51.5 23.5q-33 11 -49.5 48t-15 72.5 +t15.5 47.5q1 -31 8 -56.5t14.5 -40.5t20.5 -28.5t21 -19t21.5 -13t16.5 -9.5z" /> + <glyph glyph-name="dribble" unicode="" +d="M1024 36q-42 241 -140 498h-2l-2 -1q-16 -6 -43 -16.5t-101 -49t-137 -82t-131 -114.5t-103 -148l-15 11q184 -150 418 -150q132 0 256 52zM839 643q-21 49 -53 111q-311 -93 -673 -93q-1 -7 -1 -21q0 -124 44 -236.5t124 -201.5q50 89 123.5 166.5t142.5 124.5t130.5 81 +t99.5 48l37 13q4 1 13 3.5t13 4.5zM732 855q-120 213 -244 378q-138 -65 -234 -186t-128 -272q302 0 606 80zM1416 536q-210 60 -409 29q87 -239 128 -469q111 75 185 189.5t96 250.5zM611 1277q-1 0 -2 -1q1 1 2 1zM1201 1132q-185 164 -433 164q-76 0 -155 -19 +q131 -170 246 -382q69 26 130 60.5t96.5 61.5t65.5 57t37.5 40.5zM1424 647q-3 232 -149 410l-1 -1q-9 -12 -19 -24.5t-43.5 -44.5t-71 -60.5t-100 -65t-131.5 -64.5q25 -53 44 -95q2 -5 6.5 -17t7.5 -17q36 5 74.5 7t73.5 2t69 -1.5t64 -4t56.5 -5.5t48 -6.5t36.5 -6 +t25 -4.5zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="skype" unicode="" +d="M1173 473q0 50 -19.5 91.5t-48.5 68.5t-73 49t-82.5 34t-87.5 23l-104 24q-30 7 -44 10.5t-35 11.5t-30 16t-16.5 21t-7.5 30q0 77 144 77q43 0 77 -12t54 -28.5t38 -33.5t40 -29t48 -12q47 0 75.5 32t28.5 77q0 55 -56 99.5t-142 67.5t-182 23q-68 0 -132 -15.5 +t-119.5 -47t-89 -87t-33.5 -128.5q0 -61 19 -106.5t56 -75.5t80 -48.5t103 -32.5l146 -36q90 -22 112 -36q32 -20 32 -60q0 -39 -40 -64.5t-105 -25.5q-51 0 -91.5 16t-65 38.5t-45.5 45t-46 38.5t-54 16q-50 0 -75.5 -30t-25.5 -75q0 -92 122 -157.5t291 -65.5 +q73 0 140 18.5t122.5 53.5t88.5 93.5t33 131.5zM1536 256q0 -159 -112.5 -271.5t-271.5 -112.5q-130 0 -234 80q-77 -16 -150 -16q-143 0 -273.5 55.5t-225 150t-150 225t-55.5 273.5q0 73 16 150q-80 104 -80 234q0 159 112.5 271.5t271.5 112.5q130 0 234 -80 +q77 16 150 16q143 0 273.5 -55.5t225 -150t150 -225t55.5 -273.5q0 -73 -16 -150q80 -104 80 -234z" /> + <glyph glyph-name="foursquare" unicode="" horiz-adv-x="1280" +d="M1000 1102l37 194q5 23 -9 40t-35 17h-712q-23 0 -38.5 -17t-15.5 -37v-1101q0 -7 6 -1l291 352q23 26 38 33.5t48 7.5h239q22 0 37 14.5t18 29.5q24 130 37 191q4 21 -11.5 40t-36.5 19h-294q-29 0 -48 19t-19 48v42q0 29 19 47.5t48 18.5h346q18 0 35 13.5t20 29.5z +M1227 1324q-15 -73 -53.5 -266.5t-69.5 -350t-35 -173.5q-6 -22 -9 -32.5t-14 -32.5t-24.5 -33t-38.5 -21t-58 -10h-271q-13 0 -22 -10q-8 -9 -426 -494q-22 -25 -58.5 -28.5t-48.5 5.5q-55 22 -55 98v1410q0 55 38 102.5t120 47.5h888q95 0 127 -53t10 -159zM1227 1324 +l-158 -790q4 17 35 173.5t69.5 350t53.5 266.5z" /> + <glyph glyph-name="trello" unicode="" +d="M704 192v1024q0 14 -9 23t-23 9h-480q-14 0 -23 -9t-9 -23v-1024q0 -14 9 -23t23 -9h480q14 0 23 9t9 23zM1376 576v640q0 14 -9 23t-23 9h-480q-14 0 -23 -9t-9 -23v-640q0 -14 9 -23t23 -9h480q14 0 23 9t9 23zM1536 1344v-1408q0 -26 -19 -45t-45 -19h-1408 +q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h1408q26 0 45 -19t19 -45z" /> + <glyph glyph-name="female" unicode="" horiz-adv-x="1280" +d="M1280 480q0 -40 -28 -68t-68 -28q-51 0 -80 43l-227 341h-45v-132l247 -411q9 -15 9 -33q0 -26 -19 -45t-45 -19h-192v-272q0 -46 -33 -79t-79 -33h-160q-46 0 -79 33t-33 79v272h-192q-26 0 -45 19t-19 45q0 18 9 33l247 411v132h-45l-227 -341q-29 -43 -80 -43 +q-40 0 -68 28t-28 68q0 29 16 53l256 384q73 107 176 107h384q103 0 176 -107l256 -384q16 -24 16 -53zM864 1280q0 -93 -65.5 -158.5t-158.5 -65.5t-158.5 65.5t-65.5 158.5t65.5 158.5t158.5 65.5t158.5 -65.5t65.5 -158.5z" /> + <glyph glyph-name="male" unicode="" horiz-adv-x="1024" +d="M1024 832v-416q0 -40 -28 -68t-68 -28t-68 28t-28 68v352h-64v-912q0 -46 -33 -79t-79 -33t-79 33t-33 79v464h-64v-464q0 -46 -33 -79t-79 -33t-79 33t-33 79v912h-64v-352q0 -40 -28 -68t-68 -28t-68 28t-28 68v416q0 80 56 136t136 56h640q80 0 136 -56t56 -136z +M736 1280q0 -93 -65.5 -158.5t-158.5 -65.5t-158.5 65.5t-65.5 158.5t65.5 158.5t158.5 65.5t158.5 -65.5t65.5 -158.5z" /> + <glyph glyph-name="gittip" unicode="" +d="M773 234l350 473q16 22 24.5 59t-6 85t-61.5 79q-40 26 -83 25.5t-73.5 -17.5t-54.5 -45q-36 -40 -96 -40q-59 0 -95 40q-24 28 -54.5 45t-73.5 17.5t-84 -25.5q-46 -31 -60.5 -79t-6 -85t24.5 -59zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103 +t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="sun" unicode="" horiz-adv-x="1792" +d="M1472 640q0 117 -45.5 223.5t-123 184t-184 123t-223.5 45.5t-223.5 -45.5t-184 -123t-123 -184t-45.5 -223.5t45.5 -223.5t123 -184t184 -123t223.5 -45.5t223.5 45.5t184 123t123 184t45.5 223.5zM1748 363q-4 -15 -20 -20l-292 -96v-306q0 -16 -13 -26q-15 -10 -29 -4 +l-292 94l-180 -248q-10 -13 -26 -13t-26 13l-180 248l-292 -94q-14 -6 -29 4q-13 10 -13 26v306l-292 96q-16 5 -20 20q-5 17 4 29l180 248l-180 248q-9 13 -4 29q4 15 20 20l292 96v306q0 16 13 26q15 10 29 4l292 -94l180 248q9 12 26 12t26 -12l180 -248l292 94 +q14 6 29 -4q13 -10 13 -26v-306l292 -96q16 -5 20 -20q5 -16 -4 -29l-180 -248l180 -248q9 -12 4 -29z" /> + <glyph glyph-name="_366" unicode="" +d="M1262 233q-54 -9 -110 -9q-182 0 -337 90t-245 245t-90 337q0 192 104 357q-201 -60 -328.5 -229t-127.5 -384q0 -130 51 -248.5t136.5 -204t204 -136.5t248.5 -51q144 0 273.5 61.5t220.5 171.5zM1465 318q-94 -203 -283.5 -324.5t-413.5 -121.5q-156 0 -298 61 +t-245 164t-164 245t-61 298q0 153 57.5 292.5t156 241.5t235.5 164.5t290 68.5q44 2 61 -39q18 -41 -15 -72q-86 -78 -131.5 -181.5t-45.5 -218.5q0 -148 73 -273t198 -198t273 -73q118 0 228 51q41 18 72 -13q14 -14 17.5 -34t-4.5 -38z" /> + <glyph glyph-name="archive" unicode="" horiz-adv-x="1792" +d="M1088 704q0 26 -19 45t-45 19h-256q-26 0 -45 -19t-19 -45t19 -45t45 -19h256q26 0 45 19t19 45zM1664 896v-960q0 -26 -19 -45t-45 -19h-1408q-26 0 -45 19t-19 45v960q0 26 19 45t45 19h1408q26 0 45 -19t19 -45zM1728 1344v-256q0 -26 -19 -45t-45 -19h-1536 +q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h1536q26 0 45 -19t19 -45z" /> + <glyph glyph-name="bug" unicode="" horiz-adv-x="1664" +d="M1632 576q0 -26 -19 -45t-45 -19h-224q0 -171 -67 -290l208 -209q19 -19 19 -45t-19 -45q-18 -19 -45 -19t-45 19l-198 197q-5 -5 -15 -13t-42 -28.5t-65 -36.5t-82 -29t-97 -13v896h-128v-896q-51 0 -101.5 13.5t-87 33t-66 39t-43.5 32.5l-15 14l-183 -207 +q-20 -21 -48 -21q-24 0 -43 16q-19 18 -20.5 44.5t15.5 46.5l202 227q-58 114 -58 274h-224q-26 0 -45 19t-19 45t19 45t45 19h224v294l-173 173q-19 19 -19 45t19 45t45 19t45 -19l173 -173h844l173 173q19 19 45 19t45 -19t19 -45t-19 -45l-173 -173v-294h224q26 0 45 -19 +t19 -45zM1152 1152h-640q0 133 93.5 226.5t226.5 93.5t226.5 -93.5t93.5 -226.5z" /> + <glyph glyph-name="vk" unicode="" horiz-adv-x="1920" +d="M1917 1016q23 -64 -150 -294q-24 -32 -65 -85q-40 -51 -55 -72t-30.5 -49.5t-12 -42t13 -34.5t32.5 -43t57 -53q4 -2 5 -4q141 -131 191 -221q3 -5 6.5 -12.5t7 -26.5t-0.5 -34t-25 -27.5t-59 -12.5l-256 -4q-24 -5 -56 5t-52 22l-20 12q-30 21 -70 64t-68.5 77.5t-61 58 +t-56.5 15.5q-3 -1 -8 -3.5t-17 -14.5t-21.5 -29.5t-17 -52t-6.5 -77.5q0 -15 -3.5 -27.5t-7.5 -18.5l-4 -5q-18 -19 -53 -22h-115q-71 -4 -146 16.5t-131.5 53t-103 66t-70.5 57.5l-25 24q-10 10 -27.5 30t-71.5 91t-106 151t-122.5 211t-130.5 272q-6 16 -6 27t3 16l4 6 +q15 19 57 19l274 2q12 -2 23 -6.5t16 -8.5l5 -3q16 -11 24 -32q20 -50 46 -103.5t41 -81.5l16 -29q29 -60 56 -104t48.5 -68.5t41.5 -38.5t34 -14t27 5q2 1 5 5t12 22t13.5 47t9.5 81t0 125q-2 40 -9 73t-14 46l-6 12q-25 34 -85 43q-13 2 5 24q16 19 38 30q53 26 239 24 +q82 -1 135 -13q20 -5 33.5 -13.5t20.5 -24t10.5 -32t3.5 -45.5t-1 -55t-2.5 -70.5t-1.5 -82.5q0 -11 -1 -42t-0.5 -48t3.5 -40.5t11.5 -39t22.5 -24.5q8 -2 17 -4t26 11t38 34.5t52 67t68 107.5q60 104 107 225q4 10 10 17.5t11 10.5l4 3l5 2.5t13 3t20 0.5l288 2 +q39 5 64 -2.5t31 -16.5z" /> + <glyph glyph-name="weibo" unicode="" horiz-adv-x="1792" +d="M675 252q21 34 11 69t-45 50q-34 14 -73 1t-60 -46q-22 -34 -13 -68.5t43 -50.5t74.5 -2.5t62.5 47.5zM769 373q8 13 3.5 26.5t-17.5 18.5q-14 5 -28.5 -0.5t-21.5 -18.5q-17 -31 13 -45q14 -5 29 0.5t22 18.5zM943 266q-45 -102 -158 -150t-224 -12 +q-107 34 -147.5 126.5t6.5 187.5q47 93 151.5 139t210.5 19q111 -29 158.5 -119.5t2.5 -190.5zM1255 426q-9 96 -89 170t-208.5 109t-274.5 21q-223 -23 -369.5 -141.5t-132.5 -264.5q9 -96 89 -170t208.5 -109t274.5 -21q223 23 369.5 141.5t132.5 264.5zM1563 422 +q0 -68 -37 -139.5t-109 -137t-168.5 -117.5t-226 -83t-270.5 -31t-275 33.5t-240.5 93t-171.5 151t-65 199.5q0 115 69.5 245t197.5 258q169 169 341.5 236t246.5 -7q65 -64 20 -209q-4 -14 -1 -20t10 -7t14.5 0.5t13.5 3.5l6 2q139 59 246 59t153 -61q45 -63 0 -178 +q-2 -13 -4.5 -20t4.5 -12.5t12 -7.5t17 -6q57 -18 103 -47t80 -81.5t34 -116.5zM1489 1046q42 -47 54.5 -108.5t-6.5 -117.5q-8 -23 -29.5 -34t-44.5 -4q-23 8 -34 29.5t-4 44.5q20 63 -24 111t-107 35q-24 -5 -45 8t-25 37q-5 24 8 44.5t37 25.5q60 13 119 -5.5t101 -65.5z +M1670 1209q87 -96 112.5 -222.5t-13.5 -241.5q-9 -27 -34 -40t-52 -4t-40 34t-5 52q28 82 10 172t-80 158q-62 69 -148 95.5t-173 8.5q-28 -6 -52 9.5t-30 43.5t9.5 51.5t43.5 29.5q123 26 244 -11.5t208 -134.5z" /> + <glyph glyph-name="renren" unicode="" +d="M1133 -34q-171 -94 -368 -94q-196 0 -367 94q138 87 235.5 211t131.5 268q35 -144 132.5 -268t235.5 -211zM638 1394v-485q0 -252 -126.5 -459.5t-330.5 -306.5q-181 215 -181 495q0 187 83.5 349.5t229.5 269.5t325 137zM1536 638q0 -280 -181 -495 +q-204 99 -330.5 306.5t-126.5 459.5v485q179 -30 325 -137t229.5 -269.5t83.5 -349.5z" /> + <glyph glyph-name="_372" unicode="" horiz-adv-x="1408" +d="M1402 433q-32 -80 -76 -138t-91 -88.5t-99 -46.5t-101.5 -14.5t-96.5 8.5t-86.5 22t-69.5 27.5t-46 22.5l-17 10q-113 -228 -289.5 -359.5t-384.5 -132.5q-19 0 -32 13t-13 32t13 31.5t32 12.5q173 1 322.5 107.5t251.5 294.5q-36 -14 -72 -23t-83 -13t-91 2.5t-93 28.5 +t-92 59t-84.5 100t-74.5 146q114 47 214 57t167.5 -7.5t124.5 -56.5t88.5 -77t56.5 -82q53 131 79 291q-7 -1 -18 -2.5t-46.5 -2.5t-69.5 0.5t-81.5 10t-88.5 23t-84 42.5t-75 65t-54.5 94.5t-28.5 127.5q70 28 133.5 36.5t112.5 -1t92 -30t73.5 -50t56 -61t42 -63t27.5 -56 +t16 -39.5l4 -16q12 122 12 195q-8 6 -21.5 16t-49 44.5t-63.5 71.5t-54 93t-33 112.5t12 127t70 138.5q73 -25 127.5 -61.5t84.5 -76.5t48 -85t20.5 -89t-0.5 -85.5t-13 -76.5t-19 -62t-17 -42l-7 -15q1 -4 1 -50t-1 -72q3 7 10 18.5t30.5 43t50.5 58t71 55.5t91.5 44.5 +t112 14.5t132.5 -24q-2 -78 -21.5 -141.5t-50 -104.5t-69.5 -71.5t-81.5 -45.5t-84.5 -24t-80 -9.5t-67.5 1t-46.5 4.5l-17 3q-23 -147 -73 -283q6 7 18 18.5t49.5 41t77.5 52.5t99.5 42t117.5 20t129 -23.5t137 -77.5z" /> + <glyph glyph-name="stack_exchange" unicode="" horiz-adv-x="1280" +d="M1259 283v-66q0 -85 -57.5 -144.5t-138.5 -59.5h-57l-260 -269v269h-529q-81 0 -138.5 59.5t-57.5 144.5v66h1238zM1259 609v-255h-1238v255h1238zM1259 937v-255h-1238v255h1238zM1259 1077v-67h-1238v67q0 84 57.5 143.5t138.5 59.5h846q81 0 138.5 -59.5t57.5 -143.5z +" /> + <glyph glyph-name="_374" unicode="" +d="M1152 640q0 -14 -9 -23l-320 -320q-9 -9 -23 -9q-13 0 -22.5 9.5t-9.5 22.5v192h-352q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h352v192q0 14 9 23t23 9q12 0 24 -10l319 -319q9 -9 9 -23zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198 +t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="arrow_circle_alt_left" unicode="" +d="M1152 736v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-352v-192q0 -14 -9 -23t-23 -9q-12 0 -24 10l-319 319q-9 9 -9 23t9 23l320 320q9 9 23 9q13 0 22.5 -9.5t9.5 -22.5v-192h352q13 0 22.5 -9.5t9.5 -22.5zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198 +t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="_376" unicode="" +d="M1024 960v-640q0 -26 -19 -45t-45 -19q-20 0 -37 12l-448 320q-27 19 -27 52t27 52l448 320q17 12 37 12q26 0 45 -19t19 -45zM1280 160v960q0 13 -9.5 22.5t-22.5 9.5h-960q-13 0 -22.5 -9.5t-9.5 -22.5v-960q0 -13 9.5 -22.5t22.5 -9.5h960q13 0 22.5 9.5t9.5 22.5z +M1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="dot_circle_alt" unicode="" +d="M1024 640q0 -106 -75 -181t-181 -75t-181 75t-75 181t75 181t181 75t181 -75t75 -181zM768 1184q-148 0 -273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273t-73 273t-198 198t-273 73zM1536 640q0 -209 -103 -385.5t-279.5 -279.5 +t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="_378" unicode="" horiz-adv-x="1664" +d="M1023 349l102 -204q-58 -179 -210 -290t-339 -111q-156 0 -288.5 77.5t-210 210t-77.5 288.5q0 181 104.5 330t274.5 211l17 -131q-122 -54 -195 -165.5t-73 -244.5q0 -185 131.5 -316.5t316.5 -131.5q126 0 232.5 65t165 175.5t49.5 236.5zM1571 249l58 -114l-256 -128 +q-13 -7 -29 -7q-40 0 -57 35l-239 477h-472q-24 0 -42.5 16.5t-21.5 40.5l-96 779q-2 17 6 42q14 51 57 82.5t97 31.5q66 0 113 -47t47 -113q0 -69 -52 -117.5t-120 -41.5l37 -289h423v-128h-407l16 -128h455q40 0 57 -35l228 -455z" /> + <glyph glyph-name="vimeo_square" unicode="" +d="M1292 898q10 216 -161 222q-231 8 -312 -261q44 19 82 19q85 0 74 -96q-4 -57 -74 -167t-105 -110q-43 0 -82 169q-13 54 -45 255q-30 189 -160 177q-59 -7 -164 -100l-81 -72l-81 -72l52 -67q76 52 87 52q57 0 107 -179q15 -55 45 -164.5t45 -164.5q68 -179 164 -179 +q157 0 383 294q220 283 226 444zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="_380" unicode="" horiz-adv-x="1152" +d="M1152 704q0 -191 -94.5 -353t-256.5 -256.5t-353 -94.5h-160q-14 0 -23 9t-9 23v611l-215 -66q-3 -1 -9 -1q-10 0 -19 6q-13 10 -13 26v128q0 23 23 31l233 71v93l-215 -66q-3 -1 -9 -1q-10 0 -19 6q-13 10 -13 26v128q0 23 23 31l233 71v250q0 14 9 23t23 9h160 +q14 0 23 -9t9 -23v-181l375 116q15 5 28 -5t13 -26v-128q0 -23 -23 -31l-393 -121v-93l375 116q15 5 28 -5t13 -26v-128q0 -23 -23 -31l-393 -121v-487q188 13 318 151t130 328q0 14 9 23t23 9h160q14 0 23 -9t9 -23z" /> + <glyph glyph-name="plus_square_o" unicode="" horiz-adv-x="1408" +d="M1152 736v-64q0 -14 -9 -23t-23 -9h-352v-352q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v352h-352q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h352v352q0 14 9 23t23 9h64q14 0 23 -9t9 -23v-352h352q14 0 23 -9t9 -23zM1280 288v832q0 66 -47 113t-113 47h-832 +q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832q66 0 113 47t47 113zM1408 1120v-832q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h832q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="_382" unicode="" horiz-adv-x="2176" +d="M620 416q-110 -64 -268 -64h-128v64h-64q-13 0 -22.5 23.5t-9.5 56.5q0 24 7 49q-58 2 -96.5 10.5t-38.5 20.5t38.5 20.5t96.5 10.5q-7 25 -7 49q0 33 9.5 56.5t22.5 23.5h64v64h128q158 0 268 -64h1113q42 -7 106.5 -18t80.5 -14q89 -15 150 -40.5t83.5 -47.5t22.5 -40 +t-22.5 -40t-83.5 -47.5t-150 -40.5q-16 -3 -80.5 -14t-106.5 -18h-1113zM1739 668q53 -36 53 -92t-53 -92l81 -30q68 48 68 122t-68 122zM625 400h1015q-217 -38 -456 -80q-57 0 -113 -24t-83 -48l-28 -24l-288 -288q-26 -26 -70.5 -45t-89.5 -19h-96l-93 464h29 +q157 0 273 64zM352 816h-29l93 464h96q46 0 90 -19t70 -45l288 -288q4 -4 11 -10.5t30.5 -23t48.5 -29t61.5 -23t72.5 -10.5l456 -80h-1015q-116 64 -273 64z" /> + <glyph glyph-name="_383" unicode="" horiz-adv-x="1664" +d="M1519 760q62 0 103.5 -40.5t41.5 -101.5q0 -97 -93 -130l-172 -59l56 -167q7 -21 7 -47q0 -59 -42 -102t-101 -43q-47 0 -85.5 27t-53.5 72l-55 165l-310 -106l55 -164q8 -24 8 -47q0 -59 -42 -102t-102 -43q-47 0 -85 27t-53 72l-55 163l-153 -53q-29 -9 -50 -9 +q-61 0 -101.5 40t-40.5 101q0 47 27.5 85t71.5 53l156 53l-105 313l-156 -54q-26 -8 -48 -8q-60 0 -101 40.5t-41 100.5q0 47 27.5 85t71.5 53l157 53l-53 159q-8 24 -8 47q0 60 42 102.5t102 42.5q47 0 85 -27t53 -72l54 -160l310 105l-54 160q-8 24 -8 47q0 59 42.5 102 +t101.5 43q47 0 85.5 -27.5t53.5 -71.5l53 -161l162 55q21 6 43 6q60 0 102.5 -39.5t42.5 -98.5q0 -45 -30 -81.5t-74 -51.5l-157 -54l105 -316l164 56q24 8 46 8zM725 498l310 105l-105 315l-310 -107z" /> + <glyph glyph-name="_384" unicode="" +d="M1248 1408q119 0 203.5 -84.5t84.5 -203.5v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960zM1280 352v436q-31 -35 -64 -55q-34 -22 -132.5 -85t-151.5 -99q-98 -69 -164 -69v0v0q-66 0 -164 69 +q-47 32 -142 92.5t-142 92.5q-12 8 -33 27t-31 27v-436q0 -40 28 -68t68 -28h832q40 0 68 28t28 68zM1280 925q0 41 -27.5 70t-68.5 29h-832q-40 0 -68 -28t-28 -68q0 -37 30.5 -76.5t67.5 -64.5q47 -32 137.5 -89t129.5 -83q3 -2 17 -11.5t21 -14t21 -13t23.5 -13 +t21.5 -9.5t22.5 -7.5t20.5 -2.5t20.5 2.5t22.5 7.5t21.5 9.5t23.5 13t21 13t21 14t17 11.5l267 174q35 23 66.5 62.5t31.5 73.5z" /> + <glyph glyph-name="_385" unicode="" horiz-adv-x="1792" +d="M127 640q0 163 67 313l367 -1005q-196 95 -315 281t-119 411zM1415 679q0 -19 -2.5 -38.5t-10 -49.5t-11.5 -44t-17.5 -59t-17.5 -58l-76 -256l-278 826q46 3 88 8q19 2 26 18.5t-2.5 31t-28.5 13.5l-205 -10q-75 1 -202 10q-12 1 -20.5 -5t-11.5 -15t-1.5 -18.5t9 -16.5 +t19.5 -8l80 -8l120 -328l-168 -504l-280 832q46 3 88 8q19 2 26 18.5t-2.5 31t-28.5 13.5l-205 -10q-7 0 -23 0.5t-26 0.5q105 160 274.5 253.5t367.5 93.5q147 0 280.5 -53t238.5 -149h-10q-55 0 -92 -40.5t-37 -95.5q0 -12 2 -24t4 -21.5t8 -23t9 -21t12 -22.5t12.5 -21 +t14.5 -24t14 -23q63 -107 63 -212zM909 573l237 -647q1 -6 5 -11q-126 -44 -255 -44q-112 0 -217 32zM1570 1009q95 -174 95 -369q0 -209 -104 -385.5t-279 -278.5l235 678q59 169 59 276q0 42 -6 79zM896 1536q182 0 348 -71t286 -191t191 -286t71 -348t-71 -348t-191 -286 +t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71zM896 -215q173 0 331.5 68t273 182.5t182.5 273t68 331.5t-68 331.5t-182.5 273t-273 182.5t-331.5 68t-331.5 -68t-273 -182.5t-182.5 -273t-68 -331.5t68 -331.5t182.5 -273 +t273 -182.5t331.5 -68z" /> + <glyph glyph-name="_386" unicode="" horiz-adv-x="1792" +d="M1086 1536v-1536l-272 -128q-228 20 -414 102t-293 208.5t-107 272.5q0 140 100.5 263.5t275 205.5t391.5 108v-172q-217 -38 -356.5 -150t-139.5 -255q0 -152 154.5 -267t388.5 -145v1360zM1755 954l37 -390l-525 114l147 83q-119 70 -280 99v172q277 -33 481 -157z" /> + <glyph glyph-name="_387" unicode="" horiz-adv-x="2048" +d="M960 1536l960 -384v-128h-128q0 -26 -20.5 -45t-48.5 -19h-1526q-28 0 -48.5 19t-20.5 45h-128v128zM256 896h256v-768h128v768h256v-768h128v768h256v-768h128v768h256v-768h59q28 0 48.5 -19t20.5 -45v-64h-1664v64q0 26 20.5 45t48.5 19h59v768zM1851 -64 +q28 0 48.5 -19t20.5 -45v-128h-1920v128q0 26 20.5 45t48.5 19h1782z" /> + <glyph glyph-name="_388" unicode="" horiz-adv-x="2304" +d="M1774 700l18 -316q4 -69 -82 -128t-235 -93.5t-323 -34.5t-323 34.5t-235 93.5t-82 128l18 316l574 -181q22 -7 48 -7t48 7zM2304 1024q0 -23 -22 -31l-1120 -352q-4 -1 -10 -1t-10 1l-652 206q-43 -34 -71 -111.5t-34 -178.5q63 -36 63 -109q0 -69 -58 -107l58 -433 +q2 -14 -8 -25q-9 -11 -24 -11h-192q-15 0 -24 11q-10 11 -8 25l58 433q-58 38 -58 107q0 73 65 111q11 207 98 330l-333 104q-22 8 -22 31t22 31l1120 352q4 1 10 1t10 -1l1120 -352q22 -8 22 -31z" /> + <glyph glyph-name="_389" unicode="" +d="M859 579l13 -707q-62 11 -105 11q-41 0 -105 -11l13 707q-40 69 -168.5 295.5t-216.5 374.5t-181 287q58 -15 108 -15q44 0 111 15q63 -111 133.5 -229.5t167 -276.5t138.5 -227q37 61 109.5 177.5t117.5 190t105 176t107 189.5q54 -14 107 -14q56 0 114 14v0 +q-28 -39 -60 -88.5t-49.5 -78.5t-56.5 -96t-49 -84q-146 -248 -353 -610z" /> + <glyph glyph-name="uniF1A0" unicode="" +d="M768 750h725q12 -67 12 -128q0 -217 -91 -387.5t-259.5 -266.5t-386.5 -96q-157 0 -299 60.5t-245 163.5t-163.5 245t-60.5 299t60.5 299t163.5 245t245 163.5t299 60.5q300 0 515 -201l-209 -201q-123 119 -306 119q-129 0 -238.5 -65t-173.5 -176.5t-64 -243.5 +t64 -243.5t173.5 -176.5t238.5 -65q87 0 160 24t120 60t82 82t51.5 87t22.5 78h-436v264z" /> + <glyph glyph-name="f1a1" unicode="" horiz-adv-x="1792" +d="M1095 369q16 -16 0 -31q-62 -62 -199 -62t-199 62q-16 15 0 31q6 6 15 6t15 -6q48 -49 169 -49q120 0 169 49q6 6 15 6t15 -6zM788 550q0 -37 -26 -63t-63 -26t-63.5 26t-26.5 63q0 38 26.5 64t63.5 26t63 -26.5t26 -63.5zM1183 550q0 -37 -26.5 -63t-63.5 -26t-63 26 +t-26 63t26 63.5t63 26.5t63.5 -26t26.5 -64zM1434 670q0 49 -35 84t-85 35t-86 -36q-130 90 -311 96l63 283l200 -45q0 -37 26 -63t63 -26t63.5 26.5t26.5 63.5t-26.5 63.5t-63.5 26.5q-54 0 -80 -50l-221 49q-19 5 -25 -16l-69 -312q-180 -7 -309 -97q-35 37 -87 37 +q-50 0 -85 -35t-35 -84q0 -35 18.5 -64t49.5 -44q-6 -27 -6 -56q0 -142 140 -243t337 -101q198 0 338 101t140 243q0 32 -7 57q30 15 48 43.5t18 63.5zM1792 640q0 -182 -71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191 +t348 71t348 -71t286 -191t191 -286t71 -348z" /> + <glyph glyph-name="_392" unicode="" +d="M939 407q13 -13 0 -26q-53 -53 -171 -53t-171 53q-13 13 0 26q5 6 13 6t13 -6q42 -42 145 -42t145 42q5 6 13 6t13 -6zM676 563q0 -31 -23 -54t-54 -23t-54 23t-23 54q0 32 22.5 54.5t54.5 22.5t54.5 -22.5t22.5 -54.5zM1014 563q0 -31 -23 -54t-54 -23t-54 23t-23 54 +q0 32 22.5 54.5t54.5 22.5t54.5 -22.5t22.5 -54.5zM1229 666q0 42 -30 72t-73 30q-42 0 -73 -31q-113 78 -267 82l54 243l171 -39q1 -32 23.5 -54t53.5 -22q32 0 54.5 22.5t22.5 54.5t-22.5 54.5t-54.5 22.5q-48 0 -69 -43l-189 42q-17 5 -21 -13l-60 -268q-154 -6 -265 -83 +q-30 32 -74 32q-43 0 -73 -30t-30 -72q0 -30 16 -55t42 -38q-5 -25 -5 -48q0 -122 120 -208.5t289 -86.5q170 0 290 86.5t120 208.5q0 25 -6 49q25 13 40.5 37.5t15.5 54.5zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960 +q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="_393" unicode="" +d="M866 697l90 27v62q0 79 -58 135t-138 56t-138 -55.5t-58 -134.5v-283q0 -20 -14 -33.5t-33 -13.5t-32.5 13.5t-13.5 33.5v120h-151v-122q0 -82 57.5 -139t139.5 -57q81 0 138.5 56.5t57.5 136.5v280q0 19 13.5 33t33.5 14q19 0 32.5 -14t13.5 -33v-54zM1199 502v122h-150 +v-126q0 -20 -13.5 -33.5t-33.5 -13.5q-19 0 -32.5 14t-13.5 33v123l-90 -26l-60 28v-123q0 -80 58 -137t139 -57t138.5 57t57.5 139zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103 +t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="f1a4" unicode="" horiz-adv-x="1920" +d="M1062 824v118q0 42 -30 72t-72 30t-72 -30t-30 -72v-612q0 -175 -126 -299t-303 -124q-178 0 -303.5 125.5t-125.5 303.5v266h328v-262q0 -43 30 -72.5t72 -29.5t72 29.5t30 72.5v620q0 171 126.5 292t301.5 121q176 0 302 -122t126 -294v-136l-195 -58zM1592 602h328 +v-266q0 -178 -125.5 -303.5t-303.5 -125.5q-177 0 -303 124.5t-126 300.5v268l131 -61l195 58v-270q0 -42 30 -71.5t72 -29.5t72 29.5t30 71.5v275z" /> + <glyph glyph-name="_395" unicode="" +d="M1472 160v480h-704v704h-480q-93 0 -158.5 -65.5t-65.5 -158.5v-480h704v-704h480q93 0 158.5 65.5t65.5 158.5zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5 +t84.5 -203.5z" /> + <glyph glyph-name="_396" unicode="" horiz-adv-x="2048" +d="M328 1254h204v-983h-532v697h328v286zM328 435v369h-123v-369h123zM614 968v-697h205v697h-205zM614 1254v-204h205v204h-205zM901 968h533v-942h-533v163h328v82h-328v697zM1229 435v369h-123v-369h123zM1516 968h532v-942h-532v163h327v82h-327v697zM1843 435v369h-123 +v-369h123z" /> + <glyph glyph-name="_397" unicode="" +d="M1046 516q0 -64 -38 -109t-91 -45q-43 0 -70 15v277q28 17 70 17q53 0 91 -45.5t38 -109.5zM703 944q0 -64 -38 -109.5t-91 -45.5q-43 0 -70 15v277q28 17 70 17q53 0 91 -45t38 -109zM1265 513q0 134 -88 229t-213 95q-20 0 -39 -3q-23 -78 -78 -136q-87 -95 -211 -101 +v-636l211 41v206q51 -19 117 -19q125 0 213 95t88 229zM922 940q0 134 -88.5 229t-213.5 95q-74 0 -141 -36h-186v-840l211 41v206q55 -19 116 -19q125 0 213.5 95t88.5 229zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960 +q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="_398" unicode="" horiz-adv-x="2038" +d="M1222 607q75 3 143.5 -20.5t118 -58.5t101 -94.5t84 -108t75.5 -120.5q33 -56 78.5 -109t75.5 -80.5t99 -88.5q-48 -30 -108.5 -57.5t-138.5 -59t-114 -47.5q-44 37 -74 115t-43.5 164.5t-33 180.5t-42.5 168.5t-72.5 123t-122.5 48.5l-10 -2l-6 -4q4 -5 13 -14 +q6 -5 28 -23.5t25.5 -22t19 -18t18 -20.5t11.5 -21t10.5 -27.5t4.5 -31t4 -40.5l1 -33q1 -26 -2.5 -57.5t-7.5 -52t-12.5 -58.5t-11.5 -53q-35 1 -101 -9.5t-98 -10.5q-39 0 -72 10q-2 16 -2 47q0 74 3 96q2 13 31.5 41.5t57 59t26.5 51.5q-24 2 -43 -24 +q-36 -53 -111.5 -99.5t-136.5 -46.5q-25 0 -75.5 63t-106.5 139.5t-84 96.5q-6 4 -27 30q-482 -112 -513 -112q-16 0 -28 11t-12 27q0 15 8.5 26.5t22.5 14.5l486 106q-8 14 -8 25t5.5 17.5t16 11.5t20 7t23 4.5t18.5 4.5q4 1 15.5 7.5t17.5 6.5q15 0 28 -16t20 -33 +q163 37 172 37q17 0 29.5 -11t12.5 -28q0 -15 -8.5 -26t-23.5 -14l-182 -40l-1 -16q-1 -26 81.5 -117.5t104.5 -91.5q47 0 119 80t72 129q0 36 -23.5 53t-51 18.5t-51 11.5t-23.5 34q0 16 10 34l-68 19q43 44 43 117q0 26 -5 58q82 16 144 16q44 0 71.5 -1.5t48.5 -8.5 +t31 -13.5t20.5 -24.5t15.5 -33.5t17 -47.5t24 -60l50 25q-3 -40 -23 -60t-42.5 -21t-40 -6.5t-16.5 -20.5zM1282 842q-5 5 -13.5 15.5t-12 14.5t-10.5 11.5t-10 10.5l-8 8t-8.5 7.5t-8 5t-8.5 4.5q-7 3 -14.5 5t-20.5 2.5t-22 0.5h-32.5h-37.5q-126 0 -217 -43 +q16 30 36 46.5t54 29.5t65.5 36t46 36.5t50 55t43.5 50.5q12 -9 28 -31.5t32 -36.5t38 -13l12 1v-76l22 -1q247 95 371 190q28 21 50 39t42.5 37.5t33 31t29.5 34t24 31t24.5 37t23 38t27 47.5t29.5 53l7 9q-2 -53 -43 -139q-79 -165 -205 -264t-306 -142q-14 -3 -42 -7.5 +t-50 -9.5t-39 -14q3 -19 24.5 -46t21.5 -34q0 -11 -26 -30zM1061 -79q39 26 131.5 47.5t146.5 21.5q9 0 22.5 -15.5t28 -42.5t26 -50t24 -51t14.5 -33q-121 -45 -244 -45q-61 0 -125 11zM822 568l48 12l109 -177l-73 -48zM1323 51q3 -15 3 -16q0 -7 -17.5 -14.5t-46 -13 +t-54 -9.5t-53.5 -7.5t-32 -4.5l-7 43q21 2 60.5 8.5t72 10t60.5 3.5h14zM866 679l-96 -20l-6 17q10 1 32.5 7t34.5 6q19 0 35 -10zM1061 45h31l10 -83l-41 -12v95zM1950 1535v1v-1zM1950 1535l-1 -5l-2 -2l1 3zM1950 1535l1 1z" /> + <glyph glyph-name="_399" unicode="" +d="M1167 -50q-5 19 -24 5q-30 -22 -87 -39t-131 -17q-129 0 -193 49q-5 4 -13 4q-11 0 -26 -12q-7 -6 -7.5 -16t7.5 -20q34 -32 87.5 -46t102.5 -12.5t99 4.5q41 4 84.5 20.5t65 30t28.5 20.5q12 12 7 29zM1128 65q-19 47 -39 61q-23 15 -76 15q-47 0 -71 -10 +q-29 -12 -78 -56q-26 -24 -12 -44q9 -8 17.5 -4.5t31.5 23.5q3 2 10.5 8.5t10.5 8.5t10 7t11.5 7t12.5 5t15 4.5t16.5 2.5t20.5 1q27 0 44.5 -7.5t23 -14.5t13.5 -22q10 -17 12.5 -20t12.5 1q23 12 14 34zM1483 346q0 22 -5 44.5t-16.5 45t-34 36.5t-52.5 14 +q-33 0 -97 -41.5t-129 -83.5t-101 -42q-27 -1 -63.5 19t-76 49t-83.5 58t-100 49t-111 19q-115 -1 -197 -78.5t-84 -178.5q-2 -112 74 -164q29 -20 62.5 -28.5t103.5 -8.5q57 0 132 32.5t134 71t120 70.5t93 31q26 -1 65 -31.5t71.5 -67t68 -67.5t55.5 -32q35 -3 58.5 14 +t55.5 63q28 41 42.5 101t14.5 106zM1536 506q0 -164 -62 -304.5t-166 -236t-242.5 -149.5t-290.5 -54t-293 57.5t-247.5 157t-170.5 241.5t-64 302q0 89 19.5 172.5t49 145.5t70.5 118.5t78.5 94t78.5 69.5t64.5 46.5t42.5 24.5q14 8 51 26.5t54.5 28.5t48 30t60.5 44 +q36 28 58 72.5t30 125.5q129 -155 186 -193q44 -29 130 -68t129 -66q21 -13 39 -25t60.5 -46.5t76 -70.5t75 -95t69 -122t47 -148.5t19.5 -177.5z" /> + <glyph glyph-name="_400" unicode="" +d="M1070 463l-160 -160l-151 -152l-30 -30q-65 -64 -151.5 -87t-171.5 -2q-16 -70 -72 -115t-129 -45q-85 0 -145 60.5t-60 145.5q0 72 44.5 128t113.5 72q-22 86 1 173t88 152l12 12l151 -152l-11 -11q-37 -37 -37 -89t37 -90q37 -37 89 -37t89 37l30 30l151 152l161 160z +M729 1145l12 -12l-152 -152l-12 12q-37 37 -89 37t-89 -37t-37 -89.5t37 -89.5l29 -29l152 -152l160 -160l-151 -152l-161 160l-151 152l-30 30q-68 67 -90 159.5t5 179.5q-70 15 -115 71t-45 129q0 85 60 145.5t145 60.5q76 0 133.5 -49t69.5 -123q84 20 169.5 -3.5 +t149.5 -87.5zM1536 78q0 -85 -60 -145.5t-145 -60.5q-74 0 -131 47t-71 118q-86 -28 -179.5 -6t-161.5 90l-11 12l151 152l12 -12q37 -37 89 -37t89 37t37 89t-37 89l-30 30l-152 152l-160 160l152 152l160 -160l152 -152l29 -30q64 -64 87.5 -150.5t2.5 -171.5 +q76 -11 126.5 -68.5t50.5 -134.5zM1534 1202q0 -77 -51 -135t-127 -69q26 -85 3 -176.5t-90 -158.5l-12 -12l-151 152l12 12q37 37 37 89t-37 89t-89 37t-89 -37l-30 -30l-152 -152l-160 -160l-152 152l161 160l152 152l29 30q67 67 159 89.5t178 -3.5q11 75 68.5 126 +t135.5 51q85 0 145 -60.5t60 -145.5z" /> + <glyph glyph-name="f1ab" unicode="" +d="M654 458q-1 -3 -12.5 0.5t-31.5 11.5l-20 9q-44 20 -87 49q-7 5 -41 31.5t-38 28.5q-67 -103 -134 -181q-81 -95 -105 -110q-4 -2 -19.5 -4t-18.5 0q6 4 82 92q21 24 85.5 115t78.5 118q17 30 51 98.5t36 77.5q-8 1 -110 -33q-8 -2 -27.5 -7.5t-34.5 -9.5t-17 -5 +q-2 -2 -2 -10.5t-1 -9.5q-5 -10 -31 -15q-23 -7 -47 0q-18 4 -28 21q-4 6 -5 23q6 2 24.5 5t29.5 6q58 16 105 32q100 35 102 35q10 2 43 19.5t44 21.5q9 3 21.5 8t14.5 5.5t6 -0.5q2 -12 -1 -33q0 -2 -12.5 -27t-26.5 -53.5t-17 -33.5q-25 -50 -77 -131l64 -28 +q12 -6 74.5 -32t67.5 -28q4 -1 10.5 -25.5t4.5 -30.5zM449 944q3 -15 -4 -28q-12 -23 -50 -38q-30 -12 -60 -12q-26 3 -49 26q-14 15 -18 41l1 3q3 -3 19.5 -5t26.5 0t58 16q36 12 55 14q17 0 21 -17zM1147 815l63 -227l-139 42zM39 15l694 232v1032l-694 -233v-1031z +M1280 332l102 -31l-181 657l-100 31l-216 -536l102 -31l45 110l211 -65zM777 1294l573 -184v380zM1088 -29l158 -13l-54 -160l-40 66q-130 -83 -276 -108q-58 -12 -91 -12h-84q-79 0 -199.5 39t-183.5 85q-8 7 -8 16q0 8 5 13.5t13 5.5q4 0 18 -7.5t30.5 -16.5t20.5 -11 +q73 -37 159.5 -61.5t157.5 -24.5q95 0 167 14.5t157 50.5q15 7 30.5 15.5t34 19t28.5 16.5zM1536 1050v-1079l-774 246q-14 -6 -375 -127.5t-368 -121.5q-13 0 -18 13q0 1 -1 3v1078q3 9 4 10q5 6 20 11q107 36 149 50v384l558 -198q2 0 160.5 55t316 108.5t161.5 53.5 +q20 0 20 -21v-418z" /> + <glyph glyph-name="_402" unicode="" horiz-adv-x="1792" +d="M288 1152q66 0 113 -47t47 -113v-1088q0 -66 -47 -113t-113 -47h-128q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h128zM1664 989q58 -34 93 -93t35 -128v-768q0 -106 -75 -181t-181 -75h-864q-66 0 -113 47t-47 113v1536q0 40 28 68t68 28h672q40 0 88 -20t76 -48 +l152 -152q28 -28 48 -76t20 -88v-163zM928 0v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM928 256v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM928 512v128q0 14 -9 23 +t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1184 0v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1184 256v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128 +q14 0 23 9t9 23zM1184 512v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1440 0v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1440 256v128q0 14 -9 23t-23 9h-128 +q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1440 512v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1536 896v256h-160q-40 0 -68 28t-28 68v160h-640v-512h896z" /> + <glyph glyph-name="_403" unicode="" +d="M1344 1536q26 0 45 -19t19 -45v-1664q0 -26 -19 -45t-45 -19h-1280q-26 0 -45 19t-19 45v1664q0 26 19 45t45 19h1280zM512 1248v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23zM512 992v-64q0 -14 9 -23t23 -9h64q14 0 23 9 +t9 23v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23zM512 736v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23zM512 480v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23zM384 160v64 +q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM384 416v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM384 672v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64 +q14 0 23 9t9 23zM384 928v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM384 1184v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM896 -96v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9 +t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM896 416v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM896 672v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM896 928v64 +q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM896 1184v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1152 160v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64 +q14 0 23 9t9 23zM1152 416v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1152 672v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1152 928v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9 +t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1152 1184v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23z" /> + <glyph glyph-name="_404" unicode="" horiz-adv-x="1280" +d="M1188 988l-292 -292v-824q0 -46 -33 -79t-79 -33t-79 33t-33 79v384h-64v-384q0 -46 -33 -79t-79 -33t-79 33t-33 79v824l-292 292q-28 28 -28 68t28 68q29 28 68.5 28t67.5 -28l228 -228h368l228 228q28 28 68 28t68 -28q28 -29 28 -68.5t-28 -67.5zM864 1152 +q0 -93 -65.5 -158.5t-158.5 -65.5t-158.5 65.5t-65.5 158.5t65.5 158.5t158.5 65.5t158.5 -65.5t65.5 -158.5z" /> + <glyph glyph-name="uniF1B1" unicode="" horiz-adv-x="1664" +d="M780 1064q0 -60 -19 -113.5t-63 -92.5t-105 -39q-76 0 -138 57.5t-92 135.5t-30 151q0 60 19 113.5t63 92.5t105 39q77 0 138.5 -57.5t91.5 -135t30 -151.5zM438 581q0 -80 -42 -139t-119 -59q-76 0 -141.5 55.5t-100.5 133.5t-35 152q0 80 42 139.5t119 59.5 +q76 0 141.5 -55.5t100.5 -134t35 -152.5zM832 608q118 0 255 -97.5t229 -237t92 -254.5q0 -46 -17 -76.5t-48.5 -45t-64.5 -20t-76 -5.5q-68 0 -187.5 45t-182.5 45q-66 0 -192.5 -44.5t-200.5 -44.5q-183 0 -183 146q0 86 56 191.5t139.5 192.5t187.5 146t193 59zM1071 819 +q-61 0 -105 39t-63 92.5t-19 113.5q0 74 30 151.5t91.5 135t138.5 57.5q61 0 105 -39t63 -92.5t19 -113.5q0 -73 -30 -151t-92 -135.5t-138 -57.5zM1503 923q77 0 119 -59.5t42 -139.5q0 -74 -35 -152t-100.5 -133.5t-141.5 -55.5q-77 0 -119 59t-42 139q0 74 35 152.5 +t100.5 134t141.5 55.5z" /> + <glyph glyph-name="_406" unicode="" horiz-adv-x="768" +d="M704 1008q0 -145 -57 -243.5t-152 -135.5l45 -821q2 -26 -16 -45t-44 -19h-192q-26 0 -44 19t-16 45l45 821q-95 37 -152 135.5t-57 243.5q0 128 42.5 249.5t117.5 200t160 78.5t160 -78.5t117.5 -200t42.5 -249.5z" /> + <glyph glyph-name="_407" unicode="" horiz-adv-x="1792" +d="M896 -93l640 349v636l-640 -233v-752zM832 772l698 254l-698 254l-698 -254zM1664 1024v-768q0 -35 -18 -65t-49 -47l-704 -384q-28 -16 -61 -16t-61 16l-704 384q-31 17 -49 47t-18 65v768q0 40 23 73t61 47l704 256q22 8 44 8t44 -8l704 -256q38 -14 61 -47t23 -73z +" /> + <glyph glyph-name="_408" unicode="" horiz-adv-x="2304" +d="M640 -96l384 192v314l-384 -164v-342zM576 358l404 173l-404 173l-404 -173zM1664 -96l384 192v314l-384 -164v-342zM1600 358l404 173l-404 173l-404 -173zM1152 651l384 165v266l-384 -164v-267zM1088 1030l441 189l-441 189l-441 -189zM2176 512v-416q0 -36 -19 -67 +t-52 -47l-448 -224q-25 -14 -57 -14t-57 14l-448 224q-4 2 -7 4q-2 -2 -7 -4l-448 -224q-25 -14 -57 -14t-57 14l-448 224q-33 16 -52 47t-19 67v416q0 38 21.5 70t56.5 48l434 186v400q0 38 21.5 70t56.5 48l448 192q23 10 50 10t50 -10l448 -192q35 -16 56.5 -48t21.5 -70 +v-400l434 -186q36 -16 57 -48t21 -70z" /> + <glyph glyph-name="_409" unicode="" horiz-adv-x="2048" +d="M1848 1197h-511v-124h511v124zM1596 771q-90 0 -146 -52.5t-62 -142.5h408q-18 195 -200 195zM1612 186q63 0 122 32t76 87h221q-100 -307 -427 -307q-214 0 -340.5 132t-126.5 347q0 208 130.5 345.5t336.5 137.5q138 0 240.5 -68t153 -179t50.5 -248q0 -17 -2 -47h-658 +q0 -111 57.5 -171.5t166.5 -60.5zM277 236h296q205 0 205 167q0 180 -199 180h-302v-347zM277 773h281q78 0 123.5 36.5t45.5 113.5q0 144 -190 144h-260v-294zM0 1282h594q87 0 155 -14t126.5 -47.5t90 -96.5t31.5 -154q0 -181 -172 -263q114 -32 172 -115t58 -204 +q0 -75 -24.5 -136.5t-66 -103.5t-98.5 -71t-121 -42t-134 -13h-611v1260z" /> + <glyph glyph-name="_410" unicode="" +d="M1248 1408q119 0 203.5 -84.5t84.5 -203.5v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960zM499 1041h-371v-787h382q117 0 197 57.5t80 170.5q0 158 -143 200q107 52 107 164q0 57 -19.5 96.5 +t-56.5 60.5t-79 29.5t-97 8.5zM477 723h-176v184h163q119 0 119 -90q0 -94 -106 -94zM486 388h-185v217h189q124 0 124 -113q0 -104 -128 -104zM1136 356q-68 0 -104 38t-36 107h411q1 10 1 30q0 132 -74.5 220.5t-203.5 88.5q-128 0 -210 -86t-82 -216q0 -135 79 -217 +t213 -82q205 0 267 191h-138q-11 -34 -47.5 -54t-75.5 -20zM1126 722q113 0 124 -122h-254q4 56 39 89t91 33zM964 988h319v-77h-319v77z" /> + <glyph glyph-name="_411" unicode="" horiz-adv-x="1792" +d="M1582 954q0 -101 -71.5 -172.5t-172.5 -71.5t-172.5 71.5t-71.5 172.5t71.5 172.5t172.5 71.5t172.5 -71.5t71.5 -172.5zM812 212q0 104 -73 177t-177 73q-27 0 -54 -6l104 -42q77 -31 109.5 -106.5t1.5 -151.5q-31 -77 -107 -109t-152 -1q-21 8 -62 24.5t-61 24.5 +q32 -60 91 -96.5t130 -36.5q104 0 177 73t73 177zM1642 953q0 126 -89.5 215.5t-215.5 89.5q-127 0 -216.5 -89.5t-89.5 -215.5q0 -127 89.5 -216t216.5 -89q126 0 215.5 89t89.5 216zM1792 953q0 -189 -133.5 -322t-321.5 -133l-437 -319q-12 -129 -109 -218t-229 -89 +q-121 0 -214 76t-118 192l-230 92v429l389 -157q79 48 173 48q13 0 35 -2l284 407q2 187 135.5 319t320.5 132q188 0 321.5 -133.5t133.5 -321.5z" /> + <glyph glyph-name="_412" unicode="" +d="M1242 889q0 80 -57 136.5t-137 56.5t-136.5 -57t-56.5 -136q0 -80 56.5 -136.5t136.5 -56.5t137 56.5t57 136.5zM632 301q0 -83 -58 -140.5t-140 -57.5q-56 0 -103 29t-72 77q52 -20 98 -40q60 -24 120 1.5t85 86.5q24 60 -1.5 120t-86.5 84l-82 33q22 5 42 5 +q82 0 140 -57.5t58 -140.5zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v153l172 -69q20 -92 93.5 -152t168.5 -60q104 0 181 70t87 173l345 252q150 0 255.5 105.5t105.5 254.5q0 150 -105.5 255.5t-255.5 105.5 +q-148 0 -253 -104.5t-107 -252.5l-225 -322q-9 1 -28 1q-75 0 -137 -37l-297 119v468q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5zM1289 887q0 -100 -71 -170.5t-171 -70.5t-170.5 70.5t-70.5 170.5t70.5 171t170.5 71q101 0 171.5 -70.5t70.5 -171.5z +" /> + <glyph glyph-name="_413" unicode="" horiz-adv-x="1792" +d="M836 367l-15 -368l-2 -22l-420 29q-36 3 -67 31.5t-47 65.5q-11 27 -14.5 55t4 65t12 55t21.5 64t19 53q78 -12 509 -28zM449 953l180 -379l-147 92q-63 -72 -111.5 -144.5t-72.5 -125t-39.5 -94.5t-18.5 -63l-4 -21l-190 357q-17 26 -18 56t6 47l8 18q35 63 114 188 +l-140 86zM1680 436l-188 -359q-12 -29 -36.5 -46.5t-43.5 -20.5l-18 -4q-71 -7 -219 -12l8 -164l-230 367l211 362l7 -173q170 -16 283 -5t170 33zM895 1360q-47 -63 -265 -435l-317 187l-19 12l225 356q20 31 60 45t80 10q24 -2 48.5 -12t42 -21t41.5 -33t36 -34.5 +t36 -39.5t32 -35zM1550 1053l212 -363q18 -37 12.5 -76t-27.5 -74q-13 -20 -33 -37t-38 -28t-48.5 -22t-47 -16t-51.5 -14t-46 -12q-34 72 -265 436l313 195zM1407 1279l142 83l-220 -373l-419 20l151 86q-34 89 -75 166t-75.5 123.5t-64.5 80t-47 46.5l-17 13l405 -1 +q31 3 58 -10.5t39 -28.5l11 -15q39 -61 112 -190z" /> + <glyph glyph-name="_414" unicode="" horiz-adv-x="2048" +d="M480 448q0 66 -47 113t-113 47t-113 -47t-47 -113t47 -113t113 -47t113 47t47 113zM516 768h1016l-89 357q-2 8 -14 17.5t-21 9.5h-768q-9 0 -21 -9.5t-14 -17.5zM1888 448q0 66 -47 113t-113 47t-113 -47t-47 -113t47 -113t113 -47t113 47t47 113zM2048 544v-384 +q0 -14 -9 -23t-23 -9h-96v-128q0 -80 -56 -136t-136 -56t-136 56t-56 136v128h-1024v-128q0 -80 -56 -136t-136 -56t-136 56t-56 136v128h-96q-14 0 -23 9t-9 23v384q0 93 65.5 158.5t158.5 65.5h28l105 419q23 94 104 157.5t179 63.5h768q98 0 179 -63.5t104 -157.5 +l105 -419h28q93 0 158.5 -65.5t65.5 -158.5z" /> + <glyph glyph-name="_415" unicode="" horiz-adv-x="2048" +d="M1824 640q93 0 158.5 -65.5t65.5 -158.5v-384q0 -14 -9 -23t-23 -9h-96v-64q0 -80 -56 -136t-136 -56t-136 56t-56 136v64h-1024v-64q0 -80 -56 -136t-136 -56t-136 56t-56 136v64h-96q-14 0 -23 9t-9 23v384q0 93 65.5 158.5t158.5 65.5h28l105 419q23 94 104 157.5 +t179 63.5h128v224q0 14 9 23t23 9h448q14 0 23 -9t9 -23v-224h128q98 0 179 -63.5t104 -157.5l105 -419h28zM320 160q66 0 113 47t47 113t-47 113t-113 47t-113 -47t-47 -113t47 -113t113 -47zM516 640h1016l-89 357q-2 8 -14 17.5t-21 9.5h-768q-9 0 -21 -9.5t-14 -17.5z +M1728 160q66 0 113 47t47 113t-47 113t-113 47t-113 -47t-47 -113t47 -113t113 -47z" /> + <glyph glyph-name="_416" unicode="" +d="M1504 64q0 -26 -19 -45t-45 -19h-462q1 -17 6 -87.5t5 -108.5q0 -25 -18 -42.5t-43 -17.5h-320q-25 0 -43 17.5t-18 42.5q0 38 5 108.5t6 87.5h-462q-26 0 -45 19t-19 45t19 45l402 403h-229q-26 0 -45 19t-19 45t19 45l402 403h-197q-26 0 -45 19t-19 45t19 45l384 384 +q19 19 45 19t45 -19l384 -384q19 -19 19 -45t-19 -45t-45 -19h-197l402 -403q19 -19 19 -45t-19 -45t-45 -19h-229l402 -403q19 -19 19 -45z" /> + <glyph glyph-name="_417" unicode="" +d="M1127 326q0 32 -30 51q-193 115 -447 115q-133 0 -287 -34q-42 -9 -42 -52q0 -20 13.5 -34.5t35.5 -14.5q5 0 37 8q132 27 243 27q226 0 397 -103q19 -11 33 -11q19 0 33 13.5t14 34.5zM1223 541q0 40 -35 61q-237 141 -548 141q-153 0 -303 -42q-48 -13 -48 -64 +q0 -25 17.5 -42.5t42.5 -17.5q7 0 37 8q122 33 251 33q279 0 488 -124q24 -13 38 -13q25 0 42.5 17.5t17.5 42.5zM1331 789q0 47 -40 70q-126 73 -293 110.5t-343 37.5q-204 0 -364 -47q-23 -7 -38.5 -25.5t-15.5 -48.5q0 -31 20.5 -52t51.5 -21q11 0 40 8q133 37 307 37 +q159 0 309.5 -34t253.5 -95q21 -12 40 -12q29 0 50.5 20.5t21.5 51.5zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="_418" unicode="" horiz-adv-x="1024" +d="M1024 1233l-303 -582l24 -31h279v-415h-507l-44 -30l-142 -273l-30 -30h-301v303l303 583l-24 30h-279v415h507l44 30l142 273l30 30h301v-303z" /> + <glyph glyph-name="_419" unicode="" horiz-adv-x="2304" +d="M784 164l16 241l-16 523q-1 10 -7.5 17t-16.5 7q-9 0 -16 -7t-7 -17l-14 -523l14 -241q1 -10 7.5 -16.5t15.5 -6.5q22 0 24 23zM1080 193l11 211l-12 586q0 16 -13 24q-8 5 -16 5t-16 -5q-13 -8 -13 -24l-1 -6l-10 -579q0 -1 11 -236v-1q0 -10 6 -17q9 -11 23 -11 +q11 0 20 9q9 7 9 20zM35 533l20 -128l-20 -126q-2 -9 -9 -9t-9 9l-17 126l17 128q2 9 9 9t9 -9zM121 612l26 -207l-26 -203q-2 -9 -10 -9q-9 0 -9 10l-23 202l23 207q0 9 9 9q8 0 10 -9zM401 159zM213 650l25 -245l-25 -237q0 -11 -11 -11q-10 0 -12 11l-21 237l21 245 +q2 12 12 12q11 0 11 -12zM307 657l23 -252l-23 -244q-2 -13 -14 -13q-13 0 -13 13l-21 244l21 252q0 13 13 13q12 0 14 -13zM401 639l21 -234l-21 -246q-2 -16 -16 -16q-6 0 -10.5 4.5t-4.5 11.5l-20 246l20 234q0 6 4.5 10.5t10.5 4.5q14 0 16 -15zM784 164zM495 785 +l21 -380l-21 -246q0 -7 -5 -12.5t-12 -5.5q-16 0 -18 18l-18 246l18 380q2 18 18 18q7 0 12 -5.5t5 -12.5zM589 871l19 -468l-19 -244q0 -8 -5.5 -13.5t-13.5 -5.5q-18 0 -20 19l-16 244l16 468q2 19 20 19q8 0 13.5 -5.5t5.5 -13.5zM687 911l18 -506l-18 -242 +q-2 -21 -22 -21q-19 0 -21 21l-16 242l16 506q0 9 6.5 15.5t14.5 6.5q9 0 15 -6.5t7 -15.5zM1079 169v0v0v0zM881 915l15 -510l-15 -239q0 -10 -7.5 -17.5t-17.5 -7.5t-17 7t-8 18l-14 239l14 510q0 11 7.5 18t17.5 7t17.5 -7t7.5 -18zM980 896l14 -492l-14 -236 +q0 -11 -8 -19t-19 -8t-19 8t-9 19l-12 236l12 492q1 12 9 20t19 8t18.5 -8t8.5 -20zM1192 404l-14 -231v0q0 -13 -9 -22t-22 -9t-22 9t-10 22l-6 114l-6 117l12 636v3q2 15 12 24q9 7 20 7q8 0 15 -5q14 -8 16 -26zM2304 423q0 -117 -83 -199.5t-200 -82.5h-786 +q-13 2 -22 11t-9 22v899q0 23 28 33q85 34 181 34q195 0 338 -131.5t160 -323.5q53 22 110 22q117 0 200 -83t83 -201z" /> + <glyph glyph-name="uniF1C0" unicode="" +d="M768 768q237 0 443 43t325 127v-170q0 -69 -103 -128t-280 -93.5t-385 -34.5t-385 34.5t-280 93.5t-103 128v170q119 -84 325 -127t443 -43zM768 0q237 0 443 43t325 127v-170q0 -69 -103 -128t-280 -93.5t-385 -34.5t-385 34.5t-280 93.5t-103 128v170q119 -84 325 -127 +t443 -43zM768 384q237 0 443 43t325 127v-170q0 -69 -103 -128t-280 -93.5t-385 -34.5t-385 34.5t-280 93.5t-103 128v170q119 -84 325 -127t443 -43zM768 1536q208 0 385 -34.5t280 -93.5t103 -128v-128q0 -69 -103 -128t-280 -93.5t-385 -34.5t-385 34.5t-280 93.5 +t-103 128v128q0 69 103 128t280 93.5t385 34.5z" /> + <glyph glyph-name="uniF1C1" unicode="" +d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z +M894 465q33 -26 84 -56q59 7 117 7q147 0 177 -49q16 -22 2 -52q0 -1 -1 -2l-2 -2v-1q-6 -38 -71 -38q-48 0 -115 20t-130 53q-221 -24 -392 -83q-153 -262 -242 -262q-15 0 -28 7l-24 12q-1 1 -6 5q-10 10 -6 36q9 40 56 91.5t132 96.5q14 9 23 -6q2 -2 2 -4q52 85 107 197 +q68 136 104 262q-24 82 -30.5 159.5t6.5 127.5q11 40 42 40h21h1q23 0 35 -15q18 -21 9 -68q-2 -6 -4 -8q1 -3 1 -8v-30q-2 -123 -14 -192q55 -164 146 -238zM318 54q52 24 137 158q-51 -40 -87.5 -84t-49.5 -74zM716 974q-15 -42 -2 -132q1 7 7 44q0 3 7 43q1 4 4 8 +q-1 1 -1 2q-1 2 -1 3q-1 22 -13 36q0 -1 -1 -2v-2zM592 313q135 54 284 81q-2 1 -13 9.5t-16 13.5q-76 67 -127 176q-27 -86 -83 -197q-30 -56 -45 -83zM1238 329q-24 24 -140 24q76 -28 124 -28q14 0 18 1q0 1 -2 3z" /> + <glyph glyph-name="_422" unicode="" +d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z +M233 768v-107h70l164 -661h159l128 485q7 20 10 46q2 16 2 24h4l3 -24q1 -3 3.5 -20t5.5 -26l128 -485h159l164 661h70v107h-300v-107h90l-99 -438q-5 -20 -7 -46l-2 -21h-4q0 3 -0.5 6.5t-1.5 8t-1 6.5q-1 5 -4 21t-5 25l-144 545h-114l-144 -545q-2 -9 -4.5 -24.5 +t-3.5 -21.5l-4 -21h-4l-2 21q-2 26 -7 46l-99 438h90v107h-300z" /> + <glyph glyph-name="_423" unicode="" +d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z +M429 106v-106h281v106h-75l103 161q5 7 10 16.5t7.5 13.5t3.5 4h2q1 -4 5 -10q2 -4 4.5 -7.5t6 -8t6.5 -8.5l107 -161h-76v-106h291v106h-68l-192 273l195 282h67v107h-279v-107h74l-103 -159q-4 -7 -10 -16.5t-9 -13.5l-2 -3h-2q-1 4 -5 10q-6 11 -17 23l-106 159h76v107 +h-290v-107h68l189 -272l-194 -283h-68z" /> + <glyph glyph-name="_424" unicode="" +d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z +M416 106v-106h327v106h-93v167h137q76 0 118 15q67 23 106.5 87t39.5 146q0 81 -37 141t-100 87q-48 19 -130 19h-368v-107h92v-555h-92zM769 386h-119v268h120q52 0 83 -18q56 -33 56 -115q0 -89 -62 -120q-31 -15 -78 -15z" /> + <glyph glyph-name="_425" unicode="" +d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z +M1280 320v-320h-1024v192l192 192l128 -128l384 384zM448 512q-80 0 -136 56t-56 136t56 136t136 56t136 -56t56 -136t-56 -136t-136 -56z" /> + <glyph glyph-name="_426" unicode="" +d="M640 1152v128h-128v-128h128zM768 1024v128h-128v-128h128zM640 896v128h-128v-128h128zM768 768v128h-128v-128h128zM1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400 +v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-128v-128h-128v128h-512v-1536h1280zM781 593l107 -349q8 -27 8 -52q0 -83 -72.5 -137.5t-183.5 -54.5t-183.5 54.5t-72.5 137.5q0 25 8 52q21 63 120 396v128h128v-128h79 +q22 0 39 -13t23 -34zM640 128q53 0 90.5 19t37.5 45t-37.5 45t-90.5 19t-90.5 -19t-37.5 -45t37.5 -45t90.5 -19z" /> + <glyph glyph-name="_427" unicode="" +d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z +M620 686q20 -8 20 -30v-544q0 -22 -20 -30q-8 -2 -12 -2q-12 0 -23 9l-166 167h-131q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h131l166 167q16 15 35 7zM1037 -3q31 0 50 24q129 159 129 363t-129 363q-16 21 -43 24t-47 -14q-21 -17 -23.5 -43.5t14.5 -47.5 +q100 -123 100 -282t-100 -282q-17 -21 -14.5 -47.5t23.5 -42.5q18 -15 40 -15zM826 145q27 0 47 20q87 93 87 219t-87 219q-18 19 -45 20t-46 -17t-20 -44.5t18 -46.5q52 -57 52 -131t-52 -131q-19 -20 -18 -46.5t20 -44.5q20 -17 44 -17z" /> + <glyph glyph-name="_428" unicode="" +d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z +M768 768q52 0 90 -38t38 -90v-384q0 -52 -38 -90t-90 -38h-384q-52 0 -90 38t-38 90v384q0 52 38 90t90 38h384zM1260 766q20 -8 20 -30v-576q0 -22 -20 -30q-8 -2 -12 -2q-14 0 -23 9l-265 266v90l265 266q9 9 23 9q4 0 12 -2z" /> + <glyph glyph-name="_429" unicode="" +d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z +M480 768q8 11 21 12.5t24 -6.5l51 -38q11 -8 12.5 -21t-6.5 -24l-182 -243l182 -243q8 -11 6.5 -24t-12.5 -21l-51 -38q-11 -8 -24 -6.5t-21 12.5l-226 301q-14 19 0 38zM1282 467q14 -19 0 -38l-226 -301q-8 -11 -21 -12.5t-24 6.5l-51 38q-11 8 -12.5 21t6.5 24l182 243 +l-182 243q-8 11 -6.5 24t12.5 21l51 38q11 8 24 6.5t21 -12.5zM662 6q-13 2 -20.5 13t-5.5 24l138 831q2 13 13 20.5t24 5.5l63 -10q13 -2 20.5 -13t5.5 -24l-138 -831q-2 -13 -13 -20.5t-24 -5.5z" /> + <glyph glyph-name="_430" unicode="" +d="M1497 709v-198q-101 -23 -198 -23q-65 -136 -165.5 -271t-181.5 -215.5t-128 -106.5q-80 -45 -162 3q-28 17 -60.5 43.5t-85 83.5t-102.5 128.5t-107.5 184t-105.5 244t-91.5 314.5t-70.5 390h283q26 -218 70 -398.5t104.5 -317t121.5 -235.5t140 -195q169 169 287 406 +q-142 72 -223 220t-81 333q0 192 104 314.5t284 122.5q178 0 273 -105.5t95 -297.5q0 -159 -58 -286q-7 -1 -19.5 -3t-46 -2t-63 6t-62 25.5t-50.5 51.5q31 103 31 184q0 87 -29 132t-79 45q-53 0 -85 -49.5t-32 -140.5q0 -186 105 -293.5t267 -107.5q62 0 121 14z" /> + <glyph glyph-name="_431" unicode="" horiz-adv-x="1792" +d="M216 367l603 -402v359l-334 223zM154 511l193 129l-193 129v-258zM973 -35l603 402l-269 180l-334 -223v-359zM896 458l272 182l-272 182l-272 -182zM485 733l334 223v359l-603 -402zM1445 640l193 -129v258zM1307 733l269 180l-603 402v-359zM1792 913v-546 +q0 -41 -34 -64l-819 -546q-21 -13 -43 -13t-43 13l-819 546q-34 23 -34 64v546q0 41 34 64l819 546q21 13 43 13t43 -13l819 -546q34 -23 34 -64z" /> + <glyph glyph-name="_432" unicode="" horiz-adv-x="2048" +d="M1800 764q111 -46 179.5 -145.5t68.5 -221.5q0 -164 -118 -280.5t-285 -116.5q-4 0 -11.5 0.5t-10.5 0.5h-1209h-1h-2h-5q-170 10 -288 125.5t-118 280.5q0 110 55 203t147 147q-12 39 -12 82q0 115 82 196t199 81q95 0 172 -58q75 154 222.5 248t326.5 94 +q166 0 306 -80.5t221.5 -218.5t81.5 -301q0 -6 -0.5 -18t-0.5 -18zM468 498q0 -122 84 -193t208 -71q137 0 240 99q-16 20 -47.5 56.5t-43.5 50.5q-67 -65 -144 -65q-55 0 -93.5 33.5t-38.5 87.5q0 53 38.5 87t91.5 34q44 0 84.5 -21t73 -55t65 -75t69 -82t77 -75t97 -55 +t121.5 -21q121 0 204.5 71.5t83.5 190.5q0 121 -84 192t-207 71q-143 0 -241 -97l93 -108q66 64 142 64q52 0 92 -33t40 -84q0 -57 -37 -91.5t-94 -34.5q-43 0 -82.5 21t-72 55t-65.5 75t-69.5 82t-77.5 75t-96.5 55t-118.5 21q-122 0 -207 -70.5t-85 -189.5z" /> + <glyph glyph-name="_433" unicode="" horiz-adv-x="1792" +d="M896 1536q182 0 348 -71t286 -191t191 -286t71 -348t-71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71zM896 1408q-190 0 -361 -90l194 -194q82 28 167 28t167 -28l194 194q-171 90 -361 90zM218 279l194 194 +q-28 82 -28 167t28 167l-194 194q-90 -171 -90 -361t90 -361zM896 -128q190 0 361 90l-194 194q-82 -28 -167 -28t-167 28l-194 -194q171 -90 361 -90zM896 256q159 0 271.5 112.5t112.5 271.5t-112.5 271.5t-271.5 112.5t-271.5 -112.5t-112.5 -271.5t112.5 -271.5 +t271.5 -112.5zM1380 473l194 -194q90 171 90 361t-90 361l-194 -194q28 -82 28 -167t-28 -167z" /> + <glyph glyph-name="_434" unicode="" horiz-adv-x="1792" +d="M1760 640q0 -176 -68.5 -336t-184 -275.5t-275.5 -184t-336 -68.5t-336 68.5t-275.5 184t-184 275.5t-68.5 336q0 213 97 398.5t265 305.5t374 151v-228q-221 -45 -366.5 -221t-145.5 -406q0 -130 51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5 +t136.5 204t51 248.5q0 230 -145.5 406t-366.5 221v228q206 -31 374 -151t265 -305.5t97 -398.5z" /> + <glyph glyph-name="uniF1D0" unicode="" horiz-adv-x="1792" +d="M19 662q8 217 116 406t305 318h5q0 -1 -1 -3q-8 -8 -28 -33.5t-52 -76.5t-60 -110.5t-44.5 -135.5t-14 -150.5t39 -157.5t108.5 -154q50 -50 102 -69.5t90.5 -11.5t69.5 23.5t47 32.5l16 16q39 51 53 116.5t6.5 122.5t-21 107t-26.5 80l-14 29q-10 25 -30.5 49.5t-43 41 +t-43.5 29.5t-35 19l-13 6l104 115q39 -17 78 -52t59 -61l19 -27q1 48 -18.5 103.5t-40.5 87.5l-20 31l161 183l160 -181q-33 -46 -52.5 -102.5t-22.5 -90.5l-4 -33q22 37 61.5 72.5t67.5 52.5l28 17l103 -115q-44 -14 -85 -50t-60 -65l-19 -29q-31 -56 -48 -133.5t-7 -170 +t57 -156.5q33 -45 77.5 -60.5t85 -5.5t76 26.5t57.5 33.5l21 16q60 53 96.5 115t48.5 121.5t10 121.5t-18 118t-37 107.5t-45.5 93t-45 72t-34.5 47.5l-13 17q-14 13 -7 13l10 -3q40 -29 62.5 -46t62 -50t64 -58t58.5 -65t55.5 -77t45.5 -88t38 -103t23.5 -117t10.5 -136 +q3 -259 -108 -465t-312 -321t-456 -115q-185 0 -351 74t-283.5 198t-184 293t-60.5 353z" /> + <glyph glyph-name="uniF1D1" unicode="" horiz-adv-x="1792" +d="M874 -102v-66q-208 6 -385 109.5t-283 275.5l58 34q29 -49 73 -99l65 57q148 -168 368 -212l-17 -86q65 -12 121 -13zM276 428l-83 -28q22 -60 49 -112l-57 -33q-98 180 -98 385t98 385l57 -33q-30 -56 -49 -112l82 -28q-35 -100 -35 -212q0 -109 36 -212zM1528 251 +l58 -34q-106 -172 -283 -275.5t-385 -109.5v66q56 1 121 13l-17 86q220 44 368 212l65 -57q44 50 73 99zM1377 805l-233 -80q14 -42 14 -85t-14 -85l232 -80q-31 -92 -98 -169l-185 162q-57 -67 -147 -85l48 -241q-52 -10 -98 -10t-98 10l48 241q-90 18 -147 85l-185 -162 +q-67 77 -98 169l232 80q-14 42 -14 85t14 85l-233 80q33 93 99 169l185 -162q59 68 147 86l-48 240q44 10 98 10t98 -10l-48 -240q88 -18 147 -86l185 162q66 -76 99 -169zM874 1448v-66q-65 -2 -121 -13l17 -86q-220 -42 -368 -211l-65 56q-38 -42 -73 -98l-57 33 +q106 172 282 275.5t385 109.5zM1705 640q0 -205 -98 -385l-57 33q27 52 49 112l-83 28q36 103 36 212q0 112 -35 212l82 28q-19 56 -49 112l57 33q98 -180 98 -385zM1585 1063l-57 -33q-35 56 -73 98l-65 -56q-148 169 -368 211l17 86q-56 11 -121 13v66q209 -6 385 -109.5 +t282 -275.5zM1748 640q0 173 -67.5 331t-181.5 272t-272 181.5t-331 67.5t-331 -67.5t-272 -181.5t-181.5 -272t-67.5 -331t67.5 -331t181.5 -272t272 -181.5t331 -67.5t331 67.5t272 181.5t181.5 272t67.5 331zM1792 640q0 -182 -71 -348t-191 -286t-286 -191t-348 -71 +t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71t348 -71t286 -191t191 -286t71 -348z" /> + <glyph glyph-name="uniF1D2" unicode="" +d="M582 228q0 -66 -93 -66q-107 0 -107 63q0 64 98 64q102 0 102 -61zM546 694q0 -85 -74 -85q-77 0 -77 84q0 90 77 90q36 0 55 -25.5t19 -63.5zM712 769v125q-78 -29 -135 -29q-50 29 -110 29q-86 0 -145 -57t-59 -143q0 -50 29.5 -102t73.5 -67v-3q-38 -17 -38 -85 +q0 -53 41 -77v-3q-113 -37 -113 -139q0 -45 20 -78.5t54 -51t72 -25.5t81 -8q224 0 224 188q0 67 -48 99t-126 46q-27 5 -51.5 20.5t-24.5 39.5q0 44 49 52q77 15 122 70t45 134q0 24 -10 52q37 9 49 13zM771 350h137q-2 27 -2 82v387q0 46 2 69h-137q3 -23 3 -71v-392 +q0 -50 -3 -75zM1280 366v121q-30 -21 -68 -21q-53 0 -53 82v225h52q9 0 26.5 -1t26.5 -1v117h-105q0 82 3 102h-140q4 -24 4 -55v-47h-60v-117q36 3 37 3q3 0 11 -0.5t12 -0.5v-2h-2v-217q0 -37 2.5 -64t11.5 -56.5t24.5 -48.5t43.5 -31t66 -12q64 0 108 24zM924 1072 +q0 36 -24 63.5t-60 27.5t-60.5 -27t-24.5 -64q0 -36 25 -62.5t60 -26.5t59.5 27t24.5 62zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="_438" unicode="" horiz-adv-x="1792" +d="M595 22q0 100 -165 100q-158 0 -158 -104q0 -101 172 -101q151 0 151 105zM536 777q0 61 -30 102t-89 41q-124 0 -124 -145q0 -135 124 -135q119 0 119 137zM805 1101v-202q-36 -12 -79 -22q16 -43 16 -84q0 -127 -73 -216.5t-197 -112.5q-40 -8 -59.5 -27t-19.5 -58 +q0 -31 22.5 -51.5t58 -32t78.5 -22t86 -25.5t78.5 -37.5t58 -64t22.5 -98.5q0 -304 -363 -304q-69 0 -130 12.5t-116 41t-87.5 82t-32.5 127.5q0 165 182 225v4q-67 41 -67 126q0 109 63 137v4q-72 24 -119.5 108.5t-47.5 165.5q0 139 95 231.5t235 92.5q96 0 178 -47 +q98 0 218 47zM1123 220h-222q4 45 4 134v609q0 94 -4 128h222q-4 -33 -4 -124v-613q0 -89 4 -134zM1724 442v-196q-71 -39 -174 -39q-62 0 -107 20t-70 50t-39.5 78t-18.5 92t-4 103v351h2v4q-7 0 -19 1t-18 1q-21 0 -59 -6v190h96v76q0 54 -6 89h227q-6 -41 -6 -165h171 +v-190q-15 0 -43.5 2t-42.5 2h-85v-365q0 -131 87 -131q61 0 109 33zM1148 1389q0 -58 -39 -101.5t-96 -43.5q-58 0 -98 43.5t-40 101.5q0 59 39.5 103t98.5 44q58 0 96.5 -44.5t38.5 -102.5z" /> + <glyph glyph-name="_439" unicode="" +d="M809 532l266 499h-112l-157 -312q-24 -48 -44 -92l-42 92l-155 312h-120l263 -493v-324h101v318zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="uniF1D5" unicode="" horiz-adv-x="1280" +d="M842 964q0 -80 -57 -136.5t-136 -56.5q-60 0 -111 35q-62 -67 -115 -146q-247 -371 -202 -859q1 -22 -12.5 -38.5t-34.5 -18.5h-5q-20 0 -35 13.5t-17 33.5q-14 126 -3.5 247.5t29.5 217t54 186t69 155.5t74 125q61 90 132 165q-16 35 -16 77q0 80 56.5 136.5t136.5 56.5 +t136.5 -56.5t56.5 -136.5zM1223 953q0 -158 -78 -292t-212.5 -212t-292.5 -78q-64 0 -131 14q-21 5 -32.5 23.5t-6.5 39.5q5 20 23 31.5t39 7.5q51 -13 108 -13q97 0 186 38t153 102t102 153t38 186t-38 186t-102 153t-153 102t-186 38t-186 -38t-153 -102t-102 -153 +t-38 -186q0 -114 52 -218q10 -20 3.5 -40t-25.5 -30t-39.5 -3t-30.5 26q-64 123 -64 265q0 119 46.5 227t124.5 186t186 124t226 46q158 0 292.5 -78t212.5 -212.5t78 -292.5z" /> + <glyph glyph-name="uniF1D6" unicode="" horiz-adv-x="1792" +d="M270 730q-8 19 -8 52q0 20 11 49t24 45q-1 22 7.5 53t22.5 43q0 139 92.5 288.5t217.5 209.5q139 66 324 66q133 0 266 -55q49 -21 90 -48t71 -56t55 -68t42 -74t32.5 -84.5t25.5 -89.5t22 -98l1 -5q55 -83 55 -150q0 -14 -9 -40t-9 -38q0 -1 1.5 -3.5t3.5 -5t2 -3.5 +q77 -114 120.5 -214.5t43.5 -208.5q0 -43 -19.5 -100t-55.5 -57q-9 0 -19.5 7.5t-19 17.5t-19 26t-16 26.5t-13.5 26t-9 17.5q-1 1 -3 1l-5 -4q-59 -154 -132 -223q20 -20 61.5 -38.5t69 -41.5t35.5 -65q-2 -4 -4 -16t-7 -18q-64 -97 -302 -97q-53 0 -110.5 9t-98 20 +t-104.5 30q-15 5 -23 7q-14 4 -46 4.5t-40 1.5q-41 -45 -127.5 -65t-168.5 -20q-35 0 -69 1.5t-93 9t-101 20.5t-74.5 40t-32.5 64q0 40 10 59.5t41 48.5q11 2 40.5 13t49.5 12q4 0 14 2q2 2 2 4l-2 3q-48 11 -108 105.5t-73 156.5l-5 3q-4 0 -12 -20q-18 -41 -54.5 -74.5 +t-77.5 -37.5h-1q-4 0 -6 4.5t-5 5.5q-23 54 -23 100q0 275 252 466z" /> + <glyph glyph-name="uniF1D7" unicode="" horiz-adv-x="2048" +d="M580 1075q0 41 -25 66t-66 25q-43 0 -76 -25.5t-33 -65.5q0 -39 33 -64.5t76 -25.5q41 0 66 24.5t25 65.5zM1323 568q0 28 -25.5 50t-65.5 22q-27 0 -49.5 -22.5t-22.5 -49.5q0 -28 22.5 -50.5t49.5 -22.5q40 0 65.5 22t25.5 51zM1087 1075q0 41 -24.5 66t-65.5 25 +q-43 0 -76 -25.5t-33 -65.5q0 -39 33 -64.5t76 -25.5q41 0 65.5 24.5t24.5 65.5zM1722 568q0 28 -26 50t-65 22q-27 0 -49.5 -22.5t-22.5 -49.5q0 -28 22.5 -50.5t49.5 -22.5q39 0 65 22t26 51zM1456 965q-31 4 -70 4q-169 0 -311 -77t-223.5 -208.5t-81.5 -287.5 +q0 -78 23 -152q-35 -3 -68 -3q-26 0 -50 1.5t-55 6.5t-44.5 7t-54.5 10.5t-50 10.5l-253 -127l72 218q-290 203 -290 490q0 169 97.5 311t264 223.5t363.5 81.5q176 0 332.5 -66t262 -182.5t136.5 -260.5zM2048 404q0 -117 -68.5 -223.5t-185.5 -193.5l55 -181l-199 109 +q-150 -37 -218 -37q-169 0 -311 70.5t-223.5 191.5t-81.5 264t81.5 264t223.5 191.5t311 70.5q161 0 303 -70.5t227.5 -192t85.5 -263.5z" /> + <glyph glyph-name="_443" unicode="" horiz-adv-x="1792" +d="M1764 1525q33 -24 27 -64l-256 -1536q-5 -29 -32 -45q-14 -8 -31 -8q-11 0 -24 5l-453 185l-242 -295q-18 -23 -49 -23q-13 0 -22 4q-19 7 -30.5 23.5t-11.5 36.5v349l864 1059l-1069 -925l-395 162q-37 14 -40 55q-2 40 32 59l1664 960q15 9 32 9q20 0 36 -11z" /> + <glyph glyph-name="_444" unicode="" horiz-adv-x="1792" +d="M1764 1525q33 -24 27 -64l-256 -1536q-5 -29 -32 -45q-14 -8 -31 -8q-11 0 -24 5l-527 215l-298 -327q-18 -21 -47 -21q-14 0 -23 4q-19 7 -30 23.5t-11 36.5v452l-472 193q-37 14 -40 55q-3 39 32 59l1664 960q35 21 68 -2zM1422 26l221 1323l-1434 -827l336 -137 +l863 639l-478 -797z" /> + <glyph glyph-name="_445" unicode="" +d="M1536 640q0 -156 -61 -298t-164 -245t-245 -164t-298 -61q-172 0 -327 72.5t-264 204.5q-7 10 -6.5 22.5t8.5 20.5l137 138q10 9 25 9q16 -2 23 -12q73 -95 179 -147t225 -52q104 0 198.5 40.5t163.5 109.5t109.5 163.5t40.5 198.5t-40.5 198.5t-109.5 163.5 +t-163.5 109.5t-198.5 40.5q-98 0 -188 -35.5t-160 -101.5l137 -138q31 -30 14 -69q-17 -40 -59 -40h-448q-26 0 -45 19t-19 45v448q0 42 40 59q39 17 69 -14l130 -129q107 101 244.5 156.5t284.5 55.5q156 0 298 -61t245 -164t164 -245t61 -298zM896 928v-448q0 -14 -9 -23 +t-23 -9h-320q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h224v352q0 14 9 23t23 9h64q14 0 23 -9t9 -23z" /> + <glyph glyph-name="_446" unicode="" +d="M768 1280q-130 0 -248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5t-51 248.5t-136.5 204t-204 136.5t-248.5 51zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103 +t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="_447" unicode="" horiz-adv-x="1792" +d="M1682 -128q-44 0 -132.5 3.5t-133.5 3.5q-44 0 -132 -3.5t-132 -3.5q-24 0 -37 20.5t-13 45.5q0 31 17 46t39 17t51 7t45 15q33 21 33 140l-1 391q0 21 -1 31q-13 4 -50 4h-675q-38 0 -51 -4q-1 -10 -1 -31l-1 -371q0 -142 37 -164q16 -10 48 -13t57 -3.5t45 -15 +t20 -45.5q0 -26 -12.5 -48t-36.5 -22q-47 0 -139.5 3.5t-138.5 3.5q-43 0 -128 -3.5t-127 -3.5q-23 0 -35.5 21t-12.5 45q0 30 15.5 45t36 17.5t47.5 7.5t42 15q33 23 33 143l-1 57v813q0 3 0.5 26t0 36.5t-1.5 38.5t-3.5 42t-6.5 36.5t-11 31.5t-16 18q-15 10 -45 12t-53 2 +t-41 14t-18 45q0 26 12 48t36 22q46 0 138.5 -3.5t138.5 -3.5q42 0 126.5 3.5t126.5 3.5q25 0 37.5 -22t12.5 -48q0 -30 -17 -43.5t-38.5 -14.5t-49.5 -4t-43 -13q-35 -21 -35 -160l1 -320q0 -21 1 -32q13 -3 39 -3h699q25 0 38 3q1 11 1 32l1 320q0 139 -35 160 +q-18 11 -58.5 12.5t-66 13t-25.5 49.5q0 26 12.5 48t37.5 22q44 0 132 -3.5t132 -3.5q43 0 129 3.5t129 3.5q25 0 37.5 -22t12.5 -48q0 -30 -17.5 -44t-40 -14.5t-51.5 -3t-44 -12.5q-35 -23 -35 -161l1 -943q0 -119 34 -140q16 -10 46 -13.5t53.5 -4.5t41.5 -15.5t18 -44.5 +q0 -26 -12 -48t-36 -22z" /> + <glyph glyph-name="_448" unicode="" horiz-adv-x="1280" +d="M1278 1347v-73q0 -29 -18.5 -61t-42.5 -32q-50 0 -54 -1q-26 -6 -32 -31q-3 -11 -3 -64v-1152q0 -25 -18 -43t-43 -18h-108q-25 0 -43 18t-18 43v1218h-143v-1218q0 -25 -17.5 -43t-43.5 -18h-108q-26 0 -43.5 18t-17.5 43v496q-147 12 -245 59q-126 58 -192 179 +q-64 117 -64 259q0 166 88 286q88 118 209 159q111 37 417 37h479q25 0 43 -18t18 -43z" /> + <glyph glyph-name="_449" unicode="" +d="M352 128v-128h-352v128h352zM704 256q26 0 45 -19t19 -45v-256q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h256zM864 640v-128h-864v128h864zM224 1152v-128h-224v128h224zM1536 128v-128h-736v128h736zM576 1280q26 0 45 -19t19 -45v-256 +q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h256zM1216 768q26 0 45 -19t19 -45v-256q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h256zM1536 640v-128h-224v128h224zM1536 1152v-128h-864v128h864z" /> + <glyph glyph-name="uniF1E0" unicode="" +d="M1216 512q133 0 226.5 -93.5t93.5 -226.5t-93.5 -226.5t-226.5 -93.5t-226.5 93.5t-93.5 226.5q0 12 2 34l-360 180q-92 -86 -218 -86q-133 0 -226.5 93.5t-93.5 226.5t93.5 226.5t226.5 93.5q126 0 218 -86l360 180q-2 22 -2 34q0 133 93.5 226.5t226.5 93.5 +t226.5 -93.5t93.5 -226.5t-93.5 -226.5t-226.5 -93.5q-126 0 -218 86l-360 -180q2 -22 2 -34t-2 -34l360 -180q92 86 218 86z" /> + <glyph glyph-name="_451" unicode="" +d="M1280 341q0 88 -62.5 151t-150.5 63q-84 0 -145 -58l-241 120q2 16 2 23t-2 23l241 120q61 -58 145 -58q88 0 150.5 63t62.5 151t-62.5 150.5t-150.5 62.5t-151 -62.5t-63 -150.5q0 -7 2 -23l-241 -120q-62 57 -145 57q-88 0 -150.5 -62.5t-62.5 -150.5t62.5 -150.5 +t150.5 -62.5q83 0 145 57l241 -120q-2 -16 -2 -23q0 -88 63 -150.5t151 -62.5t150.5 62.5t62.5 150.5zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="_452" unicode="" horiz-adv-x="1792" +d="M571 947q-10 25 -34 35t-49 0q-108 -44 -191 -127t-127 -191q-10 -25 0 -49t35 -34q13 -5 24 -5q42 0 60 40q34 84 98.5 148.5t148.5 98.5q25 11 35 35t0 49zM1513 1303l46 -46l-244 -243l68 -68q19 -19 19 -45.5t-19 -45.5l-64 -64q89 -161 89 -343q0 -143 -55.5 -273.5 +t-150 -225t-225 -150t-273.5 -55.5t-273.5 55.5t-225 150t-150 225t-55.5 273.5t55.5 273.5t150 225t225 150t273.5 55.5q182 0 343 -89l64 64q19 19 45.5 19t45.5 -19l68 -68zM1521 1359q-10 -10 -22 -10q-13 0 -23 10l-91 90q-9 10 -9 23t9 23q10 9 23 9t23 -9l90 -91 +q10 -9 10 -22.5t-10 -22.5zM1751 1129q-11 -9 -23 -9t-23 9l-90 91q-10 9 -10 22.5t10 22.5q9 10 22.5 10t22.5 -10l91 -90q9 -10 9 -23t-9 -23zM1792 1312q0 -14 -9 -23t-23 -9h-96q-14 0 -23 9t-9 23t9 23t23 9h96q14 0 23 -9t9 -23zM1600 1504v-96q0 -14 -9 -23t-23 -9 +t-23 9t-9 23v96q0 14 9 23t23 9t23 -9t9 -23zM1751 1449l-91 -90q-10 -10 -22 -10q-13 0 -23 10q-10 9 -10 22.5t10 22.5l90 91q10 9 23 9t23 -9q9 -10 9 -23t-9 -23z" /> + <glyph glyph-name="_453" unicode="" horiz-adv-x="1792" +d="M609 720l287 208l287 -208l-109 -336h-355zM896 1536q182 0 348 -71t286 -191t191 -286t71 -348t-71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71zM1515 186q149 203 149 454v3l-102 -89l-240 224l63 323 +l134 -12q-150 206 -389 282l53 -124l-287 -159l-287 159l53 124q-239 -76 -389 -282l135 12l62 -323l-240 -224l-102 89v-3q0 -251 149 -454l30 132l326 -40l139 -298l-116 -69q117 -39 240 -39t240 39l-116 69l139 298l326 40z" /> + <glyph glyph-name="_454" unicode="" horiz-adv-x="1792" +d="M448 224v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM256 608v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM832 224v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23 +v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM640 608v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM66 768q-28 0 -47 19t-19 46v129h514v-129q0 -27 -19 -46t-46 -19h-383zM1216 224v-192q0 -14 -9 -23t-23 -9h-192 +q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1024 608v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1600 224v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23 +zM1408 608v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1792 1016v-13h-514v10q0 104 -382 102q-382 -1 -382 -102v-10h-514v13q0 17 8.5 43t34 64t65.5 75.5t110.5 76t160 67.5t224 47.5t293.5 18.5t293 -18.5t224 -47.5 +t160.5 -67.5t110.5 -76t65.5 -75.5t34 -64t8.5 -43zM1792 608v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1792 962v-129q0 -27 -19 -46t-46 -19h-384q-27 0 -46 19t-19 46v129h514z" /> + <glyph glyph-name="_455" unicode="" horiz-adv-x="1792" +d="M704 1216v-768q0 -26 -19 -45t-45 -19v-576q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v512l249 873q7 23 31 23h424zM1024 1216v-704h-256v704h256zM1792 320v-512q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v576q-26 0 -45 19t-19 45v768h424q24 0 31 -23z +M736 1504v-224h-352v224q0 14 9 23t23 9h288q14 0 23 -9t9 -23zM1408 1504v-224h-352v224q0 14 9 23t23 9h288q14 0 23 -9t9 -23z" /> + <glyph glyph-name="_456" unicode="" horiz-adv-x="1792" +d="M1755 1083q37 -38 37 -90.5t-37 -90.5l-401 -400l150 -150l-160 -160q-163 -163 -389.5 -186.5t-411.5 100.5l-362 -362h-181v181l362 362q-124 185 -100.5 411.5t186.5 389.5l160 160l150 -150l400 401q38 37 91 37t90 -37t37 -90.5t-37 -90.5l-400 -401l234 -234 +l401 400q38 37 91 37t90 -37z" /> + <glyph glyph-name="_457" unicode="" horiz-adv-x="1792" +d="M873 796q0 -83 -63.5 -142.5t-152.5 -59.5t-152.5 59.5t-63.5 142.5q0 84 63.5 143t152.5 59t152.5 -59t63.5 -143zM1375 796q0 -83 -63 -142.5t-153 -59.5q-89 0 -152.5 59.5t-63.5 142.5q0 84 63.5 143t152.5 59q90 0 153 -59t63 -143zM1600 616v667q0 87 -32 123.5 +t-111 36.5h-1112q-83 0 -112.5 -34t-29.5 -126v-673q43 -23 88.5 -40t81 -28t81 -18.5t71 -11t70 -4t58.5 -0.5t56.5 2t44.5 2q68 1 95 -27q6 -6 10 -9q26 -25 61 -51q7 91 118 87q5 0 36.5 -1.5t43 -2t45.5 -1t53 1t54.5 4.5t61 8.5t62 13.5t67 19.5t67.5 27t72 34.5z +M1763 621q-121 -149 -372 -252q84 -285 -23 -465q-66 -113 -183 -148q-104 -32 -182 15q-86 51 -82 164l-1 326v1q-8 2 -24.5 6t-23.5 5l-1 -338q4 -114 -83 -164q-79 -47 -183 -15q-117 36 -182 150q-105 180 -22 463q-251 103 -372 252q-25 37 -4 63t60 -1q4 -2 11.5 -7 +t10.5 -8v694q0 72 47 123t114 51h1257q67 0 114 -51t47 -123v-694l21 15q39 27 60 1t-4 -63z" /> + <glyph glyph-name="_458" unicode="" horiz-adv-x="1792" +d="M896 1102v-434h-145v434h145zM1294 1102v-434h-145v434h145zM1294 342l253 254v795h-1194v-1049h326v-217l217 217h398zM1692 1536v-1013l-434 -434h-326l-217 -217h-217v217h-398v1158l109 289h1483z" /> + <glyph glyph-name="_459" unicode="" +d="M773 217v-127q-1 -292 -6 -305q-12 -32 -51 -40q-54 -9 -181.5 38t-162.5 89q-13 15 -17 36q-1 12 4 26q4 10 34 47t181 216q1 0 60 70q15 19 39.5 24.5t49.5 -3.5q24 -10 37.5 -29t12.5 -42zM624 468q-3 -55 -52 -70l-120 -39q-275 -88 -292 -88q-35 2 -54 36 +q-12 25 -17 75q-8 76 1 166.5t30 124.5t56 32q13 0 202 -77q71 -29 115 -47l84 -34q23 -9 35.5 -30.5t11.5 -48.5zM1450 171q-7 -54 -91.5 -161t-135.5 -127q-37 -14 -63 7q-14 10 -184 287l-47 77q-14 21 -11.5 46t19.5 46q35 43 83 26q1 -1 119 -40q203 -66 242 -79.5 +t47 -20.5q28 -22 22 -61zM778 803q5 -102 -54 -122q-58 -17 -114 71l-378 598q-8 35 19 62q41 43 207.5 89.5t224.5 31.5q40 -10 49 -45q3 -18 22 -305.5t24 -379.5zM1440 695q3 -39 -26 -59q-15 -10 -329 -86q-67 -15 -91 -23l1 2q-23 -6 -46 4t-37 32q-30 47 0 87 +q1 1 75 102q125 171 150 204t34 39q28 19 65 2q48 -23 123 -133.5t81 -167.5v-3z" /> + <glyph glyph-name="_460" unicode="" horiz-adv-x="2048" +d="M1024 1024h-384v-384h384v384zM1152 384v-128h-640v128h640zM1152 1152v-640h-640v640h640zM1792 384v-128h-512v128h512zM1792 640v-128h-512v128h512zM1792 896v-128h-512v128h512zM1792 1152v-128h-512v128h512zM256 192v960h-128v-960q0 -26 19 -45t45 -19t45 19 +t19 45zM1920 192v1088h-1536v-1088q0 -33 -11 -64h1483q26 0 45 19t19 45zM2048 1408v-1216q0 -80 -56 -136t-136 -56h-1664q-80 0 -136 56t-56 136v1088h256v128h1792z" /> + <glyph glyph-name="_461" unicode="" horiz-adv-x="2048" +d="M1024 13q-20 0 -93 73.5t-73 93.5q0 32 62.5 54t103.5 22t103.5 -22t62.5 -54q0 -20 -73 -93.5t-93 -73.5zM1294 284q-2 0 -40 25t-101.5 50t-128.5 25t-128.5 -25t-101 -50t-40.5 -25q-18 0 -93.5 75t-75.5 93q0 13 10 23q78 77 196 121t233 44t233 -44t196 -121 +q10 -10 10 -23q0 -18 -75.5 -93t-93.5 -75zM1567 556q-11 0 -23 8q-136 105 -252 154.5t-268 49.5q-85 0 -170.5 -22t-149 -53t-113.5 -62t-79 -53t-31 -22q-17 0 -92 75t-75 93q0 12 10 22q132 132 320 205t380 73t380 -73t320 -205q10 -10 10 -22q0 -18 -75 -93t-92 -75z +M1838 827q-11 0 -22 9q-179 157 -371.5 236.5t-420.5 79.5t-420.5 -79.5t-371.5 -236.5q-11 -9 -22 -9q-17 0 -92.5 75t-75.5 93q0 13 10 23q187 186 445 288t527 102t527 -102t445 -288q10 -10 10 -23q0 -18 -75.5 -93t-92.5 -75z" /> + <glyph glyph-name="_462" unicode="" horiz-adv-x="1792" +d="M384 0q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM768 0q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM384 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5 +t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1152 0q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM768 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5 +t37.5 90.5zM384 768q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1152 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM768 768q0 53 -37.5 90.5t-90.5 37.5 +t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1536 0v384q0 52 -38 90t-90 38t-90 -38t-38 -90v-384q0 -52 38 -90t90 -38t90 38t38 90zM1152 768q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5z +M1536 1088v256q0 26 -19 45t-45 19h-1280q-26 0 -45 -19t-19 -45v-256q0 -26 19 -45t45 -19h1280q26 0 45 19t19 45zM1536 768q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1664 1408v-1536q0 -52 -38 -90t-90 -38 +h-1408q-52 0 -90 38t-38 90v1536q0 52 38 90t90 38h1408q52 0 90 -38t38 -90z" /> + <glyph glyph-name="_463" unicode="" +d="M1519 890q18 -84 -4 -204q-87 -444 -565 -444h-44q-25 0 -44 -16.5t-24 -42.5l-4 -19l-55 -346l-2 -15q-5 -26 -24.5 -42.5t-44.5 -16.5h-251q-21 0 -33 15t-9 36q9 56 26.5 168t26.5 168t27 167.5t27 167.5q5 37 43 37h131q133 -2 236 21q175 39 287 144q102 95 155 246 +q24 70 35 133q1 6 2.5 7.5t3.5 1t6 -3.5q79 -59 98 -162zM1347 1172q0 -107 -46 -236q-80 -233 -302 -315q-113 -40 -252 -42q0 -1 -90 -1l-90 1q-100 0 -118 -96q-2 -8 -85 -530q-1 -10 -12 -10h-295q-22 0 -36.5 16.5t-11.5 38.5l232 1471q5 29 27.5 48t51.5 19h598 +q34 0 97.5 -13t111.5 -32q107 -41 163.5 -123t56.5 -196z" /> + <glyph glyph-name="_464" unicode="" horiz-adv-x="1792" +d="M441 864q33 0 52 -26q266 -364 362 -774h-446q-127 441 -367 749q-12 16 -3 33.5t29 17.5h373zM1000 507q-49 -199 -125 -393q-79 310 -256 594q40 221 44 449q211 -340 337 -650zM1099 1216q235 -324 384.5 -698.5t184.5 -773.5h-451q-41 665 -553 1472h435zM1792 640 +q0 -424 -101 -812q-67 560 -359 1083q-25 301 -106 584q-4 16 5.5 28.5t25.5 12.5h359q21 0 38.5 -13t22.5 -33q115 -409 115 -850z" /> + <glyph glyph-name="uniF1F0" unicode="" horiz-adv-x="2304" +d="M1975 546h-138q14 37 66 179l3 9q4 10 10 26t9 26l12 -55zM531 611l-58 295q-11 54 -75 54h-268l-2 -13q311 -79 403 -336zM710 960l-162 -438l-17 89q-26 70 -85 129.5t-131 88.5l135 -510h175l261 641h-176zM849 318h166l104 642h-166zM1617 944q-69 27 -149 27 +q-123 0 -201 -59t-79 -153q-1 -102 145 -174q48 -23 67 -41t19 -39q0 -30 -30 -46t-69 -16q-86 0 -156 33l-22 11l-23 -144q74 -34 185 -34q130 -1 208.5 59t80.5 160q0 106 -140 174q-49 25 -71 42t-22 38q0 22 24.5 38.5t70.5 16.5q70 1 124 -24l15 -8zM2042 960h-128 +q-65 0 -87 -54l-246 -588h174l35 96h212q5 -22 20 -96h154zM2304 1280v-1280q0 -52 -38 -90t-90 -38h-2048q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h2048q52 0 90 -38t38 -90z" /> + <glyph glyph-name="_466" unicode="" horiz-adv-x="2304" +d="M1119 1195q-128 85 -281 85q-103 0 -197.5 -40.5t-162.5 -108.5t-108.5 -162t-40.5 -197q0 -104 40.5 -198t108.5 -162t162 -108.5t198 -40.5q153 0 281 85q-131 107 -178 265.5t0.5 316.5t177.5 265zM1152 1171q-126 -99 -172 -249.5t-0.5 -300.5t172.5 -249 +q127 99 172.5 249t-0.5 300.5t-172 249.5zM1185 1195q130 -107 177.5 -265.5t0.5 -317t-178 -264.5q128 -85 281 -85q104 0 198 40.5t162 108.5t108.5 162t40.5 198q0 103 -40.5 197t-108.5 162t-162.5 108.5t-197.5 40.5q-153 0 -281 -85zM1926 473h7v3h-17v-3h7v-17h3v17z +M1955 456h4v20h-5l-6 -13l-6 13h-5v-20h3v15l6 -13h4l5 13v-15zM1947 16v-2h-2h-3v3h3h2v-1zM1947 7h3l-4 5h2l1 1q1 1 1 3t-1 3l-1 1h-3h-6v-13h3v5h1zM685 75q0 19 11 31t30 12q18 0 29 -12.5t11 -30.5q0 -19 -11 -31t-29 -12q-19 0 -30 12t-11 31zM1158 119q30 0 35 -32 +h-70q5 32 35 32zM1514 75q0 19 11 31t29 12t29.5 -12.5t11.5 -30.5q0 -19 -11 -31t-30 -12q-18 0 -29 12t-11 31zM1786 75q0 18 11.5 30.5t29.5 12.5t29.5 -12.5t11.5 -30.5q0 -19 -11.5 -31t-29.5 -12t-29.5 12.5t-11.5 30.5zM1944 3q-2 0 -4 1q-1 0 -3 2t-2 3q-1 2 -1 4 +q0 3 1 4q0 2 2 4l1 1q2 0 2 1q2 1 4 1q3 0 4 -1l4 -2l2 -4v-1q1 -2 1 -3l-1 -1v-3t-1 -1l-1 -2q-2 -2 -4 -2q-1 -1 -4 -1zM599 7h30v85q0 24 -14.5 38.5t-39.5 15.5q-32 0 -47 -24q-14 24 -45 24q-24 0 -39 -20v16h-30v-135h30v75q0 36 33 36q30 0 30 -36v-75h29v75 +q0 36 33 36q30 0 30 -36v-75zM765 7h29v68v67h-29v-16q-17 20 -43 20q-29 0 -48 -20t-19 -51t19 -51t48 -20q28 0 43 20v-17zM943 48q0 34 -47 40l-14 2q-23 4 -23 14q0 15 25 15q23 0 43 -11l12 24q-22 14 -55 14q-26 0 -41 -12t-15 -32q0 -33 47 -39l13 -2q24 -4 24 -14 +q0 -17 -31 -17q-25 0 -45 14l-13 -23q25 -17 58 -17q29 0 45.5 12t16.5 32zM1073 14l-8 25q-13 -7 -26 -7q-19 0 -19 22v61h48v27h-48v41h-30v-41h-28v-27h28v-61q0 -50 47 -50q21 0 36 10zM1159 146q-29 0 -48 -20t-19 -51q0 -32 19.5 -51.5t49.5 -19.5q33 0 55 19l-14 22 +q-18 -15 -39 -15q-34 0 -41 33h101v12q0 32 -18 51.5t-46 19.5zM1318 146q-23 0 -35 -20v16h-30v-135h30v76q0 35 29 35q10 0 18 -4l9 28q-9 4 -21 4zM1348 75q0 -31 19.5 -51t52.5 -20q29 0 48 16l-14 24q-18 -13 -35 -12q-18 0 -29.5 12t-11.5 31t11.5 31t29.5 12 +q19 0 35 -12l14 24q-20 16 -48 16q-33 0 -52.5 -20t-19.5 -51zM1593 7h30v68v67h-30v-16q-15 20 -42 20q-29 0 -48.5 -20t-19.5 -51t19.5 -51t48.5 -20q28 0 42 20v-17zM1726 146q-23 0 -35 -20v16h-29v-135h29v76q0 35 29 35q10 0 18 -4l9 28q-8 4 -21 4zM1866 7h29v68v122 +h-29v-71q-15 20 -43 20t-47.5 -20.5t-19.5 -50.5t19.5 -50.5t47.5 -20.5q29 0 43 20v-17zM1944 27l-2 -1h-3q-2 -1 -4 -3q-3 -1 -3 -4q-1 -2 -1 -6q0 -3 1 -5q0 -2 3 -4q2 -2 4 -3t5 -1q4 0 6 1q0 1 2 2l2 1q1 1 3 4q1 2 1 5q0 4 -1 6q-1 1 -3 4q0 1 -2 2l-2 1q-1 0 -3 0.5 +t-3 0.5zM2304 1280v-1280q0 -52 -38 -90t-90 -38h-2048q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h2048q52 0 90 -38t38 -90z" /> + <glyph glyph-name="_467" unicode="" horiz-adv-x="2304" +d="M313 759q0 -51 -36 -84q-29 -26 -89 -26h-17v220h17q61 0 89 -27q36 -31 36 -83zM2089 824q0 -52 -64 -52h-19v101h20q63 0 63 -49zM380 759q0 74 -50 120.5t-129 46.5h-95v-333h95q74 0 119 38q60 51 60 128zM410 593h65v333h-65v-333zM730 694q0 40 -20.5 62t-75.5 42 +q-29 10 -39.5 19t-10.5 23q0 16 13.5 26.5t34.5 10.5q29 0 53 -27l34 44q-41 37 -98 37q-44 0 -74 -27.5t-30 -67.5q0 -35 18 -55.5t64 -36.5q37 -13 45 -19q19 -12 19 -34q0 -20 -14 -33.5t-36 -13.5q-48 0 -71 44l-42 -40q44 -64 115 -64q51 0 83 30.5t32 79.5zM1008 604 +v77q-37 -37 -78 -37q-49 0 -80.5 32.5t-31.5 82.5q0 48 31.5 81.5t77.5 33.5q43 0 81 -38v77q-40 20 -80 20q-74 0 -125.5 -50.5t-51.5 -123.5t51 -123.5t125 -50.5q42 0 81 19zM2240 0v527q-65 -40 -144.5 -84t-237.5 -117t-329.5 -137.5t-417.5 -134.5t-504 -118h1569 +q26 0 45 19t19 45zM1389 757q0 75 -53 128t-128 53t-128 -53t-53 -128t53 -128t128 -53t128 53t53 128zM1541 584l144 342h-71l-90 -224l-89 224h-71l142 -342h35zM1714 593h184v56h-119v90h115v56h-115v74h119v57h-184v-333zM2105 593h80l-105 140q76 16 76 94q0 47 -31 73 +t-87 26h-97v-333h65v133h9zM2304 1274v-1268q0 -56 -38.5 -95t-93.5 -39h-2040q-55 0 -93.5 39t-38.5 95v1268q0 56 38.5 95t93.5 39h2040q55 0 93.5 -39t38.5 -95z" /> + <glyph glyph-name="f1f3" unicode="" horiz-adv-x="2304" +d="M119 854h89l-45 108zM740 328l74 79l-70 79h-163v-49h142v-55h-142v-54h159zM898 406l99 -110v217zM1186 453q0 33 -40 33h-84v-69h83q41 0 41 36zM1475 457q0 29 -42 29h-82v-61h81q43 0 43 32zM1197 923q0 29 -42 29h-82v-60h81q43 0 43 31zM1656 854h89l-44 108z +M699 1009v-271h-66v212l-94 -212h-57l-94 212v-212h-132l-25 60h-135l-25 -60h-70l116 271h96l110 -257v257h106l85 -184l77 184h108zM1255 453q0 -20 -5.5 -35t-14 -25t-22.5 -16.5t-26 -10t-31.5 -4.5t-31.5 -1t-32.5 0.5t-29.5 0.5v-91h-126l-80 90l-83 -90h-256v271h260 +l80 -89l82 89h207q109 0 109 -89zM964 794v-56h-217v271h217v-57h-152v-49h148v-55h-148v-54h152zM2304 235v-229q0 -55 -38.5 -94.5t-93.5 -39.5h-2040q-55 0 -93.5 39.5t-38.5 94.5v678h111l25 61h55l25 -61h218v46l19 -46h113l20 47v-47h541v99l10 1q10 0 10 -14v-86h279 +v23q23 -12 55 -18t52.5 -6.5t63 0.5t51.5 1l25 61h56l25 -61h227v58l34 -58h182v378h-180v-44l-25 44h-185v-44l-23 44h-249q-69 0 -109 -22v22h-172v-22q-24 22 -73 22h-628l-43 -97l-43 97h-198v-44l-22 44h-169l-78 -179v391q0 55 38.5 94.5t93.5 39.5h2040 +q55 0 93.5 -39.5t38.5 -94.5v-678h-120q-51 0 -81 -22v22h-177q-55 0 -78 -22v22h-316v-22q-31 22 -87 22h-209v-22q-23 22 -91 22h-234l-54 -58l-50 58h-349v-378h343l55 59l52 -59h211v89h21q59 0 90 13v-102h174v99h8q8 0 10 -2t2 -10v-87h529q57 0 88 24v-24h168 +q60 0 95 17zM1546 469q0 -23 -12 -43t-34 -29q25 -9 34 -26t9 -46v-54h-65v45q0 33 -12 43.5t-46 10.5h-69v-99h-65v271h154q48 0 77 -15t29 -58zM1269 936q0 -24 -12.5 -44t-33.5 -29q26 -9 34.5 -25.5t8.5 -46.5v-53h-65q0 9 0.5 26.5t0 25t-3 18.5t-8.5 16t-17.5 8.5 +t-29.5 3.5h-70v-98h-64v271l153 -1q49 0 78 -14.5t29 -57.5zM1798 327v-56h-216v271h216v-56h-151v-49h148v-55h-148v-54zM1372 1009v-271h-66v271h66zM2065 357q0 -86 -102 -86h-126v58h126q34 0 34 25q0 16 -17 21t-41.5 5t-49.5 3.5t-42 22.5t-17 55q0 39 26 60t66 21 +h130v-57h-119q-36 0 -36 -25q0 -16 17.5 -20.5t42 -4t49 -2.5t42 -21.5t17.5 -54.5zM2304 407v-101q-24 -35 -88 -35h-125v58h125q33 0 33 25q0 13 -12.5 19t-31 5.5t-40 2t-40 8t-31 24t-12.5 48.5q0 39 26.5 60t66.5 21h129v-57h-118q-36 0 -36 -25q0 -20 29 -22t68.5 -5 +t56.5 -26zM2139 1008v-270h-92l-122 203v-203h-132l-26 60h-134l-25 -60h-75q-129 0 -129 133q0 138 133 138h63v-59q-7 0 -28 1t-28.5 0.5t-23 -2t-21.5 -6.5t-14.5 -13.5t-11.5 -23t-3 -33.5q0 -38 13.5 -58t49.5 -20h29l92 213h97l109 -256v256h99l114 -188v188h66z" /> + <glyph glyph-name="_469" unicode="" horiz-adv-x="2304" +d="M745 630q0 -37 -25.5 -61.5t-62.5 -24.5q-29 0 -46.5 16t-17.5 44q0 37 25 62.5t62 25.5q28 0 46.5 -16.5t18.5 -45.5zM1530 779q0 -42 -22 -57t-66 -15l-32 -1l17 107q2 11 13 11h18q22 0 35 -2t25 -12.5t12 -30.5zM1881 630q0 -36 -25.5 -61t-61.5 -25q-29 0 -47 16 +t-18 44q0 37 25 62.5t62 25.5q28 0 46.5 -16.5t18.5 -45.5zM513 801q0 59 -38.5 85.5t-100.5 26.5h-160q-19 0 -21 -19l-65 -408q-1 -6 3 -11t10 -5h76q20 0 22 19l18 110q1 8 7 13t15 6.5t17 1.5t19 -1t14 -1q86 0 135 48.5t49 134.5zM822 489l41 261q1 6 -3 11t-10 5h-76 +q-14 0 -17 -33q-27 40 -95 40q-72 0 -122.5 -54t-50.5 -127q0 -59 34.5 -94t92.5 -35q28 0 58 12t48 32q-4 -12 -4 -21q0 -16 13 -16h69q19 0 22 19zM1269 752q0 5 -4 9.5t-9 4.5h-77q-11 0 -18 -10l-106 -156l-44 150q-5 16 -22 16h-75q-5 0 -9 -4.5t-4 -9.5q0 -2 19.5 -59 +t42 -123t23.5 -70q-82 -112 -82 -120q0 -13 13 -13h77q11 0 18 10l255 368q2 2 2 7zM1649 801q0 59 -38.5 85.5t-100.5 26.5h-159q-20 0 -22 -19l-65 -408q-1 -6 3 -11t10 -5h82q12 0 16 13l18 116q1 8 7 13t15 6.5t17 1.5t19 -1t14 -1q86 0 135 48.5t49 134.5zM1958 489 +l41 261q1 6 -3 11t-10 5h-76q-14 0 -17 -33q-26 40 -95 40q-72 0 -122.5 -54t-50.5 -127q0 -59 34.5 -94t92.5 -35q29 0 59 12t47 32q0 -1 -2 -9t-2 -12q0 -16 13 -16h69q19 0 22 19zM2176 898v1q0 14 -13 14h-74q-11 0 -13 -11l-65 -416l-1 -2q0 -5 4 -9.5t10 -4.5h66 +q19 0 21 19zM392 764q-5 -35 -26 -46t-60 -11l-33 -1l17 107q2 11 13 11h19q40 0 58 -11.5t12 -48.5zM2304 1280v-1280q0 -52 -38 -90t-90 -38h-2048q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h2048q52 0 90 -38t38 -90z" /> + <glyph glyph-name="_470" unicode="" horiz-adv-x="2304" +d="M1597 633q0 -69 -21 -106q-19 -35 -52 -35q-23 0 -41 9v224q29 30 57 30q57 0 57 -122zM2035 669h-110q6 98 56 98q51 0 54 -98zM476 534q0 59 -33 91.5t-101 57.5q-36 13 -52 24t-16 25q0 26 38 26q58 0 124 -33l18 112q-67 32 -149 32q-77 0 -123 -38q-48 -39 -48 -109 +q0 -58 32.5 -90.5t99.5 -56.5q39 -14 54.5 -25.5t15.5 -27.5q0 -31 -48 -31q-29 0 -70 12.5t-72 30.5l-18 -113q72 -41 168 -41q81 0 129 37q51 41 51 117zM771 749l19 111h-96v135l-129 -21l-18 -114l-46 -8l-17 -103h62v-219q0 -84 44 -120q38 -30 111 -30q32 0 79 11v118 +q-32 -7 -44 -7q-42 0 -42 50v197h77zM1087 724v139q-15 3 -28 3q-32 0 -55.5 -16t-33.5 -46l-10 56h-131v-471h150v306q26 31 82 31q16 0 26 -2zM1124 389h150v471h-150v-471zM1746 638q0 122 -45 179q-40 52 -111 52q-64 0 -117 -56l-8 47h-132v-645l150 25v151 +q36 -11 68 -11q83 0 134 56q61 65 61 202zM1278 986q0 33 -23 56t-56 23t-56 -23t-23 -56t23 -56.5t56 -23.5t56 23.5t23 56.5zM2176 629q0 113 -48 176q-50 64 -144 64q-96 0 -151.5 -66t-55.5 -180q0 -128 63 -188q55 -55 161 -55q101 0 160 40l-16 103q-57 -31 -128 -31 +q-43 0 -63 19q-23 19 -28 66h248q2 14 2 52zM2304 1280v-1280q0 -52 -38 -90t-90 -38h-2048q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h2048q52 0 90 -38t38 -90z" /> + <glyph glyph-name="_471" unicode="" horiz-adv-x="2048" +d="M1558 684q61 -356 298 -556q0 -52 -38 -90t-90 -38h-448q0 -106 -75 -181t-181 -75t-180.5 74.5t-75.5 180.5zM1024 -176q16 0 16 16t-16 16q-59 0 -101.5 42.5t-42.5 101.5q0 16 -16 16t-16 -16q0 -73 51.5 -124.5t124.5 -51.5zM2026 1424q8 -10 7.5 -23.5t-10.5 -22.5 +l-1872 -1622q-10 -8 -23.5 -7t-21.5 11l-84 96q-8 10 -7.5 23.5t10.5 21.5l186 161q-19 32 -19 66q50 42 91 88t85 119.5t74.5 158.5t50 206t19.5 260q0 152 117 282.5t307 158.5q-8 19 -8 39q0 40 28 68t68 28t68 -28t28 -68q0 -20 -8 -39q124 -18 219 -82.5t148 -157.5 +l418 363q10 8 23.5 7t21.5 -11z" /> + <glyph glyph-name="_472" unicode="" horiz-adv-x="2048" +d="M1040 -160q0 16 -16 16q-59 0 -101.5 42.5t-42.5 101.5q0 16 -16 16t-16 -16q0 -73 51.5 -124.5t124.5 -51.5q16 0 16 16zM503 315l877 760q-42 88 -132.5 146.5t-223.5 58.5q-93 0 -169.5 -31.5t-121.5 -80.5t-69 -103t-24 -105q0 -384 -137 -645zM1856 128 +q0 -52 -38 -90t-90 -38h-448q0 -106 -75 -181t-181 -75t-180.5 74.5t-75.5 180.5l149 129h757q-166 187 -227 459l111 97q61 -356 298 -556zM1942 1520l84 -96q8 -10 7.5 -23.5t-10.5 -22.5l-1872 -1622q-10 -8 -23.5 -7t-21.5 11l-84 96q-8 10 -7.5 23.5t10.5 21.5l186 161 +q-19 32 -19 66q50 42 91 88t85 119.5t74.5 158.5t50 206t19.5 260q0 152 117 282.5t307 158.5q-8 19 -8 39q0 40 28 68t68 28t68 -28t28 -68q0 -20 -8 -39q124 -18 219 -82.5t148 -157.5l418 363q10 8 23.5 7t21.5 -11z" /> + <glyph glyph-name="_473" unicode="" horiz-adv-x="1408" +d="M512 160v704q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-704q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM768 160v704q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-704q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1024 160v704q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-704 +q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM480 1152h448l-48 117q-7 9 -17 11h-317q-10 -2 -17 -11zM1408 1120v-64q0 -14 -9 -23t-23 -9h-96v-948q0 -83 -47 -143.5t-113 -60.5h-832q-66 0 -113 58.5t-47 141.5v952h-96q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h309l70 167 +q15 37 54 63t79 26h320q40 0 79 -26t54 -63l70 -167h309q14 0 23 -9t9 -23z" /> + <glyph glyph-name="_474" unicode="" +d="M1150 462v-109q0 -50 -36.5 -89t-94 -60.5t-118 -32.5t-117.5 -11q-205 0 -342.5 139t-137.5 346q0 203 136 339t339 136q34 0 75.5 -4.5t93 -18t92.5 -34t69 -56.5t28 -81v-109q0 -16 -16 -16h-118q-16 0 -16 16v70q0 43 -65.5 67.5t-137.5 24.5q-140 0 -228.5 -91.5 +t-88.5 -237.5q0 -151 91.5 -249.5t233.5 -98.5q68 0 138 24t70 66v70q0 7 4.5 11.5t10.5 4.5h119q6 0 11 -4.5t5 -11.5zM768 1280q-130 0 -248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5 +t-51 248.5t-136.5 204t-204 136.5t-248.5 51zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="_475" unicode="" +d="M972 761q0 108 -53.5 169t-147.5 61q-63 0 -124 -30.5t-110 -84.5t-79.5 -137t-30.5 -180q0 -112 53.5 -173t150.5 -61q96 0 176 66.5t122.5 166t42.5 203.5zM1536 640q0 -111 -37 -197t-98.5 -135t-131.5 -74.5t-145 -27.5q-6 0 -15.5 -0.5t-16.5 -0.5q-95 0 -142 53 +q-28 33 -33 83q-52 -66 -131.5 -110t-173.5 -44q-161 0 -249.5 95.5t-88.5 269.5q0 157 66 290t179 210.5t246 77.5q87 0 155 -35.5t106 -99.5l2 19l11 56q1 6 5.5 12t9.5 6h118q5 0 13 -11q5 -5 3 -16l-120 -614q-5 -24 -5 -48q0 -39 12.5 -52t44.5 -13q28 1 57 5.5t73 24 +t77 50t57 89.5t24 137q0 292 -174 466t-466 174q-130 0 -248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51q228 0 405 144q11 9 24 8t21 -12l41 -49q8 -12 7 -24q-2 -13 -12 -22q-102 -83 -227.5 -128t-258.5 -45q-156 0 -298 61 +t-245 164t-164 245t-61 298t61 298t164 245t245 164t298 61q344 0 556 -212t212 -556z" /> + <glyph glyph-name="_476" unicode="" horiz-adv-x="1792" +d="M1698 1442q94 -94 94 -226.5t-94 -225.5l-225 -223l104 -104q10 -10 10 -23t-10 -23l-210 -210q-10 -10 -23 -10t-23 10l-105 105l-603 -603q-37 -37 -90 -37h-203l-256 -128l-64 64l128 256v203q0 53 37 90l603 603l-105 105q-10 10 -10 23t10 23l210 210q10 10 23 10 +t23 -10l104 -104l223 225q93 94 225.5 94t226.5 -94zM512 64l576 576l-192 192l-576 -576v-192h192z" /> + <glyph glyph-name="f1fc" unicode="" horiz-adv-x="1792" +d="M1615 1536q70 0 122.5 -46.5t52.5 -116.5q0 -63 -45 -151q-332 -629 -465 -752q-97 -91 -218 -91q-126 0 -216.5 92.5t-90.5 219.5q0 128 92 212l638 579q59 54 130 54zM706 502q39 -76 106.5 -130t150.5 -76l1 -71q4 -213 -129.5 -347t-348.5 -134q-123 0 -218 46.5 +t-152.5 127.5t-86.5 183t-29 220q7 -5 41 -30t62 -44.5t59 -36.5t46 -17q41 0 55 37q25 66 57.5 112.5t69.5 76t88 47.5t103 25.5t125 10.5z" /> + <glyph glyph-name="_478" unicode="" horiz-adv-x="1792" +d="M1792 128v-384h-1792v384q45 0 85 14t59 27.5t47 37.5q30 27 51.5 38t56.5 11q24 0 44 -7t31 -15t33 -27q29 -25 47 -38t58 -27t86 -14q45 0 85 14.5t58 27t48 37.5q21 19 32.5 27t31 15t43.5 7q35 0 56.5 -11t51.5 -38q28 -24 47 -37.5t59 -27.5t85 -14t85 14t59 27.5 +t47 37.5q30 27 51.5 38t56.5 11q34 0 55.5 -11t51.5 -38q28 -24 47 -37.5t59 -27.5t85 -14zM1792 448v-192q-24 0 -44 7t-31 15t-33 27q-29 25 -47 38t-58 27t-85 14q-46 0 -86 -14t-58 -27t-47 -38q-22 -19 -33 -27t-31 -15t-44 -7q-35 0 -56.5 11t-51.5 38q-29 25 -47 38 +t-58 27t-86 14q-45 0 -85 -14.5t-58 -27t-48 -37.5q-21 -19 -32.5 -27t-31 -15t-43.5 -7q-35 0 -56.5 11t-51.5 38q-28 24 -47 37.5t-59 27.5t-85 14q-46 0 -86 -14t-58 -27t-47 -38q-30 -27 -51.5 -38t-56.5 -11v192q0 80 56 136t136 56h64v448h256v-448h256v448h256v-448 +h256v448h256v-448h64q80 0 136 -56t56 -136zM512 1312q0 -77 -36 -118.5t-92 -41.5q-53 0 -90.5 37.5t-37.5 90.5q0 29 9.5 51t23.5 34t31 28t31 31.5t23.5 44.5t9.5 67q38 0 83 -74t45 -150zM1024 1312q0 -77 -36 -118.5t-92 -41.5q-53 0 -90.5 37.5t-37.5 90.5 +q0 29 9.5 51t23.5 34t31 28t31 31.5t23.5 44.5t9.5 67q38 0 83 -74t45 -150zM1536 1312q0 -77 -36 -118.5t-92 -41.5q-53 0 -90.5 37.5t-37.5 90.5q0 29 9.5 51t23.5 34t31 28t31 31.5t23.5 44.5t9.5 67q38 0 83 -74t45 -150z" /> + <glyph glyph-name="_479" unicode="" horiz-adv-x="2048" +d="M2048 0v-128h-2048v1536h128v-1408h1920zM1664 1024l256 -896h-1664v576l448 576l576 -576z" /> + <glyph glyph-name="_480" unicode="" horiz-adv-x="1792" +d="M768 646l546 -546q-106 -108 -247.5 -168t-298.5 -60q-209 0 -385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103v-762zM955 640h773q0 -157 -60 -298.5t-168 -247.5zM1664 768h-768v768q209 0 385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="_481" unicode="" horiz-adv-x="2048" +d="M2048 0v-128h-2048v1536h128v-1408h1920zM1920 1248v-435q0 -21 -19.5 -29.5t-35.5 7.5l-121 121l-633 -633q-10 -10 -23 -10t-23 10l-233 233l-416 -416l-192 192l585 585q10 10 23 10t23 -10l233 -233l464 464l-121 121q-16 16 -7.5 35.5t29.5 19.5h435q14 0 23 -9 +t9 -23z" /> + <glyph glyph-name="_482" unicode="" horiz-adv-x="1792" +d="M1292 832q0 -6 10 -41q10 -29 25 -49.5t41 -34t44 -20t55 -16.5q325 -91 325 -332q0 -146 -105.5 -242.5t-254.5 -96.5q-59 0 -111.5 18.5t-91.5 45.5t-77 74.5t-63 87.5t-53.5 103.5t-43.5 103t-39.5 106.5t-35.5 95q-32 81 -61.5 133.5t-73.5 96.5t-104 64t-142 20 +q-96 0 -183 -55.5t-138 -144.5t-51 -185q0 -160 106.5 -279.5t263.5 -119.5q177 0 258 95q56 63 83 116l84 -152q-15 -34 -44 -70l1 -1q-131 -152 -388 -152q-147 0 -269.5 79t-190.5 207.5t-68 274.5q0 105 43.5 206t116 176.5t172 121.5t204.5 46q87 0 159 -19t123.5 -50 +t95 -80t72.5 -99t58.5 -117t50.5 -124.5t50 -130.5t55 -127q96 -200 233 -200q81 0 138.5 48.5t57.5 128.5q0 42 -19 72t-50.5 46t-72.5 31.5t-84.5 27t-87.5 34t-81 52t-65 82t-39 122.5q-3 16 -3 33q0 110 87.5 192t198.5 78q78 -3 120.5 -14.5t90.5 -53.5h-1 +q12 -11 23 -24.5t26 -36t19 -27.5l-129 -99q-26 49 -54 70v1q-23 21 -97 21q-49 0 -84 -33t-35 -83z" /> + <glyph glyph-name="_483" unicode="" +d="M1432 484q0 173 -234 239q-35 10 -53 16.5t-38 25t-29 46.5q0 2 -2 8.5t-3 12t-1 7.5q0 36 24.5 59.5t60.5 23.5q54 0 71 -15h-1q20 -15 39 -51l93 71q-39 54 -49 64q-33 29 -67.5 39t-85.5 10q-80 0 -142 -57.5t-62 -137.5q0 -7 2 -23q16 -96 64.5 -140t148.5 -73 +q29 -8 49 -15.5t45 -21.5t38.5 -34.5t13.5 -46.5v-5q1 -58 -40.5 -93t-100.5 -35q-97 0 -167 144q-23 47 -51.5 121.5t-48 125.5t-54 110.5t-74 95.5t-103.5 60.5t-147 24.5q-101 0 -192 -56t-144 -148t-50 -192v-1q4 -108 50.5 -199t133.5 -147.5t196 -56.5q186 0 279 110 +q20 27 31 51l-60 109q-42 -80 -99 -116t-146 -36q-115 0 -191 87t-76 204q0 105 82 189t186 84q112 0 170 -53.5t104 -172.5q8 -21 25.5 -68.5t28.5 -76.5t31.5 -74.5t38.5 -74t45.5 -62.5t55.5 -53.5t66 -33t80 -13.5q107 0 183 69.5t76 174.5zM1536 1120v-960 +q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="_484" unicode="" horiz-adv-x="2048" +d="M1152 640q0 104 -40.5 198.5t-109.5 163.5t-163.5 109.5t-198.5 40.5t-198.5 -40.5t-163.5 -109.5t-109.5 -163.5t-40.5 -198.5t40.5 -198.5t109.5 -163.5t163.5 -109.5t198.5 -40.5t198.5 40.5t163.5 109.5t109.5 163.5t40.5 198.5zM1920 640q0 104 -40.5 198.5 +t-109.5 163.5t-163.5 109.5t-198.5 40.5h-386q119 -90 188.5 -224t69.5 -288t-69.5 -288t-188.5 -224h386q104 0 198.5 40.5t163.5 109.5t109.5 163.5t40.5 198.5zM2048 640q0 -130 -51 -248.5t-136.5 -204t-204 -136.5t-248.5 -51h-768q-130 0 -248.5 51t-204 136.5 +t-136.5 204t-51 248.5t51 248.5t136.5 204t204 136.5t248.5 51h768q130 0 248.5 -51t204 -136.5t136.5 -204t51 -248.5z" /> + <glyph glyph-name="_485" unicode="" horiz-adv-x="2048" +d="M0 640q0 130 51 248.5t136.5 204t204 136.5t248.5 51h768q130 0 248.5 -51t204 -136.5t136.5 -204t51 -248.5t-51 -248.5t-136.5 -204t-204 -136.5t-248.5 -51h-768q-130 0 -248.5 51t-204 136.5t-136.5 204t-51 248.5zM1408 128q104 0 198.5 40.5t163.5 109.5 +t109.5 163.5t40.5 198.5t-40.5 198.5t-109.5 163.5t-163.5 109.5t-198.5 40.5t-198.5 -40.5t-163.5 -109.5t-109.5 -163.5t-40.5 -198.5t40.5 -198.5t109.5 -163.5t163.5 -109.5t198.5 -40.5z" /> + <glyph glyph-name="_486" unicode="" horiz-adv-x="2304" +d="M762 384h-314q-40 0 -57.5 35t6.5 67l188 251q-65 31 -137 31q-132 0 -226 -94t-94 -226t94 -226t226 -94q115 0 203 72.5t111 183.5zM576 512h186q-18 85 -75 148zM1056 512l288 384h-480l-99 -132q105 -103 126 -252h165zM2176 448q0 132 -94 226t-226 94 +q-60 0 -121 -24l174 -260q15 -23 10 -49t-27 -40q-15 -11 -36 -11q-35 0 -53 29l-174 260q-93 -95 -93 -225q0 -132 94 -226t226 -94t226 94t94 226zM2304 448q0 -185 -131.5 -316.5t-316.5 -131.5t-316.5 131.5t-131.5 316.5q0 97 39.5 183.5t109.5 149.5l-65 98l-353 -469 +q-18 -26 -51 -26h-197q-23 -164 -149 -274t-294 -110q-185 0 -316.5 131.5t-131.5 316.5t131.5 316.5t316.5 131.5q114 0 215 -55l137 183h-224q-26 0 -45 19t-19 45t19 45t45 19h384v-128h435l-85 128h-222q-26 0 -45 19t-19 45t19 45t45 19h256q33 0 53 -28l267 -400 +q91 44 192 44q185 0 316.5 -131.5t131.5 -316.5z" /> + <glyph glyph-name="_487" unicode="" +d="M384 320q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1408 320q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1362 716l-72 384q-5 23 -22.5 37.5t-40.5 14.5 +h-918q-23 0 -40.5 -14.5t-22.5 -37.5l-72 -384q-5 -30 14 -53t49 -23h1062q30 0 49 23t14 53zM1136 1328q0 20 -14 34t-34 14h-640q-20 0 -34 -14t-14 -34t14 -34t34 -14h640q20 0 34 14t14 34zM1536 603v-603h-128v-128q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5 +t-37.5 90.5v128h-768v-128q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5v128h-128v603q0 112 25 223l103 454q9 78 97.5 137t230 89t312.5 30t312.5 -30t230 -89t97.5 -137l105 -454q23 -102 23 -223z" /> + <glyph glyph-name="_488" unicode="" horiz-adv-x="2048" +d="M1463 704q0 -35 -25 -60.5t-61 -25.5h-702q-36 0 -61 25.5t-25 60.5t25 60.5t61 25.5h702q36 0 61 -25.5t25 -60.5zM1677 704q0 86 -23 170h-982q-36 0 -61 25t-25 60q0 36 25 61t61 25h908q-88 143 -235 227t-320 84q-177 0 -327.5 -87.5t-238 -237.5t-87.5 -327 +q0 -86 23 -170h982q36 0 61 -25t25 -60q0 -36 -25 -61t-61 -25h-908q88 -143 235.5 -227t320.5 -84q132 0 253 51.5t208 139t139 208t52 253.5zM2048 959q0 -35 -25 -60t-61 -25h-131q17 -85 17 -170q0 -167 -65.5 -319.5t-175.5 -263t-262.5 -176t-319.5 -65.5 +q-246 0 -448.5 133t-301.5 350h-189q-36 0 -61 25t-25 61q0 35 25 60t61 25h132q-17 85 -17 170q0 167 65.5 319.5t175.5 263t262.5 176t320.5 65.5q245 0 447.5 -133t301.5 -350h188q36 0 61 -25t25 -61z" /> + <glyph glyph-name="_489" unicode="" horiz-adv-x="1280" +d="M953 1158l-114 -328l117 -21q165 451 165 518q0 56 -38 56q-57 0 -130 -225zM654 471l33 -88q37 42 71 67l-33 5.5t-38.5 7t-32.5 8.5zM362 1367q0 -98 159 -521q17 10 49 10q15 0 75 -5l-121 351q-75 220 -123 220q-19 0 -29 -17.5t-10 -37.5zM283 608q0 -36 51.5 -119 +t117.5 -153t100 -70q14 0 25.5 13t11.5 27q0 24 -32 102q-13 32 -32 72t-47.5 89t-61.5 81t-62 32q-20 0 -45.5 -27t-25.5 -47zM125 273q0 -41 25 -104q59 -145 183.5 -227t281.5 -82q227 0 382 170q152 169 152 427q0 43 -1 67t-11.5 62t-30.5 56q-56 49 -211.5 75.5 +t-270.5 26.5q-37 0 -49 -11q-12 -5 -12 -35q0 -34 21.5 -60t55.5 -40t77.5 -23.5t87.5 -11.5t85 -4t70 0h23q24 0 40 -19q15 -19 19 -55q-28 -28 -96 -54q-61 -22 -93 -46q-64 -46 -108.5 -114t-44.5 -137q0 -31 18.5 -88.5t18.5 -87.5l-3 -12q-4 -12 -4 -14 +q-137 10 -146 216q-8 -2 -41 -2q2 -7 2 -21q0 -53 -40.5 -89.5t-94.5 -36.5q-82 0 -166.5 78t-84.5 159q0 34 33 67q52 -64 60 -76q77 -104 133 -104q12 0 26.5 8.5t14.5 20.5q0 34 -87.5 145t-116.5 111q-43 0 -70 -44.5t-27 -90.5zM11 264q0 101 42.5 163t136.5 88 +q-28 74 -28 104q0 62 61 123t122 61q29 0 70 -15q-163 462 -163 567q0 80 41 130.5t119 50.5q131 0 325 -581q6 -17 8 -23q6 16 29 79.5t43.5 118.5t54 127.5t64.5 123t70.5 86.5t76.5 36q71 0 112 -49t41 -122q0 -108 -159 -550q61 -15 100.5 -46t58.5 -78t26 -93.5 +t7 -110.5q0 -150 -47 -280t-132 -225t-211 -150t-278 -55q-111 0 -223 42q-149 57 -258 191.5t-109 286.5z" /> + <glyph glyph-name="_490" unicode="" horiz-adv-x="2048" +d="M785 528h207q-14 -158 -98.5 -248.5t-214.5 -90.5q-162 0 -254.5 116t-92.5 316q0 194 93 311.5t233 117.5q148 0 232 -87t97 -247h-203q-5 64 -35.5 99t-81.5 35q-57 0 -88.5 -60.5t-31.5 -177.5q0 -48 5 -84t18 -69.5t40 -51.5t66 -18q95 0 109 139zM1497 528h206 +q-14 -158 -98 -248.5t-214 -90.5q-162 0 -254.5 116t-92.5 316q0 194 93 311.5t233 117.5q148 0 232 -87t97 -247h-204q-4 64 -35 99t-81 35q-57 0 -88.5 -60.5t-31.5 -177.5q0 -48 5 -84t18 -69.5t39.5 -51.5t65.5 -18q49 0 76.5 38t33.5 101zM1856 647q0 207 -15.5 307 +t-60.5 161q-6 8 -13.5 14t-21.5 15t-16 11q-86 63 -697 63q-625 0 -710 -63q-5 -4 -17.5 -11.5t-21 -14t-14.5 -14.5q-45 -60 -60 -159.5t-15 -308.5q0 -208 15 -307.5t60 -160.5q6 -8 15 -15t20.5 -14t17.5 -12q44 -33 239.5 -49t470.5 -16q610 0 697 65q5 4 17 11t20.5 14 +t13.5 16q46 60 61 159t15 309zM2048 1408v-1536h-2048v1536h2048z" /> + <glyph glyph-name="_491" unicode="" +d="M992 912v-496q0 -14 -9 -23t-23 -9h-160q-14 0 -23 9t-9 23v496q0 112 -80 192t-192 80h-272v-1152q0 -14 -9 -23t-23 -9h-160q-14 0 -23 9t-9 23v1344q0 14 9 23t23 9h464q135 0 249 -66.5t180.5 -180.5t66.5 -249zM1376 1376v-880q0 -135 -66.5 -249t-180.5 -180.5 +t-249 -66.5h-464q-14 0 -23 9t-9 23v960q0 14 9 23t23 9h160q14 0 23 -9t9 -23v-768h272q112 0 192 80t80 192v880q0 14 9 23t23 9h160q14 0 23 -9t9 -23z" /> + <glyph glyph-name="_492" unicode="" +d="M1311 694v-114q0 -24 -13.5 -38t-37.5 -14h-202q-24 0 -38 14t-14 38v114q0 24 14 38t38 14h202q24 0 37.5 -14t13.5 -38zM821 464v250q0 53 -32.5 85.5t-85.5 32.5h-133q-68 0 -96 -52q-28 52 -96 52h-130q-53 0 -85.5 -32.5t-32.5 -85.5v-250q0 -22 21 -22h55 +q22 0 22 22v230q0 24 13.5 38t38.5 14h94q24 0 38 -14t14 -38v-230q0 -22 21 -22h54q22 0 22 22v230q0 24 14 38t38 14h97q24 0 37.5 -14t13.5 -38v-230q0 -22 22 -22h55q21 0 21 22zM1410 560v154q0 53 -33 85.5t-86 32.5h-264q-53 0 -86 -32.5t-33 -85.5v-410 +q0 -21 22 -21h55q21 0 21 21v180q31 -42 94 -42h191q53 0 86 32.5t33 85.5zM1536 1176v-1072q0 -96 -68 -164t-164 -68h-1072q-96 0 -164 68t-68 164v1072q0 96 68 164t164 68h1072q96 0 164 -68t68 -164z" /> + <glyph glyph-name="_493" unicode="" +d="M915 450h-294l147 551zM1001 128h311l-324 1024h-440l-324 -1024h311l383 314zM1536 1120v-960q0 -118 -85 -203t-203 -85h-960q-118 0 -203 85t-85 203v960q0 118 85 203t203 85h960q118 0 203 -85t85 -203z" /> + <glyph glyph-name="_494" unicode="" horiz-adv-x="2048" +d="M2048 641q0 -21 -13 -36.5t-33 -19.5l-205 -356q3 -9 3 -18q0 -20 -12.5 -35.5t-32.5 -19.5l-193 -337q3 -8 3 -16q0 -23 -16.5 -40t-40.5 -17q-25 0 -41 18h-400q-17 -20 -43 -20t-43 20h-399q-17 -20 -43 -20q-23 0 -40 16.5t-17 40.5q0 8 4 20l-193 335 +q-20 4 -32.5 19.5t-12.5 35.5q0 9 3 18l-206 356q-20 5 -32.5 20.5t-12.5 35.5q0 21 13.5 36.5t33.5 19.5l199 344q0 1 -0.5 3t-0.5 3q0 36 34 51l209 363q-4 10 -4 18q0 24 17 40.5t40 16.5q26 0 44 -21h396q16 21 43 21t43 -21h398q18 21 44 21q23 0 40 -16.5t17 -40.5 +q0 -6 -4 -18l207 -358q23 -1 39 -17.5t16 -38.5q0 -13 -7 -27l187 -324q19 -4 31.5 -19.5t12.5 -35.5zM1063 -158h389l-342 354h-143l-342 -354h360q18 16 39 16t39 -16zM112 654q1 -4 1 -13q0 -10 -2 -15l208 -360l15 -6l188 199v347l-187 194q-13 -8 -29 -10zM986 1438 +h-388l190 -200l554 200h-280q-16 -16 -38 -16t-38 16zM1689 226q1 6 5 11l-64 68l-17 -79h76zM1583 226l22 105l-252 266l-296 -307l63 -64h463zM1495 -142l16 28l65 310h-427l333 -343q8 4 13 5zM578 -158h5l342 354h-373v-335l4 -6q14 -5 22 -13zM552 226h402l64 66 +l-309 321l-157 -166v-221zM359 226h163v189l-168 -177q4 -8 5 -12zM358 1051q0 -1 0.5 -2t0.5 -2q0 -16 -8 -29l171 -177v269zM552 1121v-311l153 -157l297 314l-223 236zM556 1425l-4 -8v-264l205 74l-191 201q-6 -2 -10 -3zM1447 1438h-16l-621 -224l213 -225zM1023 946 +l-297 -315l311 -319l296 307zM688 634l-136 141v-284zM1038 270l-42 -44h85zM1374 618l238 -251l132 624l-3 5l-1 1zM1718 1018q-8 13 -8 29v2l-216 376q-5 1 -13 5l-437 -463l310 -327zM522 1142v223l-163 -282zM522 196h-163l163 -283v283zM1607 196l-48 -227l130 227h-82 +zM1729 266l207 361q-2 10 -2 14q0 1 3 16l-171 296l-129 -612l77 -82q5 3 15 7z" /> + <glyph glyph-name="f210" unicode="" +d="M0 856q0 131 91.5 226.5t222.5 95.5h742l352 358v-1470q0 -132 -91.5 -227t-222.5 -95h-780q-131 0 -222.5 95t-91.5 227v790zM1232 102l-176 180v425q0 46 -32 79t-78 33h-484q-46 0 -78 -33t-32 -79v-492q0 -46 32.5 -79.5t77.5 -33.5h770z" /> + <glyph glyph-name="_496" unicode="" +d="M934 1386q-317 -121 -556 -362.5t-358 -560.5q-20 89 -20 176q0 208 102.5 384.5t278.5 279t384 102.5q82 0 169 -19zM1203 1267q93 -65 164 -155q-389 -113 -674.5 -400.5t-396.5 -676.5q-93 72 -155 162q112 386 395 671t667 399zM470 -67q115 356 379.5 622t619.5 384 +q40 -92 54 -195q-292 -120 -516 -345t-343 -518q-103 14 -194 52zM1536 -125q-193 50 -367 115q-135 -84 -290 -107q109 205 274 370.5t369 275.5q-21 -152 -101 -284q65 -175 115 -370z" /> + <glyph glyph-name="f212" unicode="" horiz-adv-x="2048" +d="M1893 1144l155 -1272q-131 0 -257 57q-200 91 -393 91q-226 0 -374 -148q-148 148 -374 148q-193 0 -393 -91q-128 -57 -252 -57h-5l155 1272q224 127 482 127q233 0 387 -106q154 106 387 106q258 0 482 -127zM1398 157q129 0 232 -28.5t260 -93.5l-124 1021 +q-171 78 -368 78q-224 0 -374 -141q-150 141 -374 141q-197 0 -368 -78l-124 -1021q105 43 165.5 65t148.5 39.5t178 17.5q202 0 374 -108q172 108 374 108zM1438 191l-55 907q-211 -4 -359 -155q-152 155 -374 155q-176 0 -336 -66l-114 -941q124 51 228.5 76t221.5 25 +q209 0 374 -102q172 107 374 102z" /> + <glyph glyph-name="_498" unicode="" horiz-adv-x="2048" +d="M1500 165v733q0 21 -15 36t-35 15h-93q-20 0 -35 -15t-15 -36v-733q0 -20 15 -35t35 -15h93q20 0 35 15t15 35zM1216 165v531q0 20 -15 35t-35 15h-101q-20 0 -35 -15t-15 -35v-531q0 -20 15 -35t35 -15h101q20 0 35 15t15 35zM924 165v429q0 20 -15 35t-35 15h-101 +q-20 0 -35 -15t-15 -35v-429q0 -20 15 -35t35 -15h101q20 0 35 15t15 35zM632 165v362q0 20 -15 35t-35 15h-101q-20 0 -35 -15t-15 -35v-362q0 -20 15 -35t35 -15h101q20 0 35 15t15 35zM2048 311q0 -166 -118 -284t-284 -118h-1244q-166 0 -284 118t-118 284 +q0 116 63 214.5t168 148.5q-10 34 -10 73q0 113 80.5 193.5t193.5 80.5q102 0 180 -67q45 183 194 300t338 117q149 0 275 -73.5t199.5 -199.5t73.5 -275q0 -66 -14 -122q135 -33 221 -142.5t86 -247.5z" /> + <glyph glyph-name="_499" unicode="" +d="M0 1536h1536v-1392l-776 -338l-760 338v1392zM1436 209v926h-1336v-926l661 -294zM1436 1235v201h-1336v-201h1336zM181 937v-115h-37v115h37zM181 789v-115h-37v115h37zM181 641v-115h-37v115h37zM181 493v-115h-37v115h37zM181 345v-115h-37v115h37zM207 202l15 34 +l105 -47l-15 -33zM343 142l15 34l105 -46l-15 -34zM478 82l15 34l105 -46l-15 -34zM614 23l15 33l104 -46l-15 -34zM797 10l105 46l15 -33l-105 -47zM932 70l105 46l15 -34l-105 -46zM1068 130l105 46l15 -34l-105 -46zM1203 189l105 47l15 -34l-105 -46zM259 1389v-36h-114 +v36h114zM421 1389v-36h-115v36h115zM583 1389v-36h-115v36h115zM744 1389v-36h-114v36h114zM906 1389v-36h-114v36h114zM1068 1389v-36h-115v36h115zM1230 1389v-36h-115v36h115zM1391 1389v-36h-114v36h114zM181 1049v-79h-37v115h115v-36h-78zM421 1085v-36h-115v36h115z +M583 1085v-36h-115v36h115zM744 1085v-36h-114v36h114zM906 1085v-36h-114v36h114zM1068 1085v-36h-115v36h115zM1230 1085v-36h-115v36h115zM1355 970v79h-78v36h115v-115h-37zM1355 822v115h37v-115h-37zM1355 674v115h37v-115h-37zM1355 526v115h37v-115h-37zM1355 378 +v115h37v-115h-37zM1355 230v115h37v-115h-37zM760 265q-129 0 -221 91.5t-92 221.5q0 129 92 221t221 92q130 0 221.5 -92t91.5 -221q0 -130 -91.5 -221.5t-221.5 -91.5zM595 646q0 -36 19.5 -56.5t49.5 -25t64 -7t64 -2t49.5 -9t19.5 -30.5q0 -49 -112 -49q-97 0 -123 51 +h-3l-31 -63q67 -42 162 -42q29 0 56.5 5t55.5 16t45.5 33t17.5 53q0 46 -27.5 69.5t-67.5 27t-79.5 3t-67 5t-27.5 25.5q0 21 20.5 33t40.5 15t41 3q34 0 70.5 -11t51.5 -34h3l30 58q-3 1 -21 8.5t-22.5 9t-19.5 7t-22 7t-20 4.5t-24 4t-23 1q-29 0 -56.5 -5t-54 -16.5 +t-43 -34t-16.5 -53.5z" /> + <glyph glyph-name="_500" unicode="" horiz-adv-x="2048" +d="M863 504q0 112 -79.5 191.5t-191.5 79.5t-191 -79.5t-79 -191.5t79 -191t191 -79t191.5 79t79.5 191zM1726 505q0 112 -79 191t-191 79t-191.5 -79t-79.5 -191q0 -113 79.5 -192t191.5 -79t191 79.5t79 191.5zM2048 1314v-1348q0 -44 -31.5 -75.5t-76.5 -31.5h-1832 +q-45 0 -76.5 31.5t-31.5 75.5v1348q0 44 31.5 75.5t76.5 31.5h431q44 0 76 -31.5t32 -75.5v-161h754v161q0 44 32 75.5t76 31.5h431q45 0 76.5 -31.5t31.5 -75.5z" /> + <glyph glyph-name="_501" unicode="" horiz-adv-x="2048" +d="M1430 953zM1690 749q148 0 253 -98.5t105 -244.5q0 -157 -109 -261.5t-267 -104.5q-85 0 -162 27.5t-138 73.5t-118 106t-109 126t-103.5 132.5t-108.5 126.5t-117 106t-136 73.5t-159 27.5q-154 0 -251.5 -91.5t-97.5 -244.5q0 -157 104 -250t263 -93q100 0 208 37.5 +t193 98.5q5 4 21 18.5t30 24t22 9.5q14 0 24.5 -10.5t10.5 -24.5q0 -24 -60 -77q-101 -88 -234.5 -142t-260.5 -54q-133 0 -245.5 58t-180 165t-67.5 241q0 205 141.5 341t347.5 136q120 0 226.5 -43.5t185.5 -113t151.5 -153t139 -167.5t133.5 -153.5t149.5 -113 +t172.5 -43.5q102 0 168.5 61.5t66.5 162.5q0 95 -64.5 159t-159.5 64q-30 0 -81.5 -18.5t-68.5 -18.5q-20 0 -35.5 15t-15.5 35q0 18 8.5 57t8.5 59q0 159 -107.5 263t-266.5 104q-58 0 -111.5 -18.5t-84 -40.5t-55.5 -40.5t-33 -18.5q-15 0 -25.5 10.5t-10.5 25.5 +q0 19 25 46q59 67 147 103.5t182 36.5q191 0 318 -125.5t127 -315.5q0 -37 -4 -66q57 15 115 15z" /> + <glyph glyph-name="_502" unicode="" horiz-adv-x="1664" +d="M1216 832q0 26 -19 45t-45 19h-128v128q0 26 -19 45t-45 19t-45 -19t-19 -45v-128h-128q-26 0 -45 -19t-19 -45t19 -45t45 -19h128v-128q0 -26 19 -45t45 -19t45 19t19 45v128h128q26 0 45 19t19 45zM640 0q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5 +t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1536 0q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1664 1088v-512q0 -24 -16 -42.5t-41 -21.5l-1044 -122q1 -7 4.5 -21.5t6 -26.5t2.5 -22q0 -16 -24 -64h920 +q26 0 45 -19t19 -45t-19 -45t-45 -19h-1024q-26 0 -45 19t-19 45q0 14 11 39.5t29.5 59.5t20.5 38l-177 823h-204q-26 0 -45 19t-19 45t19 45t45 19h256q16 0 28.5 -6.5t20 -15.5t13 -24.5t7.5 -26.5t5.5 -29.5t4.5 -25.5h1201q26 0 45 -19t19 -45z" /> + <glyph glyph-name="_503" unicode="" horiz-adv-x="1664" +d="M1280 832q0 26 -19 45t-45 19t-45 -19l-147 -146v293q0 26 -19 45t-45 19t-45 -19t-19 -45v-293l-147 146q-19 19 -45 19t-45 -19t-19 -45t19 -45l256 -256q19 -19 45 -19t45 19l256 256q19 19 19 45zM640 0q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5 +t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1536 0q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1664 1088v-512q0 -24 -16 -42.5t-41 -21.5l-1044 -122q1 -7 4.5 -21.5t6 -26.5t2.5 -22q0 -16 -24 -64h920 +q26 0 45 -19t19 -45t-19 -45t-45 -19h-1024q-26 0 -45 19t-19 45q0 14 11 39.5t29.5 59.5t20.5 38l-177 823h-204q-26 0 -45 19t-19 45t19 45t45 19h256q16 0 28.5 -6.5t20 -15.5t13 -24.5t7.5 -26.5t5.5 -29.5t4.5 -25.5h1201q26 0 45 -19t19 -45z" /> + <glyph glyph-name="_504" unicode="" horiz-adv-x="2048" +d="M212 768l623 -665l-300 665h-323zM1024 -4l349 772h-698zM538 896l204 384h-262l-288 -384h346zM1213 103l623 665h-323zM683 896h682l-204 384h-274zM1510 896h346l-288 384h-262zM1651 1382l384 -512q14 -18 13 -41.5t-17 -40.5l-960 -1024q-18 -20 -47 -20t-47 20 +l-960 1024q-16 17 -17 40.5t13 41.5l384 512q18 26 51 26h1152q33 0 51 -26z" /> + <glyph glyph-name="_505" unicode="" horiz-adv-x="2048" +d="M1811 -19q19 19 45 19t45 -19l128 -128l-90 -90l-83 83l-83 -83q-18 -19 -45 -19t-45 19l-83 83l-83 -83q-19 -19 -45 -19t-45 19l-83 83l-83 -83q-19 -19 -45 -19t-45 19l-83 83l-83 -83q-19 -19 -45 -19t-45 19l-83 83l-83 -83q-19 -19 -45 -19t-45 19l-83 83l-83 -83 +q-19 -19 -45 -19t-45 19l-83 83l-83 -83q-19 -19 -45 -19t-45 19l-128 128l90 90l83 -83l83 83q19 19 45 19t45 -19l83 -83l83 83q19 19 45 19t45 -19l83 -83l83 83q19 19 45 19t45 -19l83 -83l83 83q19 19 45 19t45 -19l83 -83l83 83q19 19 45 19t45 -19l83 -83l83 83 +q19 19 45 19t45 -19l83 -83zM237 19q-19 -19 -45 -19t-45 19l-128 128l90 90l83 -82l83 82q19 19 45 19t45 -19l83 -82l64 64v293l-210 314q-17 26 -7 56.5t40 40.5l177 58v299h128v128h256v128h256v-128h256v-128h128v-299l177 -58q30 -10 40 -40.5t-7 -56.5l-210 -314 +v-293l19 18q19 19 45 19t45 -19l83 -82l83 82q19 19 45 19t45 -19l128 -128l-90 -90l-83 83l-83 -83q-18 -19 -45 -19t-45 19l-83 83l-83 -83q-19 -19 -45 -19t-45 19l-83 83l-83 -83q-19 -19 -45 -19t-45 19l-83 83l-83 -83q-19 -19 -45 -19t-45 19l-83 83l-83 -83 +q-19 -19 -45 -19t-45 19l-83 83l-83 -83q-19 -19 -45 -19t-45 19l-83 83zM640 1152v-128l384 128l384 -128v128h-128v128h-512v-128h-128z" /> + <glyph glyph-name="_506" unicode="" +d="M576 0l96 448l-96 128l-128 64zM832 0l128 640l-128 -64l-96 -128zM992 1010q-2 4 -4 6q-10 8 -96 8q-70 0 -167 -19q-7 -2 -21 -2t-21 2q-97 19 -167 19q-86 0 -96 -8q-2 -2 -4 -6q2 -18 4 -27q2 -3 7.5 -6.5t7.5 -10.5q2 -4 7.5 -20.5t7 -20.5t7.5 -17t8.5 -17t9 -14 +t12 -13.5t14 -9.5t17.5 -8t20.5 -4t24.5 -2q36 0 59 12.5t32.5 30t14.5 34.5t11.5 29.5t17.5 12.5h12q11 0 17.5 -12.5t11.5 -29.5t14.5 -34.5t32.5 -30t59 -12.5q13 0 24.5 2t20.5 4t17.5 8t14 9.5t12 13.5t9 14t8.5 17t7.5 17t7 20.5t7.5 20.5q2 7 7.5 10.5t7.5 6.5 +q2 9 4 27zM1408 131q0 -121 -73 -190t-194 -69h-874q-121 0 -194 69t-73 190q0 61 4.5 118t19 125.5t37.5 123.5t63.5 103.5t93.5 74.5l-90 220h214q-22 64 -22 128q0 12 2 32q-194 40 -194 96q0 57 210 99q17 62 51.5 134t70.5 114q32 37 76 37q30 0 84 -31t84 -31t84 31 +t84 31q44 0 76 -37q36 -42 70.5 -114t51.5 -134q210 -42 210 -99q0 -56 -194 -96q7 -81 -20 -160h214l-82 -225q63 -33 107.5 -96.5t65.5 -143.5t29 -151.5t8 -148.5z" /> + <glyph glyph-name="_507" unicode="" horiz-adv-x="2304" +d="M2301 500q12 -103 -22 -198.5t-99 -163.5t-158.5 -106t-196.5 -31q-161 11 -279.5 125t-134.5 274q-12 111 27.5 210.5t118.5 170.5l-71 107q-96 -80 -151 -194t-55 -244q0 -27 -18.5 -46.5t-45.5 -19.5h-256h-69q-23 -164 -149 -274t-294 -110q-185 0 -316.5 131.5 +t-131.5 316.5t131.5 316.5t316.5 131.5q76 0 152 -27l24 45q-123 110 -304 110h-64q-26 0 -45 19t-19 45t19 45t45 19h128q78 0 145 -13.5t116.5 -38.5t71.5 -39.5t51 -36.5h512h115l-85 128h-222q-30 0 -49 22.5t-14 52.5q4 23 23 38t43 15h253q33 0 53 -28l70 -105 +l114 114q19 19 46 19h101q26 0 45 -19t19 -45v-128q0 -26 -19 -45t-45 -19h-179l115 -172q131 63 275 36q143 -26 244 -134.5t118 -253.5zM448 128q115 0 203 72.5t111 183.5h-314q-35 0 -55 31q-18 32 -1 63l147 277q-47 13 -91 13q-132 0 -226 -94t-94 -226t94 -226 +t226 -94zM1856 128q132 0 226 94t94 226t-94 226t-226 94q-60 0 -121 -24l174 -260q15 -23 10 -49t-27 -40q-15 -11 -36 -11q-35 0 -53 29l-174 260q-93 -95 -93 -225q0 -132 94 -226t226 -94z" /> + <glyph glyph-name="_508" unicode="" +d="M1408 0q0 -63 -61.5 -113.5t-164 -81t-225 -46t-253.5 -15.5t-253.5 15.5t-225 46t-164 81t-61.5 113.5q0 49 33 88.5t91 66.5t118 44.5t131 29.5q26 5 48 -10.5t26 -41.5q5 -26 -10.5 -48t-41.5 -26q-58 -10 -106 -23.5t-76.5 -25.5t-48.5 -23.5t-27.5 -19.5t-8.5 -12 +q3 -11 27 -26.5t73 -33t114 -32.5t160.5 -25t201.5 -10t201.5 10t160.5 25t114 33t73 33.5t27 27.5q-1 4 -8.5 11t-27.5 19t-48.5 23.5t-76.5 25t-106 23.5q-26 4 -41.5 26t-10.5 48q4 26 26 41.5t48 10.5q71 -12 131 -29.5t118 -44.5t91 -66.5t33 -88.5zM1024 896v-384 +q0 -26 -19 -45t-45 -19h-64v-384q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v384h-64q-26 0 -45 19t-19 45v384q0 53 37.5 90.5t90.5 37.5h384q53 0 90.5 -37.5t37.5 -90.5zM928 1280q0 -93 -65.5 -158.5t-158.5 -65.5t-158.5 65.5t-65.5 158.5t65.5 158.5t158.5 65.5 +t158.5 -65.5t65.5 -158.5z" /> + <glyph glyph-name="_509" unicode="" horiz-adv-x="1792" +d="M1280 512h305q-5 -6 -10 -10.5t-9 -7.5l-3 -4l-623 -600q-18 -18 -44 -18t-44 18l-624 602q-5 2 -21 20h369q22 0 39.5 13.5t22.5 34.5l70 281l190 -667q6 -20 23 -33t39 -13q21 0 38 13t23 33l146 485l56 -112q18 -35 57 -35zM1792 940q0 -145 -103 -300h-369l-111 221 +q-8 17 -25.5 27t-36.5 8q-45 -5 -56 -46l-129 -430l-196 686q-6 20 -23.5 33t-39.5 13t-39 -13.5t-22 -34.5l-116 -464h-423q-103 155 -103 300q0 220 127 344t351 124q62 0 126.5 -21.5t120 -58t95.5 -68.5t76 -68q36 36 76 68t95.5 68.5t120 58t126.5 21.5q224 0 351 -124 +t127 -344z" /> + <glyph glyph-name="venus" unicode="" horiz-adv-x="1280" +d="M1152 960q0 -221 -147.5 -384.5t-364.5 -187.5v-260h224q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-224v-224q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v224h-224q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h224v260q-150 16 -271.5 103t-186 224t-52.5 292 +q11 134 80.5 249t182 188t245.5 88q170 19 319 -54t236 -212t87 -306zM128 960q0 -185 131.5 -316.5t316.5 -131.5t316.5 131.5t131.5 316.5t-131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5z" /> + <glyph glyph-name="_511" unicode="" +d="M1472 1408q26 0 45 -19t19 -45v-416q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v262l-382 -383q126 -156 126 -359q0 -117 -45.5 -223.5t-123 -184t-184 -123t-223.5 -45.5t-223.5 45.5t-184 123t-123 184t-45.5 223.5t45.5 223.5t123 184t184 123t223.5 45.5 +q203 0 359 -126l382 382h-261q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h416zM576 0q185 0 316.5 131.5t131.5 316.5t-131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5z" /> + <glyph glyph-name="_512" unicode="" horiz-adv-x="1280" +d="M830 1220q145 -72 233.5 -210.5t88.5 -305.5q0 -221 -147.5 -384.5t-364.5 -187.5v-132h96q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-96v-96q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v96h-96q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h96v132q-217 24 -364.5 187.5 +t-147.5 384.5q0 167 88.5 305.5t233.5 210.5q-165 96 -228 273q-6 16 3.5 29.5t26.5 13.5h69q21 0 29 -20q44 -106 140 -171t214 -65t214 65t140 171q8 20 37 20h61q17 0 26.5 -13.5t3.5 -29.5q-63 -177 -228 -273zM576 256q185 0 316.5 131.5t131.5 316.5t-131.5 316.5 +t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5z" /> + <glyph glyph-name="_513" unicode="" +d="M1024 1504q0 14 9 23t23 9h288q26 0 45 -19t19 -45v-288q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v134l-254 -255q126 -158 126 -359q0 -221 -147.5 -384.5t-364.5 -187.5v-132h96q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-96v-96q0 -14 -9 -23t-23 -9h-64 +q-14 0 -23 9t-9 23v96h-96q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h96v132q-149 16 -270.5 103t-186.5 223.5t-53 291.5q16 204 160 353.5t347 172.5q118 14 228 -19t198 -103l255 254h-134q-14 0 -23 9t-9 23v64zM576 256q185 0 316.5 131.5t131.5 316.5t-131.5 316.5 +t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5z" /> + <glyph glyph-name="_514" unicode="" horiz-adv-x="1792" +d="M1280 1504q0 14 9 23t23 9h288q26 0 45 -19t19 -45v-288q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v134l-254 -255q126 -158 126 -359q0 -221 -147.5 -384.5t-364.5 -187.5v-132h96q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-96v-96q0 -14 -9 -23t-23 -9h-64 +q-14 0 -23 9t-9 23v96h-96q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h96v132q-217 24 -364.5 187.5t-147.5 384.5q0 201 126 359l-52 53l-101 -111q-9 -10 -22 -10.5t-23 7.5l-48 44q-10 8 -10.5 21.5t8.5 23.5l105 115l-111 112v-134q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9 +t-9 23v288q0 26 19 45t45 19h288q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-133l106 -107l86 94q9 10 22 10.5t23 -7.5l48 -44q10 -8 10.5 -21.5t-8.5 -23.5l-90 -99l57 -56q158 126 359 126t359 -126l255 254h-134q-14 0 -23 9t-9 23v64zM832 256q185 0 316.5 131.5 +t131.5 316.5t-131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5z" /> + <glyph glyph-name="_515" unicode="" horiz-adv-x="1792" +d="M1790 1007q12 -155 -52.5 -292t-186 -224t-271.5 -103v-260h224q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-224v-224q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v224h-512v-224q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v224h-224q-14 0 -23 9t-9 23v64q0 14 9 23 +t23 9h224v260q-150 16 -271.5 103t-186 224t-52.5 292q17 206 164.5 356.5t352.5 169.5q206 21 377 -94q171 115 377 94q205 -19 352.5 -169.5t164.5 -356.5zM896 647q128 131 128 313t-128 313q-128 -131 -128 -313t128 -313zM576 512q115 0 218 57q-154 165 -154 391 +q0 224 154 391q-103 57 -218 57q-185 0 -316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5zM1152 128v260q-137 15 -256 94q-119 -79 -256 -94v-260h512zM1216 512q185 0 316.5 131.5t131.5 316.5t-131.5 316.5t-316.5 131.5q-115 0 -218 -57q154 -167 154 -391 +q0 -226 -154 -391q103 -57 218 -57z" /> + <glyph glyph-name="_516" unicode="" horiz-adv-x="1920" +d="M1536 1120q0 14 9 23t23 9h288q26 0 45 -19t19 -45v-288q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v134l-254 -255q76 -95 107.5 -214t9.5 -247q-31 -182 -166 -312t-318 -156q-210 -29 -384.5 80t-241.5 300q-117 6 -221 57.5t-177.5 133t-113.5 192.5t-32 230 +q9 135 78 252t182 191.5t248 89.5q118 14 227.5 -19t198.5 -103l255 254h-134q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h288q26 0 45 -19t19 -45v-288q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v134l-254 -255q59 -74 93 -169q182 -9 328 -124l255 254h-134q-14 0 -23 9 +t-9 23v64zM1024 704q0 20 -4 58q-162 -25 -271 -150t-109 -292q0 -20 4 -58q162 25 271 150t109 292zM128 704q0 -168 111 -294t276 -149q-3 29 -3 59q0 210 135 369.5t338 196.5q-53 120 -163.5 193t-245.5 73q-185 0 -316.5 -131.5t-131.5 -316.5zM1088 -128 +q185 0 316.5 131.5t131.5 316.5q0 168 -111 294t-276 149q3 -28 3 -59q0 -210 -135 -369.5t-338 -196.5q53 -120 163.5 -193t245.5 -73z" /> + <glyph glyph-name="_517" unicode="" horiz-adv-x="2048" +d="M1664 1504q0 14 9 23t23 9h288q26 0 45 -19t19 -45v-288q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v134l-254 -255q76 -95 107.5 -214t9.5 -247q-32 -180 -164.5 -310t-313.5 -157q-223 -34 -409 90q-117 -78 -256 -93v-132h96q14 0 23 -9t9 -23v-64q0 -14 -9 -23 +t-23 -9h-96v-96q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v96h-96q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h96v132q-155 17 -279.5 109.5t-187 237.5t-39.5 307q25 187 159.5 322.5t320.5 164.5q224 34 410 -90q146 97 320 97q201 0 359 -126l255 254h-134q-14 0 -23 9 +t-9 23v64zM896 391q128 131 128 313t-128 313q-128 -131 -128 -313t128 -313zM128 704q0 -185 131.5 -316.5t316.5 -131.5q117 0 218 57q-154 167 -154 391t154 391q-101 57 -218 57q-185 0 -316.5 -131.5t-131.5 -316.5zM1216 256q185 0 316.5 131.5t131.5 316.5 +t-131.5 316.5t-316.5 131.5q-117 0 -218 -57q154 -167 154 -391t-154 -391q101 -57 218 -57z" /> + <glyph glyph-name="_518" unicode="" +d="M1472 1408q26 0 45 -19t19 -45v-416q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v262l-213 -214l140 -140q9 -10 9 -23t-9 -22l-46 -46q-9 -9 -22 -9t-23 9l-140 141l-78 -79q126 -156 126 -359q0 -117 -45.5 -223.5t-123 -184t-184 -123t-223.5 -45.5t-223.5 45.5 +t-184 123t-123 184t-45.5 223.5t45.5 223.5t123 184t184 123t223.5 45.5q203 0 359 -126l78 78l-172 172q-9 10 -9 23t9 22l46 46q9 9 22 9t23 -9l172 -172l213 213h-261q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h416zM576 0q185 0 316.5 131.5t131.5 316.5t-131.5 316.5 +t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5z" /> + <glyph glyph-name="_519" unicode="" horiz-adv-x="1280" +d="M640 892q217 -24 364.5 -187.5t147.5 -384.5q0 -167 -87 -306t-236 -212t-319 -54q-133 15 -245.5 88t-182 188t-80.5 249q-12 155 52.5 292t186 224t271.5 103v132h-160q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h160v165l-92 -92q-10 -9 -23 -9t-22 9l-46 46q-9 9 -9 22 +t9 23l202 201q19 19 45 19t45 -19l202 -201q9 -10 9 -23t-9 -22l-46 -46q-9 -9 -22 -9t-23 9l-92 92v-165h160q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-160v-132zM576 -128q185 0 316.5 131.5t131.5 316.5t-131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5 +t131.5 -316.5t316.5 -131.5z" /> + <glyph glyph-name="_520" unicode="" horiz-adv-x="2048" +d="M1901 621q19 -19 19 -45t-19 -45l-294 -294q-9 -10 -22.5 -10t-22.5 10l-45 45q-10 9 -10 22.5t10 22.5l185 185h-294v-224q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v224h-132q-24 -217 -187.5 -364.5t-384.5 -147.5q-167 0 -306 87t-212 236t-54 319q15 133 88 245.5 +t188 182t249 80.5q155 12 292 -52.5t224 -186t103 -271.5h132v224q0 14 9 23t23 9h64q14 0 23 -9t9 -23v-224h294l-185 185q-10 9 -10 22.5t10 22.5l45 45q9 10 22.5 10t22.5 -10zM576 128q185 0 316.5 131.5t131.5 316.5t-131.5 316.5t-316.5 131.5t-316.5 -131.5 +t-131.5 -316.5t131.5 -316.5t316.5 -131.5z" /> + <glyph glyph-name="_521" unicode="" horiz-adv-x="1280" +d="M1152 960q0 -221 -147.5 -384.5t-364.5 -187.5v-612q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v612q-217 24 -364.5 187.5t-147.5 384.5q0 117 45.5 223.5t123 184t184 123t223.5 45.5t223.5 -45.5t184 -123t123 -184t45.5 -223.5zM576 512q185 0 316.5 131.5 +t131.5 316.5t-131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5z" /> + <glyph glyph-name="_522" unicode="" horiz-adv-x="1280" +d="M1024 576q0 185 -131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5t316.5 131.5t131.5 316.5zM1152 576q0 -117 -45.5 -223.5t-123 -184t-184 -123t-223.5 -45.5t-223.5 45.5t-184 123t-123 184t-45.5 223.5t45.5 223.5t123 184t184 123 +t223.5 45.5t223.5 -45.5t184 -123t123 -184t45.5 -223.5z" /> + <glyph glyph-name="_523" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="_524" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="_525" unicode="" +d="M1451 1408q35 0 60 -25t25 -60v-1366q0 -35 -25 -60t-60 -25h-391v595h199l30 232h-229v148q0 56 23.5 84t91.5 28l122 1v207q-63 9 -178 9q-136 0 -217.5 -80t-81.5 -226v-171h-200v-232h200v-595h-735q-35 0 -60 25t-25 60v1366q0 35 25 60t60 25h1366z" /> + <glyph glyph-name="_526" unicode="" horiz-adv-x="1280" +d="M0 939q0 108 37.5 203.5t103.5 166.5t152 123t185 78t202 26q158 0 294 -66.5t221 -193.5t85 -287q0 -96 -19 -188t-60 -177t-100 -149.5t-145 -103t-189 -38.5q-68 0 -135 32t-96 88q-10 -39 -28 -112.5t-23.5 -95t-20.5 -71t-26 -71t-32 -62.5t-46 -77.5t-62 -86.5 +l-14 -5l-9 10q-15 157 -15 188q0 92 21.5 206.5t66.5 287.5t52 203q-32 65 -32 169q0 83 52 156t132 73q61 0 95 -40.5t34 -102.5q0 -66 -44 -191t-44 -187q0 -63 45 -104.5t109 -41.5q55 0 102 25t78.5 68t56 95t38 110.5t20 111t6.5 99.5q0 173 -109.5 269.5t-285.5 96.5 +q-200 0 -334 -129.5t-134 -328.5q0 -44 12.5 -85t27 -65t27 -45.5t12.5 -30.5q0 -28 -15 -73t-37 -45q-2 0 -17 3q-51 15 -90.5 56t-61 94.5t-32.5 108t-11 106.5z" /> + <glyph glyph-name="_527" unicode="" +d="M985 562q13 0 97.5 -44t89.5 -53q2 -5 2 -15q0 -33 -17 -76q-16 -39 -71 -65.5t-102 -26.5q-57 0 -190 62q-98 45 -170 118t-148 185q-72 107 -71 194v8q3 91 74 158q24 22 52 22q6 0 18 -1.5t19 -1.5q19 0 26.5 -6.5t15.5 -27.5q8 -20 33 -88t25 -75q0 -21 -34.5 -57.5 +t-34.5 -46.5q0 -7 5 -15q34 -73 102 -137q56 -53 151 -101q12 -7 22 -7q15 0 54 48.5t52 48.5zM782 32q127 0 243.5 50t200.5 134t134 200.5t50 243.5t-50 243.5t-134 200.5t-200.5 134t-243.5 50t-243.5 -50t-200.5 -134t-134 -200.5t-50 -243.5q0 -203 120 -368l-79 -233 +l242 77q158 -104 345 -104zM782 1414q153 0 292.5 -60t240.5 -161t161 -240.5t60 -292.5t-60 -292.5t-161 -240.5t-240.5 -161t-292.5 -60q-195 0 -365 94l-417 -134l136 405q-108 178 -108 389q0 153 60 292.5t161 240.5t240.5 161t292.5 60z" /> + <glyph glyph-name="_528" unicode="" horiz-adv-x="1792" +d="M128 128h1024v128h-1024v-128zM128 640h1024v128h-1024v-128zM1696 192q0 40 -28 68t-68 28t-68 -28t-28 -68t28 -68t68 -28t68 28t28 68zM128 1152h1024v128h-1024v-128zM1696 704q0 40 -28 68t-68 28t-68 -28t-28 -68t28 -68t68 -28t68 28t28 68zM1696 1216 +q0 40 -28 68t-68 28t-68 -28t-28 -68t28 -68t68 -28t68 28t28 68zM1792 384v-384h-1792v384h1792zM1792 896v-384h-1792v384h1792zM1792 1408v-384h-1792v384h1792z" /> + <glyph glyph-name="_529" unicode="" horiz-adv-x="2048" +d="M704 640q-159 0 -271.5 112.5t-112.5 271.5t112.5 271.5t271.5 112.5t271.5 -112.5t112.5 -271.5t-112.5 -271.5t-271.5 -112.5zM1664 512h352q13 0 22.5 -9.5t9.5 -22.5v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-352v-352q0 -13 -9.5 -22.5t-22.5 -9.5h-192q-13 0 -22.5 9.5 +t-9.5 22.5v352h-352q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h352v352q0 13 9.5 22.5t22.5 9.5h192q13 0 22.5 -9.5t9.5 -22.5v-352zM928 288q0 -52 38 -90t90 -38h256v-238q-68 -50 -171 -50h-874q-121 0 -194 69t-73 190q0 53 3.5 103.5t14 109t26.5 108.5 +t43 97.5t62 81t85.5 53.5t111.5 20q19 0 39 -17q79 -61 154.5 -91.5t164.5 -30.5t164.5 30.5t154.5 91.5q20 17 39 17q132 0 217 -96h-223q-52 0 -90 -38t-38 -90v-192z" /> + <glyph glyph-name="_530" unicode="" horiz-adv-x="2048" +d="M704 640q-159 0 -271.5 112.5t-112.5 271.5t112.5 271.5t271.5 112.5t271.5 -112.5t112.5 -271.5t-112.5 -271.5t-271.5 -112.5zM1781 320l249 -249q9 -9 9 -23q0 -13 -9 -22l-136 -136q-9 -9 -22 -9q-14 0 -23 9l-249 249l-249 -249q-9 -9 -23 -9q-13 0 -22 9l-136 136 +q-9 9 -9 22q0 14 9 23l249 249l-249 249q-9 9 -9 23q0 13 9 22l136 136q9 9 22 9q14 0 23 -9l249 -249l249 249q9 9 23 9q13 0 22 -9l136 -136q9 -9 9 -22q0 -14 -9 -23zM1283 320l-181 -181q-37 -37 -37 -91q0 -53 37 -90l83 -83q-21 -3 -44 -3h-874q-121 0 -194 69 +t-73 190q0 53 3.5 103.5t14 109t26.5 108.5t43 97.5t62 81t85.5 53.5t111.5 20q19 0 39 -17q154 -122 319 -122t319 122q20 17 39 17q28 0 57 -6q-28 -27 -41 -50t-13 -56q0 -54 37 -91z" /> + <glyph glyph-name="_531" unicode="" horiz-adv-x="2048" +d="M256 512h1728q26 0 45 -19t19 -45v-448h-256v256h-1536v-256h-256v1216q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-704zM832 832q0 106 -75 181t-181 75t-181 -75t-75 -181t75 -181t181 -75t181 75t75 181zM2048 576v64q0 159 -112.5 271.5t-271.5 112.5h-704 +q-26 0 -45 -19t-19 -45v-384h1152z" /> + <glyph glyph-name="_532" unicode="" +d="M1536 1536l-192 -448h192v-192h-274l-55 -128h329v-192h-411l-357 -832l-357 832h-411v192h329l-55 128h-274v192h192l-192 448h256l323 -768h378l323 768h256zM768 320l108 256h-216z" /> + <glyph glyph-name="_533" unicode="" +d="M1088 1536q185 0 316.5 -93.5t131.5 -226.5v-896q0 -130 -125.5 -222t-305.5 -97l213 -202q16 -15 8 -35t-30 -20h-1056q-22 0 -30 20t8 35l213 202q-180 5 -305.5 97t-125.5 222v896q0 133 131.5 226.5t316.5 93.5h640zM768 192q80 0 136 56t56 136t-56 136t-136 56 +t-136 -56t-56 -136t56 -136t136 -56zM1344 768v512h-1152v-512h1152z" /> + <glyph glyph-name="_534" unicode="" +d="M1088 1536q185 0 316.5 -93.5t131.5 -226.5v-896q0 -130 -125.5 -222t-305.5 -97l213 -202q16 -15 8 -35t-30 -20h-1056q-22 0 -30 20t8 35l213 202q-180 5 -305.5 97t-125.5 222v896q0 133 131.5 226.5t316.5 93.5h640zM288 224q66 0 113 47t47 113t-47 113t-113 47 +t-113 -47t-47 -113t47 -113t113 -47zM704 768v512h-544v-512h544zM1248 224q66 0 113 47t47 113t-47 113t-113 47t-113 -47t-47 -113t47 -113t113 -47zM1408 768v512h-576v-512h576z" /> + <glyph glyph-name="_535" unicode="" horiz-adv-x="1792" +d="M597 1115v-1173q0 -25 -12.5 -42.5t-36.5 -17.5q-17 0 -33 8l-465 233q-21 10 -35.5 33.5t-14.5 46.5v1140q0 20 10 34t29 14q14 0 44 -15l511 -256q3 -3 3 -5zM661 1014l534 -866l-534 266v600zM1792 996v-1054q0 -25 -14 -40.5t-38 -15.5t-47 13l-441 220zM1789 1116 +q0 -3 -256.5 -419.5t-300.5 -487.5l-390 634l324 527q17 28 52 28q14 0 26 -6l541 -270q4 -2 4 -6z" /> + <glyph glyph-name="_536" unicode="" +d="M809 532l266 499h-112l-157 -312q-24 -48 -44 -92l-42 92l-155 312h-120l263 -493v-324h101v318zM1536 1408v-1536h-1536v1536h1536z" /> + <glyph glyph-name="_537" unicode="" horiz-adv-x="2296" +d="M478 -139q-8 -16 -27 -34.5t-37 -25.5q-25 -9 -51.5 3.5t-28.5 31.5q-1 22 40 55t68 38q23 4 34 -21.5t2 -46.5zM1819 -139q7 -16 26 -34.5t38 -25.5q25 -9 51.5 3.5t27.5 31.5q2 22 -39.5 55t-68.5 38q-22 4 -33 -21.5t-2 -46.5zM1867 -30q13 -27 56.5 -59.5t77.5 -41.5 +q45 -13 82 4.5t37 50.5q0 46 -67.5 100.5t-115.5 59.5q-40 5 -63.5 -37.5t-6.5 -76.5zM428 -30q-13 -27 -56 -59.5t-77 -41.5q-45 -13 -82 4.5t-37 50.5q0 46 67.5 100.5t115.5 59.5q40 5 63 -37.5t6 -76.5zM1158 1094h1q-41 0 -76 -15q27 -8 44 -30.5t17 -49.5 +q0 -35 -27 -60t-65 -25q-52 0 -80 43q-5 -23 -5 -42q0 -74 56 -126.5t135 -52.5q80 0 136 52.5t56 126.5t-56 126.5t-136 52.5zM1462 1312q-99 109 -220.5 131.5t-245.5 -44.5q27 60 82.5 96.5t118 39.5t121.5 -17t99.5 -74.5t44.5 -131.5zM2212 73q8 -11 -11 -42 +q7 -23 7 -40q1 -56 -44.5 -112.5t-109.5 -91.5t-118 -37q-48 -2 -92 21.5t-66 65.5q-687 -25 -1259 0q-23 -41 -66.5 -65t-92.5 -22q-86 3 -179.5 80.5t-92.5 160.5q2 22 7 40q-19 31 -11 42q6 10 31 1q14 22 41 51q-7 29 2 38q11 10 39 -4q29 20 59 34q0 29 13 37 +q23 12 51 -16q35 5 61 -2q18 -4 38 -19v73q-11 0 -18 2q-53 10 -97 44.5t-55 87.5q-9 38 0 81q15 62 93 95q2 17 19 35.5t36 23.5t33 -7.5t19 -30.5h13q46 -5 60 -23q3 -3 5 -7q10 1 30.5 3.5t30.5 3.5q-15 11 -30 17q-23 40 -91 43q0 6 1 10q-62 2 -118.5 18.5t-84.5 47.5 +q-32 36 -42.5 92t-2.5 112q16 126 90 179q23 16 52 4.5t32 -40.5q0 -1 1.5 -14t2.5 -21t3 -20t5.5 -19t8.5 -10q27 -14 76 -12q48 46 98 74q-40 4 -162 -14l47 46q61 58 163 111q145 73 282 86q-20 8 -41 15.5t-47 14t-42.5 10.5t-47.5 11t-43 10q595 126 904 -139 +q98 -84 158 -222q85 -10 121 9h1q5 3 8.5 10t5.5 19t3 19.5t3 21.5l1 14q3 28 32 40t52 -5q73 -52 91 -178q7 -57 -3.5 -113t-42.5 -91q-28 -32 -83.5 -48.5t-115.5 -18.5v-10q-71 -2 -95 -43q-14 -5 -31 -17q11 -1 32 -3.5t30 -3.5q1 5 5 8q16 18 60 23h13q5 18 19 30t33 8 +t36 -23t19 -36q79 -32 93 -95q9 -40 1 -81q-12 -53 -56 -88t-97 -44q-10 -2 -17 -2q0 -49 -1 -73q20 15 38 19q26 7 61 2q28 28 51 16q14 -9 14 -37q33 -16 59 -34q27 13 38 4q10 -10 2 -38q28 -30 41 -51q23 8 31 -1zM1937 1025q0 -29 -9 -54q82 -32 112 -132 +q4 37 -9.5 98.5t-41.5 90.5q-20 19 -36 17t-16 -20zM1859 925q35 -42 47.5 -108.5t-0.5 -124.5q67 13 97 45q13 14 18 28q-3 64 -31 114.5t-79 66.5q-15 -15 -52 -21zM1822 921q-30 0 -44 1q42 -115 53 -239q21 0 43 3q16 68 1 135t-53 100zM258 839q30 100 112 132 +q-9 25 -9 54q0 18 -16.5 20t-35.5 -17q-28 -29 -41.5 -90.5t-9.5 -98.5zM294 737q29 -31 97 -45q-13 58 -0.5 124.5t47.5 108.5v0q-37 6 -52 21q-51 -16 -78.5 -66t-31.5 -115q9 -17 18 -28zM471 683q14 124 73 235q-19 -4 -55 -18l-45 -19v1q-46 -89 -20 -196q25 -3 47 -3z +M1434 644q8 -38 16.5 -108.5t11.5 -89.5q3 -18 9.5 -21.5t23.5 4.5q40 20 62 85.5t23 125.5q-24 2 -146 4zM1152 1285q-116 0 -199 -82.5t-83 -198.5q0 -117 83 -199.5t199 -82.5t199 82.5t83 199.5q0 116 -83 198.5t-199 82.5zM1380 646q-105 2 -211 0v1q-1 -27 2.5 -86 +t13.5 -66q29 -14 93.5 -14.5t95.5 10.5q9 3 11 39t-0.5 69.5t-4.5 46.5zM1112 447q8 4 9.5 48t-0.5 88t-4 63v1q-212 -3 -214 -3q-4 -20 -7 -62t0 -83t14 -46q34 -15 101 -16t101 10zM718 636q-16 -59 4.5 -118.5t77.5 -84.5q15 -8 24 -5t12 21q3 16 8 90t10 103 +q-69 -2 -136 -6zM591 510q3 -23 -34 -36q132 -141 271.5 -240t305.5 -154q172 49 310.5 146t293.5 250q-33 13 -30 34q0 2 0.5 3.5t1.5 3t1 2.5v1v-1q-17 2 -50 5.5t-48 4.5q-26 -90 -82 -132q-51 -38 -82 1q-5 6 -9 14q-7 13 -17 62q-2 -5 -5 -9t-7.5 -7t-8 -5.5t-9.5 -4 +l-10 -2.5t-12 -2l-12 -1.5t-13.5 -1t-13.5 -0.5q-106 -9 -163 11q-4 -17 -10 -26.5t-21 -15t-23 -7t-36 -3.5q-6 -1 -9 -1q-179 -17 -203 40q-2 -63 -56 -54q-47 8 -91 54q-12 13 -20 26q-17 29 -26 65q-58 -6 -87 -10q1 -2 4 -10zM507 -118q3 14 3 30q-17 71 -51 130 +t-73 70q-41 12 -101.5 -14.5t-104.5 -80t-39 -107.5q35 -53 100 -93t119 -42q51 -2 94 28t53 79zM510 53q23 -63 27 -119q195 113 392 174q-98 52 -180.5 120t-179.5 165q-6 -4 -29 -13q0 -1 -1 -4t-1 -5q31 -18 22 -37q-12 -23 -56 -34q-10 -13 -29 -24h-1q-2 -83 1 -150 +q19 -34 35 -73zM579 -113q532 -21 1145 0q-254 147 -428 196q-76 -35 -156 -57q-8 -3 -16 0q-65 21 -129 49q-208 -60 -416 -188h-1v-1q1 0 1 1zM1763 -67q4 54 28 120q14 38 33 71l-1 -1q3 77 3 153q-15 8 -30 25q-42 9 -56 33q-9 20 22 38q-2 4 -2 9q-16 4 -28 12 +q-204 -190 -383 -284q198 -59 414 -176zM2155 -90q5 54 -39 107.5t-104 80t-102 14.5q-38 -11 -72.5 -70.5t-51.5 -129.5q0 -16 3 -30q10 -49 53 -79t94 -28q54 2 119 42t100 93z" /> + <glyph glyph-name="_538" unicode="" horiz-adv-x="2304" +d="M1524 -25q0 -68 -48 -116t-116 -48t-116.5 48t-48.5 116t48.5 116.5t116.5 48.5t116 -48.5t48 -116.5zM775 -25q0 -68 -48.5 -116t-116.5 -48t-116 48t-48 116t48 116.5t116 48.5t116.5 -48.5t48.5 -116.5zM0 1469q57 -60 110.5 -104.5t121 -82t136 -63t166 -45.5 +t200 -31.5t250 -18.5t304 -9.5t372.5 -2.5q139 0 244.5 -5t181 -16.5t124 -27.5t71 -39.5t24 -51.5t-19.5 -64t-56.5 -76.5t-89.5 -91t-116 -104.5t-139 -119q-185 -157 -286 -247q29 51 76.5 109t94 105.5t94.5 98.5t83 91.5t54 80.5t13 70t-45.5 55.5t-116.5 41t-204 23.5 +t-304 5q-168 -2 -314 6t-256 23t-204.5 41t-159.5 51.5t-122.5 62.5t-91.5 66.5t-68 71.5t-50.5 69.5t-40 68t-36.5 59.5z" /> + <glyph glyph-name="_539" unicode="" horiz-adv-x="1792" +d="M896 1472q-169 0 -323 -66t-265.5 -177.5t-177.5 -265.5t-66 -323t66 -323t177.5 -265.5t265.5 -177.5t323 -66t323 66t265.5 177.5t177.5 265.5t66 323t-66 323t-177.5 265.5t-265.5 177.5t-323 66zM896 1536q182 0 348 -71t286 -191t191 -286t71 -348t-71 -348 +t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71zM496 704q16 0 16 -16v-480q0 -16 -16 -16h-32q-16 0 -16 16v480q0 16 16 16h32zM896 640q53 0 90.5 -37.5t37.5 -90.5q0 -35 -17.5 -64t-46.5 -46v-114q0 -14 -9 -23 +t-23 -9h-64q-14 0 -23 9t-9 23v114q-29 17 -46.5 46t-17.5 64q0 53 37.5 90.5t90.5 37.5zM896 1408q209 0 385.5 -103t279.5 -279.5t103 -385.5t-103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103zM544 928v-96 +q0 -14 9 -23t23 -9h64q14 0 23 9t9 23v96q0 93 65.5 158.5t158.5 65.5t158.5 -65.5t65.5 -158.5v-96q0 -14 9 -23t23 -9h64q14 0 23 9t9 23v96q0 146 -103 249t-249 103t-249 -103t-103 -249zM1408 192v512q0 26 -19 45t-45 19h-896q-26 0 -45 -19t-19 -45v-512 +q0 -26 19 -45t45 -19h896q26 0 45 19t19 45z" /> + <glyph glyph-name="_540" unicode="" horiz-adv-x="2304" +d="M1920 1024v-768h-1664v768h1664zM2048 448h128v384h-128v288q0 14 -9 23t-23 9h-1856q-14 0 -23 -9t-9 -23v-960q0 -14 9 -23t23 -9h1856q14 0 23 9t9 23v288zM2304 832v-384q0 -53 -37.5 -90.5t-90.5 -37.5v-160q0 -66 -47 -113t-113 -47h-1856q-66 0 -113 47t-47 113 +v960q0 66 47 113t113 47h1856q66 0 113 -47t47 -113v-160q53 0 90.5 -37.5t37.5 -90.5z" /> + <glyph glyph-name="_541" unicode="" horiz-adv-x="2304" +d="M256 256v768h1280v-768h-1280zM2176 960q53 0 90.5 -37.5t37.5 -90.5v-384q0 -53 -37.5 -90.5t-90.5 -37.5v-160q0 -66 -47 -113t-113 -47h-1856q-66 0 -113 47t-47 113v960q0 66 47 113t113 47h1856q66 0 113 -47t47 -113v-160zM2176 448v384h-128v288q0 14 -9 23t-23 9 +h-1856q-14 0 -23 -9t-9 -23v-960q0 -14 9 -23t23 -9h1856q14 0 23 9t9 23v288h128z" /> + <glyph glyph-name="_542" unicode="" horiz-adv-x="2304" +d="M256 256v768h896v-768h-896zM2176 960q53 0 90.5 -37.5t37.5 -90.5v-384q0 -53 -37.5 -90.5t-90.5 -37.5v-160q0 -66 -47 -113t-113 -47h-1856q-66 0 -113 47t-47 113v960q0 66 47 113t113 47h1856q66 0 113 -47t47 -113v-160zM2176 448v384h-128v288q0 14 -9 23t-23 9 +h-1856q-14 0 -23 -9t-9 -23v-960q0 -14 9 -23t23 -9h1856q14 0 23 9t9 23v288h128z" /> + <glyph glyph-name="_543" unicode="" horiz-adv-x="2304" +d="M256 256v768h512v-768h-512zM2176 960q53 0 90.5 -37.5t37.5 -90.5v-384q0 -53 -37.5 -90.5t-90.5 -37.5v-160q0 -66 -47 -113t-113 -47h-1856q-66 0 -113 47t-47 113v960q0 66 47 113t113 47h1856q66 0 113 -47t47 -113v-160zM2176 448v384h-128v288q0 14 -9 23t-23 9 +h-1856q-14 0 -23 -9t-9 -23v-960q0 -14 9 -23t23 -9h1856q14 0 23 9t9 23v288h128z" /> + <glyph glyph-name="_544" unicode="" horiz-adv-x="2304" +d="M2176 960q53 0 90.5 -37.5t37.5 -90.5v-384q0 -53 -37.5 -90.5t-90.5 -37.5v-160q0 -66 -47 -113t-113 -47h-1856q-66 0 -113 47t-47 113v960q0 66 47 113t113 47h1856q66 0 113 -47t47 -113v-160zM2176 448v384h-128v288q0 14 -9 23t-23 9h-1856q-14 0 -23 -9t-9 -23 +v-960q0 -14 9 -23t23 -9h1856q14 0 23 9t9 23v288h128z" /> + <glyph glyph-name="_545" unicode="" horiz-adv-x="1280" +d="M1133 493q31 -30 14 -69q-17 -40 -59 -40h-382l201 -476q10 -25 0 -49t-34 -35l-177 -75q-25 -10 -49 0t-35 34l-191 452l-312 -312q-19 -19 -45 -19q-12 0 -24 5q-40 17 -40 59v1504q0 42 40 59q12 5 24 5q27 0 45 -19z" /> + <glyph glyph-name="_546" unicode="" horiz-adv-x="1024" +d="M832 1408q-320 0 -320 -224v-416h128v-128h-128v-544q0 -224 320 -224h64v-128h-64q-272 0 -384 146q-112 -146 -384 -146h-64v128h64q320 0 320 224v544h-128v128h128v416q0 224 -320 224h-64v128h64q272 0 384 -146q112 146 384 146h64v-128h-64z" /> + <glyph glyph-name="_547" unicode="" horiz-adv-x="2048" +d="M2048 1152h-128v-1024h128v-384h-384v128h-1280v-128h-384v384h128v1024h-128v384h384v-128h1280v128h384v-384zM1792 1408v-128h128v128h-128zM128 1408v-128h128v128h-128zM256 -128v128h-128v-128h128zM1664 0v128h128v1024h-128v128h-1280v-128h-128v-1024h128v-128 +h1280zM1920 -128v128h-128v-128h128zM1280 896h384v-768h-896v256h-384v768h896v-256zM512 512h640v512h-640v-512zM1536 256v512h-256v-384h-384v-128h640z" /> + <glyph glyph-name="_548" unicode="" horiz-adv-x="2304" +d="M2304 768h-128v-640h128v-384h-384v128h-896v-128h-384v384h128v128h-384v-128h-384v384h128v640h-128v384h384v-128h896v128h384v-384h-128v-128h384v128h384v-384zM2048 1024v-128h128v128h-128zM1408 1408v-128h128v128h-128zM128 1408v-128h128v128h-128zM256 256 +v128h-128v-128h128zM1536 384h-128v-128h128v128zM384 384h896v128h128v640h-128v128h-896v-128h-128v-640h128v-128zM896 -128v128h-128v-128h128zM2176 -128v128h-128v-128h128zM2048 128v640h-128v128h-384v-384h128v-384h-384v128h-384v-128h128v-128h896v128h128z" /> + <glyph glyph-name="_549" unicode="" +d="M1024 288v-416h-928q-40 0 -68 28t-28 68v1344q0 40 28 68t68 28h1344q40 0 68 -28t28 -68v-928h-416q-40 0 -68 -28t-28 -68zM1152 256h381q-15 -82 -65 -132l-184 -184q-50 -50 -132 -65v381z" /> + <glyph glyph-name="_550" unicode="" +d="M1400 256h-248v-248q29 10 41 22l185 185q12 12 22 41zM1120 384h288v896h-1280v-1280h896v288q0 40 28 68t68 28zM1536 1312v-1024q0 -40 -20 -88t-48 -76l-184 -184q-28 -28 -76 -48t-88 -20h-1024q-40 0 -68 28t-28 68v1344q0 40 28 68t68 28h1344q40 0 68 -28t28 -68 +z" /> + <glyph glyph-name="_551" unicode="" horiz-adv-x="2304" +d="M1951 538q0 -26 -15.5 -44.5t-38.5 -23.5q-8 -2 -18 -2h-153v140h153q10 0 18 -2q23 -5 38.5 -23.5t15.5 -44.5zM1933 751q0 -25 -15 -42t-38 -21q-3 -1 -15 -1h-139v129h139q3 0 8.5 -0.5t6.5 -0.5q23 -4 38 -21.5t15 -42.5zM728 587v308h-228v-308q0 -58 -38 -94.5 +t-105 -36.5q-108 0 -229 59v-112q53 -15 121 -23t109 -9l42 -1q328 0 328 217zM1442 403v113q-99 -52 -200 -59q-108 -8 -169 41t-61 142t61 142t169 41q101 -7 200 -58v112q-48 12 -100 19.5t-80 9.5l-28 2q-127 6 -218.5 -14t-140.5 -60t-71 -88t-22 -106t22 -106t71 -88 +t140.5 -60t218.5 -14q101 4 208 31zM2176 518q0 54 -43 88.5t-109 39.5v3q57 8 89 41.5t32 79.5q0 55 -41 88t-107 36q-3 0 -12 0.5t-14 0.5h-455v-510h491q74 0 121.5 36.5t47.5 96.5zM2304 1280v-1280q0 -52 -38 -90t-90 -38h-2048q-52 0 -90 38t-38 90v1280q0 52 38 90 +t90 38h2048q52 0 90 -38t38 -90z" /> + <glyph glyph-name="_552" unicode="" horiz-adv-x="2304" +d="M858 295v693q-106 -41 -172 -135.5t-66 -211.5t66 -211.5t172 -134.5zM1362 641q0 117 -66 211.5t-172 135.5v-694q106 41 172 135.5t66 211.5zM1577 641q0 -159 -78.5 -294t-213.5 -213.5t-294 -78.5q-119 0 -227.5 46.5t-187 125t-125 187t-46.5 227.5q0 159 78.5 294 +t213.5 213.5t294 78.5t294 -78.5t213.5 -213.5t78.5 -294zM1960 634q0 139 -55.5 261.5t-147.5 205.5t-213.5 131t-252.5 48h-301q-176 0 -323.5 -81t-235 -230t-87.5 -335q0 -171 87 -317.5t236 -231.5t323 -85h301q129 0 251.5 50.5t214.5 135t147.5 202.5t55.5 246z +M2304 1280v-1280q0 -52 -38 -90t-90 -38h-2048q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h2048q52 0 90 -38t38 -90z" /> + <glyph glyph-name="_553" unicode="" horiz-adv-x="1792" +d="M1664 -96v1088q0 13 -9.5 22.5t-22.5 9.5h-1088q-13 0 -22.5 -9.5t-9.5 -22.5v-1088q0 -13 9.5 -22.5t22.5 -9.5h1088q13 0 22.5 9.5t9.5 22.5zM1792 992v-1088q0 -66 -47 -113t-113 -47h-1088q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h1088q66 0 113 -47t47 -113 +zM1408 1376v-160h-128v160q0 13 -9.5 22.5t-22.5 9.5h-1088q-13 0 -22.5 -9.5t-9.5 -22.5v-1088q0 -13 9.5 -22.5t22.5 -9.5h160v-128h-160q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h1088q66 0 113 -47t47 -113z" /> + <glyph glyph-name="_554" unicode="" horiz-adv-x="2304" +d="M1728 1088l-384 -704h768zM448 1088l-384 -704h768zM1269 1280q-14 -40 -45.5 -71.5t-71.5 -45.5v-1291h608q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-1344q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h608v1291q-40 14 -71.5 45.5t-45.5 71.5h-491q-14 0 -23 9t-9 23v64 +q0 14 9 23t23 9h491q21 57 70 92.5t111 35.5t111 -35.5t70 -92.5h491q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-491zM1088 1264q33 0 56.5 23.5t23.5 56.5t-23.5 56.5t-56.5 23.5t-56.5 -23.5t-23.5 -56.5t23.5 -56.5t56.5 -23.5zM2176 384q0 -73 -46.5 -131t-117.5 -91 +t-144.5 -49.5t-139.5 -16.5t-139.5 16.5t-144.5 49.5t-117.5 91t-46.5 131q0 11 35 81t92 174.5t107 195.5t102 184t56 100q18 33 56 33t56 -33q4 -7 56 -100t102 -184t107 -195.5t92 -174.5t35 -81zM896 384q0 -73 -46.5 -131t-117.5 -91t-144.5 -49.5t-139.5 -16.5 +t-139.5 16.5t-144.5 49.5t-117.5 91t-46.5 131q0 11 35 81t92 174.5t107 195.5t102 184t56 100q18 33 56 33t56 -33q4 -7 56 -100t102 -184t107 -195.5t92 -174.5t35 -81z" /> + <glyph glyph-name="_555" unicode="" +d="M1408 1408q0 -261 -106.5 -461.5t-266.5 -306.5q160 -106 266.5 -306.5t106.5 -461.5h96q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-1472q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h96q0 261 106.5 461.5t266.5 306.5q-160 106 -266.5 306.5t-106.5 461.5h-96q-14 0 -23 9 +t-9 23v64q0 14 9 23t23 9h1472q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-96zM874 700q77 29 149 92.5t129.5 152.5t92.5 210t35 253h-1024q0 -132 35 -253t92.5 -210t129.5 -152.5t149 -92.5q19 -7 30.5 -23.5t11.5 -36.5t-11.5 -36.5t-30.5 -23.5q-77 -29 -149 -92.5 +t-129.5 -152.5t-92.5 -210t-35 -253h1024q0 132 -35 253t-92.5 210t-129.5 152.5t-149 92.5q-19 7 -30.5 23.5t-11.5 36.5t11.5 36.5t30.5 23.5z" /> + <glyph glyph-name="_556" unicode="" +d="M1408 1408q0 -261 -106.5 -461.5t-266.5 -306.5q160 -106 266.5 -306.5t106.5 -461.5h96q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-1472q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h96q0 261 106.5 461.5t266.5 306.5q-160 106 -266.5 306.5t-106.5 461.5h-96q-14 0 -23 9 +t-9 23v64q0 14 9 23t23 9h1472q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-96zM1280 1408h-1024q0 -66 9 -128h1006q9 61 9 128zM1280 -128q0 130 -34 249.5t-90.5 208t-126.5 152t-146 94.5h-230q-76 -31 -146 -94.5t-126.5 -152t-90.5 -208t-34 -249.5h1024z" /> + <glyph glyph-name="_557" unicode="" +d="M1408 1408q0 -261 -106.5 -461.5t-266.5 -306.5q160 -106 266.5 -306.5t106.5 -461.5h96q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-1472q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h96q0 261 106.5 461.5t266.5 306.5q-160 106 -266.5 306.5t-106.5 461.5h-96q-14 0 -23 9 +t-9 23v64q0 14 9 23t23 9h1472q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-96zM1280 1408h-1024q0 -206 85 -384h854q85 178 85 384zM1223 192q-54 141 -145.5 241.5t-194.5 142.5h-230q-103 -42 -194.5 -142.5t-145.5 -241.5h910z" /> + <glyph glyph-name="_558" unicode="" +d="M1408 1408q0 -261 -106.5 -461.5t-266.5 -306.5q160 -106 266.5 -306.5t106.5 -461.5h96q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-1472q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h96q0 261 106.5 461.5t266.5 306.5q-160 106 -266.5 306.5t-106.5 461.5h-96q-14 0 -23 9 +t-9 23v64q0 14 9 23t23 9h1472q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-96zM874 700q77 29 149 92.5t129.5 152.5t92.5 210t35 253h-1024q0 -132 35 -253t92.5 -210t129.5 -152.5t149 -92.5q19 -7 30.5 -23.5t11.5 -36.5t-11.5 -36.5t-30.5 -23.5q-137 -51 -244 -196 +h700q-107 145 -244 196q-19 7 -30.5 23.5t-11.5 36.5t11.5 36.5t30.5 23.5z" /> + <glyph glyph-name="_559" unicode="" +d="M1504 -64q14 0 23 -9t9 -23v-128q0 -14 -9 -23t-23 -9h-1472q-14 0 -23 9t-9 23v128q0 14 9 23t23 9h1472zM130 0q3 55 16 107t30 95t46 87t53.5 76t64.5 69.5t66 60t70.5 55t66.5 47.5t65 43q-43 28 -65 43t-66.5 47.5t-70.5 55t-66 60t-64.5 69.5t-53.5 76t-46 87 +t-30 95t-16 107h1276q-3 -55 -16 -107t-30 -95t-46 -87t-53.5 -76t-64.5 -69.5t-66 -60t-70.5 -55t-66.5 -47.5t-65 -43q43 -28 65 -43t66.5 -47.5t70.5 -55t66 -60t64.5 -69.5t53.5 -76t46 -87t30 -95t16 -107h-1276zM1504 1536q14 0 23 -9t9 -23v-128q0 -14 -9 -23t-23 -9 +h-1472q-14 0 -23 9t-9 23v128q0 14 9 23t23 9h1472z" /> + <glyph glyph-name="_560" unicode="" +d="M768 1152q-53 0 -90.5 -37.5t-37.5 -90.5v-128h-32v93q0 48 -32 81.5t-80 33.5q-46 0 -79 -33t-33 -79v-429l-32 30v172q0 48 -32 81.5t-80 33.5q-46 0 -79 -33t-33 -79v-224q0 -47 35 -82l310 -296q39 -39 39 -102q0 -26 19 -45t45 -19h640q26 0 45 19t19 45v25 +q0 41 10 77l108 436q10 36 10 77v246q0 48 -32 81.5t-80 33.5q-46 0 -79 -33t-33 -79v-32h-32v125q0 40 -25 72.5t-64 40.5q-14 2 -23 2q-46 0 -79 -33t-33 -79v-128h-32v122q0 51 -32.5 89.5t-82.5 43.5q-5 1 -13 1zM768 1280q84 0 149 -50q57 34 123 34q59 0 111 -27 +t86 -76q27 7 59 7q100 0 170 -71.5t70 -171.5v-246q0 -51 -13 -108l-109 -436q-6 -24 -6 -71q0 -80 -56 -136t-136 -56h-640q-84 0 -138 58.5t-54 142.5l-308 296q-76 73 -76 175v224q0 99 70.5 169.5t169.5 70.5q11 0 16 -1q6 95 75.5 160t164.5 65q52 0 98 -21 +q72 69 174 69z" /> + <glyph glyph-name="_561" unicode="" horiz-adv-x="1792" +d="M880 1408q-46 0 -79 -33t-33 -79v-656h-32v528q0 46 -33 79t-79 33t-79 -33t-33 -79v-528v-256l-154 205q-38 51 -102 51q-53 0 -90.5 -37.5t-37.5 -90.5q0 -43 26 -77l384 -512q38 -51 102 -51h688q34 0 61 22t34 56l76 405q5 32 5 59v498q0 46 -33 79t-79 33t-79 -33 +t-33 -79v-272h-32v528q0 46 -33 79t-79 33t-79 -33t-33 -79v-528h-32v656q0 46 -33 79t-79 33zM880 1536q68 0 125.5 -35.5t88.5 -96.5q19 4 42 4q99 0 169.5 -70.5t70.5 -169.5v-17q105 6 180.5 -64t75.5 -175v-498q0 -40 -8 -83l-76 -404q-14 -79 -76.5 -131t-143.5 -52 +h-688q-60 0 -114.5 27.5t-90.5 74.5l-384 512q-51 68 -51 154q0 106 75 181t181 75q78 0 128 -34v434q0 99 70.5 169.5t169.5 70.5q23 0 42 -4q31 61 88.5 96.5t125.5 35.5z" /> + <glyph glyph-name="_562" unicode="" horiz-adv-x="1792" +d="M1073 -128h-177q-163 0 -226 141q-23 49 -23 102v5q-62 30 -98.5 88.5t-36.5 127.5q0 38 5 48h-261q-106 0 -181 75t-75 181t75 181t181 75h113l-44 17q-74 28 -119.5 93.5t-45.5 145.5q0 106 75 181t181 75q46 0 91 -17l628 -239h401q106 0 181 -75t75 -181v-668 +q0 -88 -54 -157.5t-140 -90.5l-339 -85q-92 -23 -186 -23zM1024 583l-155 -71l-163 -74q-30 -14 -48 -41.5t-18 -60.5q0 -46 33 -79t79 -33q26 0 46 10l338 154q-49 10 -80.5 50t-31.5 90v55zM1344 272q0 46 -33 79t-79 33q-26 0 -46 -10l-290 -132q-28 -13 -37 -17 +t-30.5 -17t-29.5 -23.5t-16 -29t-8 -40.5q0 -50 31.5 -82t81.5 -32q20 0 38 9l352 160q30 14 48 41.5t18 60.5zM1112 1024l-650 248q-24 8 -46 8q-53 0 -90.5 -37.5t-37.5 -90.5q0 -40 22.5 -73t59.5 -47l526 -200v-64h-640q-53 0 -90.5 -37.5t-37.5 -90.5t37.5 -90.5 +t90.5 -37.5h535l233 106v198q0 63 46 106l111 102h-69zM1073 0q82 0 155 19l339 85q43 11 70 45.5t27 78.5v668q0 53 -37.5 90.5t-90.5 37.5h-308l-136 -126q-36 -33 -36 -82v-296q0 -46 33 -77t79 -31t79 35t33 81v208h32v-208q0 -70 -57 -114q52 -8 86.5 -48.5t34.5 -93.5 +q0 -42 -23 -78t-61 -53l-310 -141h91z" /> + <glyph glyph-name="_563" unicode="" horiz-adv-x="2048" +d="M1151 1536q61 0 116 -28t91 -77l572 -781q118 -159 118 -359v-355q0 -80 -56 -136t-136 -56h-384q-80 0 -136 56t-56 136v177l-286 143h-546q-80 0 -136 56t-56 136v32q0 119 84.5 203.5t203.5 84.5h420l42 128h-686q-100 0 -173.5 67.5t-81.5 166.5q-65 79 -65 182v32 +q0 80 56 136t136 56h959zM1920 -64v355q0 157 -93 284l-573 781q-39 52 -103 52h-959q-26 0 -45 -19t-19 -45q0 -32 1.5 -49.5t9.5 -40.5t25 -43q10 31 35.5 50t56.5 19h832v-32h-832q-26 0 -45 -19t-19 -45q0 -44 3 -58q8 -44 44 -73t81 -29h640h91q40 0 68 -28t28 -68 +q0 -15 -5 -30l-64 -192q-10 -29 -35 -47.5t-56 -18.5h-443q-66 0 -113 -47t-47 -113v-32q0 -26 19 -45t45 -19h561q16 0 29 -7l317 -158q24 -13 38.5 -36t14.5 -50v-197q0 -26 19 -45t45 -19h384q26 0 45 19t19 45z" /> + <glyph glyph-name="_564" unicode="" horiz-adv-x="2048" +d="M459 -256q-77 0 -137.5 47.5t-79.5 122.5l-101 401q-13 57 -13 108q0 45 -5 67l-116 477q-7 27 -7 57q0 93 62 161t155 78q17 85 82.5 139t152.5 54q83 0 148 -51.5t85 -132.5l83 -348l103 428q20 81 85 132.5t148 51.5q89 0 155.5 -57.5t80.5 -144.5q92 -10 152 -79 +t60 -162q0 -24 -7 -59l-123 -512q10 7 37.5 28.5t38.5 29.5t35 23t41 20.5t41.5 11t49.5 5.5q105 0 180 -74t75 -179q0 -62 -28.5 -118t-78.5 -94l-507 -380q-68 -51 -153 -51h-694zM1104 1408q-38 0 -68.5 -24t-39.5 -62l-164 -682h-127l-145 602q-9 38 -39.5 62t-68.5 24 +q-48 0 -80 -33t-32 -80q0 -15 3 -28l132 -547h-26l-99 408q-9 37 -40 62.5t-69 25.5q-47 0 -80 -33t-33 -79q0 -14 3 -26l116 -478q7 -28 9 -86t10 -88l100 -401q8 -32 34 -52.5t59 -20.5h694q42 0 76 26l507 379q56 43 56 110q0 52 -37.5 88.5t-89.5 36.5q-43 0 -77 -26 +l-307 -230v227q0 4 32 138t68 282t39 161q4 18 4 29q0 47 -32 81t-79 34q-39 0 -69.5 -24t-39.5 -62l-116 -482h-26l150 624q3 14 3 28q0 48 -31.5 82t-79.5 34z" /> + <glyph glyph-name="_565" unicode="" horiz-adv-x="1792" +d="M640 1408q-53 0 -90.5 -37.5t-37.5 -90.5v-512v-384l-151 202q-41 54 -107 54q-52 0 -89 -38t-37 -90q0 -43 26 -77l384 -512q38 -51 102 -51h718q22 0 39.5 13.5t22.5 34.5l92 368q24 96 24 194v217q0 41 -28 71t-68 30t-68 -28t-28 -68h-32v61q0 48 -32 81.5t-80 33.5 +q-46 0 -79 -33t-33 -79v-64h-32v90q0 55 -37 94.5t-91 39.5q-53 0 -90.5 -37.5t-37.5 -90.5v-96h-32v570q0 55 -37 94.5t-91 39.5zM640 1536q107 0 181.5 -77.5t74.5 -184.5v-220q22 2 32 2q99 0 173 -69q47 21 99 21q113 0 184 -87q27 7 56 7q94 0 159 -67.5t65 -161.5 +v-217q0 -116 -28 -225l-92 -368q-16 -64 -68 -104.5t-118 -40.5h-718q-60 0 -114.5 27.5t-90.5 74.5l-384 512q-51 68 -51 154q0 105 74.5 180.5t179.5 75.5q71 0 130 -35v547q0 106 75 181t181 75zM768 128v384h-32v-384h32zM1024 128v384h-32v-384h32zM1280 128v384h-32 +v-384h32z" /> + <glyph glyph-name="_566" unicode="" +d="M1288 889q60 0 107 -23q141 -63 141 -226v-177q0 -94 -23 -186l-85 -339q-21 -86 -90.5 -140t-157.5 -54h-668q-106 0 -181 75t-75 181v401l-239 628q-17 45 -17 91q0 106 75 181t181 75q80 0 145.5 -45.5t93.5 -119.5l17 -44v113q0 106 75 181t181 75t181 -75t75 -181 +v-261q27 5 48 5q69 0 127.5 -36.5t88.5 -98.5zM1072 896q-33 0 -60.5 -18t-41.5 -48l-74 -163l-71 -155h55q50 0 90 -31.5t50 -80.5l154 338q10 20 10 46q0 46 -33 79t-79 33zM1293 761q-22 0 -40.5 -8t-29 -16t-23.5 -29.5t-17 -30.5t-17 -37l-132 -290q-10 -20 -10 -46 +q0 -46 33 -79t79 -33q33 0 60.5 18t41.5 48l160 352q9 18 9 38q0 50 -32 81.5t-82 31.5zM128 1120q0 -22 8 -46l248 -650v-69l102 111q43 46 106 46h198l106 233v535q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5v-640h-64l-200 526q-14 37 -47 59.5t-73 22.5 +q-53 0 -90.5 -37.5t-37.5 -90.5zM1180 -128q44 0 78.5 27t45.5 70l85 339q19 73 19 155v91l-141 -310q-17 -38 -53 -61t-78 -23q-53 0 -93.5 34.5t-48.5 86.5q-44 -57 -114 -57h-208v32h208q46 0 81 33t35 79t-31 79t-77 33h-296q-49 0 -82 -36l-126 -136v-308 +q0 -53 37.5 -90.5t90.5 -37.5h668z" /> + <glyph glyph-name="_567" unicode="" horiz-adv-x="1973" +d="M857 992v-117q0 -13 -9.5 -22t-22.5 -9h-298v-812q0 -13 -9 -22.5t-22 -9.5h-135q-13 0 -22.5 9t-9.5 23v812h-297q-13 0 -22.5 9t-9.5 22v117q0 14 9 23t23 9h793q13 0 22.5 -9.5t9.5 -22.5zM1895 995l77 -961q1 -13 -8 -24q-10 -10 -23 -10h-134q-12 0 -21 8.5 +t-10 20.5l-46 588l-189 -425q-8 -19 -29 -19h-120q-20 0 -29 19l-188 427l-45 -590q-1 -12 -10 -20.5t-21 -8.5h-135q-13 0 -23 10q-9 10 -9 24l78 961q1 12 10 20.5t21 8.5h142q20 0 29 -19l220 -520q10 -24 20 -51q3 7 9.5 24.5t10.5 26.5l221 520q9 19 29 19h141 +q13 0 22 -8.5t10 -20.5z" /> + <glyph glyph-name="_568" unicode="" horiz-adv-x="1792" +d="M1042 833q0 88 -60 121q-33 18 -117 18h-123v-281h162q66 0 102 37t36 105zM1094 548l205 -373q8 -17 -1 -31q-8 -16 -27 -16h-152q-20 0 -28 17l-194 365h-155v-350q0 -14 -9 -23t-23 -9h-134q-14 0 -23 9t-9 23v960q0 14 9 23t23 9h294q128 0 190 -24q85 -31 134 -109 +t49 -180q0 -92 -42.5 -165.5t-115.5 -109.5q6 -10 9 -16zM896 1376q-150 0 -286 -58.5t-234.5 -157t-157 -234.5t-58.5 -286t58.5 -286t157 -234.5t234.5 -157t286 -58.5t286 58.5t234.5 157t157 234.5t58.5 286t-58.5 286t-157 234.5t-234.5 157t-286 58.5zM1792 640 +q0 -182 -71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71t348 -71t286 -191t191 -286t71 -348z" /> + <glyph glyph-name="_569" unicode="" horiz-adv-x="1792" +d="M605 303q153 0 257 104q14 18 3 36l-45 82q-6 13 -24 17q-16 2 -27 -11l-4 -3q-4 -4 -11.5 -10t-17.5 -13.5t-23.5 -14.5t-28.5 -13t-33.5 -9.5t-37.5 -3.5q-76 0 -125 50t-49 127q0 76 48 125.5t122 49.5q37 0 71.5 -14t50.5 -28l16 -14q11 -11 26 -10q16 2 24 14l53 78 +q13 20 -2 39q-3 4 -11 12t-30 23.5t-48.5 28t-67.5 22.5t-86 10q-148 0 -246 -96.5t-98 -240.5q0 -146 97 -241.5t247 -95.5zM1235 303q153 0 257 104q14 18 4 36l-45 82q-8 14 -25 17q-16 2 -27 -11l-4 -3q-4 -4 -11.5 -10t-17.5 -13.5t-23.5 -14.5t-28.5 -13t-33.5 -9.5 +t-37.5 -3.5q-76 0 -125 50t-49 127q0 76 48 125.5t122 49.5q37 0 71.5 -14t50.5 -28l16 -14q11 -11 26 -10q16 2 24 14l53 78q13 20 -2 39q-3 4 -11 12t-30 23.5t-48.5 28t-67.5 22.5t-86 10q-147 0 -245.5 -96.5t-98.5 -240.5q0 -146 97 -241.5t247 -95.5zM896 1376 +q-150 0 -286 -58.5t-234.5 -157t-157 -234.5t-58.5 -286t58.5 -286t157 -234.5t234.5 -157t286 -58.5t286 58.5t234.5 157t157 234.5t58.5 286t-58.5 286t-157 234.5t-234.5 157t-286 58.5zM896 1536q182 0 348 -71t286 -191t191 -286t71 -348t-71 -348t-191 -286t-286 -191 +t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71z" /> + <glyph glyph-name="f260" unicode="" horiz-adv-x="2048" +d="M736 736l384 -384l-384 -384l-672 672l672 672l168 -168l-96 -96l-72 72l-480 -480l480 -480l193 193l-289 287zM1312 1312l672 -672l-672 -672l-168 168l96 96l72 -72l480 480l-480 480l-193 -193l289 -287l-96 -96l-384 384z" /> + <glyph glyph-name="f261" unicode="" horiz-adv-x="1792" +d="M717 182l271 271l-279 279l-88 -88l192 -191l-96 -96l-279 279l279 279l40 -40l87 87l-127 128l-454 -454zM1075 190l454 454l-454 454l-271 -271l279 -279l88 88l-192 191l96 96l279 -279l-279 -279l-40 40l-87 -88zM1792 640q0 -182 -71 -348t-191 -286t-286 -191 +t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71t348 -71t286 -191t191 -286t71 -348z" /> + <glyph glyph-name="_572" unicode="" horiz-adv-x="2304" +d="M651 539q0 -39 -27.5 -66.5t-65.5 -27.5q-39 0 -66.5 27.5t-27.5 66.5q0 38 27.5 65.5t66.5 27.5q38 0 65.5 -27.5t27.5 -65.5zM1805 540q0 -39 -27.5 -66.5t-66.5 -27.5t-66.5 27.5t-27.5 66.5t27.5 66t66.5 27t66.5 -27t27.5 -66zM765 539q0 79 -56.5 136t-136.5 57 +t-136.5 -56.5t-56.5 -136.5t56.5 -136.5t136.5 -56.5t136.5 56.5t56.5 136.5zM1918 540q0 80 -56.5 136.5t-136.5 56.5q-79 0 -136 -56.5t-57 -136.5t56.5 -136.5t136.5 -56.5t136.5 56.5t56.5 136.5zM850 539q0 -116 -81.5 -197.5t-196.5 -81.5q-116 0 -197.5 82t-81.5 197 +t82 196.5t197 81.5t196.5 -81.5t81.5 -196.5zM2004 540q0 -115 -81.5 -196.5t-197.5 -81.5q-115 0 -196.5 81.5t-81.5 196.5t81.5 196.5t196.5 81.5q116 0 197.5 -81.5t81.5 -196.5zM1040 537q0 191 -135.5 326.5t-326.5 135.5q-125 0 -231 -62t-168 -168.5t-62 -231.5 +t62 -231.5t168 -168.5t231 -62q191 0 326.5 135.5t135.5 326.5zM1708 1110q-254 111 -556 111q-319 0 -573 -110q117 0 223 -45.5t182.5 -122.5t122 -183t45.5 -223q0 115 43.5 219.5t118 180.5t177.5 123t217 50zM2187 537q0 191 -135 326.5t-326 135.5t-326.5 -135.5 +t-135.5 -326.5t135.5 -326.5t326.5 -135.5t326 135.5t135 326.5zM1921 1103h383q-44 -51 -75 -114.5t-40 -114.5q110 -151 110 -337q0 -156 -77 -288t-209 -208.5t-287 -76.5q-133 0 -249 56t-196 155q-47 -56 -129 -179q-11 22 -53.5 82.5t-74.5 97.5 +q-80 -99 -196.5 -155.5t-249.5 -56.5q-155 0 -287 76.5t-209 208.5t-77 288q0 186 110 337q-9 51 -40 114.5t-75 114.5h365q149 100 355 156.5t432 56.5q224 0 421 -56t348 -157z" /> + <glyph glyph-name="f263" unicode="" horiz-adv-x="1280" +d="M640 629q-188 0 -321 133t-133 320q0 188 133 321t321 133t321 -133t133 -321q0 -187 -133 -320t-321 -133zM640 1306q-92 0 -157.5 -65.5t-65.5 -158.5q0 -92 65.5 -157.5t157.5 -65.5t157.5 65.5t65.5 157.5q0 93 -65.5 158.5t-157.5 65.5zM1163 574q13 -27 15 -49.5 +t-4.5 -40.5t-26.5 -38.5t-42.5 -37t-61.5 -41.5q-115 -73 -315 -94l73 -72l267 -267q30 -31 30 -74t-30 -73l-12 -13q-31 -30 -74 -30t-74 30q-67 68 -267 268l-267 -268q-31 -30 -74 -30t-73 30l-12 13q-31 30 -31 73t31 74l267 267l72 72q-203 21 -317 94 +q-39 25 -61.5 41.5t-42.5 37t-26.5 38.5t-4.5 40.5t15 49.5q10 20 28 35t42 22t56 -2t65 -35q5 -4 15 -11t43 -24.5t69 -30.5t92 -24t113 -11q91 0 174 25.5t120 50.5l38 25q33 26 65 35t56 2t42 -22t28 -35z" /> + <glyph glyph-name="_574" unicode="" +d="M927 956q0 -66 -46.5 -112.5t-112.5 -46.5t-112.5 46.5t-46.5 112.5t46.5 112.5t112.5 46.5t112.5 -46.5t46.5 -112.5zM1141 593q-10 20 -28 32t-47.5 9.5t-60.5 -27.5q-10 -8 -29 -20t-81 -32t-127 -20t-124 18t-86 36l-27 18q-31 25 -60.5 27.5t-47.5 -9.5t-28 -32 +q-22 -45 -2 -74.5t87 -73.5q83 -53 226 -67l-51 -52q-142 -142 -191 -190q-22 -22 -22 -52.5t22 -52.5l9 -9q22 -22 52.5 -22t52.5 22l191 191q114 -115 191 -191q22 -22 52.5 -22t52.5 22l9 9q22 22 22 52.5t-22 52.5l-191 190l-52 52q141 14 225 67q67 44 87 73.5t-2 74.5 +zM1092 956q0 134 -95 229t-229 95t-229 -95t-95 -229t95 -229t229 -95t229 95t95 229zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="_575" unicode="" horiz-adv-x="1720" +d="M1565 1408q65 0 110 -45.5t45 -110.5v-519q0 -176 -68 -336t-182.5 -275t-274 -182.5t-334.5 -67.5q-176 0 -335.5 67.5t-274.5 182.5t-183 275t-68 336v519q0 64 46 110t110 46h1409zM861 344q47 0 82 33l404 388q37 35 37 85q0 49 -34.5 83.5t-83.5 34.5q-47 0 -82 -33 +l-323 -310l-323 310q-35 33 -81 33q-49 0 -83.5 -34.5t-34.5 -83.5q0 -51 36 -85l405 -388q33 -33 81 -33z" /> + <glyph glyph-name="_576" unicode="" horiz-adv-x="2304" +d="M1494 -103l-295 695q-25 -49 -158.5 -305.5t-198.5 -389.5q-1 -1 -27.5 -0.5t-26.5 1.5q-82 193 -255.5 587t-259.5 596q-21 50 -66.5 107.5t-103.5 100.5t-102 43q0 5 -0.5 24t-0.5 27h583v-50q-39 -2 -79.5 -16t-66.5 -43t-10 -64q26 -59 216.5 -499t235.5 -540 +q31 61 140 266.5t131 247.5q-19 39 -126 281t-136 295q-38 69 -201 71v50l513 -1v-47q-60 -2 -93.5 -25t-12.5 -69q33 -70 87 -189.5t86 -187.5q110 214 173 363q24 55 -10 79.5t-129 26.5q1 7 1 25v24q64 0 170.5 0.5t180 1t92.5 0.5v-49q-62 -2 -119 -33t-90 -81 +l-213 -442q13 -33 127.5 -290t121.5 -274l441 1017q-14 38 -49.5 62.5t-65 31.5t-55.5 8v50l460 -4l1 -2l-1 -44q-139 -4 -201 -145q-526 -1216 -559 -1291h-49z" /> + <glyph glyph-name="_577" unicode="" horiz-adv-x="1792" +d="M949 643q0 -26 -16.5 -45t-41.5 -19q-26 0 -45 16.5t-19 41.5q0 26 17 45t42 19t44 -16.5t19 -41.5zM964 585l350 581q-9 -8 -67.5 -62.5t-125.5 -116.5t-136.5 -127t-117 -110.5t-50.5 -51.5l-349 -580q7 7 67 62t126 116.5t136 127t117 111t50 50.5zM1611 640 +q0 -201 -104 -371q-3 2 -17 11t-26.5 16.5t-16.5 7.5q-13 0 -13 -13q0 -10 59 -44q-74 -112 -184.5 -190.5t-241.5 -110.5l-16 67q-1 10 -15 10q-5 0 -8 -5.5t-2 -9.5l16 -68q-72 -15 -146 -15q-199 0 -372 105q1 2 13 20.5t21.5 33.5t9.5 19q0 13 -13 13q-6 0 -17 -14.5 +t-22.5 -34.5t-13.5 -23q-113 75 -192 187.5t-110 244.5l69 15q10 3 10 15q0 5 -5.5 8t-10.5 2l-68 -15q-14 72 -14 139q0 206 109 379q2 -1 18.5 -12t30 -19t17.5 -8q13 0 13 12q0 6 -12.5 15.5t-32.5 21.5l-20 12q77 112 189 189t244 107l15 -67q2 -10 15 -10q5 0 8 5.5 +t2 10.5l-15 66q71 13 134 13q204 0 379 -109q-39 -56 -39 -65q0 -13 12 -13q11 0 48 64q111 -75 187.5 -186t107.5 -241l-56 -12q-10 -2 -10 -16q0 -5 5.5 -8t9.5 -2l57 13q14 -72 14 -140zM1696 640q0 163 -63.5 311t-170.5 255t-255 170.5t-311 63.5t-311 -63.5 +t-255 -170.5t-170.5 -255t-63.5 -311t63.5 -311t170.5 -255t255 -170.5t311 -63.5t311 63.5t255 170.5t170.5 255t63.5 311zM1792 640q0 -182 -71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71t348 -71t286 -191 +t191 -286t71 -348z" /> + <glyph glyph-name="_578" unicode="" horiz-adv-x="1792" +d="M893 1536q240 2 451 -120q232 -134 352 -372l-742 39q-160 9 -294 -74.5t-185 -229.5l-276 424q128 159 311 245.5t383 87.5zM146 1131l337 -663q72 -143 211 -217t293 -45l-230 -451q-212 33 -385 157.5t-272.5 316t-99.5 411.5q0 267 146 491zM1732 962 +q58 -150 59.5 -310.5t-48.5 -306t-153 -272t-246 -209.5q-230 -133 -498 -119l405 623q88 131 82.5 290.5t-106.5 277.5zM896 942q125 0 213.5 -88.5t88.5 -213.5t-88.5 -213.5t-213.5 -88.5t-213.5 88.5t-88.5 213.5t88.5 213.5t213.5 88.5z" /> + <glyph glyph-name="_579" unicode="" horiz-adv-x="1792" +d="M903 -256q-283 0 -504.5 150.5t-329.5 398.5q-58 131 -67 301t26 332.5t111 312t179 242.5l-11 -281q11 14 68 15.5t70 -15.5q42 81 160.5 138t234.5 59q-54 -45 -119.5 -148.5t-58.5 -163.5q25 -8 62.5 -13.5t63 -7.5t68 -4t50.5 -3q15 -5 9.5 -45.5t-30.5 -75.5 +q-5 -7 -16.5 -18.5t-56.5 -35.5t-101 -34l15 -189l-139 67q-18 -43 -7.5 -81.5t36 -66.5t65.5 -41.5t81 -6.5q51 9 98 34.5t83.5 45t73.5 17.5q61 -4 89.5 -33t19.5 -65q-1 -2 -2.5 -5.5t-8.5 -12.5t-18 -15.5t-31.5 -10.5t-46.5 -1q-60 -95 -144.5 -135.5t-209.5 -29.5 +q74 -61 162.5 -82.5t168.5 -6t154.5 52t128 87.5t80.5 104q43 91 39 192.5t-37.5 188.5t-78.5 125q87 -38 137 -79.5t77 -112.5q15 170 -57.5 343t-209.5 284q265 -77 412 -279.5t151 -517.5q2 -127 -40.5 -255t-123.5 -238t-189 -196t-247.5 -135.5t-288.5 -49.5z" /> + <glyph glyph-name="_580" unicode="" horiz-adv-x="1792" +d="M1493 1308q-165 110 -359 110q-155 0 -293 -73t-240 -200q-75 -93 -119.5 -218t-48.5 -266v-42q4 -141 48.5 -266t119.5 -218q102 -127 240 -200t293 -73q194 0 359 110q-121 -108 -274.5 -168t-322.5 -60q-29 0 -43 1q-175 8 -333 82t-272 193t-181 281t-67 339 +q0 182 71 348t191 286t286 191t348 71h3q168 -1 320.5 -60.5t273.5 -167.5zM1792 640q0 -192 -77 -362.5t-213 -296.5q-104 -63 -222 -63q-137 0 -255 84q154 56 253.5 233t99.5 405q0 227 -99 404t-253 234q119 83 254 83q119 0 226 -65q135 -125 210.5 -295t75.5 -361z +" /> + <glyph glyph-name="_581" unicode="" horiz-adv-x="1792" +d="M1792 599q0 -56 -7 -104h-1151q0 -146 109.5 -244.5t257.5 -98.5q99 0 185.5 46.5t136.5 130.5h423q-56 -159 -170.5 -281t-267.5 -188.5t-321 -66.5q-187 0 -356 83q-228 -116 -394 -116q-237 0 -237 263q0 115 45 275q17 60 109 229q199 360 475 606 +q-184 -79 -427 -354q63 274 283.5 449.5t501.5 175.5q30 0 45 -1q255 117 433 117q64 0 116 -13t94.5 -40.5t66.5 -76.5t24 -115q0 -116 -75 -286q101 -182 101 -390zM1722 1239q0 83 -53 132t-137 49q-108 0 -254 -70q121 -47 222.5 -131.5t170.5 -195.5q51 135 51 216z +M128 2q0 -86 48.5 -132.5t134.5 -46.5q115 0 266 83q-122 72 -213.5 183t-137.5 245q-98 -205 -98 -332zM632 715h728q-5 142 -113 237t-251 95q-144 0 -251.5 -95t-112.5 -237z" /> + <glyph glyph-name="_582" unicode="" horiz-adv-x="2048" +d="M1792 288v960q0 13 -9.5 22.5t-22.5 9.5h-1600q-13 0 -22.5 -9.5t-9.5 -22.5v-960q0 -13 9.5 -22.5t22.5 -9.5h1600q13 0 22.5 9.5t9.5 22.5zM1920 1248v-960q0 -66 -47 -113t-113 -47h-736v-128h352q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-832q-14 0 -23 9t-9 23 +v64q0 14 9 23t23 9h352v128h-736q-66 0 -113 47t-47 113v960q0 66 47 113t113 47h1600q66 0 113 -47t47 -113z" /> + <glyph glyph-name="_583" unicode="" horiz-adv-x="1792" +d="M138 1408h197q-70 -64 -126 -149q-36 -56 -59 -115t-30 -125.5t-8.5 -120t10.5 -132t21 -126t28 -136.5q4 -19 6 -28q51 -238 81 -329q57 -171 152 -275h-272q-48 0 -82 34t-34 82v1304q0 48 34 82t82 34zM1346 1408h308q48 0 82 -34t34 -82v-1304q0 -48 -34 -82t-82 -34 +h-178q212 210 196 565l-469 -101q-2 -45 -12 -82t-31 -72t-59.5 -59.5t-93.5 -36.5q-123 -26 -199 40q-32 27 -53 61t-51.5 129t-64.5 258q-35 163 -45.5 263t-5.5 139t23 77q20 41 62.5 73t102.5 45q45 12 83.5 6.5t67 -17t54 -35t43 -48t34.5 -56.5l468 100 +q-68 175 -180 287z" /> + <glyph glyph-name="_584" unicode="" +d="M1401 -11l-6 -6q-113 -113 -259 -175q-154 -64 -317 -64q-165 0 -317 64q-148 63 -259 175q-113 112 -175 258q-42 103 -54 189q-4 28 48 36q51 8 56 -20q1 -1 1 -4q18 -90 46 -159q50 -124 152 -226q98 -98 226 -152q132 -56 276 -56q143 0 276 56q128 55 225 152l6 6 +q10 10 25 6q12 -3 33 -22q36 -37 17 -58zM929 604l-66 -66l63 -63q21 -21 -7 -49q-17 -17 -32 -17q-10 0 -19 10l-62 61l-66 -66q-5 -5 -15 -5q-15 0 -31 16l-2 2q-18 15 -18 29q0 7 8 17l66 65l-66 66q-16 16 14 45q18 18 31 18q6 0 13 -5l65 -66l65 65q18 17 48 -13 +q27 -27 11 -44zM1400 547q0 -118 -46 -228q-45 -105 -126 -186q-80 -80 -187 -126t-228 -46t-228 46t-187 126q-82 82 -125 186q-15 33 -15 40h-1q-9 27 43 44q50 16 60 -12q37 -99 97 -167h1v339v2q3 136 102 232q105 103 253 103q147 0 251 -103t104 -249 +q0 -147 -104.5 -251t-250.5 -104q-58 0 -112 16q-28 11 -13 61q16 51 44 43l14 -3q14 -3 33 -6t30 -3q104 0 176 71.5t72 174.5q0 101 -72 171q-71 71 -175 71q-107 0 -178 -80q-64 -72 -64 -160v-413q110 -67 242 -67q96 0 185 36.5t156 103.5t103.5 155t36.5 183 +q0 198 -141 339q-140 140 -339 140q-200 0 -340 -140q-53 -53 -77 -87l-2 -2q-8 -11 -13 -15.5t-21.5 -9.5t-38.5 3q-21 5 -36.5 16.5t-15.5 26.5v680q0 15 10.5 26.5t27.5 11.5h877q30 0 30 -55t-30 -55h-811v-483h1q40 42 102 84t108 61q109 46 231 46q121 0 228 -46 +t187 -126q81 -81 126 -186q46 -112 46 -229zM1369 1128q9 -8 9 -18t-5.5 -18t-16.5 -21q-26 -26 -39 -26q-9 0 -16 7q-106 91 -207 133q-128 56 -276 56q-133 0 -262 -49q-27 -10 -45 37q-9 25 -8 38q3 16 16 20q130 57 299 57q164 0 316 -64q137 -58 235 -152z" /> + <glyph glyph-name="_585" unicode="" horiz-adv-x="1792" +d="M1551 60q15 6 26 3t11 -17.5t-15 -33.5q-13 -16 -44 -43.5t-95.5 -68t-141 -74t-188 -58t-229.5 -24.5q-119 0 -238 31t-209 76.5t-172.5 104t-132.5 105t-84 87.5q-8 9 -10 16.5t1 12t8 7t11.5 2t11.5 -4.5q192 -117 300 -166q389 -176 799 -90q190 40 391 135z +M1758 175q11 -16 2.5 -69.5t-28.5 -102.5q-34 -83 -85 -124q-17 -14 -26 -9t0 24q21 45 44.5 121.5t6.5 98.5q-5 7 -15.5 11.5t-27 6t-29.5 2.5t-35 0t-31.5 -2t-31 -3t-22.5 -2q-6 -1 -13 -1.5t-11 -1t-8.5 -1t-7 -0.5h-5.5h-4.5t-3 0.5t-2 1.5l-1.5 3q-6 16 47 40t103 30 +q46 7 108 1t76 -24zM1364 618q0 -31 13.5 -64t32 -58t37.5 -46t33 -32l13 -11l-227 -224q-40 37 -79 75.5t-58 58.5l-19 20q-11 11 -25 33q-38 -59 -97.5 -102.5t-127.5 -63.5t-140 -23t-137.5 21t-117.5 65.5t-83 113t-31 162.5q0 84 28 154t72 116.5t106.5 83t122.5 57 +t130 34.5t119.5 18.5t99.5 6.5v127q0 65 -21 97q-34 53 -121 53q-6 0 -16.5 -1t-40.5 -12t-56 -29.5t-56 -59.5t-48 -96l-294 27q0 60 22 119t67 113t108 95t151.5 65.5t190.5 24.5q100 0 181 -25t129.5 -61.5t81 -83t45 -86t12.5 -73.5v-589zM692 597q0 -86 70 -133 +q66 -44 139 -22q84 25 114 123q14 45 14 101v162q-59 -2 -111 -12t-106.5 -33.5t-87 -71t-32.5 -114.5z" /> + <glyph glyph-name="_586" unicode="" horiz-adv-x="1792" +d="M1536 1280q52 0 90 -38t38 -90v-1280q0 -52 -38 -90t-90 -38h-1408q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h128v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h384v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h128zM1152 1376v-288q0 -14 9 -23t23 -9 +h64q14 0 23 9t9 23v288q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23zM384 1376v-288q0 -14 9 -23t23 -9h64q14 0 23 9t9 23v288q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23zM1536 -128v1024h-1408v-1024h1408zM896 448h224q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-224 +v-224q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v224h-224q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h224v224q0 14 9 23t23 9h64q14 0 23 -9t9 -23v-224z" /> + <glyph glyph-name="_587" unicode="" horiz-adv-x="1792" +d="M1152 416v-64q0 -14 -9 -23t-23 -9h-576q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h576q14 0 23 -9t9 -23zM128 -128h1408v1024h-1408v-1024zM512 1088v288q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-288q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1280 1088v288q0 14 -9 23 +t-23 9h-64q-14 0 -23 -9t-9 -23v-288q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1664 1152v-1280q0 -52 -38 -90t-90 -38h-1408q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h128v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h384v96q0 66 47 113t113 47h64q66 0 113 -47 +t47 -113v-96h128q52 0 90 -38t38 -90z" /> + <glyph glyph-name="_588" unicode="" horiz-adv-x="1792" +d="M1111 151l-46 -46q-9 -9 -22 -9t-23 9l-188 189l-188 -189q-10 -9 -23 -9t-22 9l-46 46q-9 9 -9 22t9 23l189 188l-189 188q-9 10 -9 23t9 22l46 46q9 9 22 9t23 -9l188 -188l188 188q10 9 23 9t22 -9l46 -46q9 -9 9 -22t-9 -23l-188 -188l188 -188q9 -10 9 -23t-9 -22z +M128 -128h1408v1024h-1408v-1024zM512 1088v288q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-288q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1280 1088v288q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-288q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1664 1152v-1280 +q0 -52 -38 -90t-90 -38h-1408q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h128v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h384v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h128q52 0 90 -38t38 -90z" /> + <glyph glyph-name="_589" unicode="" horiz-adv-x="1792" +d="M1303 572l-512 -512q-10 -9 -23 -9t-23 9l-288 288q-9 10 -9 23t9 22l46 46q9 9 22 9t23 -9l220 -220l444 444q10 9 23 9t22 -9l46 -46q9 -9 9 -22t-9 -23zM128 -128h1408v1024h-1408v-1024zM512 1088v288q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-288q0 -14 9 -23 +t23 -9h64q14 0 23 9t9 23zM1280 1088v288q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-288q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1664 1152v-1280q0 -52 -38 -90t-90 -38h-1408q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h128v96q0 66 47 113t113 47h64q66 0 113 -47 +t47 -113v-96h384v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h128q52 0 90 -38t38 -90z" /> + <glyph glyph-name="_590" unicode="" horiz-adv-x="1792" +d="M448 1536q26 0 45 -19t19 -45v-891l536 429q17 14 40 14q26 0 45 -19t19 -45v-379l536 429q17 14 40 14q26 0 45 -19t19 -45v-1152q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v1664q0 26 19 45t45 19h384z" /> + <glyph glyph-name="_591" unicode="" horiz-adv-x="1024" +d="M512 448q66 0 128 15v-655q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v655q62 -15 128 -15zM512 1536q212 0 362 -150t150 -362t-150 -362t-362 -150t-362 150t-150 362t150 362t362 150zM512 1312q14 0 23 9t9 23t-9 23t-23 9q-146 0 -249 -103t-103 -249 +q0 -14 9 -23t23 -9t23 9t9 23q0 119 84.5 203.5t203.5 84.5z" /> + <glyph glyph-name="_592" unicode="" horiz-adv-x="1792" +d="M1745 1239q10 -10 10 -23t-10 -23l-141 -141q-28 -28 -68 -28h-1344q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h576v64q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-64h512q40 0 68 -28zM768 320h256v-512q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v512zM1600 768 +q26 0 45 -19t19 -45v-256q0 -26 -19 -45t-45 -19h-1344q-40 0 -68 28l-141 141q-10 10 -10 23t10 23l141 141q28 28 68 28h512v192h256v-192h576z" /> + <glyph glyph-name="_593" unicode="" horiz-adv-x="2048" +d="M2020 1525q28 -20 28 -53v-1408q0 -20 -11 -36t-29 -23l-640 -256q-24 -11 -48 0l-616 246l-616 -246q-10 -5 -24 -5q-19 0 -36 11q-28 20 -28 53v1408q0 20 11 36t29 23l640 256q24 11 48 0l616 -246l616 246q32 13 60 -6zM736 1390v-1270l576 -230v1270zM128 1173 +v-1270l544 217v1270zM1920 107v1270l-544 -217v-1270z" /> + <glyph glyph-name="_594" unicode="" horiz-adv-x="1792" +d="M512 1536q13 0 22.5 -9.5t9.5 -22.5v-1472q0 -20 -17 -28l-480 -256q-7 -4 -15 -4q-13 0 -22.5 9.5t-9.5 22.5v1472q0 20 17 28l480 256q7 4 15 4zM1760 1536q13 0 22.5 -9.5t9.5 -22.5v-1472q0 -20 -17 -28l-480 -256q-7 -4 -15 -4q-13 0 -22.5 9.5t-9.5 22.5v1472 +q0 20 17 28l480 256q7 4 15 4zM640 1536q8 0 14 -3l512 -256q18 -10 18 -29v-1472q0 -13 -9.5 -22.5t-22.5 -9.5q-8 0 -14 3l-512 256q-18 10 -18 29v1472q0 13 9.5 22.5t22.5 9.5z" /> + <glyph glyph-name="_595" unicode="" horiz-adv-x="1792" +d="M640 640q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1024 640q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1408 640q0 53 -37.5 90.5t-90.5 37.5 +t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1792 640q0 -174 -120 -321.5t-326 -233t-450 -85.5q-110 0 -211 18q-173 -173 -435 -229q-52 -10 -86 -13q-12 -1 -22 6t-13 18q-4 15 20 37q5 5 23.5 21.5t25.5 23.5t23.5 25.5t24 31.5t20.5 37 +t20 48t14.5 57.5t12.5 72.5q-146 90 -229.5 216.5t-83.5 269.5q0 174 120 321.5t326 233t450 85.5t450 -85.5t326 -233t120 -321.5z" /> + <glyph glyph-name="_596" unicode="" horiz-adv-x="1792" +d="M640 640q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1024 640q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1408 640q0 -53 -37.5 -90.5t-90.5 -37.5 +t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM896 1152q-204 0 -381.5 -69.5t-282 -187.5t-104.5 -255q0 -112 71.5 -213.5t201.5 -175.5l87 -50l-27 -96q-24 -91 -70 -172q152 63 275 171l43 38l57 -6q69 -8 130 -8q204 0 381.5 69.5t282 187.5 +t104.5 255t-104.5 255t-282 187.5t-381.5 69.5zM1792 640q0 -174 -120 -321.5t-326 -233t-450 -85.5q-70 0 -145 8q-198 -175 -460 -242q-49 -14 -114 -22h-5q-15 0 -27 10.5t-16 27.5v1q-3 4 -0.5 12t2 10t4.5 9.5l6 9t7 8.5t8 9q7 8 31 34.5t34.5 38t31 39.5t32.5 51 +t27 59t26 76q-157 89 -247.5 220t-90.5 281q0 130 71 248.5t191 204.5t286 136.5t348 50.5t348 -50.5t286 -136.5t191 -204.5t71 -248.5z" /> + <glyph glyph-name="_597" unicode="" horiz-adv-x="1024" +d="M512 345l512 295v-591l-512 -296v592zM0 640v-591l512 296zM512 1527v-591l-512 -296v591zM512 936l512 295v-591z" /> + <glyph glyph-name="_598" unicode="" horiz-adv-x="1792" +d="M1709 1018q-10 -236 -332 -651q-333 -431 -562 -431q-142 0 -240 263q-44 160 -132 482q-72 262 -157 262q-18 0 -127 -76l-77 98q24 21 108 96.5t130 115.5q156 138 241 146q95 9 153 -55.5t81 -203.5q44 -287 66 -373q55 -249 120 -249q51 0 154 161q101 161 109 246 +q13 139 -109 139q-57 0 -121 -26q120 393 459 382q251 -8 236 -326z" /> + <glyph glyph-name="f27e" unicode="" +d="M0 1408h1536v-1536h-1536v1536zM1085 293l-221 631l221 297h-634l221 -297l-221 -631l317 -304z" /> + <glyph glyph-name="uniF280" unicode="" +d="M0 1408h1536v-1536h-1536v1536zM908 1088l-12 -33l75 -83l-31 -114l25 -25l107 57l107 -57l25 25l-31 114l75 83l-12 33h-95l-53 96h-32l-53 -96h-95zM641 925q32 0 44.5 -16t11.5 -63l174 21q0 55 -17.5 92.5t-50.5 56t-69 25.5t-85 7q-133 0 -199 -57.5t-66 -182.5v-72 +h-96v-128h76q20 0 20 -8v-382q0 -14 -5 -20t-18 -7l-73 -7v-88h448v86l-149 14q-6 1 -8.5 1.5t-3.5 2.5t-0.5 4t1 7t0.5 10v387h191l38 128h-231q-6 0 -2 6t4 9v80q0 27 1.5 40.5t7.5 28t19.5 20t36.5 5.5zM1248 96v86l-54 9q-7 1 -9.5 2.5t-2.5 3t1 7.5t1 12v520h-275 +l-23 -101l83 -22q23 -7 23 -27v-370q0 -14 -6 -18.5t-20 -6.5l-70 -9v-86h352z" /> + <glyph glyph-name="uniF281" unicode="" horiz-adv-x="1792" +d="M1792 690q0 -58 -29.5 -105.5t-79.5 -72.5q12 -46 12 -96q0 -155 -106.5 -287t-290.5 -208.5t-400 -76.5t-399.5 76.5t-290 208.5t-106.5 287q0 47 11 94q-51 25 -82 73.5t-31 106.5q0 82 58 140.5t141 58.5q85 0 145 -63q218 152 515 162l116 521q3 13 15 21t26 5 +l369 -81q18 37 54 59.5t79 22.5q62 0 106 -43.5t44 -105.5t-44 -106t-106 -44t-105.5 43.5t-43.5 105.5l-334 74l-104 -472q300 -9 519 -160q58 61 143 61q83 0 141 -58.5t58 -140.5zM418 491q0 -62 43.5 -106t105.5 -44t106 44t44 106t-44 105.5t-106 43.5q-61 0 -105 -44 +t-44 -105zM1228 136q11 11 11 26t-11 26q-10 10 -25 10t-26 -10q-41 -42 -121 -62t-160 -20t-160 20t-121 62q-11 10 -26 10t-25 -10q-11 -10 -11 -25.5t11 -26.5q43 -43 118.5 -68t122.5 -29.5t91 -4.5t91 4.5t122.5 29.5t118.5 68zM1225 341q62 0 105.5 44t43.5 106 +q0 61 -44 105t-105 44q-62 0 -106 -43.5t-44 -105.5t44 -106t106 -44z" /> + <glyph glyph-name="_602" unicode="" horiz-adv-x="1792" +d="M69 741h1q16 126 58.5 241.5t115 217t167.5 176t223.5 117.5t276.5 43q231 0 414 -105.5t294 -303.5q104 -187 104 -442v-188h-1125q1 -111 53.5 -192.5t136.5 -122.5t189.5 -57t213 -3t208 46.5t173.5 84.5v-377q-92 -55 -229.5 -92t-312.5 -38t-316 53 +q-189 73 -311.5 249t-124.5 372q-3 242 111 412t325 268q-48 -60 -78 -125.5t-46 -159.5h635q8 77 -8 140t-47 101.5t-70.5 66.5t-80.5 41t-75 20.5t-56 8.5l-22 1q-135 -5 -259.5 -44.5t-223.5 -104.5t-176 -140.5t-138 -163.5z" /> + <glyph glyph-name="_603" unicode="" horiz-adv-x="2304" +d="M0 32v608h2304v-608q0 -66 -47 -113t-113 -47h-1984q-66 0 -113 47t-47 113zM640 256v-128h384v128h-384zM256 256v-128h256v128h-256zM2144 1408q66 0 113 -47t47 -113v-224h-2304v224q0 66 47 113t113 47h1984z" /> + <glyph glyph-name="_604" unicode="" horiz-adv-x="1792" +d="M1584 246l-218 111q-74 -120 -196.5 -189t-263.5 -69q-147 0 -271 72t-196 196t-72 270q0 110 42.5 209.5t115 172t172 115t209.5 42.5q131 0 247.5 -60.5t192.5 -168.5l215 125q-110 169 -286.5 265t-378.5 96q-161 0 -308 -63t-253 -169t-169 -253t-63 -308t63 -308 +t169 -253t253 -169t308 -63q213 0 397.5 107t290.5 292zM1030 643l693 -352q-116 -253 -334.5 -400t-492.5 -147q-182 0 -348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71q260 0 470.5 -133.5t335.5 -366.5zM1543 640h-39v-160h-96v352h136q32 0 54.5 -20 +t28.5 -48t1 -56t-27.5 -48t-57.5 -20z" /> + <glyph glyph-name="uniF285" unicode="" horiz-adv-x="1792" +d="M1427 827l-614 386l92 151h855zM405 562l-184 116v858l1183 -743zM1424 697l147 -95v-858l-532 335zM1387 718l-500 -802h-855l356 571z" /> + <glyph glyph-name="uniF286" unicode="" horiz-adv-x="1792" +d="M640 528v224q0 16 -16 16h-96q-16 0 -16 -16v-224q0 -16 16 -16h96q16 0 16 16zM1152 528v224q0 16 -16 16h-96q-16 0 -16 -16v-224q0 -16 16 -16h96q16 0 16 16zM1664 496v-752h-640v320q0 80 -56 136t-136 56t-136 -56t-56 -136v-320h-640v752q0 16 16 16h96 +q16 0 16 -16v-112h128v624q0 16 16 16h96q16 0 16 -16v-112h128v112q0 16 16 16h96q16 0 16 -16v-112h128v112q0 6 2.5 9.5t8.5 5t9.5 2t11.5 0t9 -0.5v391q-32 15 -32 50q0 23 16.5 39t38.5 16t38.5 -16t16.5 -39q0 -35 -32 -50v-17q45 10 83 10q21 0 59.5 -7.5t54.5 -7.5 +q17 0 47 7.5t37 7.5q16 0 16 -16v-210q0 -15 -35 -21.5t-62 -6.5q-18 0 -54.5 7.5t-55.5 7.5q-40 0 -90 -12v-133q1 0 9 0.5t11.5 0t9.5 -2t8.5 -5t2.5 -9.5v-112h128v112q0 16 16 16h96q16 0 16 -16v-112h128v112q0 16 16 16h96q16 0 16 -16v-624h128v112q0 16 16 16h96 +q16 0 16 -16z" /> + <glyph glyph-name="_607" unicode="" horiz-adv-x="2304" +d="M2288 731q16 -8 16 -27t-16 -27l-320 -192q-8 -5 -16 -5q-9 0 -16 4q-16 10 -16 28v128h-858q37 -58 83 -165q16 -37 24.5 -55t24 -49t27 -47t27 -34t31.5 -26t33 -8h96v96q0 14 9 23t23 9h320q14 0 23 -9t9 -23v-320q0 -14 -9 -23t-23 -9h-320q-14 0 -23 9t-9 23v96h-96 +q-32 0 -61 10t-51 23.5t-45 40.5t-37 46t-33.5 57t-28.5 57.5t-28 60.5q-23 53 -37 81.5t-36 65t-44.5 53.5t-46.5 17h-360q-22 -84 -91 -138t-157 -54q-106 0 -181 75t-75 181t75 181t181 75q88 0 157 -54t91 -138h104q24 0 46.5 17t44.5 53.5t36 65t37 81.5q19 41 28 60.5 +t28.5 57.5t33.5 57t37 46t45 40.5t51 23.5t61 10h107q21 57 70 92.5t111 35.5q80 0 136 -56t56 -136t-56 -136t-136 -56q-62 0 -111 35.5t-70 92.5h-107q-17 0 -33 -8t-31.5 -26t-27 -34t-27 -47t-24 -49t-24.5 -55q-46 -107 -83 -165h1114v128q0 18 16 28t32 -1z" /> + <glyph glyph-name="_608" unicode="" horiz-adv-x="1792" +d="M1150 774q0 -56 -39.5 -95t-95.5 -39h-253v269h253q56 0 95.5 -39.5t39.5 -95.5zM1329 774q0 130 -91.5 222t-222.5 92h-433v-896h180v269h253q130 0 222 91.5t92 221.5zM1792 640q0 -182 -71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348 +t71 348t191 286t286 191t348 71t348 -71t286 -191t191 -286t71 -348z" /> + <glyph glyph-name="_609" unicode="" horiz-adv-x="2304" +d="M1645 438q0 59 -34 106.5t-87 68.5q-7 -45 -23 -92q-7 -24 -27.5 -38t-44.5 -14q-12 0 -24 3q-31 10 -45 38.5t-4 58.5q23 71 23 143q0 123 -61 227.5t-166 165.5t-228 61q-134 0 -247 -73t-167 -194q108 -28 188 -106q22 -23 22 -55t-22 -54t-54 -22t-55 22 +q-75 75 -180 75q-106 0 -181 -74.5t-75 -180.5t75 -180.5t181 -74.5h1046q79 0 134.5 55.5t55.5 133.5zM1798 438q0 -142 -100.5 -242t-242.5 -100h-1046q-169 0 -289 119.5t-120 288.5q0 153 100 267t249 136q62 184 221 298t354 114q235 0 408.5 -158.5t196.5 -389.5 +q116 -25 192.5 -118.5t76.5 -214.5zM2048 438q0 -175 -97 -319q-23 -33 -64 -33q-24 0 -43 13q-26 17 -32 48.5t12 57.5q71 104 71 233t-71 233q-18 26 -12 57t32 49t57.5 11.5t49.5 -32.5q97 -142 97 -318zM2304 438q0 -244 -134 -443q-23 -34 -64 -34q-23 0 -42 13 +q-26 18 -32.5 49t11.5 57q108 164 108 358q0 195 -108 357q-18 26 -11.5 57.5t32.5 48.5q26 18 57 12t49 -33q134 -198 134 -442z" /> + <glyph glyph-name="_610" unicode="" +d="M1500 -13q0 -89 -63 -152.5t-153 -63.5t-153.5 63.5t-63.5 152.5q0 90 63.5 153.5t153.5 63.5t153 -63.5t63 -153.5zM1267 268q-115 -15 -192.5 -102.5t-77.5 -205.5q0 -74 33 -138q-146 -78 -379 -78q-109 0 -201 21t-153.5 54.5t-110.5 76.5t-76 85t-44.5 83 +t-23.5 66.5t-6 39.5q0 19 4.5 42.5t18.5 56t36.5 58t64 43.5t94.5 18t94 -17.5t63 -41t35.5 -53t17.5 -49t4 -33.5q0 -34 -23 -81q28 -27 82 -42t93 -17l40 -1q115 0 190 51t75 133q0 26 -9 48.5t-31.5 44.5t-49.5 41t-74 44t-93.5 47.5t-119.5 56.5q-28 13 -43 20 +q-116 55 -187 100t-122.5 102t-72 125.5t-20.5 162.5q0 78 20.5 150t66 137.5t112.5 114t166.5 77t221.5 28.5q120 0 220 -26t164.5 -67t109.5 -94t64 -105.5t19 -103.5q0 -46 -15 -82.5t-36.5 -58t-48.5 -36t-49 -19.5t-39 -5h-8h-32t-39 5t-44 14t-41 28t-37 46t-24 70.5 +t-10 97.5q-15 16 -59 25.5t-81 10.5l-37 1q-68 0 -117.5 -31t-70.5 -70t-21 -76q0 -24 5 -43t24 -46t53 -51t97 -53.5t150 -58.5q76 -25 138.5 -53.5t109 -55.5t83 -59t60.5 -59.5t41 -62.5t26.5 -62t14.5 -63.5t6 -62t1 -62.5z" /> + <glyph glyph-name="_611" unicode="" +d="M704 352v576q0 14 -9 23t-23 9h-256q-14 0 -23 -9t-9 -23v-576q0 -14 9 -23t23 -9h256q14 0 23 9t9 23zM1152 352v576q0 14 -9 23t-23 9h-256q-14 0 -23 -9t-9 -23v-576q0 -14 9 -23t23 -9h256q14 0 23 9t9 23zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103 +t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="_612" unicode="" +d="M768 1408q209 0 385.5 -103t279.5 -279.5t103 -385.5t-103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103zM768 96q148 0 273 73t198 198t73 273t-73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273 +t73 -273t198 -198t273 -73zM864 320q-14 0 -23 9t-9 23v576q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-576q0 -14 -9 -23t-23 -9h-192zM480 320q-14 0 -23 9t-9 23v576q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-576q0 -14 -9 -23t-23 -9h-192z" /> + <glyph glyph-name="_613" unicode="" +d="M1088 352v576q0 14 -9 23t-23 9h-576q-14 0 -23 -9t-9 -23v-576q0 -14 9 -23t23 -9h576q14 0 23 9t9 23zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5 +t103 -385.5z" /> + <glyph glyph-name="_614" unicode="" +d="M768 1408q209 0 385.5 -103t279.5 -279.5t103 -385.5t-103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103zM768 96q148 0 273 73t198 198t73 273t-73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273 +t73 -273t198 -198t273 -73zM480 320q-14 0 -23 9t-9 23v576q0 14 9 23t23 9h576q14 0 23 -9t9 -23v-576q0 -14 -9 -23t-23 -9h-576z" /> + <glyph glyph-name="_615" unicode="" horiz-adv-x="1792" +d="M1757 128l35 -313q3 -28 -16 -50q-19 -21 -48 -21h-1664q-29 0 -48 21q-19 22 -16 50l35 313h1722zM1664 967l86 -775h-1708l86 775q3 24 21 40.5t43 16.5h256v-128q0 -53 37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5v128h384v-128q0 -53 37.5 -90.5t90.5 -37.5 +t90.5 37.5t37.5 90.5v128h256q25 0 43 -16.5t21 -40.5zM1280 1152v-256q0 -26 -19 -45t-45 -19t-45 19t-19 45v256q0 106 -75 181t-181 75t-181 -75t-75 -181v-256q0 -26 -19 -45t-45 -19t-45 19t-19 45v256q0 159 112.5 271.5t271.5 112.5t271.5 -112.5t112.5 -271.5z" /> + <glyph glyph-name="_616" unicode="" horiz-adv-x="2048" +d="M1920 768q53 0 90.5 -37.5t37.5 -90.5t-37.5 -90.5t-90.5 -37.5h-15l-115 -662q-8 -46 -44 -76t-82 -30h-1280q-46 0 -82 30t-44 76l-115 662h-15q-53 0 -90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5h1792zM485 -32q26 2 43.5 22.5t15.5 46.5l-32 416q-2 26 -22.5 43.5 +t-46.5 15.5t-43.5 -22.5t-15.5 -46.5l32 -416q2 -25 20.5 -42t43.5 -17h5zM896 32v416q0 26 -19 45t-45 19t-45 -19t-19 -45v-416q0 -26 19 -45t45 -19t45 19t19 45zM1280 32v416q0 26 -19 45t-45 19t-45 -19t-19 -45v-416q0 -26 19 -45t45 -19t45 19t19 45zM1632 27l32 416 +q2 26 -15.5 46.5t-43.5 22.5t-46.5 -15.5t-22.5 -43.5l-32 -416q-2 -26 15.5 -46.5t43.5 -22.5h5q25 0 43.5 17t20.5 42zM476 1244l-93 -412h-132l101 441q19 88 89 143.5t160 55.5h167q0 26 19 45t45 19h384q26 0 45 -19t19 -45h167q90 0 160 -55.5t89 -143.5l101 -441 +h-132l-93 412q-11 44 -45.5 72t-79.5 28h-167q0 -26 -19 -45t-45 -19h-384q-26 0 -45 19t-19 45h-167q-45 0 -79.5 -28t-45.5 -72z" /> + <glyph glyph-name="_617" unicode="" horiz-adv-x="1792" +d="M991 512l64 256h-254l-64 -256h254zM1759 1016l-56 -224q-7 -24 -31 -24h-327l-64 -256h311q15 0 25 -12q10 -14 6 -28l-56 -224q-5 -24 -31 -24h-327l-81 -328q-7 -24 -31 -24h-224q-16 0 -26 12q-9 12 -6 28l78 312h-254l-81 -328q-7 -24 -31 -24h-225q-15 0 -25 12 +q-9 12 -6 28l78 312h-311q-15 0 -25 12q-9 12 -6 28l56 224q7 24 31 24h327l64 256h-311q-15 0 -25 12q-10 14 -6 28l56 224q5 24 31 24h327l81 328q7 24 32 24h224q15 0 25 -12q9 -12 6 -28l-78 -312h254l81 328q7 24 32 24h224q15 0 25 -12q9 -12 6 -28l-78 -312h311 +q15 0 25 -12q9 -12 6 -28z" /> + <glyph glyph-name="_618" unicode="" +d="M841 483l148 -148l-149 -149zM840 1094l149 -149l-148 -148zM710 -130l464 464l-306 306l306 306l-464 464v-611l-255 255l-93 -93l320 -321l-320 -321l93 -93l255 255v-611zM1429 640q0 -209 -32 -365.5t-87.5 -257t-140.5 -162.5t-181.5 -86.5t-219.5 -24.5 +t-219.5 24.5t-181.5 86.5t-140.5 162.5t-87.5 257t-32 365.5t32 365.5t87.5 257t140.5 162.5t181.5 86.5t219.5 24.5t219.5 -24.5t181.5 -86.5t140.5 -162.5t87.5 -257t32 -365.5z" /> + <glyph glyph-name="_619" unicode="" horiz-adv-x="1024" +d="M596 113l173 172l-173 172v-344zM596 823l173 172l-173 172v-344zM628 640l356 -356l-539 -540v711l-297 -296l-108 108l372 373l-372 373l108 108l297 -296v711l539 -540z" /> + <glyph glyph-name="_620" unicode="" +d="M1280 256q0 52 -38 90t-90 38t-90 -38t-38 -90t38 -90t90 -38t90 38t38 90zM512 1024q0 52 -38 90t-90 38t-90 -38t-38 -90t38 -90t90 -38t90 38t38 90zM1536 256q0 -159 -112.5 -271.5t-271.5 -112.5t-271.5 112.5t-112.5 271.5t112.5 271.5t271.5 112.5t271.5 -112.5 +t112.5 -271.5zM1440 1344q0 -20 -13 -38l-1056 -1408q-19 -26 -51 -26h-160q-26 0 -45 19t-19 45q0 20 13 38l1056 1408q19 26 51 26h160q26 0 45 -19t19 -45zM768 1024q0 -159 -112.5 -271.5t-271.5 -112.5t-271.5 112.5t-112.5 271.5t112.5 271.5t271.5 112.5 +t271.5 -112.5t112.5 -271.5z" /> + <glyph glyph-name="_621" unicode="" horiz-adv-x="1792" +d="M104 830l792 -1015l-868 630q-18 13 -25 34.5t0 42.5l101 308v0zM566 830h660l-330 -1015v0zM368 1442l198 -612h-462l198 612q8 23 33 23t33 -23zM1688 830l101 -308q7 -21 0 -42.5t-25 -34.5l-868 -630l792 1015v0zM1688 830h-462l198 612q8 23 33 23t33 -23z" /> + <glyph glyph-name="_622" unicode="" horiz-adv-x="1792" +d="M384 704h160v224h-160v-224zM1221 372v92q-104 -36 -243 -38q-135 -1 -259.5 46.5t-220.5 122.5l1 -96q88 -80 212 -128.5t272 -47.5q129 0 238 49zM640 704h640v224h-640v-224zM1792 736q0 -187 -99 -352q89 -102 89 -229q0 -157 -129.5 -268t-313.5 -111 +q-122 0 -225 52.5t-161 140.5q-19 -1 -57 -1t-57 1q-58 -88 -161 -140.5t-225 -52.5q-184 0 -313.5 111t-129.5 268q0 127 89 229q-99 165 -99 352q0 209 120 385.5t326.5 279.5t449.5 103t449.5 -103t326.5 -279.5t120 -385.5z" /> + <glyph glyph-name="_623" unicode="" +d="M515 625v-128h-252v128h252zM515 880v-127h-252v127h252zM1273 369v-128h-341v128h341zM1273 625v-128h-672v128h672zM1273 880v-127h-672v127h672zM1408 20v1240q0 8 -6 14t-14 6h-32l-378 -256l-210 171l-210 -171l-378 256h-32q-8 0 -14 -6t-6 -14v-1240q0 -8 6 -14 +t14 -6h1240q8 0 14 6t6 14zM553 1130l185 150h-406zM983 1130l221 150h-406zM1536 1260v-1240q0 -62 -43 -105t-105 -43h-1240q-62 0 -105 43t-43 105v1240q0 62 43 105t105 43h1240q62 0 105 -43t43 -105z" /> + <glyph glyph-name="_624" unicode="" horiz-adv-x="1792" +d="M896 720q-104 196 -160 278q-139 202 -347 318q-34 19 -70 36q-89 40 -94 32t34 -38l39 -31q62 -43 112.5 -93.5t94.5 -116.5t70.5 -113t70.5 -131q9 -17 13 -25q44 -84 84 -153t98 -154t115.5 -150t131 -123.5t148.5 -90.5q153 -66 154 -60q1 3 -49 37q-53 36 -81 57 +q-77 58 -179 211t-185 310zM549 177q-76 60 -132.5 125t-98 143.5t-71 154.5t-58.5 186t-52 209t-60.5 252t-76.5 289q273 0 497.5 -36t379 -92t271 -144.5t185.5 -172.5t110 -198.5t56 -199.5t12.5 -198.5t-9.5 -173t-20 -143.5t-13 -107l323 -327h-104l-281 285 +q-22 -2 -91.5 -14t-121.5 -19t-138 -6t-160.5 17t-167.5 59t-179 111z" /> + <glyph glyph-name="_625" unicode="" horiz-adv-x="1792" +d="M1374 879q-6 26 -28.5 39.5t-48.5 7.5q-261 -62 -401 -62t-401 62q-26 6 -48.5 -7.5t-28.5 -39.5t7.5 -48.5t39.5 -28.5q194 -46 303 -58q-2 -158 -15.5 -269t-26.5 -155.5t-41 -115.5l-9 -21q-10 -25 1 -49t36 -34q9 -4 23 -4q44 0 60 41l8 20q54 139 71 259h42 +q17 -120 71 -259l8 -20q16 -41 60 -41q14 0 23 4q25 10 36 34t1 49l-9 21q-28 71 -41 115.5t-26.5 155.5t-15.5 269q109 12 303 58q26 6 39.5 28.5t7.5 48.5zM1024 1024q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5z +M1600 640q0 -143 -55.5 -273.5t-150 -225t-225 -150t-273.5 -55.5t-273.5 55.5t-225 150t-150 225t-55.5 273.5t55.5 273.5t150 225t225 150t273.5 55.5t273.5 -55.5t225 -150t150 -225t55.5 -273.5zM896 1408q-156 0 -298 -61t-245 -164t-164 -245t-61 -298t61 -298 +t164 -245t245 -164t298 -61t298 61t245 164t164 245t61 298t-61 298t-164 245t-245 164t-298 61zM1792 640q0 -182 -71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71t348 -71t286 -191t191 -286t71 -348z" /> + <glyph glyph-name="_626" unicode="" +d="M1438 723q34 -35 29 -82l-44 -551q-4 -42 -34.5 -70t-71.5 -28q-6 0 -9 1q-44 3 -72.5 36.5t-25.5 77.5l35 429l-143 -8q55 -113 55 -240q0 -216 -148 -372l-137 137q91 101 91 235q0 145 -102.5 248t-247.5 103q-134 0 -236 -92l-137 138q120 114 284 141l264 300 +l-149 87l-181 -161q-33 -30 -77 -27.5t-73 35.5t-26.5 77t34.5 73l239 213q26 23 60 26.5t64 -14.5l488 -283q36 -21 48 -68q17 -67 -26 -117l-205 -232l371 20q49 3 83 -32zM1240 1180q-74 0 -126 52t-52 126t52 126t126 52t126.5 -52t52.5 -126t-52.5 -126t-126.5 -52z +M613 -62q106 0 196 61l139 -139q-146 -116 -335 -116q-148 0 -273.5 73t-198.5 198t-73 273q0 188 116 336l139 -139q-60 -88 -60 -197q0 -145 102.5 -247.5t247.5 -102.5z" /> + <glyph glyph-name="_627" unicode="" +d="M880 336v-160q0 -14 -9 -23t-23 -9h-160q-14 0 -23 9t-9 23v160q0 14 9 23t23 9h160q14 0 23 -9t9 -23zM1136 832q0 -50 -15 -90t-45.5 -69t-52 -44t-59.5 -36q-32 -18 -46.5 -28t-26 -24t-11.5 -29v-32q0 -14 -9 -23t-23 -9h-160q-14 0 -23 9t-9 23v68q0 35 10.5 64.5 +t24 47.5t39 35.5t41 25.5t44.5 21q53 25 75 43t22 49q0 42 -43.5 71.5t-95.5 29.5q-56 0 -95 -27q-29 -20 -80 -83q-9 -12 -25 -12q-11 0 -19 6l-108 82q-10 7 -12 20t5 23q122 192 349 192q129 0 238.5 -89.5t109.5 -214.5zM768 1280q-130 0 -248.5 -51t-204 -136.5 +t-136.5 -204t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5t-51 248.5t-136.5 204t-204 136.5t-248.5 51zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5 +t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="_628" unicode="" horiz-adv-x="1408" +d="M366 1225q-64 0 -110 45.5t-46 110.5q0 64 46 109.5t110 45.5t109.5 -45.5t45.5 -109.5q0 -65 -45.5 -110.5t-109.5 -45.5zM917 583q0 -50 -30 -67.5t-63.5 -6.5t-47.5 34l-367 438q-7 12 -14 15.5t-11 1.5l-3 -3q-7 -8 4 -21l122 -139l1 -354l-161 -457 +q-67 -192 -92 -234q-15 -26 -28 -32q-50 -26 -103 -1q-29 13 -41.5 43t-9.5 57q2 17 197 618l5 416l-85 -164l35 -222q4 -24 -1 -42t-14 -27.5t-19 -16t-17 -7.5l-7 -2q-19 -3 -34.5 3t-24 16t-14 22t-7.5 19.5t-2 9.5l-46 299l211 381q23 34 113 34q75 0 107 -40l424 -521 +q7 -5 14 -17l3 -3l-1 -1q7 -13 7 -29zM514 433q43 -113 88.5 -225t69.5 -168l24 -55q36 -93 42 -125q11 -70 -36 -97q-35 -22 -66 -16t-51 22t-29 35h-1q-6 16 -8 25l-124 351zM1338 -159q31 -49 31 -57q0 -5 -3 -7q-9 -5 -14.5 0.5t-15.5 26t-16 30.5q-114 172 -423 661 +q3 -1 7 1t7 4l3 2q11 9 11 17z" /> + <glyph glyph-name="_629" unicode="" horiz-adv-x="2304" +d="M504 542h171l-1 265zM1530 641q0 87 -50.5 140t-146.5 53h-54v-388h52q91 0 145 57t54 138zM956 1018l1 -756q0 -14 -9.5 -24t-23.5 -10h-216q-14 0 -23.5 10t-9.5 24v62h-291l-55 -81q-10 -15 -28 -15h-267q-21 0 -30.5 18t3.5 35l556 757q9 14 27 14h332q14 0 24 -10 +t10 -24zM1783 641q0 -193 -125.5 -303t-324.5 -110h-270q-14 0 -24 10t-10 24v756q0 14 10 24t24 10h268q200 0 326 -109t126 -302zM1939 640q0 -11 -0.5 -29t-8 -71.5t-21.5 -102t-44.5 -108t-73.5 -102.5h-51q38 45 66.5 104.5t41.5 112t21 98t9 72.5l1 27q0 8 -0.5 22.5 +t-7.5 60t-20 91.5t-41 111.5t-66 124.5h43q41 -47 72 -107t45.5 -111.5t23 -96t10.5 -70.5zM2123 640q0 -11 -0.5 -29t-8 -71.5t-21.5 -102t-45 -108t-74 -102.5h-51q38 45 66.5 104.5t41.5 112t21 98t9 72.5l1 27q0 8 -0.5 22.5t-7.5 60t-19.5 91.5t-40.5 111.5t-66 124.5 +h43q41 -47 72 -107t45.5 -111.5t23 -96t10.5 -70.5zM2304 640q0 -11 -0.5 -29t-8 -71.5t-21.5 -102t-44.5 -108t-73.5 -102.5h-51q38 45 66 104.5t41 112t21 98t9 72.5l1 27q0 8 -0.5 22.5t-7.5 60t-19.5 91.5t-40.5 111.5t-66 124.5h43q41 -47 72 -107t45.5 -111.5t23 -96 +t9.5 -70.5z" /> + <glyph glyph-name="uniF2A0" unicode="" horiz-adv-x="1408" +d="M617 -153q0 11 -13 58t-31 107t-20 69q-1 4 -5 26.5t-8.5 36t-13.5 21.5q-15 14 -51 14q-23 0 -70 -5.5t-71 -5.5q-34 0 -47 11q-6 5 -11 15.5t-7.5 20t-6.5 24t-5 18.5q-37 128 -37 255t37 255q1 4 5 18.5t6.5 24t7.5 20t11 15.5q13 11 47 11q24 0 71 -5.5t70 -5.5 +q36 0 51 14q9 8 13.5 21.5t8.5 36t5 26.5q2 9 20 69t31 107t13 58q0 22 -43.5 52.5t-75.5 42.5q-20 8 -45 8q-34 0 -98 -18q-57 -17 -96.5 -40.5t-71 -66t-46 -70t-45.5 -94.5q-6 -12 -9 -19q-49 -107 -68 -216t-19 -244t19 -244t68 -216q56 -122 83 -161q63 -91 179 -127 +l6 -2q64 -18 98 -18q25 0 45 8q32 12 75.5 42.5t43.5 52.5zM776 760q-26 0 -45 19t-19 45.5t19 45.5q37 37 37 90q0 52 -37 91q-19 19 -19 45t19 45t45 19t45 -19q75 -75 75 -181t-75 -181q-21 -19 -45 -19zM957 579q-27 0 -45 19q-19 19 -19 45t19 45q112 114 112 272 +t-112 272q-19 19 -19 45t19 45t45 19t45 -19q150 -150 150 -362t-150 -362q-18 -19 -45 -19zM1138 398q-27 0 -45 19q-19 19 -19 45t19 45q90 91 138.5 208t48.5 245t-48.5 245t-138.5 208q-19 19 -19 45t19 45t45 19t45 -19q109 -109 167 -249t58 -294t-58 -294t-167 -249 +q-18 -19 -45 -19z" /> + <glyph glyph-name="uniF2A1" unicode="" horiz-adv-x="2176" +d="M192 352q-66 0 -113 -47t-47 -113t47 -113t113 -47t113 47t47 113t-47 113t-113 47zM704 352q-66 0 -113 -47t-47 -113t47 -113t113 -47t113 47t47 113t-47 113t-113 47zM704 864q-66 0 -113 -47t-47 -113t47 -113t113 -47t113 47t47 113t-47 113t-113 47zM1472 352 +q-66 0 -113 -47t-47 -113t47 -113t113 -47t113 47t47 113t-47 113t-113 47zM1984 352q-66 0 -113 -47t-47 -113t47 -113t113 -47t113 47t47 113t-47 113t-113 47zM1472 864q-66 0 -113 -47t-47 -113t47 -113t113 -47t113 47t47 113t-47 113t-113 47zM1984 864 +q-66 0 -113 -47t-47 -113t47 -113t113 -47t113 47t47 113t-47 113t-113 47zM1984 1376q-66 0 -113 -47t-47 -113t47 -113t113 -47t113 47t47 113t-47 113t-113 47zM384 192q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM896 192q0 -80 -56 -136 +t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM384 704q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM896 704q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM384 1216q0 -80 -56 -136t-136 -56 +t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM1664 192q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM896 1216q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM2176 192q0 -80 -56 -136t-136 -56t-136 56 +t-56 136t56 136t136 56t136 -56t56 -136zM1664 704q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM2176 704q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM1664 1216q0 -80 -56 -136t-136 -56t-136 56t-56 136 +t56 136t136 56t136 -56t56 -136zM2176 1216q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136z" /> + <glyph glyph-name="uniF2A2" unicode="" horiz-adv-x="1792" +d="M128 -192q0 -26 -19 -45t-45 -19t-45 19t-19 45t19 45t45 19t45 -19t19 -45zM320 0q0 -26 -19 -45t-45 -19t-45 19t-19 45t19 45t45 19t45 -19t19 -45zM365 365l256 -256l-90 -90l-256 256zM704 384q0 -26 -19 -45t-45 -19t-45 19t-19 45t19 45t45 19t45 -19t19 -45z +M1411 704q0 -59 -11.5 -108.5t-37.5 -93.5t-44 -67.5t-53 -64.5q-31 -35 -45.5 -54t-33.5 -50t-26.5 -64t-7.5 -74q0 -159 -112.5 -271.5t-271.5 -112.5q-26 0 -45 19t-19 45t19 45t45 19q106 0 181 75t75 181q0 57 11.5 105.5t37 91t43.5 66.5t52 63q40 46 59.5 72 +t37.5 74.5t18 103.5q0 185 -131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5q0 -26 -19 -45t-45 -19t-45 19t-19 45q0 117 45.5 223.5t123 184t184 123t223.5 45.5t223.5 -45.5t184 -123t123 -184t45.5 -223.5zM896 576q0 -26 -19 -45t-45 -19t-45 19t-19 45t19 45 +t45 19t45 -19t19 -45zM1184 704q0 -26 -19 -45t-45 -19t-45 19t-19 45q0 93 -65.5 158.5t-158.5 65.5q-92 0 -158 -65.5t-66 -158.5q0 -26 -19 -45t-45 -19t-45 19t-19 45q0 146 103 249t249 103t249 -103t103 -249zM1578 993q10 -25 -1 -49t-36 -34q-9 -4 -23 -4 +q-19 0 -35.5 11t-23.5 30q-68 178 -224 295q-21 16 -25 42t12 47q17 21 43 25t47 -12q183 -137 266 -351zM1788 1074q9 -25 -1.5 -49t-35.5 -34q-11 -4 -23 -4q-44 0 -60 41q-92 238 -297 393q-22 16 -25.5 42t12.5 47q16 22 42 25.5t47 -12.5q235 -175 341 -449z" /> + <glyph glyph-name="uniF2A3" unicode="" horiz-adv-x="2304" +d="M1032 576q-59 2 -84 55q-17 34 -48 53.5t-68 19.5q-53 0 -90.5 -37.5t-37.5 -90.5q0 -56 36 -89l10 -8q34 -31 82 -31q37 0 68 19.5t48 53.5q25 53 84 55zM1600 704q0 56 -36 89l-10 8q-34 31 -82 31q-37 0 -68 -19.5t-48 -53.5q-25 -53 -84 -55q59 -2 84 -55 +q17 -34 48 -53.5t68 -19.5q53 0 90.5 37.5t37.5 90.5zM1174 925q-17 -35 -55 -48t-73 4q-62 31 -134 31q-51 0 -99 -17q3 0 9.5 0.5t9.5 0.5q92 0 170.5 -50t118.5 -133q17 -36 3.5 -73.5t-49.5 -54.5q-18 -9 -39 -9q21 0 39 -9q36 -17 49.5 -54.5t-3.5 -73.5 +q-40 -83 -118.5 -133t-170.5 -50h-6q-16 2 -44 4l-290 27l-239 -120q-14 -7 -29 -7q-40 0 -57 35l-160 320q-11 23 -4 47.5t29 37.5l209 119l148 267q17 155 91.5 291.5t195.5 236.5q31 25 70.5 21.5t64.5 -34.5t21.5 -70t-34.5 -65q-70 -59 -117 -128q123 84 267 101 +q40 5 71.5 -19t35.5 -64q5 -40 -19 -71.5t-64 -35.5q-84 -10 -159 -55q46 10 99 10q115 0 218 -50q36 -18 49 -55.5t-5 -73.5zM2137 1085l160 -320q11 -23 4 -47.5t-29 -37.5l-209 -119l-148 -267q-17 -155 -91.5 -291.5t-195.5 -236.5q-26 -22 -61 -22q-45 0 -74 35 +q-25 31 -21.5 70t34.5 65q70 59 117 128q-123 -84 -267 -101q-4 -1 -12 -1q-36 0 -63.5 24t-31.5 60q-5 40 19 71.5t64 35.5q84 10 159 55q-46 -10 -99 -10q-115 0 -218 50q-36 18 -49 55.5t5 73.5q17 35 55 48t73 -4q62 -31 134 -31q51 0 99 17q-3 0 -9.5 -0.5t-9.5 -0.5 +q-92 0 -170.5 50t-118.5 133q-17 36 -3.5 73.5t49.5 54.5q18 9 39 9q-21 0 -39 9q-36 17 -49.5 54.5t3.5 73.5q40 83 118.5 133t170.5 50h6h1q14 -2 42 -4l291 -27l239 120q14 7 29 7q40 0 57 -35z" /> + <glyph glyph-name="uniF2A4" unicode="" horiz-adv-x="1792" +d="M1056 704q0 -26 19 -45t45 -19t45 19t19 45q0 146 -103 249t-249 103t-249 -103t-103 -249q0 -26 19 -45t45 -19t45 19t19 45q0 93 66 158.5t158 65.5t158 -65.5t66 -158.5zM835 1280q-117 0 -223.5 -45.5t-184 -123t-123 -184t-45.5 -223.5q0 -26 19 -45t45 -19t45 19 +t19 45q0 185 131.5 316.5t316.5 131.5t316.5 -131.5t131.5 -316.5q0 -55 -18 -103.5t-37.5 -74.5t-59.5 -72q-34 -39 -52 -63t-43.5 -66.5t-37 -91t-11.5 -105.5q0 -106 -75 -181t-181 -75q-26 0 -45 -19t-19 -45t19 -45t45 -19q159 0 271.5 112.5t112.5 271.5q0 41 7.5 74 +t26.5 64t33.5 50t45.5 54q35 41 53 64.5t44 67.5t37.5 93.5t11.5 108.5q0 117 -45.5 223.5t-123 184t-184 123t-223.5 45.5zM591 561l226 -226l-579 -579q-12 -12 -29 -12t-29 12l-168 168q-12 12 -12 29t12 29zM1612 1524l168 -168q12 -12 12 -29t-12 -30l-233 -233 +l-26 -25l-71 -71q-66 153 -195 258l91 91l207 207q13 12 30 12t29 -12z" /> + <glyph glyph-name="uniF2A5" unicode="" +d="M866 1021q0 -27 -13 -94q-11 -50 -31.5 -150t-30.5 -150q-2 -11 -4.5 -12.5t-13.5 -2.5q-20 -2 -31 -2q-58 0 -84 49.5t-26 113.5q0 88 35 174t103 124q28 14 51 14q28 0 36.5 -16.5t8.5 -47.5zM1352 597q0 14 -39 75.5t-52 66.5q-21 8 -34 8q-91 0 -226 -77l-2 2 +q3 22 27.5 135t24.5 178q0 233 -242 233q-24 0 -68 -6q-94 -17 -168.5 -89.5t-111.5 -166.5t-37 -189q0 -146 80.5 -225t227.5 -79q25 0 25 -3t-1 -5q-4 -34 -26 -117q-14 -52 -51.5 -101t-82.5 -49q-42 0 -42 47q0 24 10.5 47.5t25 39.5t29.5 28.5t26 20t11 8.5q0 3 -7 10 +q-24 22 -58.5 36.5t-65.5 14.5q-35 0 -63.5 -34t-41 -75t-12.5 -75q0 -88 51.5 -142t138.5 -54q82 0 155 53t117.5 126t65.5 153q6 22 15.5 66.5t14.5 66.5q3 12 14 18q118 60 227 60q48 0 127 -18q1 -1 4 -1q5 0 9.5 4.5t4.5 8.5zM1536 1120v-960q0 -119 -84.5 -203.5 +t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="uniF2A6" unicode="" horiz-adv-x="1535" +d="M744 1231q0 24 -2 38.5t-8.5 30t-21 23t-37.5 7.5q-39 0 -78 -23q-105 -58 -159 -190.5t-54 -269.5q0 -44 8.5 -85.5t26.5 -80.5t52.5 -62.5t81.5 -23.5q4 0 18 -0.5t20 0t16 3t15 8.5t7 16q16 77 48 231.5t48 231.5q19 91 19 146zM1498 575q0 -7 -7.5 -13.5t-15.5 -6.5 +l-6 1q-22 3 -62 11t-72 12.5t-63 4.5q-167 0 -351 -93q-15 -8 -21 -27q-10 -36 -24.5 -105.5t-22.5 -100.5q-23 -91 -70 -179.5t-112.5 -164.5t-154.5 -123t-185 -47q-135 0 -214.5 83.5t-79.5 219.5q0 53 19.5 117t63 116.5t97.5 52.5q38 0 120 -33.5t83 -61.5 +q0 -1 -16.5 -12.5t-39.5 -31t-46 -44.5t-39 -61t-16 -74q0 -33 16.5 -53t48.5 -20q45 0 85 31.5t66.5 78t48 105.5t32.5 107t16 90v9q0 2 -3.5 3.5t-8.5 1.5h-10t-10 -0.5t-6 -0.5q-227 0 -352 122.5t-125 348.5q0 108 34.5 221t96 210t156 167.5t204.5 89.5q52 9 106 9 +q374 0 374 -360q0 -98 -38 -273t-43 -211l3 -3q101 57 182.5 88t167.5 31q22 0 53 -13q19 -7 80 -102.5t61 -116.5z" /> + <glyph glyph-name="uniF2A7" unicode="" horiz-adv-x="1664" +d="M831 863q32 0 59 -18l222 -148q61 -40 110 -97l146 -170q40 -46 29 -106l-72 -413q-6 -32 -29.5 -53.5t-55.5 -25.5l-527 -56l-352 -32h-9q-39 0 -67.5 28t-28.5 68q0 37 27 64t65 32l260 32h-448q-41 0 -69.5 30t-26.5 71q2 39 32 65t69 26l442 1l-521 64q-41 5 -66 37 +t-19 73q6 35 34.5 57.5t65.5 22.5h10l481 -60l-351 94q-38 10 -62 41.5t-18 68.5q6 36 33 58.5t62 22.5q6 0 20 -2l448 -96l217 -37q1 0 3 -0.5t3 -0.5q23 0 30.5 23t-12.5 36l-186 125q-35 23 -42 63.5t18 73.5q27 38 76 38zM761 661l186 -125l-218 37l-5 2l-36 38 +l-238 262q-1 1 -2.5 3.5t-2.5 3.5q-24 31 -18.5 70t37.5 64q31 23 68 17.5t64 -33.5l142 -147q-2 -1 -5 -3.5t-4 -4.5q-32 -45 -23 -99t55 -85zM1648 1115l15 -266q4 -73 -11 -147l-48 -219q-12 -59 -67 -87l-106 -54q2 62 -39 109l-146 170q-53 61 -117 103l-222 148 +q-34 23 -76 23q-51 0 -88 -37l-235 312q-25 33 -18 73.5t41 63.5q33 22 71.5 14t62.5 -40l266 -352l-262 455q-21 35 -10.5 75t47.5 59q35 18 72.5 6t57.5 -46l241 -420l-136 337q-15 35 -4.5 74t44.5 56q37 19 76 6t56 -51l193 -415l101 -196q8 -15 23 -17.5t27 7.5t11 26 +l-12 224q-2 41 26 71t69 31q39 0 67 -28.5t30 -67.5z" /> + <glyph glyph-name="uniF2A8" unicode="" horiz-adv-x="1792" +d="M335 180q-2 0 -6 2q-86 57 -168.5 145t-139.5 180q-21 30 -21 69q0 9 2 19t4 18t7 18t8.5 16t10.5 17t10 15t12 15.5t11 14.5q184 251 452 365q-110 198 -110 211q0 19 17 29q116 64 128 64q18 0 28 -16l124 -229q92 19 192 19q266 0 497.5 -137.5t378.5 -369.5 +q20 -31 20 -69t-20 -69q-91 -142 -218.5 -253.5t-278.5 -175.5q110 -198 110 -211q0 -20 -17 -29q-116 -64 -127 -64q-19 0 -29 16l-124 229l-64 119l-444 820l7 7q-58 -24 -99 -47q3 -5 127 -234t243 -449t119 -223q0 -7 -9 -9q-13 -3 -72 -3q-57 0 -60 7l-456 841 +q-39 -28 -82 -68q24 -43 214 -393.5t190 -354.5q0 -10 -11 -10q-14 0 -82.5 22t-72.5 28l-106 197l-224 413q-44 -53 -78 -106q2 -3 18 -25t23 -34l176 -327q0 -10 -10 -10zM1165 282l49 -91q273 111 450 385q-180 277 -459 389q67 -64 103 -148.5t36 -176.5 +q0 -106 -47 -200.5t-132 -157.5zM848 896q0 -20 14 -34t34 -14q86 0 147 -61t61 -147q0 -20 14 -34t34 -14t34 14t14 34q0 126 -89 215t-215 89q-20 0 -34 -14t-14 -34zM1214 961l-9 4l7 -7z" /> + <glyph glyph-name="uniF2A9" unicode="" horiz-adv-x="1280" +d="M1050 430q0 -215 -147 -374q-148 -161 -378 -161q-232 0 -378 161q-147 159 -147 374q0 147 68 270.5t189 196.5t268 73q96 0 182 -31q-32 -62 -39 -126q-66 28 -143 28q-167 0 -280.5 -123t-113.5 -291q0 -170 112.5 -288.5t281.5 -118.5t281 118.5t112 288.5 +q0 89 -32 166q66 13 123 49q41 -98 41 -212zM846 619q0 -192 -79.5 -345t-238.5 -253l-14 -1q-29 0 -62 5q83 32 146.5 102.5t99.5 154.5t58.5 189t30 192.5t7.5 178.5q0 69 -3 103q55 -160 55 -326zM791 947v-2q-73 214 -206 440q88 -59 142.5 -186.5t63.5 -251.5z +M1035 744q-83 0 -160 75q218 120 290 247q19 37 21 56q-42 -94 -139.5 -166.5t-204.5 -97.5q-35 54 -35 113q0 37 17 79t43 68q46 44 157 74q59 16 106 58.5t74 100.5q74 -105 74 -253q0 -109 -24 -170q-32 -77 -88.5 -130.5t-130.5 -53.5z" /> + <glyph glyph-name="uniF2AA" unicode="" +d="M1050 495q0 78 -28 147q-41 -25 -85 -34q22 -50 22 -114q0 -117 -77 -198.5t-193 -81.5t-193.5 81.5t-77.5 198.5q0 115 78 199.5t193 84.5q53 0 98 -19q4 43 27 87q-60 21 -125 21q-154 0 -257.5 -108.5t-103.5 -263.5t103.5 -261t257.5 -106t257.5 106.5t103.5 260.5z +M872 850q2 -24 2 -71q0 -63 -5 -123t-20.5 -132.5t-40.5 -130t-68.5 -106t-100.5 -70.5q21 -3 42 -3h10q219 139 219 411q0 116 -38 225zM872 850q-4 80 -44 171.5t-98 130.5q92 -156 142 -302zM1207 955q0 102 -51 174q-41 -86 -124 -109q-69 -19 -109 -53.5t-40 -99.5 +q0 -40 24 -77q74 17 140.5 67t95.5 115q-4 -52 -74.5 -111.5t-138.5 -97.5q52 -52 110 -52q51 0 90 37t60 90q17 42 17 117zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5 +t84.5 -203.5z" /> + <glyph glyph-name="uniF2AB" unicode="" +d="M1279 388q0 22 -22 27q-67 15 -118 59t-80 108q-7 19 -7 25q0 15 19.5 26t43 17t43 20.5t19.5 36.5q0 19 -18.5 31.5t-38.5 12.5q-12 0 -32 -8t-31 -8q-4 0 -12 2q5 95 5 114q0 79 -17 114q-36 78 -103 121.5t-152 43.5q-199 0 -275 -165q-17 -35 -17 -114q0 -19 5 -114 +q-4 -2 -14 -2q-12 0 -32 7.5t-30 7.5q-21 0 -38.5 -12t-17.5 -32q0 -21 19.5 -35.5t43 -20.5t43 -17t19.5 -26q0 -6 -7 -25q-64 -138 -198 -167q-22 -5 -22 -27q0 -46 137 -68q2 -5 6 -26t11.5 -30.5t23.5 -9.5q12 0 37.5 4.5t39.5 4.5q35 0 67 -15t54 -32.5t57.5 -32.5 +t76.5 -15q43 0 79 15t57.5 32.5t53.5 32.5t67 15q14 0 39.5 -4t38.5 -4q16 0 23 10t11 30t6 25q137 22 137 68zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5 +t103 -385.5z" /> + <glyph glyph-name="uniF2AC" unicode="" horiz-adv-x="1664" +d="M848 1408q134 1 240.5 -68.5t163.5 -192.5q27 -58 27 -179q0 -47 -9 -191q14 -7 28 -7q18 0 51 13.5t51 13.5q29 0 56 -18t27 -46q0 -32 -31.5 -54t-69 -31.5t-69 -29t-31.5 -47.5q0 -15 12 -43q37 -82 102.5 -150t144.5 -101q28 -12 80 -23q28 -6 28 -35 +q0 -70 -219 -103q-7 -11 -11 -39t-14 -46.5t-33 -18.5q-20 0 -62 6.5t-64 6.5q-37 0 -62 -5q-32 -5 -63 -22.5t-58 -38t-58 -40.5t-76 -33.5t-99 -13.5q-52 0 -96.5 13.5t-75 33.5t-57.5 40.5t-58 38t-62 22.5q-26 5 -63 5q-24 0 -65.5 -7.5t-58.5 -7.5q-25 0 -35 18.5 +t-14 47.5t-11 40q-219 33 -219 103q0 29 28 35q52 11 80 23q78 32 144.5 101t102.5 150q12 28 12 43q0 28 -31.5 47.5t-69.5 29.5t-69.5 31.5t-31.5 52.5q0 27 26 45.5t55 18.5q15 0 48 -13t53 -13q18 0 32 7q-9 142 -9 190q0 122 27 180q64 137 172 198t264 63z" /> + <glyph glyph-name="uniF2AD" unicode="" +d="M1280 388q0 22 -22 27q-67 14 -118 58t-80 109q-7 14 -7 25q0 15 19.5 26t42.5 17t42.5 20.5t19.5 36.5q0 19 -18.5 31.5t-38.5 12.5q-11 0 -31 -8t-32 -8q-4 0 -12 2q5 63 5 115q0 78 -17 114q-36 78 -102.5 121.5t-152.5 43.5q-198 0 -275 -165q-18 -38 -18 -115 +q0 -38 6 -114q-10 -2 -15 -2q-11 0 -31.5 8t-30.5 8q-20 0 -37.5 -12.5t-17.5 -32.5q0 -21 19.5 -35.5t42.5 -20.5t42.5 -17t19.5 -26q0 -11 -7 -25q-64 -138 -198 -167q-22 -5 -22 -27q0 -47 138 -69q2 -5 6 -26t11 -30.5t23 -9.5q13 0 38.5 5t38.5 5q35 0 67.5 -15 +t54.5 -32.5t57.5 -32.5t76.5 -15q43 0 79 15t57.5 32.5t54 32.5t67.5 15q13 0 39 -4.5t39 -4.5q15 0 22.5 9.5t11.5 31t5 24.5q138 22 138 69zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960 +q119 0 203.5 -84.5t84.5 -203.5z" /> + <glyph glyph-name="uniF2AE" unicode="" horiz-adv-x="2304" +d="M2304 1536q-69 -46 -125 -92t-89 -81t-59.5 -71.5t-37.5 -57.5t-22 -44.5t-14 -29.5q-10 -18 -35.5 -136.5t-48.5 -164.5q-15 -29 -50 -60.5t-67.5 -50.5t-72.5 -41t-48 -28q-47 -31 -151 -231q-341 14 -630 -158q-92 -53 -303 -179q47 16 86 31t55 22l15 7 +q71 27 163 64.5t133.5 53.5t108 34.5t142.5 31.5q186 31 465 -7q1 0 10 -3q11 -6 14 -17t-3 -22l-194 -345q-15 -29 -47 -22q-128 24 -354 24q-146 0 -402 -44.5t-392 -46.5q-82 -1 -149 13t-107 37t-61 40t-33 34l-1 1v2q0 6 6 6q138 0 371 55q192 366 374.5 524t383.5 158 +q5 0 14.5 -0.5t38 -5t55 -12t61.5 -24.5t63 -39.5t54 -59t40 -82.5l102 177q2 4 21 42.5t44.5 86.5t61 109.5t84 133.5t100.5 137q66 82 128 141.5t121.5 96.5t92.5 53.5t88 39.5z" /> + <glyph glyph-name="uniF2B0" unicode="" +d="M1322 640q0 -45 -5 -76l-236 14l224 -78q-19 -73 -58 -141l-214 103l177 -158q-44 -61 -107 -108l-157 178l103 -215q-61 -37 -140 -59l-79 228l14 -240q-38 -6 -76 -6t-76 6l14 238l-78 -226q-74 19 -140 59l103 215l-157 -178q-59 43 -108 108l178 158l-214 -104 +q-39 69 -58 141l224 79l-237 -14q-5 42 -5 76q0 35 5 77l238 -14l-225 79q19 73 58 140l214 -104l-177 159q46 61 107 108l158 -178l-103 215q67 39 140 58l77 -224l-13 236q36 6 75 6q38 0 76 -6l-14 -237l78 225q74 -19 140 -59l-103 -214l158 178q61 -47 107 -108 +l-177 -159l213 104q37 -62 58 -141l-224 -78l237 14q5 -31 5 -77zM1352 640q0 160 -78.5 295.5t-213 214t-292.5 78.5q-119 0 -227 -46.5t-186.5 -125t-124.5 -187.5t-46 -229q0 -119 46 -228t124.5 -187.5t186.5 -125t227 -46.5q158 0 292.5 78.5t213 214t78.5 294.5z +M1425 1023v-766l-657 -383l-657 383v766l657 383zM768 -183l708 412v823l-708 411l-708 -411v-823zM1536 1088v-896l-768 -448l-768 448v896l768 448z" /> + <glyph glyph-name="uniF2B1" unicode="" horiz-adv-x="1664" +d="M339 1318h691l-26 -72h-665q-110 0 -188.5 -79t-78.5 -189v-771q0 -95 60.5 -169.5t153.5 -93.5q23 -5 98 -5v-72h-45q-140 0 -239.5 100t-99.5 240v771q0 140 99.5 240t239.5 100zM1190 1536h247l-482 -1294q-23 -61 -40.5 -103.5t-45 -98t-54 -93.5t-64.5 -78.5 +t-79.5 -65t-95.5 -41t-116 -18.5v195q163 26 220 182q20 52 20 105q0 54 -20 106l-285 733h228l187 -585zM1664 978v-1111h-795q37 55 45 73h678v1038q0 85 -49.5 155t-129.5 99l25 67q101 -34 163.5 -123.5t62.5 -197.5z" /> + <glyph glyph-name="uniF2B2" unicode="" horiz-adv-x="1792" +d="M852 1227q0 -29 -17 -52.5t-45 -23.5t-45 23.5t-17 52.5t17 52.5t45 23.5t45 -23.5t17 -52.5zM688 -149v114q0 30 -20.5 51.5t-50.5 21.5t-50 -21.5t-20 -51.5v-114q0 -30 20.5 -52t49.5 -22q30 0 50.5 22t20.5 52zM860 -149v114q0 30 -20 51.5t-50 21.5t-50.5 -21.5 +t-20.5 -51.5v-114q0 -30 20.5 -52t50.5 -22q29 0 49.5 22t20.5 52zM1034 -149v114q0 30 -20.5 51.5t-50.5 21.5t-50.5 -21.5t-20.5 -51.5v-114q0 -30 20.5 -52t50.5 -22t50.5 22t20.5 52zM1208 -149v114q0 30 -20.5 51.5t-50.5 21.5t-50.5 -21.5t-20.5 -51.5v-114 +q0 -30 20.5 -52t50.5 -22t50.5 22t20.5 52zM1476 535q-84 -160 -232 -259.5t-323 -99.5q-123 0 -229.5 51.5t-178.5 137t-113 197.5t-41 232q0 88 21 174q-104 -175 -104 -390q0 -162 65 -312t185 -251q30 57 91 57q56 0 86 -50q32 50 87 50q56 0 86 -50q32 50 87 50t87 -50 +q30 50 86 50q28 0 52.5 -15.5t37.5 -40.5q112 94 177 231.5t73 287.5zM1326 564q0 75 -72 75q-17 0 -47 -6q-95 -19 -149 -19q-226 0 -226 243q0 86 30 204q-83 -127 -83 -275q0 -150 89 -260.5t235 -110.5q111 0 210 70q13 48 13 79zM884 1223q0 50 -32 89.5t-81 39.5 +t-81 -39.5t-32 -89.5q0 -51 31.5 -90.5t81.5 -39.5t81.5 39.5t31.5 90.5zM1513 884q0 96 -37.5 179t-113 137t-173.5 54q-77 0 -149 -35t-127 -94q-48 -159 -48 -268q0 -104 45.5 -157t147.5 -53q53 0 142 19q36 6 53 6q51 0 77.5 -28t26.5 -80q0 -26 -4 -46 +q75 68 117.5 165.5t42.5 200.5zM1792 667q0 -111 -33.5 -249.5t-93.5 -204.5q-58 -64 -195 -142.5t-228 -104.5l-4 -1v-114q0 -43 -29.5 -75t-72.5 -32q-56 0 -86 50q-32 -50 -87 -50t-87 50q-30 -50 -86 -50q-55 0 -87 50q-30 -50 -86 -50q-47 0 -75 33.5t-28 81.5 +q-90 -68 -198 -68q-118 0 -211 80q54 1 106 20q-113 31 -182 127q32 -7 71 -7q89 0 164 46q-192 192 -240 306q-24 56 -24 160q0 57 9 125.5t31.5 146.5t55 141t86.5 105t120 42q59 0 81 -52q19 29 42 54q2 3 12 13t13 16q10 15 23 38t25 42t28 39q87 111 211.5 177 +t260.5 66q35 0 62 -4q59 64 146 64q83 0 140 -57q5 -5 5 -12q0 -5 -6 -13.5t-12.5 -16t-16 -17l-10.5 -10.5q17 -6 36 -18t19 -24q0 -6 -16 -25q157 -138 197 -378q25 30 60 30q45 0 100 -49q90 -80 90 -279z" /> + <glyph glyph-name="uniF2B3" unicode="" +d="M917 631q0 33 -6 64h-362v-132h217q-12 -76 -74.5 -120.5t-142.5 -44.5q-99 0 -169 71.5t-70 170.5t70 170.5t169 71.5q93 0 153 -59l104 101q-108 100 -257 100q-160 0 -272 -112.5t-112 -271.5t112 -271.5t272 -112.5q165 0 266.5 105t101.5 270zM1262 585h109v110 +h-109v110h-110v-110h-110v-110h110v-110h110v110zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> + <glyph glyph-name="uniF2B4" unicode="" +d="M1536 1024v-839q0 -48 -49 -62q-174 -52 -338 -52q-73 0 -215.5 29.5t-227.5 29.5q-164 0 -370 -48v-338h-160v1368q-63 25 -101 81t-38 124q0 91 64 155t155 64t155 -64t64 -155q0 -68 -38 -124t-101 -81v-68q190 44 343 44q99 0 198 -15q14 -2 111.5 -22.5t149.5 -20.5 +q77 0 165 18q11 2 80 21t89 19q26 0 45 -19t19 -45z" /> + <glyph glyph-name="uniF2B5" unicode="" horiz-adv-x="2304" +d="M192 384q40 0 56 32t0 64t-56 32t-56 -32t0 -64t56 -32zM1665 442q-10 13 -38.5 50t-41.5 54t-38 49t-42.5 53t-40.5 47t-45 49l-125 -140q-83 -94 -208.5 -92t-205.5 98q-57 69 -56.5 158t58.5 157l177 206q-22 11 -51 16.5t-47.5 6t-56.5 -0.5t-49 -1q-92 0 -158 -66 +l-158 -158h-155v-544q5 0 21 0.5t22 0t19.5 -2t20.5 -4.5t17.5 -8.5t18.5 -13.5l297 -292q115 -111 227 -111q78 0 125 47q57 -20 112.5 8t72.5 85q74 -6 127 44q20 18 36 45.5t14 50.5q10 -10 43 -10q43 0 77 21t49.5 53t12 71.5t-30.5 73.5zM1824 384h96v512h-93l-157 180 +q-66 76 -169 76h-167q-89 0 -146 -67l-209 -243q-28 -33 -28 -75t27 -75q43 -51 110 -52t111 49l193 218q25 23 53.5 21.5t47 -27t8.5 -56.5q16 -19 56 -63t60 -68q29 -36 82.5 -105.5t64.5 -84.5q52 -66 60 -140zM2112 384q40 0 56 32t0 64t-56 32t-56 -32t0 -64t56 -32z +M2304 960v-640q0 -26 -19 -45t-45 -19h-434q-27 -65 -82 -106.5t-125 -51.5q-33 -48 -80.5 -81.5t-102.5 -45.5q-42 -53 -104.5 -81.5t-128.5 -24.5q-60 -34 -126 -39.5t-127.5 14t-117 53.5t-103.5 81l-287 282h-358q-26 0 -45 19t-19 45v672q0 26 19 45t45 19h421 +q14 14 47 48t47.5 48t44 40t50.5 37.5t51 25.5t62 19.5t68 5.5h117q99 0 181 -56q82 56 181 56h167q35 0 67 -6t56.5 -14.5t51.5 -26.5t44.5 -31t43 -39.5t39 -42t41 -48t41.5 -48.5h355q26 0 45 -19t19 -45z" /> + <glyph glyph-name="uniF2B6" unicode="" horiz-adv-x="1792" +d="M1792 882v-978q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v978q0 15 11 24q8 7 39 34.5t41.5 36t45.5 37.5t70 55.5t96 73t143.5 107t192.5 140.5q5 4 52.5 40t71.5 52.5t64 35t69 18.5t69 -18.5t65 -35.5t71 -52t52 -40q110 -80 192.5 -140.5t143.5 -107 +t96 -73t70 -55.5t45.5 -37.5t41.5 -36t39 -34.5q11 -9 11 -24zM1228 297q263 191 345 252q11 8 12.5 20.5t-6.5 23.5l-38 52q-8 11 -21 12.5t-24 -6.5q-231 -169 -343 -250q-5 -3 -52 -39t-71.5 -52.5t-64.5 -35t-69 -18.5t-69 18.5t-64.5 35t-71.5 52.5t-52 39 +q-186 134 -343 250q-11 8 -24 6.5t-21 -12.5l-38 -52q-8 -11 -6.5 -23.5t12.5 -20.5q82 -61 345 -252q10 -8 50 -38t65 -47t64 -39.5t77.5 -33.5t75.5 -11t75.5 11t79 34.5t64.5 39.5t65 47.5t48 36.5z" /> + <glyph glyph-name="uniF2B7" unicode="" horiz-adv-x="1792" +d="M1474 623l39 -51q8 -11 6.5 -23.5t-11.5 -20.5q-43 -34 -126.5 -98.5t-146.5 -113t-67 -51.5q-39 -32 -60 -48t-60.5 -41t-76.5 -36.5t-74 -11.5h-1h-1q-37 0 -74 11.5t-76 36.5t-61 41.5t-60 47.5q-5 4 -65 50.5t-143.5 111t-122.5 94.5q-11 8 -12.5 20.5t6.5 23.5 +l37 52q8 11 21.5 13t24.5 -7q94 -73 306 -236q5 -4 43.5 -35t60.5 -46.5t56.5 -32.5t58.5 -17h1h1q24 0 58.5 17t56.5 32.5t60.5 46.5t43.5 35q258 198 313 242q11 8 24 6.5t21 -12.5zM1664 -96v928q-90 83 -159 139q-91 74 -389 304q-3 2 -43 35t-61 48t-56 32.5t-59 17.5 +h-1h-1q-24 0 -59 -17.5t-56 -32.5t-61 -48t-43 -35q-215 -166 -315.5 -245.5t-129.5 -104t-82 -74.5q-14 -12 -21 -19v-928q0 -13 9.5 -22.5t22.5 -9.5h1472q13 0 22.5 9.5t9.5 22.5zM1792 832v-928q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v928q0 56 41 94 +q123 114 350 290.5t233 181.5q36 30 59 47.5t61.5 42t76 36.5t74.5 12h1h1q37 0 74.5 -12t76 -36.5t61.5 -42t59 -47.5q43 -36 156 -122t226 -177t201 -173q41 -38 41 -94z" /> + <glyph glyph-name="uniF2B8" unicode="" +d="M330 1l202 -214l-34 236l-216 213zM556 -225l274 218l-11 245l-300 -215zM245 413l227 -213l-48 327l-245 204zM495 189l317 214l-14 324l-352 -200zM843 178l95 -80l-2 239l-103 79q0 -1 1 -8.5t0 -12t-5 -7.5l-78 -52l85 -70q7 -6 7 -88zM138 930l256 -200l-68 465 +l-279 173zM1173 267l15 234l-230 -164l2 -240zM417 722l373 194l-19 441l-423 -163zM1270 357l20 233l-226 142l-2 -105l144 -95q6 -4 4 -9l-7 -119zM1461 496l30 222l-179 -128l-20 -228zM1273 329l-71 49l-8 -117q0 -5 -4 -8l-234 -187q-7 -5 -14 0l-98 83l7 -161 +q0 -5 -4 -8l-293 -234q-4 -2 -6 -2q-8 2 -8 3l-228 242q-4 4 -59 277q-2 7 5 11l61 37q-94 86 -95 92l-72 351q-2 7 6 12l94 45q-133 100 -135 108l-96 466q-2 10 7 13l433 135q5 0 8 -1l317 -153q6 -4 6 -9l20 -463q0 -7 -6 -10l-118 -61l126 -85q5 -2 5 -8l5 -123l121 74 +q5 4 11 0l84 -56l3 110q0 6 5 9l206 126q6 3 11 0l245 -135q4 -4 5 -7t-6.5 -60t-17.5 -124.5t-10 -70.5q0 -5 -4 -7l-191 -153q-6 -5 -13 0z" /> + <glyph glyph-name="uniF2B9" unicode="" horiz-adv-x="1664" +d="M1201 298q0 57 -5.5 107t-21 100.5t-39.5 86t-64 58t-91 22.5q-6 -4 -33.5 -20.5t-42.5 -24.5t-40.5 -20t-49 -17t-46.5 -5t-46.5 5t-49 17t-40.5 20t-42.5 24.5t-33.5 20.5q-51 0 -91 -22.5t-64 -58t-39.5 -86t-21 -100.5t-5.5 -107q0 -73 42 -121.5t103 -48.5h576 +q61 0 103 48.5t42 121.5zM1028 892q0 108 -76.5 184t-183.5 76t-183.5 -76t-76.5 -184q0 -107 76.5 -183t183.5 -76t183.5 76t76.5 183zM1664 352v-192q0 -14 -9 -23t-23 -9h-96v-224q0 -66 -47 -113t-113 -47h-1216q-66 0 -113 47t-47 113v1472q0 66 47 113t113 47h1216 +q66 0 113 -47t47 -113v-224h96q14 0 23 -9t9 -23v-192q0 -14 -9 -23t-23 -9h-96v-128h96q14 0 23 -9t9 -23v-192q0 -14 -9 -23t-23 -9h-96v-128h96q14 0 23 -9t9 -23z" /> + <glyph glyph-name="uniF2BA" unicode="" horiz-adv-x="1664" +d="M1028 892q0 -107 -76.5 -183t-183.5 -76t-183.5 76t-76.5 183q0 108 76.5 184t183.5 76t183.5 -76t76.5 -184zM980 672q46 0 82.5 -17t60 -47.5t39.5 -67t24 -81t11.5 -82.5t3.5 -79q0 -67 -39.5 -118.5t-105.5 -51.5h-576q-66 0 -105.5 51.5t-39.5 118.5q0 48 4.5 93.5 +t18.5 98.5t36.5 91.5t63 64.5t93.5 26h5q7 -4 32 -19.5t35.5 -21t33 -17t37 -16t35 -9t39.5 -4.5t39.5 4.5t35 9t37 16t33 17t35.5 21t32 19.5zM1664 928q0 -13 -9.5 -22.5t-22.5 -9.5h-96v-128h96q13 0 22.5 -9.5t9.5 -22.5v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-96v-128h96 +q13 0 22.5 -9.5t9.5 -22.5v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-96v-224q0 -66 -47 -113t-113 -47h-1216q-66 0 -113 47t-47 113v1472q0 66 47 113t113 47h1216q66 0 113 -47t47 -113v-224h96q13 0 22.5 -9.5t9.5 -22.5v-192zM1408 -96v1472q0 13 -9.5 22.5t-22.5 9.5h-1216 +q-13 0 -22.5 -9.5t-9.5 -22.5v-1472q0 -13 9.5 -22.5t22.5 -9.5h1216q13 0 22.5 9.5t9.5 22.5z" /> + <glyph glyph-name="uniF2BB" unicode="" horiz-adv-x="2048" +d="M1024 405q0 64 -9 117.5t-29.5 103t-60.5 78t-97 28.5q-6 -4 -30 -18t-37.5 -21.5t-35.5 -17.5t-43 -14.5t-42 -4.5t-42 4.5t-43 14.5t-35.5 17.5t-37.5 21.5t-30 18q-57 0 -97 -28.5t-60.5 -78t-29.5 -103t-9 -117.5t37 -106.5t91 -42.5h512q54 0 91 42.5t37 106.5z +M867 925q0 94 -66.5 160.5t-160.5 66.5t-160.5 -66.5t-66.5 -160.5t66.5 -160.5t160.5 -66.5t160.5 66.5t66.5 160.5zM1792 416v64q0 14 -9 23t-23 9h-576q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h576q14 0 23 9t9 23zM1792 676v56q0 15 -10.5 25.5t-25.5 10.5h-568 +q-15 0 -25.5 -10.5t-10.5 -25.5v-56q0 -15 10.5 -25.5t25.5 -10.5h568q15 0 25.5 10.5t10.5 25.5zM1792 928v64q0 14 -9 23t-23 9h-576q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h576q14 0 23 9t9 23zM2048 1248v-1216q0 -66 -47 -113t-113 -47h-352v96q0 14 -9 23t-23 9 +h-64q-14 0 -23 -9t-9 -23v-96h-768v96q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-96h-352q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1728q66 0 113 -47t47 -113z" /> + <glyph glyph-name="uniF2BC" unicode="" horiz-adv-x="2048" +d="M1024 405q0 -64 -37 -106.5t-91 -42.5h-512q-54 0 -91 42.5t-37 106.5t9 117.5t29.5 103t60.5 78t97 28.5q6 -4 30 -18t37.5 -21.5t35.5 -17.5t43 -14.5t42 -4.5t42 4.5t43 14.5t35.5 17.5t37.5 21.5t30 18q57 0 97 -28.5t60.5 -78t29.5 -103t9 -117.5zM867 925 +q0 -94 -66.5 -160.5t-160.5 -66.5t-160.5 66.5t-66.5 160.5t66.5 160.5t160.5 66.5t160.5 -66.5t66.5 -160.5zM1792 480v-64q0 -14 -9 -23t-23 -9h-576q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h576q14 0 23 -9t9 -23zM1792 732v-56q0 -15 -10.5 -25.5t-25.5 -10.5h-568 +q-15 0 -25.5 10.5t-10.5 25.5v56q0 15 10.5 25.5t25.5 10.5h568q15 0 25.5 -10.5t10.5 -25.5zM1792 992v-64q0 -14 -9 -23t-23 -9h-576q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h576q14 0 23 -9t9 -23zM1920 32v1216q0 13 -9.5 22.5t-22.5 9.5h-1728q-13 0 -22.5 -9.5 +t-9.5 -22.5v-1216q0 -13 9.5 -22.5t22.5 -9.5h352v96q0 14 9 23t23 9h64q14 0 23 -9t9 -23v-96h768v96q0 14 9 23t23 9h64q14 0 23 -9t9 -23v-96h352q13 0 22.5 9.5t9.5 22.5zM2048 1248v-1216q0 -66 -47 -113t-113 -47h-1728q-66 0 -113 47t-47 113v1216q0 66 47 113 +t113 47h1728q66 0 113 -47t47 -113z" /> + <glyph glyph-name="uniF2BD" unicode="" horiz-adv-x="1792" +d="M1523 197q-22 155 -87.5 257.5t-184.5 118.5q-67 -74 -159.5 -115.5t-195.5 -41.5t-195.5 41.5t-159.5 115.5q-119 -16 -184.5 -118.5t-87.5 -257.5q106 -150 271 -237.5t356 -87.5t356 87.5t271 237.5zM1280 896q0 159 -112.5 271.5t-271.5 112.5t-271.5 -112.5 +t-112.5 -271.5t112.5 -271.5t271.5 -112.5t271.5 112.5t112.5 271.5zM1792 640q0 -182 -71 -347.5t-190.5 -286t-285.5 -191.5t-349 -71q-182 0 -348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71t348 -71t286 -191t191 -286t71 -348z" /> + <glyph glyph-name="uniF2BE" unicode="" horiz-adv-x="1792" +d="M896 1536q182 0 348 -71t286 -191t191 -286t71 -348q0 -181 -70.5 -347t-190.5 -286t-286 -191.5t-349 -71.5t-349 71t-285.5 191.5t-190.5 286t-71 347.5t71 348t191 286t286 191t348 71zM1515 185q149 205 149 455q0 156 -61 298t-164 245t-245 164t-298 61t-298 -61 +t-245 -164t-164 -245t-61 -298q0 -250 149 -455q66 327 306 327q131 -128 313 -128t313 128q240 0 306 -327zM1280 832q0 159 -112.5 271.5t-271.5 112.5t-271.5 -112.5t-112.5 -271.5t112.5 -271.5t271.5 -112.5t271.5 112.5t112.5 271.5z" /> + <glyph glyph-name="uniF2C0" unicode="" +d="M1201 752q47 -14 89.5 -38t89 -73t79.5 -115.5t55 -172t22 -236.5q0 -154 -100 -263.5t-241 -109.5h-854q-141 0 -241 109.5t-100 263.5q0 131 22 236.5t55 172t79.5 115.5t89 73t89.5 38q-79 125 -79 272q0 104 40.5 198.5t109.5 163.5t163.5 109.5t198.5 40.5 +t198.5 -40.5t163.5 -109.5t109.5 -163.5t40.5 -198.5q0 -147 -79 -272zM768 1408q-159 0 -271.5 -112.5t-112.5 -271.5t112.5 -271.5t271.5 -112.5t271.5 112.5t112.5 271.5t-112.5 271.5t-271.5 112.5zM1195 -128q88 0 150.5 71.5t62.5 173.5q0 239 -78.5 377t-225.5 145 +q-145 -127 -336 -127t-336 127q-147 -7 -225.5 -145t-78.5 -377q0 -102 62.5 -173.5t150.5 -71.5h854z" /> + <glyph glyph-name="uniF2C1" unicode="" horiz-adv-x="1280" +d="M1024 278q0 -64 -37 -107t-91 -43h-512q-54 0 -91 43t-37 107t9 118t29.5 104t61 78.5t96.5 28.5q80 -75 188 -75t188 75q56 0 96.5 -28.5t61 -78.5t29.5 -104t9 -118zM870 797q0 -94 -67.5 -160.5t-162.5 -66.5t-162.5 66.5t-67.5 160.5t67.5 160.5t162.5 66.5 +t162.5 -66.5t67.5 -160.5zM1152 -96v1376h-1024v-1376q0 -13 9.5 -22.5t22.5 -9.5h960q13 0 22.5 9.5t9.5 22.5zM1280 1376v-1472q0 -66 -47 -113t-113 -47h-960q-66 0 -113 47t-47 113v1472q0 66 47 113t113 47h352v-96q0 -14 9 -23t23 -9h192q14 0 23 9t9 23v96h352 +q66 0 113 -47t47 -113z" /> + <glyph glyph-name="uniF2C2" unicode="" horiz-adv-x="2048" +d="M896 324q0 54 -7.5 100.5t-24.5 90t-51 68.5t-81 25q-64 -64 -156 -64t-156 64q-47 0 -81 -25t-51 -68.5t-24.5 -90t-7.5 -100.5q0 -55 31.5 -93.5t75.5 -38.5h426q44 0 75.5 38.5t31.5 93.5zM768 768q0 80 -56 136t-136 56t-136 -56t-56 -136t56 -136t136 -56t136 56 +t56 136zM1792 288v64q0 14 -9 23t-23 9h-704q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h704q14 0 23 9t9 23zM1408 544v64q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1792 544v64q0 14 -9 23t-23 9h-192q-14 0 -23 -9t-9 -23 +v-64q0 -14 9 -23t23 -9h192q14 0 23 9t9 23zM1792 800v64q0 14 -9 23t-23 9h-704q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h704q14 0 23 9t9 23zM128 1152h1792v96q0 14 -9 23t-23 9h-1728q-14 0 -23 -9t-9 -23v-96zM2048 1248v-1216q0 -66 -47 -113t-113 -47h-1728 +q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1728q66 0 113 -47t47 -113z" /> + <glyph glyph-name="uniF2C3" unicode="" horiz-adv-x="2048" +d="M896 324q0 -55 -31.5 -93.5t-75.5 -38.5h-426q-44 0 -75.5 38.5t-31.5 93.5q0 54 7.5 100.5t24.5 90t51 68.5t81 25q64 -64 156 -64t156 64q47 0 81 -25t51 -68.5t24.5 -90t7.5 -100.5zM768 768q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136z +M1792 352v-64q0 -14 -9 -23t-23 -9h-704q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h704q14 0 23 -9t9 -23zM1408 608v-64q0 -14 -9 -23t-23 -9h-320q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h320q14 0 23 -9t9 -23zM1792 608v-64q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v64 +q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1792 864v-64q0 -14 -9 -23t-23 -9h-704q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h704q14 0 23 -9t9 -23zM1920 32v1120h-1792v-1120q0 -13 9.5 -22.5t22.5 -9.5h1728q13 0 22.5 9.5t9.5 22.5zM2048 1248v-1216q0 -66 -47 -113t-113 -47 +h-1728q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1728q66 0 113 -47t47 -113z" /> + <glyph glyph-name="uniF2C4" unicode="" horiz-adv-x="1792" +d="M1255 749q0 318 -105 474.5t-330 156.5q-222 0 -326 -157t-104 -474q0 -316 104 -471.5t326 -155.5q74 0 131 17q-22 43 -39 73t-44 65t-53.5 56.5t-63 36t-77.5 14.5q-46 0 -79 -16l-49 97q105 91 276 91q132 0 215.5 -54t150.5 -155q67 149 67 402zM1645 117h117 +q3 -27 -2 -67t-26.5 -95t-58 -100.5t-107 -78t-162.5 -32.5q-71 0 -130.5 19t-105.5 56t-79 78t-66 96q-97 -27 -205 -27q-150 0 -292.5 58t-253 158.5t-178 249t-67.5 317.5q0 170 67.5 319.5t178.5 250.5t253.5 159t291.5 58q121 0 238.5 -36t217 -106t176 -164.5 +t119.5 -219t43 -261.5q0 -190 -80.5 -347.5t-218.5 -264.5q47 -70 93.5 -106.5t104.5 -36.5q61 0 94 37.5t38 85.5z" /> + <glyph glyph-name="uniF2C5" unicode="" horiz-adv-x="2304" +d="M453 -101q0 -21 -16 -37.5t-37 -16.5q-1 0 -13 3q-63 15 -162 140q-225 284 -225 676q0 341 213 614q39 51 95 103.5t94 52.5q19 0 35 -13.5t16 -32.5q0 -27 -63 -90q-98 -102 -147 -184q-119 -199 -119 -449q0 -281 123 -491q50 -85 136 -173q2 -3 14.5 -16t19.5 -21 +t17 -20.5t14.5 -23.5t4.5 -21zM1796 33q0 -29 -17.5 -48.5t-46.5 -19.5h-1081q-26 0 -45 19t-19 45q0 29 17.5 48.5t46.5 19.5h1081q26 0 45 -19t19 -45zM1581 644q0 -134 -67 -233q-25 -38 -69.5 -78.5t-83.5 -60.5q-16 -10 -27 -10q-7 0 -15 6t-8 12q0 9 19 30t42 46 +t42 67.5t19 88.5q0 76 -35 130q-29 42 -46 42q-3 0 -3 -5q0 -12 7.5 -35.5t7.5 -36.5q0 -22 -21.5 -35t-44.5 -13q-66 0 -66 76q0 15 1.5 44t1.5 44q0 25 -10 46q-13 25 -42 53.5t-51 28.5q-5 0 -7 -0.5t-3.5 -2.5t-1.5 -6q0 -2 16 -26t16 -54q0 -37 -19 -68t-46 -54 +t-53.5 -46t-45.5 -54t-19 -68q0 -98 42 -160q29 -43 79 -63q16 -5 17 -10q1 -2 1 -5q0 -16 -18 -16q-6 0 -33 11q-119 43 -195 139.5t-76 218.5q0 55 24.5 115.5t60 115t70.5 108.5t59.5 113.5t24.5 111.5q0 53 -25 94q-29 48 -56 64q-19 9 -19 21q0 20 41 20q50 0 110 -29 +q41 -19 71 -44.5t49.5 -51t33.5 -62.5t22 -69t16 -80q0 -1 3 -17.5t4.5 -25t5.5 -25t9 -27t11 -21.5t14.5 -16.5t18.5 -5.5q23 0 37 14t14 37q0 25 -20 67t-20 52t10 10q27 0 93 -70q72 -76 102.5 -156t30.5 -186zM2304 615q0 -274 -138 -503q-19 -32 -48 -72t-68 -86.5 +t-81 -77t-74 -30.5q-16 0 -31 15.5t-15 31.5q0 15 29 50.5t68.5 77t48.5 52.5q183 230 183 531q0 131 -20.5 235t-72.5 211q-58 119 -163 228q-2 3 -13 13.5t-16.5 16.5t-15 17.5t-15 20t-9.5 18.5t-4 19q0 19 16 35.5t35 16.5q70 0 196 -169q98 -131 146 -273t60 -314 +q2 -42 2 -64z" /> + <glyph glyph-name="uniF2C6" unicode="" horiz-adv-x="1792" +d="M1189 229l147 693q9 44 -10.5 63t-51.5 7l-864 -333q-29 -11 -39.5 -25t-2.5 -26.5t32 -19.5l221 -69l513 323q21 14 32 6q7 -5 -4 -15l-415 -375v0v0l-16 -228q23 0 45 22l108 104l224 -165q64 -36 81 38zM1792 640q0 -182 -71 -348t-191 -286t-286 -191t-348 -71 +t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71t348 -71t286 -191t191 -286t71 -348z" /> + <glyph glyph-name="uniF2C7" unicode="" horiz-adv-x="1024" +d="M640 192q0 -80 -56 -136t-136 -56t-136 56t-56 136q0 60 35 110t93 71v907h128v-907q58 -21 93 -71t35 -110zM768 192q0 77 -34 144t-94 112v768q0 80 -56 136t-136 56t-136 -56t-56 -136v-768q-60 -45 -94 -112t-34 -144q0 -133 93.5 -226.5t226.5 -93.5t226.5 93.5 +t93.5 226.5zM896 192q0 -185 -131.5 -316.5t-316.5 -131.5t-316.5 131.5t-131.5 316.5q0 182 128 313v711q0 133 93.5 226.5t226.5 93.5t226.5 -93.5t93.5 -226.5v-711q128 -131 128 -313zM1024 768v-128h-192v128h192zM1024 1024v-128h-192v128h192zM1024 1280v-128h-192 +v128h192z" /> + <glyph glyph-name="uniF2C8" unicode="" horiz-adv-x="1024" +d="M640 192q0 -80 -56 -136t-136 -56t-136 56t-56 136q0 60 35 110t93 71v651h128v-651q58 -21 93 -71t35 -110zM768 192q0 77 -34 144t-94 112v768q0 80 -56 136t-136 56t-136 -56t-56 -136v-768q-60 -45 -94 -112t-34 -144q0 -133 93.5 -226.5t226.5 -93.5t226.5 93.5 +t93.5 226.5zM896 192q0 -185 -131.5 -316.5t-316.5 -131.5t-316.5 131.5t-131.5 316.5q0 182 128 313v711q0 133 93.5 226.5t226.5 93.5t226.5 -93.5t93.5 -226.5v-711q128 -131 128 -313zM1024 768v-128h-192v128h192zM1024 1024v-128h-192v128h192zM1024 1280v-128h-192 +v128h192z" /> + <glyph glyph-name="uniF2C9" unicode="" horiz-adv-x="1024" +d="M640 192q0 -80 -56 -136t-136 -56t-136 56t-56 136q0 60 35 110t93 71v395h128v-395q58 -21 93 -71t35 -110zM768 192q0 77 -34 144t-94 112v768q0 80 -56 136t-136 56t-136 -56t-56 -136v-768q-60 -45 -94 -112t-34 -144q0 -133 93.5 -226.5t226.5 -93.5t226.5 93.5 +t93.5 226.5zM896 192q0 -185 -131.5 -316.5t-316.5 -131.5t-316.5 131.5t-131.5 316.5q0 182 128 313v711q0 133 93.5 226.5t226.5 93.5t226.5 -93.5t93.5 -226.5v-711q128 -131 128 -313zM1024 768v-128h-192v128h192zM1024 1024v-128h-192v128h192zM1024 1280v-128h-192 +v128h192z" /> + <glyph glyph-name="uniF2CA" unicode="" horiz-adv-x="1024" +d="M640 192q0 -80 -56 -136t-136 -56t-136 56t-56 136q0 60 35 110t93 71v139h128v-139q58 -21 93 -71t35 -110zM768 192q0 77 -34 144t-94 112v768q0 80 -56 136t-136 56t-136 -56t-56 -136v-768q-60 -45 -94 -112t-34 -144q0 -133 93.5 -226.5t226.5 -93.5t226.5 93.5 +t93.5 226.5zM896 192q0 -185 -131.5 -316.5t-316.5 -131.5t-316.5 131.5t-131.5 316.5q0 182 128 313v711q0 133 93.5 226.5t226.5 93.5t226.5 -93.5t93.5 -226.5v-711q128 -131 128 -313zM1024 768v-128h-192v128h192zM1024 1024v-128h-192v128h192zM1024 1280v-128h-192 +v128h192z" /> + <glyph glyph-name="uniF2CB" unicode="" horiz-adv-x="1024" +d="M640 192q0 -80 -56 -136t-136 -56t-136 56t-56 136q0 79 56 135.5t136 56.5t136 -56.5t56 -135.5zM768 192q0 77 -34 144t-94 112v768q0 80 -56 136t-136 56t-136 -56t-56 -136v-768q-60 -45 -94 -112t-34 -144q0 -133 93.5 -226.5t226.5 -93.5t226.5 93.5t93.5 226.5z +M896 192q0 -185 -131.5 -316.5t-316.5 -131.5t-316.5 131.5t-131.5 316.5q0 182 128 313v711q0 133 93.5 226.5t226.5 93.5t226.5 -93.5t93.5 -226.5v-711q128 -131 128 -313zM1024 768v-128h-192v128h192zM1024 1024v-128h-192v128h192zM1024 1280v-128h-192v128h192z" /> + <glyph glyph-name="uniF2CC" unicode="" horiz-adv-x="1920" +d="M1433 1287q10 -10 10 -23t-10 -23l-626 -626q-10 -10 -23 -10t-23 10l-82 82q-10 10 -10 23t10 23l44 44q-72 91 -81.5 207t46.5 215q-74 71 -176 71q-106 0 -181 -75t-75 -181v-1280h-256v1280q0 104 40.5 198.5t109.5 163.5t163.5 109.5t198.5 40.5q106 0 201 -41 +t166 -115q94 39 197 24.5t185 -79.5l44 44q10 10 23 10t23 -10zM1344 1024q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19zM1600 896q-26 0 -45 19t-19 45t19 45t45 19t45 -19t19 -45t-19 -45t-45 -19zM1856 1024q26 0 45 -19t19 -45t-19 -45t-45 -19 +t-45 19t-19 45t19 45t45 19zM1216 896q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19zM1408 832q0 26 19 45t45 19t45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45zM1728 896q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19zM1088 768 +q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19zM1344 640q-26 0 -45 19t-19 45t19 45t45 19t45 -19t19 -45t-19 -45t-45 -19zM1600 768q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19zM1216 512q-26 0 -45 19t-19 45t19 45t45 19t45 -19 +t19 -45t-19 -45t-45 -19zM1472 640q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19zM1088 512q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19zM1344 512q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19zM1216 384 +q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19zM1088 256q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19z" /> + <glyph glyph-name="uniF2CD" unicode="" horiz-adv-x="1792" +d="M1664 448v-192q0 -169 -128 -286v-194q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v118q-63 -22 -128 -22h-768q-65 0 -128 22v-110q0 -17 -9.5 -28.5t-22.5 -11.5h-64q-13 0 -22.5 11.5t-9.5 28.5v186q-128 117 -128 286v192h1536zM704 864q0 -14 -9 -23t-23 -9t-23 9 +t-9 23t9 23t23 9t23 -9t9 -23zM768 928q0 -14 -9 -23t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM704 992q0 -14 -9 -23t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM832 992q0 -14 -9 -23t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM768 1056q0 -14 -9 -23t-23 -9t-23 9 +t-9 23t9 23t23 9t23 -9t9 -23zM704 1120q0 -14 -9 -23t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM1792 608v-64q0 -14 -9 -23t-23 -9h-1728q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h96v640q0 106 75 181t181 75q108 0 184 -78q46 19 98 12t93 -39l22 22q11 11 22 0l42 -42 +q11 -11 0 -22l-314 -314q-11 -11 -22 0l-42 42q-11 11 0 22l22 22q-36 46 -40.5 104t23.5 108q-37 35 -88 35q-53 0 -90.5 -37.5t-37.5 -90.5v-640h1504q14 0 23 -9t9 -23zM896 1056q0 -14 -9 -23t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM832 1120q0 -14 -9 -23t-23 -9 +t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM768 1184q0 -14 -9 -23t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM960 1120q0 -14 -9 -23t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM896 1184q0 -14 -9 -23t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM832 1248q0 -14 -9 -23 +t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM1024 1184q0 -14 -9 -23t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM960 1248q0 -14 -9 -23t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23zM1088 1248q0 -14 -9 -23t-23 -9t-23 9t-9 23t9 23t23 9t23 -9t9 -23z" /> + <glyph glyph-name="uniF2CE" unicode="" +d="M994 344q0 -86 -17 -197q-31 -215 -55 -313q-22 -90 -152 -90t-152 90q-24 98 -55 313q-17 110 -17 197q0 168 224 168t224 -168zM1536 768q0 -240 -134 -434t-350 -280q-8 -3 -15 3t-6 15q7 48 10 66q4 32 6 47q1 9 9 12q159 81 255.5 234t96.5 337q0 180 -91 330.5 +t-247 234.5t-337 74q-124 -7 -237 -61t-193.5 -140.5t-128 -202t-46.5 -240.5q1 -184 99 -336.5t257 -231.5q7 -3 9 -12q3 -21 6 -45q1 -9 5 -32.5t6 -35.5q1 -9 -6.5 -15t-15.5 -2q-148 58 -261 169.5t-173.5 264t-52.5 319.5q7 143 66 273.5t154.5 227t225 157.5t272.5 70 +q164 10 315.5 -46.5t261 -160.5t175 -250.5t65.5 -308.5zM994 800q0 -93 -65.5 -158.5t-158.5 -65.5t-158.5 65.5t-65.5 158.5t65.5 158.5t158.5 65.5t158.5 -65.5t65.5 -158.5zM1282 768q0 -122 -53.5 -228.5t-146.5 -177.5q-8 -6 -16 -2t-10 14q-6 52 -29 92q-7 10 3 20 +q58 54 91 127t33 155q0 111 -58.5 204t-157.5 141.5t-212 36.5q-133 -15 -229 -113t-109 -231q-10 -92 23.5 -176t98.5 -144q10 -10 3 -20q-24 -41 -29 -93q-2 -9 -10 -13t-16 2q-95 74 -148.5 183t-51.5 234q3 131 69 244t177 181.5t241 74.5q144 7 268 -60t196.5 -187.5 +t72.5 -263.5z" /> + <glyph glyph-name="uniF2D0" unicode="" horiz-adv-x="1792" +d="M256 128h1280v768h-1280v-768zM1792 1248v-1216q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1472q66 0 113 -47t47 -113z" /> + <glyph glyph-name="uniF2D1" unicode="" horiz-adv-x="1792" +d="M1792 224v-192q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v192q0 66 47 113t113 47h1472q66 0 113 -47t47 -113z" /> + <glyph glyph-name="uniF2D2" unicode="" horiz-adv-x="2048" +d="M256 0h768v512h-768v-512zM1280 512h512v768h-768v-256h96q66 0 113 -47t47 -113v-352zM2048 1376v-960q0 -66 -47 -113t-113 -47h-608v-352q0 -66 -47 -113t-113 -47h-960q-66 0 -113 47t-47 113v960q0 66 47 113t113 47h608v352q0 66 47 113t113 47h960q66 0 113 -47 +t47 -113z" /> + <glyph glyph-name="uniF2D3" unicode="" horiz-adv-x="1792" +d="M1175 215l146 146q10 10 10 23t-10 23l-233 233l233 233q10 10 10 23t-10 23l-146 146q-10 10 -23 10t-23 -10l-233 -233l-233 233q-10 10 -23 10t-23 -10l-146 -146q-10 -10 -10 -23t10 -23l233 -233l-233 -233q-10 -10 -10 -23t10 -23l146 -146q10 -10 23 -10t23 10 +l233 233l233 -233q10 -10 23 -10t23 10zM1792 1248v-1216q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1472q66 0 113 -47t47 -113z" /> + <glyph glyph-name="uniF2D4" unicode="" horiz-adv-x="1792" +d="M1257 425l-146 -146q-10 -10 -23 -10t-23 10l-169 169l-169 -169q-10 -10 -23 -10t-23 10l-146 146q-10 10 -10 23t10 23l169 169l-169 169q-10 10 -10 23t10 23l146 146q10 10 23 10t23 -10l169 -169l169 169q10 10 23 10t23 -10l146 -146q10 -10 10 -23t-10 -23 +l-169 -169l169 -169q10 -10 10 -23t-10 -23zM256 128h1280v1024h-1280v-1024zM1792 1248v-1216q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1472q66 0 113 -47t47 -113z" /> + <glyph glyph-name="uniF2D5" unicode="" horiz-adv-x="1792" +d="M1070 358l306 564h-654l-306 -564h654zM1792 640q0 -182 -71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71t348 -71t286 -191t191 -286t71 -348z" /> + <glyph glyph-name="uniF2D6" unicode="" horiz-adv-x="1794" +d="M1291 1060q-15 17 -35 8.5t-26 -28.5t5 -38q14 -17 40 -14.5t34 20.5t-18 52zM895 814q-8 -8 -19.5 -8t-18.5 8q-8 8 -8 19t8 18q7 8 18.5 8t19.5 -8q7 -7 7 -18t-7 -19zM1060 740l-35 -35q-12 -13 -29.5 -13t-30.5 13l-38 38q-12 13 -12 30t12 30l35 35q12 12 29.5 12 +t30.5 -12l38 -39q12 -12 12 -29.5t-12 -29.5zM951 870q-7 -8 -18.5 -8t-19.5 8q-7 8 -7 19t7 19q8 8 19 8t19 -8t8 -19t-8 -19zM1354 968q-34 -64 -107.5 -85.5t-127.5 16.5q-38 28 -61 66.5t-21 87.5t39 92t75.5 53t70.5 -5t70 -51q2 -2 13 -12.5t14.5 -13.5t13 -13.5 +t12.5 -15.5t10 -15.5t8.5 -18t4 -18.5t1 -21t-5 -22t-9.5 -24zM1555 486q3 20 -8.5 34.5t-27.5 21.5t-33 17t-23 20q-40 71 -84 98.5t-113 11.5q19 13 40 18.5t33 4.5l12 -1q2 45 -34 90q6 20 6.5 40.5t-2.5 30.5l-3 10q43 24 71 65t34 91q10 84 -43 150.5t-137 76.5 +q-60 7 -114 -18.5t-82 -74.5q-30 -51 -33.5 -101t14.5 -87t43.5 -64t56.5 -42q-45 4 -88 36t-57 88q-28 108 32 222q-16 21 -29 32q-50 0 -89 -19q19 24 42 37t36 14l13 1q0 50 -13 78q-10 21 -32.5 28.5t-47 -3.5t-37.5 -40q2 4 4 7q-7 -28 -6.5 -75.5t19 -117t48.5 -122.5 +q-25 -14 -47 -36q-35 -16 -85.5 -70.5t-84.5 -101.5l-33 -46q-90 -34 -181 -125.5t-75 -162.5q1 -16 11 -27q-15 -12 -30 -30q-21 -25 -21 -54t21.5 -40t63.5 6q41 19 77 49.5t55 60.5q-2 2 -6.5 5t-20.5 7.5t-33 3.5q23 5 51 12.5t40 10t27.5 6t26 4t23.5 0.5q14 -7 22 34 +q7 37 7 90q0 102 -40 150q106 -103 101 -219q-1 -29 -15 -50t-27 -27l-13 -6q-4 -7 -19 -32t-26 -45.5t-26.5 -52t-25 -61t-17 -63t-6.5 -66.5t10 -63q-35 54 -37 80q-22 -24 -34.5 -39t-33.5 -42t-30.5 -46t-16.5 -41t-0.5 -38t25.5 -27q45 -25 144 64t190.5 221.5 +t122.5 228.5q86 52 145 115.5t86 119.5q47 -93 154 -178q104 -83 167 -80q39 2 46 43zM1794 640q0 -182 -71 -348t-191 -286t-286.5 -191t-348.5 -71t-348.5 71t-286.5 191t-191 286t-71 348t71 348t191 286t286.5 191t348.5 71t348.5 -71t286.5 -191t191 -286t71 -348z" /> + <glyph glyph-name="uniF2D7" unicode="" +d="M518 1353v-655q103 -1 191.5 1.5t125.5 5.5l37 3q68 2 90.5 24.5t39.5 94.5l33 142h103l-14 -322l7 -319h-103l-29 127q-15 68 -45 93t-84 26q-87 8 -352 8v-556q0 -78 43.5 -115.5t133.5 -37.5h357q35 0 59.5 2t55 7.5t54 18t48.5 32t46 50.5t39 73l93 216h89 +q-6 -37 -31.5 -252t-30.5 -276q-146 5 -263.5 8t-162.5 4h-44h-628l-376 -12v102l127 25q67 13 91.5 37t25.5 79l8 643q3 402 -8 645q-2 61 -25.5 84t-91.5 36l-127 24v102l376 -12h702q139 0 374 27q-6 -68 -14 -194.5t-12 -219.5l-5 -92h-93l-32 124q-31 121 -74 179.5 +t-113 58.5h-548q-28 0 -35.5 -8.5t-7.5 -30.5z" /> + <glyph glyph-name="uniF2D8" unicode="" +d="M922 739v-182q0 -4 0.5 -15t0 -15l-1.5 -12t-3.5 -11.5t-6.5 -7.5t-11 -5.5t-16 -1.5v309q9 0 16 -1t11 -5t6.5 -5.5t3.5 -9.5t1 -10.5v-13.5v-14zM1238 643v-121q0 -1 0.5 -12.5t0 -15.5t-2.5 -11.5t-7.5 -10.5t-13.5 -3q-9 0 -14 9q-4 10 -4 165v7v8.5v9t1.5 8.5l3.5 7 +t5 5.5t8 1.5q6 0 10 -1.5t6.5 -4.5t4 -6t2 -8.5t0.5 -8v-9.5v-9zM180 407h122v472h-122v-472zM614 407h106v472h-159l-28 -221q-20 148 -32 221h-158v-472h107v312l45 -312h76l43 319v-319zM1039 712q0 67 -5 90q-3 16 -11 28.5t-17 20.5t-25 14t-26.5 8.5t-31 4t-29 1.5 +h-29.5h-12h-91v-472h56q169 -1 197 24.5t25 180.5q-1 62 -1 100zM1356 515v133q0 29 -2 45t-9.5 33.5t-24.5 25t-46 7.5q-46 0 -77 -34v154h-117v-472h110l7 30q30 -36 77 -36q50 0 66 30.5t16 83.5zM1536 1248v-1216q0 -66 -47 -113t-113 -47h-1216q-66 0 -113 47t-47 113 +v1216q0 66 47 113t113 47h1216q66 0 113 -47t47 -113z" /> + <glyph glyph-name="uniF2D9" unicode="" horiz-adv-x="2176" +d="M1143 -197q-6 1 -11 4q-13 8 -36 23t-86 65t-116.5 104.5t-112 140t-89.5 172.5q-17 3 -175 37q66 -213 235 -362t391 -184zM502 409l168 -28q-25 76 -41 167.5t-19 145.5l-4 53q-84 -82 -121 -224q5 -65 17 -114zM612 1018q-43 -64 -77 -148q44 46 74 68zM2049 584 +q0 161 -62 307t-167.5 252t-250.5 168.5t-304 62.5q-147 0 -281 -52.5t-240 -148.5q-30 -58 -45 -160q60 51 143 83.5t158.5 43t143 13.5t108.5 -1l40 -3q33 -1 53 -15.5t24.5 -33t6.5 -37t-1 -28.5q-126 11 -227.5 0.5t-183 -43.5t-142.5 -71.5t-131 -98.5 +q4 -36 11.5 -92.5t35.5 -178t62 -179.5q123 -6 247.5 14.5t214.5 53.5t162.5 67t109.5 59l37 24q22 16 39.5 20.5t30.5 -5t17 -34.5q14 -97 -39 -121q-208 -97 -467 -134q-135 -20 -317 -16q41 -96 110 -176.5t137 -127t130.5 -79t101.5 -43.5l39 -12q143 -23 263 15 +q195 99 314 289t119 418zM2123 621q-14 -135 -40 -212q-70 -208 -181.5 -346.5t-318.5 -253.5q-48 -33 -82 -44q-72 -26 -163 -16q-36 -3 -73 -3q-283 0 -504.5 173t-295.5 442q-1 0 -4 0.5t-5 0.5q-6 -50 2.5 -112.5t26 -115t36 -98t31.5 -71.5l14 -26q8 -12 54 -82 +q-71 38 -124.5 106.5t-78.5 140t-39.5 137t-17.5 107.5l-2 42q-5 2 -33.5 12.5t-48.5 18t-53 20.5t-57.5 25t-50 25.5t-42.5 27t-25 25.5q19 -10 50.5 -25.5t113 -45.5t145.5 -38l2 32q11 149 94 290q41 202 176 365q28 115 81 214q15 28 32 45t49 32q158 74 303.5 104 +t302 11t306.5 -97q220 -115 333 -336t87 -474z" /> + <glyph glyph-name="uniF2DA" unicode="" horiz-adv-x="1792" +d="M1341 752q29 44 -6.5 129.5t-121.5 142.5q-58 39 -125.5 53.5t-118 4.5t-68.5 -37q-12 -23 -4.5 -28t42.5 -10q23 -3 38.5 -5t44.5 -9.5t56 -17.5q36 -13 67.5 -31.5t53 -37t40 -38.5t30.5 -38t22 -34.5t16.5 -28.5t12 -18.5t10.5 -6t11 9.5zM1704 178 +q-52 -127 -148.5 -220t-214.5 -141.5t-253 -60.5t-266 13.5t-251 91t-210 161.5t-141.5 235.5t-46.5 303.5q1 41 8.5 84.5t12.5 64t24 80.5t23 73q-51 -208 1 -397t173 -318t291 -206t346 -83t349 74.5t289 244.5q20 27 18 14q0 -4 -4 -14zM1465 627q0 -104 -40.5 -199 +t-108.5 -164t-162 -109.5t-198 -40.5t-198 40.5t-162 109.5t-108.5 164t-40.5 199t40.5 199t108.5 164t162 109.5t198 40.5t198 -40.5t162 -109.5t108.5 -164t40.5 -199zM1752 915q-65 147 -180.5 251t-253 153.5t-292 53.5t-301 -36.5t-275.5 -129t-220 -211.5t-131 -297 +t-10 -373q-49 161 -51.5 311.5t35.5 272.5t109 227t165.5 180.5t207 126t232 71t242.5 9t236 -54t216 -124.5t178 -197q33 -50 62 -121t31 -112zM1690 573q12 244 -136.5 416t-396.5 240q-8 0 -10 5t24 8q125 -4 230 -50t173 -120t116 -168.5t58.5 -199t-1 -208 +t-61.5 -197.5t-122.5 -167t-185 -117.5t-248.5 -46.5q108 30 201.5 80t174 123t129.5 176.5t55 225.5z" /> + <glyph glyph-name="uniF2DB" unicode="" +d="M192 256v-128h-112q-16 0 -16 16v16h-48q-16 0 -16 16v32q0 16 16 16h48v16q0 16 16 16h112zM192 512v-128h-112q-16 0 -16 16v16h-48q-16 0 -16 16v32q0 16 16 16h48v16q0 16 16 16h112zM192 768v-128h-112q-16 0 -16 16v16h-48q-16 0 -16 16v32q0 16 16 16h48v16 +q0 16 16 16h112zM192 1024v-128h-112q-16 0 -16 16v16h-48q-16 0 -16 16v32q0 16 16 16h48v16q0 16 16 16h112zM192 1280v-128h-112q-16 0 -16 16v16h-48q-16 0 -16 16v32q0 16 16 16h48v16q0 16 16 16h112zM1280 1440v-1472q0 -40 -28 -68t-68 -28h-832q-40 0 -68 28 +t-28 68v1472q0 40 28 68t68 28h832q40 0 68 -28t28 -68zM1536 208v-32q0 -16 -16 -16h-48v-16q0 -16 -16 -16h-112v128h112q16 0 16 -16v-16h48q16 0 16 -16zM1536 464v-32q0 -16 -16 -16h-48v-16q0 -16 -16 -16h-112v128h112q16 0 16 -16v-16h48q16 0 16 -16zM1536 720v-32 +q0 -16 -16 -16h-48v-16q0 -16 -16 -16h-112v128h112q16 0 16 -16v-16h48q16 0 16 -16zM1536 976v-32q0 -16 -16 -16h-48v-16q0 -16 -16 -16h-112v128h112q16 0 16 -16v-16h48q16 0 16 -16zM1536 1232v-32q0 -16 -16 -16h-48v-16q0 -16 -16 -16h-112v128h112q16 0 16 -16v-16 +h48q16 0 16 -16z" /> + <glyph glyph-name="uniF2DC" unicode="" horiz-adv-x="1664" +d="M1566 419l-167 -33l186 -107q23 -13 29.5 -38.5t-6.5 -48.5q-14 -23 -39 -29.5t-48 6.5l-186 106l55 -160q13 -38 -12 -63.5t-60.5 -20.5t-48.5 42l-102 300l-271 156v-313l208 -238q16 -18 17 -39t-11 -36.5t-28.5 -25t-37 -5.5t-36.5 22l-112 128v-214q0 -26 -19 -45 +t-45 -19t-45 19t-19 45v214l-112 -128q-16 -18 -36.5 -22t-37 5.5t-28.5 25t-11 36.5t17 39l208 238v313l-271 -156l-102 -300q-13 -37 -48.5 -42t-60.5 20.5t-12 63.5l55 160l-186 -106q-23 -13 -48 -6.5t-39 29.5q-13 23 -6.5 48.5t29.5 38.5l186 107l-167 33 +q-29 6 -42 29t-8.5 46.5t25.5 40t50 10.5l310 -62l271 157l-271 157l-310 -62q-4 -1 -13 -1q-27 0 -44 18t-19 40t11 43t40 26l167 33l-186 107q-23 13 -29.5 38.5t6.5 48.5t39 30t48 -7l186 -106l-55 160q-13 38 12 63.5t60.5 20.5t48.5 -42l102 -300l271 -156v313 +l-208 238q-16 18 -17 39t11 36.5t28.5 25t37 5.5t36.5 -22l112 -128v214q0 26 19 45t45 19t45 -19t19 -45v-214l112 128q16 18 36.5 22t37 -5.5t28.5 -25t11 -36.5t-17 -39l-208 -238v-313l271 156l102 300q13 37 48.5 42t60.5 -20.5t12 -63.5l-55 -160l186 106 +q23 13 48 6.5t39 -29.5q13 -23 6.5 -48.5t-29.5 -38.5l-186 -107l167 -33q27 -5 40 -26t11 -43t-19 -40t-44 -18q-9 0 -13 1l-310 62l-271 -157l271 -157l310 62q29 6 50 -10.5t25.5 -40t-8.5 -46.5t-42 -29z" /> + <glyph glyph-name="uniF2DD" unicode="" horiz-adv-x="1792" +d="M1473 607q7 118 -33 226.5t-113 189t-177 131t-221 57.5q-116 7 -225.5 -32t-192 -110.5t-135 -175t-59.5 -220.5q-7 -118 33 -226.5t113 -189t177.5 -131t221.5 -57.5q155 -9 293 59t224 195.5t94 283.5zM1792 1536l-349 -348q120 -117 180.5 -272t50.5 -321 +q-11 -183 -102 -339t-241 -255.5t-332 -124.5l-999 -132l347 347q-120 116 -180.5 271.5t-50.5 321.5q11 184 102 340t241.5 255.5t332.5 124.5q167 22 500 66t500 66z" /> + <glyph glyph-name="uniF2DE" unicode="" horiz-adv-x="1792" +d="M948 508l163 -329h-51l-175 350l-171 -350h-49l179 374l-78 33l21 49l240 -102l-21 -50zM563 1100l304 -130l-130 -304l-304 130zM907 915l240 -103l-103 -239l-239 102zM1188 765l191 -81l-82 -190l-190 81zM1680 640q0 159 -62 304t-167.5 250.5t-250.5 167.5t-304 62 +t-304 -62t-250.5 -167.5t-167.5 -250.5t-62 -304t62 -304t167.5 -250.5t250.5 -167.5t304 -62t304 62t250.5 167.5t167.5 250.5t62 304zM1792 640q0 -182 -71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71t348 -71 +t286 -191t191 -286t71 -348z" /> + <glyph glyph-name="uniF2E0" unicode="" horiz-adv-x="1920" +d="M1334 302q-4 24 -27.5 34t-49.5 10.5t-48.5 12.5t-25.5 38q-5 47 33 139.5t75 181t32 127.5q-14 101 -117 103q-45 1 -75 -16l-3 -2l-5 -2.5t-4.5 -2t-5 -2t-5 -0.5t-6 1.5t-6 3.5t-6.5 5q-3 2 -9 8.5t-9 9t-8.5 7.5t-9.5 7.5t-9.5 5.5t-11 4.5t-11.5 2.5q-30 5 -48 -3 +t-45 -31q-1 -1 -9 -8.5t-12.5 -11t-15 -10t-16.5 -5.5t-17 3q-54 27 -84 40q-41 18 -94 -5t-76 -65q-16 -28 -41 -98.5t-43.5 -132.5t-40 -134t-21.5 -73q-22 -69 18.5 -119t110.5 -46q30 2 50.5 15t38.5 46q7 13 79 199.5t77 194.5q6 11 21.5 18t29.5 0q27 -15 21 -53 +q-2 -18 -51 -139.5t-50 -132.5q-6 -38 19.5 -56.5t60.5 -7t55 49.5q4 8 45.5 92t81.5 163.5t46 88.5q20 29 41 28q29 0 25 -38q-2 -16 -65.5 -147.5t-70.5 -159.5q-12 -53 13 -103t74 -74q17 -9 51 -15.5t71.5 -8t62.5 14t20 48.5zM383 86q3 -15 -5 -27.5t-23 -15.5 +q-14 -3 -26.5 5t-15.5 23q-3 14 5 27t22 16t27 -5t16 -23zM953 -177q12 -17 8.5 -37.5t-20.5 -32.5t-37.5 -8t-32.5 21q-11 17 -7.5 37.5t20.5 32.5t37.5 8t31.5 -21zM177 635q-18 -27 -49.5 -33t-57.5 13q-26 18 -32 50t12 58q18 27 49.5 33t57.5 -12q26 -19 32 -50.5 +t-12 -58.5zM1467 -42q19 -28 13 -61.5t-34 -52.5t-60.5 -13t-51.5 34t-13 61t33 53q28 19 60.5 13t52.5 -34zM1579 562q69 -113 42.5 -244.5t-134.5 -207.5q-90 -63 -199 -60q-20 -80 -84.5 -127t-143.5 -44.5t-140 57.5q-12 -9 -13 -10q-103 -71 -225 -48.5t-193 126.5 +q-50 73 -53 164q-83 14 -142.5 70.5t-80.5 128t-2 152t81 138.5q-36 60 -38 128t24.5 125t79.5 98.5t121 50.5q32 85 99 148t146.5 91.5t168 17t159.5 -66.5q72 21 140 17.5t128.5 -36t104.5 -80t67.5 -115t17.5 -140.5q52 -16 87 -57t45.5 -89t-5.5 -99.5t-58 -87.5z +M455 1222q14 -20 9.5 -44.5t-24.5 -38.5q-19 -14 -43.5 -9.5t-37.5 24.5q-14 20 -9.5 44.5t24.5 38.5q19 14 43.5 9.5t37.5 -24.5zM614 1503q4 -16 -5 -30.5t-26 -18.5t-31 5.5t-18 26.5q-3 17 6.5 31t25.5 18q17 4 31 -5.5t17 -26.5zM1800 555q4 -20 -6.5 -37t-30.5 -21 +q-19 -4 -36 6.5t-21 30.5t6.5 37t30.5 22q20 4 36.5 -7.5t20.5 -30.5zM1136 1448q16 -27 8.5 -58.5t-35.5 -47.5q-27 -16 -57.5 -8.5t-46.5 34.5q-16 28 -8.5 59t34.5 48t58 9t47 -36zM1882 792q4 -15 -4 -27.5t-23 -16.5q-15 -3 -27.5 5.5t-15.5 22.5q-3 15 5 28t23 16 +q14 3 26.5 -5t15.5 -23zM1691 1033q15 -22 10.5 -49t-26.5 -43q-22 -15 -49 -10t-42 27t-10 49t27 43t48.5 11t41.5 -28z" /> + <glyph glyph-name="uniF2E1" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2E2" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2E3" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2E4" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2E5" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2E6" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2E7" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="_698" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2E9" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2EA" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2EB" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2EC" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2ED" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="uniF2EE" unicode="" horiz-adv-x="1792" + /> + <glyph glyph-name="lessequal" unicode="" horiz-adv-x="1792" + /> + </font> +</defs></svg> diff --git a/python/vespa/docs/css/fonts/fontawesome-webfont.ttf b/python/vespa/docs/css/fonts/fontawesome-webfont.ttf Binary files differnew file mode 100644 index 00000000000..35acda2fa11 --- /dev/null +++ b/python/vespa/docs/css/fonts/fontawesome-webfont.ttf diff --git a/python/vespa/docs/css/fonts/fontawesome-webfont.woff b/python/vespa/docs/css/fonts/fontawesome-webfont.woff Binary files differnew file mode 100644 index 00000000000..400014a4b06 --- /dev/null +++ b/python/vespa/docs/css/fonts/fontawesome-webfont.woff diff --git a/python/vespa/docs/css/fonts/fontawesome-webfont.woff2 b/python/vespa/docs/css/fonts/fontawesome-webfont.woff2 Binary files differnew file mode 100644 index 00000000000..4d13fc60404 --- /dev/null +++ b/python/vespa/docs/css/fonts/fontawesome-webfont.woff2 diff --git a/python/vespa/docs/css/modern-business.css b/python/vespa/docs/css/modern-business.css new file mode 100755 index 00000000000..b0323c5cdf0 --- /dev/null +++ b/python/vespa/docs/css/modern-business.css @@ -0,0 +1,89 @@ +/*! + * Start Bootstrap - Modern Business HTML Template (http://startbootstrap.com) + * Code licensed under the Apache License v2.0. + * For details, see http://www.apache.org/licenses/LICENSE-2.0. + */ + +/* Global Styles */ + +html, +body { + height: 100%; +} + +.img-portfolio { + margin-bottom: 30px; +} + +.img-hover:hover { + opacity: 0.8; +} + +/* Home Page Carousel */ + +header.carousel { + height: 50%; +} + +header.carousel .item, +header.carousel .item.active, +header.carousel .carousel-inner { + height: 100%; +} + +header.carousel .fill { + width: 100%; + height: 100%; + background-position: center; + background-size: cover; +} + +/* 404 Page Styles */ + +.error-404 { + font-size: 100px; +} + +/* Pricing Page Styles */ + +.price { + display: block; + font-size: 50px; + line-height: 50px; +} + +.price sup { + top: -20px; + left: 2px; + font-size: 20px; +} + +.period { + display: block; + font-style: italic; +} + +/* Footer Styles */ + +footer { + margin: 50px 0; +} + +/* Responsive Styles */ + +@media(max-width:991px) { + .client-img, + .img-related { + margin-bottom: 30px; + } +} + +@media(max-width:767px) { + .img-portfolio { + margin-bottom: 15px; + } + + header.carousel .carousel { + height: 70%; + } +} diff --git a/python/vespa/docs/css/printstyles.css b/python/vespa/docs/css/printstyles.css new file mode 100644 index 00000000000..64d0f631011 --- /dev/null +++ b/python/vespa/docs/css/printstyles.css @@ -0,0 +1,159 @@ + +/*body.print .container {max-width: 650px;}*/ + +body { + font-size:14px; +} +.nav ul li a {border-top:0px; background-color:transparent; color: #808080; } +#navig a[href] {color: #595959 !important;} +table .table {max-width:650px;} + +#navig li.sectionHead {font-weight: bold; font-size: 18px; color: #595959 !important; } +#navig li {font-weight: normal; } + +#navig a[href]::after { content: leader(".") target-counter(attr(href), page); } + +a[href]::after { + content: " (page " target-counter(attr(href), page) ")" +} + +a[href^="http:"]::after, a[href^="https:"]::after { + content: ""; +} + +a[href] { + color: blue !important; +} +a[href*="mailto"]::after, a[data-toggle="tooltip"]::after, a[href].noCrossRef::after { + content: ""; +} + + +@page { + margin: 60pt 90pt 60pt 90pt; + font-family: sans-serif; + font-style:none; + color: gray; + +} + +.printTitle { + line-height:30pt; + font-size:27pt; + font-weight: bold; + letter-spacing: -.5px; + margin-bottom:25px; +} + +.printSubtitle { + font-size: 19pt; + color: #cccccc !important; + font-family: "Grotesque MT Light"; + line-height: 22pt; + letter-spacing: -.5px; + margin-bottom:20px; +} +.printTitleArea hr { + color: #999999 !important; + height: 2px; + width: 100%; +} + +.printTitleImage { + max-width:300px; + margin-bottom:200px; +} + + +.printTitleImage { + max-width: 250px; +} + +#navig { + /*page-break-before: always;*/ +} + +.copyrightBoilerplate { + page-break-before:always; + font-size:14px; +} + +.lastGeneratedDate { + font-style: italic; + font-size:14px; + color: gray; +} + +.alert a { + text-decoration: none !important; +} + + +body.title { page: title } + +@page title { + @top-left { + content: " "; + } + @top-right { + content: " " + } + @bottom-right { + content: " "; + } + @bottom-left { + content: " "; + } +} + +body.frontmatter { page: frontmatter } +body.frontmatter {counter-reset: page 1} + + +@page frontmatter { + @top-left { + content: prince-script(guideName); + } + @top-right { + content: prince-script(datestamp); + } + @bottom-right { + content: counter(page, lower-roman); + } + @bottom-left { + content: "youremail@domain.com"; } +} + +body.first_page {counter-reset: page 1} + +h1 { string-set: doctitle content() } + +@page { + @top-left { + content: string(doctitle); + font-size: 11px; + font-style: italic; + } + @top-right { + content: prince-script(datestamp); + font-size: 11px; + } + + @bottom-right { + content: "Page " counter(page); + font-size: 11px; + } + @bottom-left { + content: prince-script(guideName); + font-size: 11px; + } +} +.alert { + background-color: #fafafa !important; + border-color: #dedede !important; + color: black; +} + +pre { + background-color: #fafafa; +} diff --git a/python/vespa/docs/css/syntax.css b/python/vespa/docs/css/syntax.css new file mode 100644 index 00000000000..1e651cf79db --- /dev/null +++ b/python/vespa/docs/css/syntax.css @@ -0,0 +1,60 @@ +.highlight { background: #ffffff; } +.highlight .c { color: #999988; font-style: italic } /* Comment */ +.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ +.highlight .k { font-weight: bold } /* Keyword */ +.highlight .o { font-weight: bold } /* Operator */ +.highlight .cm { color: #999988; font-style: italic } /* Comment.Multiline */ +.highlight .cp { color: #999999; font-weight: bold } /* Comment.Preproc */ +.highlight .c1 { color: #999988; font-style: italic } /* Comment.Single */ +.highlight .cs { color: #999999; font-weight: bold; font-style: italic } /* Comment.Special */ +.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ +.highlight .gd .x { color: #000000; background-color: #ffaaaa } /* Generic.Deleted.Specific */ +.highlight .ge { font-style: italic } /* Generic.Emph */ +.highlight .gr { color: #aa0000 } /* Generic.Error */ +.highlight .gh { color: #999999 } /* Generic.Heading */ +.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ +.highlight .gi .x { color: #000000; background-color: #aaffaa } /* Generic.Inserted.Specific */ +.highlight .go { color: #888888 } /* Generic.Output */ +.highlight .gp { color: #555555 } /* Generic.Prompt */ +.highlight .gs { font-weight: bold } /* Generic.Strong */ +.highlight .gu { color: #aaaaaa } /* Generic.Subheading */ +.highlight .gt { color: #aa0000 } /* Generic.Traceback */ +.highlight .kc { font-weight: bold } /* Keyword.Constant */ +.highlight .kd { font-weight: bold } /* Keyword.Declaration */ +.highlight .kp { font-weight: bold } /* Keyword.Pseudo */ +.highlight .kr { font-weight: bold } /* Keyword.Reserved */ +.highlight .kt { color: #445588; font-weight: bold } /* Keyword.Type */ +.highlight .m { color: #009999 } /* Literal.Number */ +.highlight .s { color: #d14 } /* Literal.String */ +.highlight .na { color: #008080 } /* Name.Attribute */ +.highlight .nb { color: #0086B3 } /* Name.Builtin */ +.highlight .nc { color: #445588; font-weight: bold } /* Name.Class */ +.highlight .no { color: #008080 } /* Name.Constant */ +.highlight .ni { color: #800080 } /* Name.Entity */ +.highlight .ne { color: #990000; font-weight: bold } /* Name.Exception */ +.highlight .nf { color: #990000; font-weight: bold } /* Name.Function */ +.highlight .nn { color: #555555 } /* Name.Namespace */ +.highlight .nt { color: #000080 } /* Name.Tag */ +.highlight .nv { color: #008080 } /* Name.Variable */ +.highlight .ow { font-weight: bold } /* Operator.Word */ +.highlight .w { color: #bbbbbb } /* Text.Whitespace */ +.highlight .mf { color: #009999 } /* Literal.Number.Float */ +.highlight .mh { color: #009999 } /* Literal.Number.Hex */ +.highlight .mi { color: #009999 } /* Literal.Number.Integer */ +.highlight .mo { color: #009999 } /* Literal.Number.Oct */ +.highlight .sb { color: #d14 } /* Literal.String.Backtick */ +.highlight .sc { color: #d14 } /* Literal.String.Char */ +.highlight .sd { color: #d14 } /* Literal.String.Doc */ +.highlight .s2 { color: #d14 } /* Literal.String.Double */ +.highlight .se { color: #d14 } /* Literal.String.Escape */ +.highlight .sh { color: #d14 } /* Literal.String.Heredoc */ +.highlight .si { color: #d14 } /* Literal.String.Interpol */ +.highlight .sx { color: #d14 } /* Literal.String.Other */ +.highlight .sr { color: #009926 } /* Literal.String.Regex */ +.highlight .s1 { color: #d14 } /* Literal.String.Single */ +.highlight .ss { color: #990073 } /* Literal.String.Symbol */ +.highlight .bp { color: #999999 } /* Name.Builtin.Pseudo */ +.highlight .vc { color: #008080 } /* Name.Variable.Class */ +.highlight .vg { color: #008080 } /* Name.Variable.Global */ +.highlight .vi { color: #008080 } /* Name.Variable.Instance */ +.highlight .il { color: #009999 } /* Literal.Number.Integer.Long */
\ No newline at end of file diff --git a/python/vespa/docs/css/theme-blue.css b/python/vespa/docs/css/theme-blue.css new file mode 100644 index 00000000000..8027c8273f2 --- /dev/null +++ b/python/vespa/docs/css/theme-blue.css @@ -0,0 +1,121 @@ +.summary { + color: #808080; + border-left: 5px solid #ED1951; + font-size:16px; +} + + +h3 {color: #000000; } +h4 {color: #000000; } + +.nav-tabs > li.active > a, .nav-tabs > li.active > a:hover, .nav-tabs > li.active > a:focus { + background-color: #248ec2; + color: white; +} + +.nav > li.active > a { + background-color: #347DBE; +} + +.nav > li > a:hover { + background-color: #248ec2; +} + +div.navbar-collapse .dropdown-menu > li > a:hover { + background-color: #347DBE; +} + +.nav li.thirdlevel > a { + background-color: #FAFAFA !important; + color: #248EC2; + font-weight: bold; +} + +a[data-toggle="tooltip"] { + color: #649345; + font-style: italic; + cursor: default; +} + +.navbar-inverse { + background-color: #347DBE; + border-color: #015CAE; +} +.navbar-inverse .navbar-nav>li>a, .navbar-inverse .navbar-brand { + color: white; +} + +.navbar-inverse .navbar-nav>li>a:hover, a.fa.fa-home.fa-lg.navbar-brand:hover { + color: #f0f0f0; +} + +a.navbar-brand:hover { + color: #f0f0f0; +} + +.navbar-inverse .navbar-nav > .open > a, .navbar-inverse .navbar-nav > .open > a:hover, .navbar-inverse .navbar-nav > .open > a:focus { + color: #015CAE; +} + +.navbar-inverse .navbar-nav > .open > a, .navbar-inverse .navbar-nav > .open > a:hover, .navbar-inverse .navbar-nav > .open > a:focus { + background-color: #015CAE; + color: #ffffff; +} + +.navbar-inverse .navbar-collapse, .navbar-inverse .navbar-form { + border-color: #248ec2 !important; +} + +.btn-primary { + color: #ffffff; + background-color: #347DBE; + border-color: #347DBE; +} + +.navbar-inverse .navbar-nav > .active > a, .navbar-inverse .navbar-nav > .active > a:hover, .navbar-inverse .navbar-nav > .active > a:focus { + background-color: #347DBE; +} + +.btn-primary:hover, +.btn-primary:focus, +.btn-primary:active, +.btn-primary.active, +.open .dropdown-toggle.btn-primary { + background-color: #248ec2; + border-color: #347DBE; +} + +.printTitle { + color: #015CAE !important; +} + +body.print h1 {color: #015CAE !important; font-size:28px !important;} +body.print h2 {color: #595959 !important; font-size:20px !important;} +body.print h3 {color: #E50E51 !important; font-size:14px !important;} +body.print h4 {color: #679DCE !important; font-size:14px; font-style: italic !important;} + +.anchorjs-link:hover { + color: #216f9b; +} + +div.sidebarTitle { + color: #015CAE; +} + +li.sidebarTitle { + margin-top:20px; + font-weight:normal; + font-size:130%; + color: #ED1951; + margin-bottom:10px; + margin-left: 5px; + +} + +.navbar-inverse .navbar-toggle:focus, .navbar-inverse .navbar-toggle:hover { + background-color: #015CAE; +} + +.navbar-inverse .navbar-toggle { + border-color: #015CAE; +} diff --git a/python/vespa/docs/css/theme-green.css b/python/vespa/docs/css/theme-green.css new file mode 100644 index 00000000000..4991586b5a6 --- /dev/null +++ b/python/vespa/docs/css/theme-green.css @@ -0,0 +1,110 @@ +.summary { + color: #808080; + border-left: 5px solid #E50E51; + font-size:16px; +} + + +h3 {color: #E50E51; } +h4 {color: #808080; } + +.nav-tabs > li.active > a, .nav-tabs > li.active > a:hover, .nav-tabs > li.active > a:focus { + background-color: #248ec2; + color: white; +} + +.nav > li.active > a { + background-color: #72ac4a; +} + +.nav > li > a:hover { + background-color: #72ac4a; +} + +div.navbar-collapse .dropdown-menu > li > a:hover { + background-color: #72ac4a; +} + +.navbar-inverse .navbar-nav>li>a, .navbar-inverse .navbar-brand { + color: white; +} + +.navbar-inverse .navbar-nav>li>a:hover, a.fa.fa-home.fa-lg.navbar-brand:hover { + color: #f0f0f0; +} + +.nav li.thirdlevel > a { + background-color: #FAFAFA !important; + color: #72ac4a; + font-weight: bold; +} + +a[data-toggle="tooltip"] { + color: #649345; + font-style: italic; + cursor: default; +} + +.navbar-inverse { + background-color: #72ac4a; + border-color: #5b893c; +} + +.navbar-inverse .navbar-nav > .open > a, .navbar-inverse .navbar-nav > .open > a:hover, .navbar-inverse .navbar-nav > .open > a:focus { + color: #5b893c; +} + +.navbar-inverse .navbar-nav > .open > a, .navbar-inverse .navbar-nav > .open > a:hover, .navbar-inverse .navbar-nav > .open > a:focus { + background-color: #5b893c; + color: #ffffff; +} + +/* not sure if using this ...*/ +.navbar-inverse .navbar-collapse, .navbar-inverse .navbar-form { + border-color: #72ac4a !important; +} + +.btn-primary { + color: #ffffff; + background-color: #5b893c; + border-color: #5b893c; +} + +.btn-primary:hover, +.btn-primary:focus, +.btn-primary:active, +.btn-primary.active, +.open .dropdown-toggle.btn-primary { + background-color: #72ac4a; + border-color: #5b893c; +} + +.printTitle { + color: #5b893c !important; +} + +body.print h1 {color: #5b893c !important; font-size:28px;} +body.print h2 {color: #595959 !important; font-size:24px;} +body.print h3 {color: #E50E51 !important; font-size:14px;} +body.print h4 {color: #679DCE !important; font-size:14px; font-style: italic;} + +.anchorjs-link:hover { + color: #4f7233; +} + +div.sidebarTitle { + color: #E50E51; +} + +li.sidebarTitle { + margin-top:20px; + font-weight:normal; + font-size:130%; + color: #ED1951; + margin-bottom:10px; + margin-left: 5px; +} + +.navbar-inverse .navbar-toggle:focus, .navbar-inverse .navbar-toggle:hover { + background-color: #E50E51; +} diff --git a/python/vespa/docs/evaluation.html b/python/vespa/docs/evaluation.html new file mode 100644 index 00000000000..132b6fbfe89 --- /dev/null +++ b/python/vespa/docs/evaluation.html @@ -0,0 +1,321 @@ +--- + +title: Vespa - Evaluate query models + +keywords: fastai +sidebar: home_sidebar + +summary: "Define metrics and evaluate query models" +description: "Define metrics and evaluate query models" +--- +<!-- + +################################################# +### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ### +################################################# +# file to edit: notebooks/evaluation.ipynb +# command to build the docs after a change: nbdev_build_docs + +--> + +<div class="container" id="notebook-container"> + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Example-setup">Example setup<a class="anchor-link" href="#Example-setup"> </a></h2> +</div> +</div> +</div> +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>Connect to the application and define a query model.</p> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.application</span> <span class="k">import</span> <span class="n">Vespa</span> +<span class="kn">from</span> <span class="nn">vespa.query</span> <span class="k">import</span> <span class="n">Query</span><span class="p">,</span> <span class="n">RankProfile</span><span class="p">,</span> <span class="n">OR</span> + +<span class="n">app</span> <span class="o">=</span> <span class="n">Vespa</span><span class="p">(</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"https://api.cord19.vespa.ai"</span><span class="p">)</span> +<span class="n">query_model</span> <span class="o">=</span> <span class="n">Query</span><span class="p">(</span> + <span class="n">match_phase</span> <span class="o">=</span> <span class="n">OR</span><span class="p">(),</span> + <span class="n">rank_profile</span> <span class="o">=</span> <span class="n">RankProfile</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"bm25"</span><span class="p">,</span> <span class="n">list_features</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>Define some labelled data.</p> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">labelled_data</span> <span class="o">=</span> <span class="p">[</span> + <span class="p">{</span> + <span class="s2">"query_id"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> + <span class="s2">"query"</span><span class="p">:</span> <span class="s2">"Intrauterine virus infections and congenital heart disease"</span><span class="p">,</span> + <span class="s2">"relevant_docs"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span> <span class="p">{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">}]</span> + <span class="p">},</span> + <span class="p">{</span> + <span class="s2">"query_id"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> + <span class="s2">"query"</span><span class="p">:</span> <span class="s2">"Clinical and immunologic studies in identical twins discordant for systemic lupus erythematosus"</span><span class="p">,</span> + <span class="s2">"relevant_docs"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span> <span class="p">{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">}]</span> + <span class="p">}</span> +<span class="p">]</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Define-metrics">Define metrics<a class="anchor-link" href="#Define-metrics"> </a></h2> +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.evaluation</span> <span class="k">import</span> <span class="n">MatchRatio</span><span class="p">,</span> <span class="n">Recall</span><span class="p">,</span> <span class="n">ReciprocalRank</span> + +<span class="n">eval_metrics</span> <span class="o">=</span> <span class="p">[</span><span class="n">MatchRatio</span><span class="p">(),</span> <span class="n">Recall</span><span class="p">(</span><span class="n">at</span><span class="o">=</span><span class="mi">10</span><span class="p">),</span> <span class="n">ReciprocalRank</span><span class="p">(</span><span class="n">at</span><span class="o">=</span><span class="mi">10</span><span class="p">)]</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Evaluate-in-batch">Evaluate in batch<a class="anchor-link" href="#Evaluate-in-batch"> </a></h2> +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">evaluation</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">evaluate</span><span class="p">(</span> + <span class="n">labelled_data</span> <span class="o">=</span> <span class="n">labelled_data</span><span class="p">,</span> + <span class="n">eval_metrics</span> <span class="o">=</span> <span class="n">eval_metrics</span><span class="p">,</span> + <span class="n">query_model</span> <span class="o">=</span> <span class="n">query_model</span><span class="p">,</span> + <span class="n">id_field</span> <span class="o">=</span> <span class="s2">"id"</span><span class="p">,</span> +<span class="p">)</span> +<span class="n">evaluation</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + +<div class="output_html rendered_html output_subarea output_execute_result"> +<div> +<style scoped> + .dataframe tbody tr th:only-of-type { + vertical-align: middle; + } + + .dataframe tbody tr th { + vertical-align: top; + } + + .dataframe thead th { + text-align: right; + } +</style> +<table border="1" class="dataframe"> + <thead> + <tr style="text-align: right;"> + <th></th> + <th>query_id</th> + <th>match_ratio_retrieved_docs</th> + <th>match_ratio_docs_available</th> + <th>match_ratio_value</th> + <th>recall_10_value</th> + <th>reciprocal_rank_10_value</th> + </tr> + </thead> + <tbody> + <tr> + <th>0</th> + <td>0</td> + <td>52526</td> + <td>58692</td> + <td>0.894943</td> + <td>0</td> + <td>0</td> + </tr> + <tr> + <th>1</th> + <td>1</td> + <td>54048</td> + <td>58692</td> + <td>0.920875</td> + <td>0</td> + <td>0</td> + </tr> + </tbody> +</table> +</div> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Evaluate-specific-query">Evaluate specific query<a class="anchor-link" href="#Evaluate-specific-query"> </a></h2><blockquote><p>You can have finer control with the <code>evaluate_query</code> method.</p> +</blockquote> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">pandas</span> <span class="k">import</span> <span class="n">concat</span><span class="p">,</span> <span class="n">DataFrame</span> + +<span class="n">evaluation</span> <span class="o">=</span> <span class="p">[]</span> +<span class="k">for</span> <span class="n">query_data</span> <span class="ow">in</span> <span class="n">labelled_data</span><span class="p">:</span> + <span class="n">query_evaluation</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">evaluate_query</span><span class="p">(</span> + <span class="n">eval_metrics</span> <span class="o">=</span> <span class="n">eval_metrics</span><span class="p">,</span> + <span class="n">query_model</span> <span class="o">=</span> <span class="n">query_model</span><span class="p">,</span> + <span class="n">query_id</span> <span class="o">=</span> <span class="n">query_data</span><span class="p">[</span><span class="s2">"query_id"</span><span class="p">],</span> + <span class="n">query</span> <span class="o">=</span> <span class="n">query_data</span><span class="p">[</span><span class="s2">"query"</span><span class="p">],</span> + <span class="n">id_field</span> <span class="o">=</span> <span class="s2">"id"</span><span class="p">,</span> + <span class="n">relevant_docs</span> <span class="o">=</span> <span class="n">query_data</span><span class="p">[</span><span class="s2">"relevant_docs"</span><span class="p">],</span> + <span class="n">default_score</span> <span class="o">=</span> <span class="mi">0</span> + <span class="p">)</span> + <span class="n">evaluation</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">query_evaluation</span><span class="p">)</span> +<span class="n">evaluation</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">from_records</span><span class="p">(</span><span class="n">evaluation</span><span class="p">)</span> +<span class="n">evaluation</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + +<div class="output_html rendered_html output_subarea output_execute_result"> +<div> +<style scoped> + .dataframe tbody tr th:only-of-type { + vertical-align: middle; + } + + .dataframe tbody tr th { + vertical-align: top; + } + + .dataframe thead th { + text-align: right; + } +</style> +<table border="1" class="dataframe"> + <thead> + <tr style="text-align: right;"> + <th></th> + <th>query_id</th> + <th>match_ratio_retrieved_docs</th> + <th>match_ratio_docs_available</th> + <th>match_ratio_value</th> + <th>recall_10_value</th> + <th>reciprocal_rank_10_value</th> + </tr> + </thead> + <tbody> + <tr> + <th>0</th> + <td>0</td> + <td>52526</td> + <td>58692</td> + <td>0.894943</td> + <td>0</td> + <td>0</td> + </tr> + <tr> + <th>1</th> + <td>1</td> + <td>54048</td> + <td>58692</td> + <td>0.920875</td> + <td>0</td> + <td>0</td> + </tr> + </tbody> +</table> +</div> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +</div> + + diff --git a/python/vespa/docs/feed.xml b/python/vespa/docs/feed.xml new file mode 100644 index 00000000000..d8d6ac999d6 --- /dev/null +++ b/python/vespa/docs/feed.xml @@ -0,0 +1,32 @@ +--- +search: exclude +layout: none +--- + +<?xml version="1.0" encoding="UTF-8"?> +<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"> + <channel> + <title>{{ site.title | xml_escape }}</title> + <description>{{ site.description | xml_escape }}</description> + <link>{{ site.url }}/</link> + <atom:link href="{{ "/feed.xml" | prepend: site.url }}" rel="self" type="application/rss+xml"/> + <pubDate>{{ site.time | date_to_rfc822 }}</pubDate> + <lastBuildDate>{{ site.time | date_to_rfc822 }}</lastBuildDate> + <generator>Jekyll v{{ jekyll.version }}</generator> + {% for post in site.posts limit:10 %} + <item> + <title>{{ post.title | xml_escape }}</title> + <description>{{ post.content | xml_escape }}</description> + <pubDate>{{ post.date | date_to_rfc822 }}</pubDate> + <link>{{ post.url | prepend: site.url }}</link> + <guid isPermaLink="true">{{ post.url | prepend: site.url }}</guid> + {% for tag in post.tags %} + <category>{{ tag | xml_escape }}</category> + {% endfor %} + {% for tag in page.tags %} + <category>{{ cat | xml_escape }}</category> + {% endfor %} + </item> + {% endfor %} + </channel> +</rss> diff --git a/python/vespa/docs/fonts/FontAwesome.otf b/python/vespa/docs/fonts/FontAwesome.otf Binary files differnew file mode 100644 index 00000000000..81c9ad949b4 --- /dev/null +++ b/python/vespa/docs/fonts/FontAwesome.otf diff --git a/python/vespa/docs/fonts/fontawesome-webfont.eot b/python/vespa/docs/fonts/fontawesome-webfont.eot Binary files differnew file mode 100644 index 00000000000..84677bc0c5f --- /dev/null +++ b/python/vespa/docs/fonts/fontawesome-webfont.eot diff --git a/python/vespa/docs/fonts/fontawesome-webfont.svg b/python/vespa/docs/fonts/fontawesome-webfont.svg new file mode 100644 index 00000000000..d907b25ae60 --- /dev/null +++ b/python/vespa/docs/fonts/fontawesome-webfont.svg @@ -0,0 +1,520 @@ +<?xml version="1.0" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd" > +<svg xmlns="http://www.w3.org/2000/svg"> +<metadata></metadata> +<defs> +<font id="fontawesomeregular" horiz-adv-x="1536" > +<font-face units-per-em="1792" ascent="1536" descent="-256" /> +<missing-glyph horiz-adv-x="448" /> +<glyph unicode=" " horiz-adv-x="448" /> +<glyph unicode="	" horiz-adv-x="448" /> +<glyph unicode=" " horiz-adv-x="448" /> +<glyph unicode="¨" horiz-adv-x="1792" /> +<glyph unicode="©" horiz-adv-x="1792" /> +<glyph unicode="®" horiz-adv-x="1792" /> +<glyph unicode="´" horiz-adv-x="1792" /> +<glyph unicode="Æ" horiz-adv-x="1792" /> +<glyph unicode="Ø" horiz-adv-x="1792" /> +<glyph unicode=" " horiz-adv-x="768" /> +<glyph unicode=" " horiz-adv-x="1537" /> +<glyph unicode=" " horiz-adv-x="768" /> +<glyph unicode=" " horiz-adv-x="1537" /> +<glyph unicode=" " horiz-adv-x="512" /> +<glyph unicode=" " horiz-adv-x="384" /> +<glyph unicode=" " horiz-adv-x="256" /> +<glyph unicode=" " horiz-adv-x="256" /> +<glyph unicode=" " horiz-adv-x="192" /> +<glyph unicode=" " horiz-adv-x="307" /> +<glyph unicode=" " horiz-adv-x="85" /> +<glyph unicode=" " horiz-adv-x="307" /> +<glyph unicode=" " horiz-adv-x="384" /> +<glyph unicode="™" horiz-adv-x="1792" /> +<glyph unicode="∞" horiz-adv-x="1792" /> +<glyph unicode="≠" horiz-adv-x="1792" /> +<glyph unicode="◼" horiz-adv-x="500" d="M0 0z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1699 1350q0 -35 -43 -78l-632 -632v-768h320q26 0 45 -19t19 -45t-19 -45t-45 -19h-896q-26 0 -45 19t-19 45t19 45t45 19h320v768l-632 632q-43 43 -43 78q0 23 18 36.5t38 17.5t43 4h1408q23 0 43 -4t38 -17.5t18 -36.5z" /> +<glyph unicode="" d="M1536 1312v-1120q0 -50 -34 -89t-86 -60.5t-103.5 -32t-96.5 -10.5t-96.5 10.5t-103.5 32t-86 60.5t-34 89t34 89t86 60.5t103.5 32t96.5 10.5q105 0 192 -39v537l-768 -237v-709q0 -50 -34 -89t-86 -60.5t-103.5 -32t-96.5 -10.5t-96.5 10.5t-103.5 32t-86 60.5t-34 89 t34 89t86 60.5t103.5 32t96.5 10.5q105 0 192 -39v967q0 31 19 56.5t49 35.5l832 256q12 4 28 4q40 0 68 -28t28 -68z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1152 704q0 185 -131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5t316.5 131.5t131.5 316.5zM1664 -128q0 -52 -38 -90t-90 -38q-54 0 -90 38l-343 342q-179 -124 -399 -124q-143 0 -273.5 55.5t-225 150t-150 225t-55.5 273.5 t55.5 273.5t150 225t225 150t273.5 55.5t273.5 -55.5t225 -150t150 -225t55.5 -273.5q0 -220 -124 -399l343 -343q37 -37 37 -90z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1664 32v768q-32 -36 -69 -66q-268 -206 -426 -338q-51 -43 -83 -67t-86.5 -48.5t-102.5 -24.5h-1h-1q-48 0 -102.5 24.5t-86.5 48.5t-83 67q-158 132 -426 338q-37 30 -69 66v-768q0 -13 9.5 -22.5t22.5 -9.5h1472q13 0 22.5 9.5t9.5 22.5zM1664 1083v11v13.5t-0.5 13 t-3 12.5t-5.5 9t-9 7.5t-14 2.5h-1472q-13 0 -22.5 -9.5t-9.5 -22.5q0 -168 147 -284q193 -152 401 -317q6 -5 35 -29.5t46 -37.5t44.5 -31.5t50.5 -27.5t43 -9h1h1q20 0 43 9t50.5 27.5t44.5 31.5t46 37.5t35 29.5q208 165 401 317q54 43 100.5 115.5t46.5 131.5z M1792 1120v-1088q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h1472q66 0 113 -47t47 -113z" /> +<glyph unicode="" horiz-adv-x="1792" d="M896 -128q-26 0 -44 18l-624 602q-10 8 -27.5 26t-55.5 65.5t-68 97.5t-53.5 121t-23.5 138q0 220 127 344t351 124q62 0 126.5 -21.5t120 -58t95.5 -68.5t76 -68q36 36 76 68t95.5 68.5t120 58t126.5 21.5q224 0 351 -124t127 -344q0 -221 -229 -450l-623 -600 q-18 -18 -44 -18z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1664 889q0 -22 -26 -48l-363 -354l86 -500q1 -7 1 -20q0 -21 -10.5 -35.5t-30.5 -14.5q-19 0 -40 12l-449 236l-449 -236q-22 -12 -40 -12q-21 0 -31.5 14.5t-10.5 35.5q0 6 2 20l86 500l-364 354q-25 27 -25 48q0 37 56 46l502 73l225 455q19 41 49 41t49 -41l225 -455 l502 -73q56 -9 56 -46z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1137 532l306 297l-422 62l-189 382l-189 -382l-422 -62l306 -297l-73 -421l378 199l377 -199zM1664 889q0 -22 -26 -48l-363 -354l86 -500q1 -7 1 -20q0 -50 -41 -50q-19 0 -40 12l-449 236l-449 -236q-22 -12 -40 -12q-21 0 -31.5 14.5t-10.5 35.5q0 6 2 20l86 500 l-364 354q-25 27 -25 48q0 37 56 46l502 73l225 455q19 41 49 41t49 -41l225 -455l502 -73q56 -9 56 -46z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1408 131q0 -120 -73 -189.5t-194 -69.5h-874q-121 0 -194 69.5t-73 189.5q0 53 3.5 103.5t14 109t26.5 108.5t43 97.5t62 81t85.5 53.5t111.5 20q9 0 42 -21.5t74.5 -48t108 -48t133.5 -21.5t133.5 21.5t108 48t74.5 48t42 21.5q61 0 111.5 -20t85.5 -53.5t62 -81 t43 -97.5t26.5 -108.5t14 -109t3.5 -103.5zM1088 1024q0 -159 -112.5 -271.5t-271.5 -112.5t-271.5 112.5t-112.5 271.5t112.5 271.5t271.5 112.5t271.5 -112.5t112.5 -271.5z" /> +<glyph unicode="" horiz-adv-x="1920" d="M384 -64v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM384 320v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM384 704v128q0 26 -19 45t-45 19h-128 q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1408 -64v512q0 26 -19 45t-45 19h-768q-26 0 -45 -19t-19 -45v-512q0 -26 19 -45t45 -19h768q26 0 45 19t19 45zM384 1088v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45 t45 -19h128q26 0 45 19t19 45zM1792 -64v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1408 704v512q0 26 -19 45t-45 19h-768q-26 0 -45 -19t-19 -45v-512q0 -26 19 -45t45 -19h768q26 0 45 19t19 45zM1792 320v128 q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1792 704v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1792 1088v128q0 26 -19 45t-45 19h-128q-26 0 -45 -19 t-19 -45v-128q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1920 1248v-1344q0 -66 -47 -113t-113 -47h-1600q-66 0 -113 47t-47 113v1344q0 66 47 113t113 47h1600q66 0 113 -47t47 -113z" /> +<glyph unicode="" horiz-adv-x="1664" d="M768 512v-384q0 -52 -38 -90t-90 -38h-512q-52 0 -90 38t-38 90v384q0 52 38 90t90 38h512q52 0 90 -38t38 -90zM768 1280v-384q0 -52 -38 -90t-90 -38h-512q-52 0 -90 38t-38 90v384q0 52 38 90t90 38h512q52 0 90 -38t38 -90zM1664 512v-384q0 -52 -38 -90t-90 -38 h-512q-52 0 -90 38t-38 90v384q0 52 38 90t90 38h512q52 0 90 -38t38 -90zM1664 1280v-384q0 -52 -38 -90t-90 -38h-512q-52 0 -90 38t-38 90v384q0 52 38 90t90 38h512q52 0 90 -38t38 -90z" /> +<glyph unicode="" horiz-adv-x="1792" d="M512 288v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM512 800v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1152 288v-192q0 -40 -28 -68t-68 -28h-320 q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM512 1312v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1152 800v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28 h320q40 0 68 -28t28 -68zM1792 288v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1152 1312v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1792 800v-192 q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1792 1312v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68z" /> +<glyph unicode="" horiz-adv-x="1792" d="M512 288v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM512 800v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1792 288v-192q0 -40 -28 -68t-68 -28h-960 q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h960q40 0 68 -28t28 -68zM512 1312v-192q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h320q40 0 68 -28t28 -68zM1792 800v-192q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v192q0 40 28 68t68 28 h960q40 0 68 -28t28 -68zM1792 1312v-192q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h960q40 0 68 -28t28 -68z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1671 970q0 -40 -28 -68l-724 -724l-136 -136q-28 -28 -68 -28t-68 28l-136 136l-362 362q-28 28 -28 68t28 68l136 136q28 28 68 28t68 -28l294 -295l656 657q28 28 68 28t68 -28l136 -136q28 -28 28 -68z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1298 214q0 -40 -28 -68l-136 -136q-28 -28 -68 -28t-68 28l-294 294l-294 -294q-28 -28 -68 -28t-68 28l-136 136q-28 28 -28 68t28 68l294 294l-294 294q-28 28 -28 68t28 68l136 136q28 28 68 28t68 -28l294 -294l294 294q28 28 68 28t68 -28l136 -136q28 -28 28 -68 t-28 -68l-294 -294l294 -294q28 -28 28 -68z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1024 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-224v-224q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v224h-224q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h224v224q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5v-224h224 q13 0 22.5 -9.5t9.5 -22.5zM1152 704q0 185 -131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5t316.5 131.5t131.5 316.5zM1664 -128q0 -53 -37.5 -90.5t-90.5 -37.5q-54 0 -90 38l-343 342q-179 -124 -399 -124q-143 0 -273.5 55.5 t-225 150t-150 225t-55.5 273.5t55.5 273.5t150 225t225 150t273.5 55.5t273.5 -55.5t225 -150t150 -225t55.5 -273.5q0 -220 -124 -399l343 -343q37 -37 37 -90z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1024 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-576q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h576q13 0 22.5 -9.5t9.5 -22.5zM1152 704q0 185 -131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5t316.5 131.5t131.5 316.5z M1664 -128q0 -53 -37.5 -90.5t-90.5 -37.5q-54 0 -90 38l-343 342q-179 -124 -399 -124q-143 0 -273.5 55.5t-225 150t-150 225t-55.5 273.5t55.5 273.5t150 225t225 150t273.5 55.5t273.5 -55.5t225 -150t150 -225t55.5 -273.5q0 -220 -124 -399l343 -343q37 -37 37 -90z " /> +<glyph unicode="" d="M1536 640q0 -156 -61 -298t-164 -245t-245 -164t-298 -61t-298 61t-245 164t-164 245t-61 298q0 182 80.5 343t226.5 270q43 32 95.5 25t83.5 -50q32 -42 24.5 -94.5t-49.5 -84.5q-98 -74 -151.5 -181t-53.5 -228q0 -104 40.5 -198.5t109.5 -163.5t163.5 -109.5 t198.5 -40.5t198.5 40.5t163.5 109.5t109.5 163.5t40.5 198.5q0 121 -53.5 228t-151.5 181q-42 32 -49.5 84.5t24.5 94.5q31 43 84 50t95 -25q146 -109 226.5 -270t80.5 -343zM896 1408v-640q0 -52 -38 -90t-90 -38t-90 38t-38 90v640q0 52 38 90t90 38t90 -38t38 -90z" /> +<glyph unicode="" horiz-adv-x="1792" d="M256 96v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM640 224v-320q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v320q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1024 480v-576q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23 v576q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1408 864v-960q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v960q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1792 1376v-1472q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v1472q0 14 9 23t23 9h192q14 0 23 -9t9 -23z" /> +<glyph unicode="" d="M1024 640q0 106 -75 181t-181 75t-181 -75t-75 -181t75 -181t181 -75t181 75t75 181zM1536 749v-222q0 -12 -8 -23t-20 -13l-185 -28q-19 -54 -39 -91q35 -50 107 -138q10 -12 10 -25t-9 -23q-27 -37 -99 -108t-94 -71q-12 0 -26 9l-138 108q-44 -23 -91 -38 q-16 -136 -29 -186q-7 -28 -36 -28h-222q-14 0 -24.5 8.5t-11.5 21.5l-28 184q-49 16 -90 37l-141 -107q-10 -9 -25 -9q-14 0 -25 11q-126 114 -165 168q-7 10 -7 23q0 12 8 23q15 21 51 66.5t54 70.5q-27 50 -41 99l-183 27q-13 2 -21 12.5t-8 23.5v222q0 12 8 23t19 13 l186 28q14 46 39 92q-40 57 -107 138q-10 12 -10 24q0 10 9 23q26 36 98.5 107.5t94.5 71.5q13 0 26 -10l138 -107q44 23 91 38q16 136 29 186q7 28 36 28h222q14 0 24.5 -8.5t11.5 -21.5l28 -184q49 -16 90 -37l142 107q9 9 24 9q13 0 25 -10q129 -119 165 -170q7 -8 7 -22 q0 -12 -8 -23q-15 -21 -51 -66.5t-54 -70.5q26 -50 41 -98l183 -28q13 -2 21 -12.5t8 -23.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M512 800v-576q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v576q0 14 9 23t23 9h64q14 0 23 -9t9 -23zM768 800v-576q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v576q0 14 9 23t23 9h64q14 0 23 -9t9 -23zM1024 800v-576q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v576 q0 14 9 23t23 9h64q14 0 23 -9t9 -23zM1152 76v948h-896v-948q0 -22 7 -40.5t14.5 -27t10.5 -8.5h832q3 0 10.5 8.5t14.5 27t7 40.5zM480 1152h448l-48 117q-7 9 -17 11h-317q-10 -2 -17 -11zM1408 1120v-64q0 -14 -9 -23t-23 -9h-96v-948q0 -83 -47 -143.5t-113 -60.5h-832 q-66 0 -113 58.5t-47 141.5v952h-96q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h309l70 167q15 37 54 63t79 26h320q40 0 79 -26t54 -63l70 -167h309q14 0 23 -9t9 -23z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1408 544v-480q0 -26 -19 -45t-45 -19h-384v384h-256v-384h-384q-26 0 -45 19t-19 45v480q0 1 0.5 3t0.5 3l575 474l575 -474q1 -2 1 -6zM1631 613l-62 -74q-8 -9 -21 -11h-3q-13 0 -21 7l-692 577l-692 -577q-12 -8 -24 -7q-13 2 -21 11l-62 74q-8 10 -7 23.5t11 21.5 l719 599q32 26 76 26t76 -26l244 -204v195q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-408l219 -182q10 -8 11 -21.5t-7 -23.5z" /> +<glyph unicode="" d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z " /> +<glyph unicode="" d="M896 992v-448q0 -14 -9 -23t-23 -9h-320q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h224v352q0 14 9 23t23 9h64q14 0 23 -9t9 -23zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640 q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1111 540v4l-24 320q-1 13 -11 22.5t-23 9.5h-186q-13 0 -23 -9.5t-11 -22.5l-24 -320v-4q-1 -12 8 -20t21 -8h244q12 0 21 8t8 20zM1870 73q0 -73 -46 -73h-704q13 0 22 9.5t8 22.5l-20 256q-1 13 -11 22.5t-23 9.5h-272q-13 0 -23 -9.5t-11 -22.5l-20 -256 q-1 -13 8 -22.5t22 -9.5h-704q-46 0 -46 73q0 54 26 116l417 1044q8 19 26 33t38 14h339q-13 0 -23 -9.5t-11 -22.5l-15 -192q-1 -14 8 -23t22 -9h166q13 0 22 9t8 23l-15 192q-1 13 -11 22.5t-23 9.5h339q20 0 38 -14t26 -33l417 -1044q26 -62 26 -116z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1280 192q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1536 192q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1664 416v-320q0 -40 -28 -68t-68 -28h-1472q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h465l135 -136 q58 -56 136 -56t136 56l136 136h464q40 0 68 -28t28 -68zM1339 985q17 -41 -14 -70l-448 -448q-18 -19 -45 -19t-45 19l-448 448q-31 29 -14 70q17 39 59 39h256v448q0 26 19 45t45 19h256q26 0 45 -19t19 -45v-448h256q42 0 59 -39z" /> +<glyph unicode="" d="M1120 608q0 -12 -10 -24l-319 -319q-11 -9 -23 -9t-23 9l-320 320q-15 16 -7 35q8 20 30 20h192v352q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-352h192q14 0 23 -9t9 -23zM768 1184q-148 0 -273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273 t-73 273t-198 198t-273 73zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1118 660q-8 -20 -30 -20h-192v-352q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v352h-192q-14 0 -23 9t-9 23q0 12 10 24l319 319q11 9 23 9t23 -9l320 -320q15 -16 7 -35zM768 1184q-148 0 -273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198 t73 273t-73 273t-198 198t-273 73zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1023 576h316q-1 3 -2.5 8t-2.5 8l-212 496h-708l-212 -496q-1 -2 -2.5 -8t-2.5 -8h316l95 -192h320zM1536 546v-482q0 -26 -19 -45t-45 -19h-1408q-26 0 -45 19t-19 45v482q0 62 25 123l238 552q10 25 36.5 42t52.5 17h832q26 0 52.5 -17t36.5 -42l238 -552 q25 -61 25 -123z" /> +<glyph unicode="" d="M1184 640q0 -37 -32 -55l-544 -320q-15 -9 -32 -9q-16 0 -32 8q-32 19 -32 56v640q0 37 32 56q33 18 64 -1l544 -320q32 -18 32 -55zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640 q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1536 1280v-448q0 -26 -19 -45t-45 -19h-448q-42 0 -59 40q-17 39 14 69l138 138q-148 137 -349 137q-104 0 -198.5 -40.5t-163.5 -109.5t-109.5 -163.5t-40.5 -198.5t40.5 -198.5t109.5 -163.5t163.5 -109.5t198.5 -40.5q119 0 225 52t179 147q7 10 23 12q14 0 25 -9 l137 -138q9 -8 9.5 -20.5t-7.5 -22.5q-109 -132 -264 -204.5t-327 -72.5q-156 0 -298 61t-245 164t-164 245t-61 298t61 298t164 245t245 164t298 61q147 0 284.5 -55.5t244.5 -156.5l130 129q29 31 70 14q39 -17 39 -59z" /> +<glyph unicode="" d="M1511 480q0 -5 -1 -7q-64 -268 -268 -434.5t-478 -166.5q-146 0 -282.5 55t-243.5 157l-129 -129q-19 -19 -45 -19t-45 19t-19 45v448q0 26 19 45t45 19h448q26 0 45 -19t19 -45t-19 -45l-137 -137q71 -66 161 -102t187 -36q134 0 250 65t186 179q11 17 53 117 q8 23 30 23h192q13 0 22.5 -9.5t9.5 -22.5zM1536 1280v-448q0 -26 -19 -45t-45 -19h-448q-26 0 -45 19t-19 45t19 45l138 138q-148 137 -349 137q-134 0 -250 -65t-186 -179q-11 -17 -53 -117q-8 -23 -30 -23h-199q-13 0 -22.5 9.5t-9.5 22.5v7q65 268 270 434.5t480 166.5 q146 0 284 -55.5t245 -156.5l130 129q19 19 45 19t45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1792" d="M384 352v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 608v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M384 864v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM1536 352v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-960q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h960q13 0 22.5 -9.5t9.5 -22.5z M1536 608v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-960q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h960q13 0 22.5 -9.5t9.5 -22.5zM1536 864v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-960q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h960q13 0 22.5 -9.5 t9.5 -22.5zM1664 160v832q0 13 -9.5 22.5t-22.5 9.5h-1472q-13 0 -22.5 -9.5t-9.5 -22.5v-832q0 -13 9.5 -22.5t22.5 -9.5h1472q13 0 22.5 9.5t9.5 22.5zM1792 1248v-1088q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h1472q66 0 113 -47 t47 -113z" /> +<glyph unicode="" horiz-adv-x="1152" d="M320 768h512v192q0 106 -75 181t-181 75t-181 -75t-75 -181v-192zM1152 672v-576q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v576q0 40 28 68t68 28h32v192q0 184 132 316t316 132t316 -132t132 -316v-192h32q40 0 68 -28t28 -68z" /> +<glyph unicode="" horiz-adv-x="1792" d="M320 1280q0 -72 -64 -110v-1266q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v1266q-64 38 -64 110q0 53 37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1792 1216v-763q0 -25 -12.5 -38.5t-39.5 -27.5q-215 -116 -369 -116q-61 0 -123.5 22t-108.5 48 t-115.5 48t-142.5 22q-192 0 -464 -146q-17 -9 -33 -9q-26 0 -45 19t-19 45v742q0 32 31 55q21 14 79 43q236 120 421 120q107 0 200 -29t219 -88q38 -19 88 -19q54 0 117.5 21t110 47t88 47t54.5 21q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1664 650q0 -166 -60 -314l-20 -49l-185 -33q-22 -83 -90.5 -136.5t-156.5 -53.5v-32q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v576q0 14 9 23t23 9h64q14 0 23 -9t9 -23v-32q71 0 130 -35.5t93 -95.5l68 12q29 95 29 193q0 148 -88 279t-236.5 209t-315.5 78 t-315.5 -78t-236.5 -209t-88 -279q0 -98 29 -193l68 -12q34 60 93 95.5t130 35.5v32q0 14 9 23t23 9h64q14 0 23 -9t9 -23v-576q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v32q-88 0 -156.5 53.5t-90.5 136.5l-185 33l-20 49q-60 148 -60 314q0 151 67 291t179 242.5 t266 163.5t320 61t320 -61t266 -163.5t179 -242.5t67 -291z" /> +<glyph unicode="" horiz-adv-x="768" d="M768 1184v-1088q0 -26 -19 -45t-45 -19t-45 19l-333 333h-262q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h262l333 333q19 19 45 19t45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1152" d="M768 1184v-1088q0 -26 -19 -45t-45 -19t-45 19l-333 333h-262q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h262l333 333q19 19 45 19t45 -19t19 -45zM1152 640q0 -76 -42.5 -141.5t-112.5 -93.5q-10 -5 -25 -5q-26 0 -45 18.5t-19 45.5q0 21 12 35.5t29 25t34 23t29 35.5 t12 57t-12 57t-29 35.5t-34 23t-29 25t-12 35.5q0 27 19 45.5t45 18.5q15 0 25 -5q70 -27 112.5 -93t42.5 -142z" /> +<glyph unicode="" horiz-adv-x="1664" d="M768 1184v-1088q0 -26 -19 -45t-45 -19t-45 19l-333 333h-262q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h262l333 333q19 19 45 19t45 -19t19 -45zM1152 640q0 -76 -42.5 -141.5t-112.5 -93.5q-10 -5 -25 -5q-26 0 -45 18.5t-19 45.5q0 21 12 35.5t29 25t34 23t29 35.5 t12 57t-12 57t-29 35.5t-34 23t-29 25t-12 35.5q0 27 19 45.5t45 18.5q15 0 25 -5q70 -27 112.5 -93t42.5 -142zM1408 640q0 -153 -85 -282.5t-225 -188.5q-13 -5 -25 -5q-27 0 -46 19t-19 45q0 39 39 59q56 29 76 44q74 54 115.5 135.5t41.5 173.5t-41.5 173.5 t-115.5 135.5q-20 15 -76 44q-39 20 -39 59q0 26 19 45t45 19q13 0 26 -5q140 -59 225 -188.5t85 -282.5zM1664 640q0 -230 -127 -422.5t-338 -283.5q-13 -5 -26 -5q-26 0 -45 19t-19 45q0 36 39 59q7 4 22.5 10.5t22.5 10.5q46 25 82 51q123 91 192 227t69 289t-69 289 t-192 227q-36 26 -82 51q-7 4 -22.5 10.5t-22.5 10.5q-39 23 -39 59q0 26 19 45t45 19q13 0 26 -5q211 -91 338 -283.5t127 -422.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M384 384v-128h-128v128h128zM384 1152v-128h-128v128h128zM1152 1152v-128h-128v128h128zM128 129h384v383h-384v-383zM128 896h384v384h-384v-384zM896 896h384v384h-384v-384zM640 640v-640h-640v640h640zM1152 128v-128h-128v128h128zM1408 128v-128h-128v128h128z M1408 640v-384h-384v128h-128v-384h-128v640h384v-128h128v128h128zM640 1408v-640h-640v640h640zM1408 1408v-640h-640v640h640z" /> +<glyph unicode="" horiz-adv-x="1792" d="M63 0h-63v1408h63v-1408zM126 1h-32v1407h32v-1407zM220 1h-31v1407h31v-1407zM377 1h-31v1407h31v-1407zM534 1h-62v1407h62v-1407zM660 1h-31v1407h31v-1407zM723 1h-31v1407h31v-1407zM786 1h-31v1407h31v-1407zM943 1h-63v1407h63v-1407zM1100 1h-63v1407h63v-1407z M1226 1h-63v1407h63v-1407zM1352 1h-63v1407h63v-1407zM1446 1h-63v1407h63v-1407zM1635 1h-94v1407h94v-1407zM1698 1h-32v1407h32v-1407zM1792 0h-63v1408h63v-1408z" /> +<glyph unicode="" d="M448 1088q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1515 512q0 -53 -37 -90l-491 -492q-39 -37 -91 -37q-53 0 -90 37l-715 716q-38 37 -64.5 101t-26.5 117v416q0 52 38 90t90 38h416q53 0 117 -26.5t102 -64.5 l715 -714q37 -39 37 -91z" /> +<glyph unicode="" horiz-adv-x="1920" d="M448 1088q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1515 512q0 -53 -37 -90l-491 -492q-39 -37 -91 -37q-53 0 -90 37l-715 716q-38 37 -64.5 101t-26.5 117v416q0 52 38 90t90 38h416q53 0 117 -26.5t102 -64.5 l715 -714q37 -39 37 -91zM1899 512q0 -53 -37 -90l-491 -492q-39 -37 -91 -37q-36 0 -59 14t-53 45l470 470q37 37 37 90q0 52 -37 91l-715 714q-38 38 -102 64.5t-117 26.5h224q53 0 117 -26.5t102 -64.5l715 -714q37 -39 37 -91z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1639 1058q40 -57 18 -129l-275 -906q-19 -64 -76.5 -107.5t-122.5 -43.5h-923q-77 0 -148.5 53.5t-99.5 131.5q-24 67 -2 127q0 4 3 27t4 37q1 8 -3 21.5t-3 19.5q2 11 8 21t16.5 23.5t16.5 23.5q23 38 45 91.5t30 91.5q3 10 0.5 30t-0.5 28q3 11 17 28t17 23 q21 36 42 92t25 90q1 9 -2.5 32t0.5 28q4 13 22 30.5t22 22.5q19 26 42.5 84.5t27.5 96.5q1 8 -3 25.5t-2 26.5q2 8 9 18t18 23t17 21q8 12 16.5 30.5t15 35t16 36t19.5 32t26.5 23.5t36 11.5t47.5 -5.5l-1 -3q38 9 51 9h761q74 0 114 -56t18 -130l-274 -906 q-36 -119 -71.5 -153.5t-128.5 -34.5h-869q-27 0 -38 -15q-11 -16 -1 -43q24 -70 144 -70h923q29 0 56 15.5t35 41.5l300 987q7 22 5 57q38 -15 59 -43zM575 1056q-4 -13 2 -22.5t20 -9.5h608q13 0 25.5 9.5t16.5 22.5l21 64q4 13 -2 22.5t-20 9.5h-608q-13 0 -25.5 -9.5 t-16.5 -22.5zM492 800q-4 -13 2 -22.5t20 -9.5h608q13 0 25.5 9.5t16.5 22.5l21 64q4 13 -2 22.5t-20 9.5h-608q-13 0 -25.5 -9.5t-16.5 -22.5z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1164 1408q23 0 44 -9q33 -13 52.5 -41t19.5 -62v-1289q0 -34 -19.5 -62t-52.5 -41q-19 -8 -44 -8q-48 0 -83 32l-441 424l-441 -424q-36 -33 -83 -33q-23 0 -44 9q-33 13 -52.5 41t-19.5 62v1289q0 34 19.5 62t52.5 41q21 9 44 9h1048z" /> +<glyph unicode="" horiz-adv-x="1664" d="M384 0h896v256h-896v-256zM384 640h896v384h-160q-40 0 -68 28t-28 68v160h-640v-640zM1536 576q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1664 576v-416q0 -13 -9.5 -22.5t-22.5 -9.5h-224v-160q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68 v160h-224q-13 0 -22.5 9.5t-9.5 22.5v416q0 79 56.5 135.5t135.5 56.5h64v544q0 40 28 68t68 28h672q40 0 88 -20t76 -48l152 -152q28 -28 48 -76t20 -88v-256h64q79 0 135.5 -56.5t56.5 -135.5z" /> +<glyph unicode="" horiz-adv-x="1920" d="M960 864q119 0 203.5 -84.5t84.5 -203.5t-84.5 -203.5t-203.5 -84.5t-203.5 84.5t-84.5 203.5t84.5 203.5t203.5 84.5zM1664 1280q106 0 181 -75t75 -181v-896q0 -106 -75 -181t-181 -75h-1408q-106 0 -181 75t-75 181v896q0 106 75 181t181 75h224l51 136 q19 49 69.5 84.5t103.5 35.5h512q53 0 103.5 -35.5t69.5 -84.5l51 -136h224zM960 128q185 0 316.5 131.5t131.5 316.5t-131.5 316.5t-316.5 131.5t-316.5 -131.5t-131.5 -316.5t131.5 -316.5t316.5 -131.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M725 977l-170 -450q33 0 136.5 -2t160.5 -2q19 0 57 2q-87 253 -184 452zM0 -128l2 79q23 7 56 12.5t57 10.5t49.5 14.5t44.5 29t31 50.5l237 616l280 724h75h53q8 -14 11 -21l205 -480q33 -78 106 -257.5t114 -274.5q15 -34 58 -144.5t72 -168.5q20 -45 35 -57 q19 -15 88 -29.5t84 -20.5q6 -38 6 -57q0 -4 -0.5 -13t-0.5 -13q-63 0 -190 8t-191 8q-76 0 -215 -7t-178 -8q0 43 4 78l131 28q1 0 12.5 2.5t15.5 3.5t14.5 4.5t15 6.5t11 8t9 11t2.5 14q0 16 -31 96.5t-72 177.5t-42 100l-450 2q-26 -58 -76.5 -195.5t-50.5 -162.5 q0 -22 14 -37.5t43.5 -24.5t48.5 -13.5t57 -8.5t41 -4q1 -19 1 -58q0 -9 -2 -27q-58 0 -174.5 10t-174.5 10q-8 0 -26.5 -4t-21.5 -4q-80 -14 -188 -14z" /> +<glyph unicode="" horiz-adv-x="1408" d="M555 15q74 -32 140 -32q376 0 376 335q0 114 -41 180q-27 44 -61.5 74t-67.5 46.5t-80.5 25t-84 10.5t-94.5 2q-73 0 -101 -10q0 -53 -0.5 -159t-0.5 -158q0 -8 -1 -67.5t-0.5 -96.5t4.5 -83.5t12 -66.5zM541 761q42 -7 109 -7q82 0 143 13t110 44.5t74.5 89.5t25.5 142 q0 70 -29 122.5t-79 82t-108 43.5t-124 14q-50 0 -130 -13q0 -50 4 -151t4 -152q0 -27 -0.5 -80t-0.5 -79q0 -46 1 -69zM0 -128l2 94q15 4 85 16t106 27q7 12 12.5 27t8.5 33.5t5.5 32.5t3 37.5t0.5 34v35.5v30q0 982 -22 1025q-4 8 -22 14.5t-44.5 11t-49.5 7t-48.5 4.5 t-30.5 3l-4 83q98 2 340 11.5t373 9.5q23 0 68.5 -0.5t67.5 -0.5q70 0 136.5 -13t128.5 -42t108 -71t74 -104.5t28 -137.5q0 -52 -16.5 -95.5t-39 -72t-64.5 -57.5t-73 -45t-84 -40q154 -35 256.5 -134t102.5 -248q0 -100 -35 -179.5t-93.5 -130.5t-138 -85.5t-163.5 -48.5 t-176 -14q-44 0 -132 3t-132 3q-106 0 -307 -11t-231 -12z" /> +<glyph unicode="" horiz-adv-x="1024" d="M0 -126l17 85q6 2 81.5 21.5t111.5 37.5q28 35 41 101q1 7 62 289t114 543.5t52 296.5v25q-24 13 -54.5 18.5t-69.5 8t-58 5.5l19 103q33 -2 120 -6.5t149.5 -7t120.5 -2.5q48 0 98.5 2.5t121 7t98.5 6.5q-5 -39 -19 -89q-30 -10 -101.5 -28.5t-108.5 -33.5 q-8 -19 -14 -42.5t-9 -40t-7.5 -45.5t-6.5 -42q-27 -148 -87.5 -419.5t-77.5 -355.5q-2 -9 -13 -58t-20 -90t-16 -83.5t-6 -57.5l1 -18q17 -4 185 -31q-3 -44 -16 -99q-11 0 -32.5 -1.5t-32.5 -1.5q-29 0 -87 10t-86 10q-138 2 -206 2q-51 0 -143 -9t-121 -11z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1744 128q33 0 42 -18.5t-11 -44.5l-126 -162q-20 -26 -49 -26t-49 26l-126 162q-20 26 -11 44.5t42 18.5h80v1024h-80q-33 0 -42 18.5t11 44.5l126 162q20 26 49 26t49 -26l126 -162q20 -26 11 -44.5t-42 -18.5h-80v-1024h80zM81 1407l54 -27q12 -5 211 -5q44 0 132 2 t132 2q36 0 107.5 -0.5t107.5 -0.5h293q6 0 21 -0.5t20.5 0t16 3t17.5 9t15 17.5l42 1q4 0 14 -0.5t14 -0.5q2 -112 2 -336q0 -80 -5 -109q-39 -14 -68 -18q-25 44 -54 128q-3 9 -11 48t-14.5 73.5t-7.5 35.5q-6 8 -12 12.5t-15.5 6t-13 2.5t-18 0.5t-16.5 -0.5 q-17 0 -66.5 0.5t-74.5 0.5t-64 -2t-71 -6q-9 -81 -8 -136q0 -94 2 -388t2 -455q0 -16 -2.5 -71.5t0 -91.5t12.5 -69q40 -21 124 -42.5t120 -37.5q5 -40 5 -50q0 -14 -3 -29l-34 -1q-76 -2 -218 8t-207 10q-50 0 -151 -9t-152 -9q-3 51 -3 52v9q17 27 61.5 43t98.5 29t78 27 q19 42 19 383q0 101 -3 303t-3 303v117q0 2 0.5 15.5t0.5 25t-1 25.5t-3 24t-5 14q-11 12 -162 12q-33 0 -93 -12t-80 -26q-19 -13 -34 -72.5t-31.5 -111t-42.5 -53.5q-42 26 -56 44v383z" /> +<glyph unicode="" d="M81 1407l54 -27q12 -5 211 -5q44 0 132 2t132 2q70 0 246.5 1t304.5 0.5t247 -4.5q33 -1 56 31l42 1q4 0 14 -0.5t14 -0.5q2 -112 2 -336q0 -80 -5 -109q-39 -14 -68 -18q-25 44 -54 128q-3 9 -11 47.5t-15 73.5t-7 36q-10 13 -27 19q-5 2 -66 2q-30 0 -93 1t-103 1 t-94 -2t-96 -7q-9 -81 -8 -136l1 -152v52q0 -55 1 -154t1.5 -180t0.5 -153q0 -16 -2.5 -71.5t0 -91.5t12.5 -69q40 -21 124 -42.5t120 -37.5q5 -40 5 -50q0 -14 -3 -29l-34 -1q-76 -2 -218 8t-207 10q-50 0 -151 -9t-152 -9q-3 51 -3 52v9q17 27 61.5 43t98.5 29t78 27 q7 16 11.5 74t6 145.5t1.5 155t-0.5 153.5t-0.5 89q0 7 -2.5 21.5t-2.5 22.5q0 7 0.5 44t1 73t0 76.5t-3 67.5t-6.5 32q-11 12 -162 12q-41 0 -163 -13.5t-138 -24.5q-19 -12 -34 -71.5t-31.5 -111.5t-42.5 -54q-42 26 -56 44v383zM1310 125q12 0 42 -19.5t57.5 -41.5 t59.5 -49t36 -30q26 -21 26 -49t-26 -49q-4 -3 -36 -30t-59.5 -49t-57.5 -41.5t-42 -19.5q-13 0 -20.5 10.5t-10 28.5t-2.5 33.5t1.5 33t1.5 19.5h-1024q0 -2 1.5 -19.5t1.5 -33t-2.5 -33.5t-10 -28.5t-20.5 -10.5q-12 0 -42 19.5t-57.5 41.5t-59.5 49t-36 30q-26 21 -26 49 t26 49q4 3 36 30t59.5 49t57.5 41.5t42 19.5q13 0 20.5 -10.5t10 -28.5t2.5 -33.5t-1.5 -33t-1.5 -19.5h1024q0 2 -1.5 19.5t-1.5 33t2.5 33.5t10 28.5t20.5 10.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 192v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1408 576v-128q0 -26 -19 -45t-45 -19h-1280q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1280q26 0 45 -19t19 -45zM1664 960v-128q0 -26 -19 -45 t-45 -19h-1536q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1536q26 0 45 -19t19 -45zM1280 1344v-128q0 -26 -19 -45t-45 -19h-1152q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1152q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 192v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1408 576v-128q0 -26 -19 -45t-45 -19h-896q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h896q26 0 45 -19t19 -45zM1664 960v-128q0 -26 -19 -45t-45 -19 h-1408q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1408q26 0 45 -19t19 -45zM1280 1344v-128q0 -26 -19 -45t-45 -19h-640q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h640q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 192v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 576v-128q0 -26 -19 -45t-45 -19h-1280q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1280q26 0 45 -19t19 -45zM1792 960v-128q0 -26 -19 -45 t-45 -19h-1536q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1536q26 0 45 -19t19 -45zM1792 1344v-128q0 -26 -19 -45t-45 -19h-1152q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1152q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 192v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 576v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 960v-128q0 -26 -19 -45 t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 1344v-128q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1664q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1792" d="M256 224v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-192q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h192q13 0 22.5 -9.5t9.5 -22.5zM256 608v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-192q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h192q13 0 22.5 -9.5 t9.5 -22.5zM256 992v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-192q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h192q13 0 22.5 -9.5t9.5 -22.5zM1792 224v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1344q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1344 q13 0 22.5 -9.5t9.5 -22.5zM256 1376v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-192q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h192q13 0 22.5 -9.5t9.5 -22.5zM1792 608v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1344q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5 t22.5 9.5h1344q13 0 22.5 -9.5t9.5 -22.5zM1792 992v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1344q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1344q13 0 22.5 -9.5t9.5 -22.5zM1792 1376v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1344q-13 0 -22.5 9.5t-9.5 22.5v192 q0 13 9.5 22.5t22.5 9.5h1344q13 0 22.5 -9.5t9.5 -22.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M384 992v-576q0 -13 -9.5 -22.5t-22.5 -9.5q-14 0 -23 9l-288 288q-9 9 -9 23t9 23l288 288q9 9 23 9q13 0 22.5 -9.5t9.5 -22.5zM1792 224v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1728q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1728q13 0 22.5 -9.5 t9.5 -22.5zM1792 608v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1088q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1088q13 0 22.5 -9.5t9.5 -22.5zM1792 992v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1088q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1088 q13 0 22.5 -9.5t9.5 -22.5zM1792 1376v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1728q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1728q13 0 22.5 -9.5t9.5 -22.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M352 704q0 -14 -9 -23l-288 -288q-9 -9 -23 -9q-13 0 -22.5 9.5t-9.5 22.5v576q0 13 9.5 22.5t22.5 9.5q14 0 23 -9l288 -288q9 -9 9 -23zM1792 224v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1728q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1728q13 0 22.5 -9.5 t9.5 -22.5zM1792 608v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1088q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1088q13 0 22.5 -9.5t9.5 -22.5zM1792 992v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1088q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1088 q13 0 22.5 -9.5t9.5 -22.5zM1792 1376v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1728q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1728q13 0 22.5 -9.5t9.5 -22.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 1184v-1088q0 -42 -39 -59q-13 -5 -25 -5q-27 0 -45 19l-403 403v-166q0 -119 -84.5 -203.5t-203.5 -84.5h-704q-119 0 -203.5 84.5t-84.5 203.5v704q0 119 84.5 203.5t203.5 84.5h704q119 0 203.5 -84.5t84.5 -203.5v-165l403 402q18 19 45 19q12 0 25 -5 q39 -17 39 -59z" /> +<glyph unicode="" horiz-adv-x="1920" d="M640 960q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM1664 576v-448h-1408v192l320 320l160 -160l512 512zM1760 1280h-1600q-13 0 -22.5 -9.5t-9.5 -22.5v-1216q0 -13 9.5 -22.5t22.5 -9.5h1600q13 0 22.5 9.5t9.5 22.5v1216 q0 13 -9.5 22.5t-22.5 9.5zM1920 1248v-1216q0 -66 -47 -113t-113 -47h-1600q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1600q66 0 113 -47t47 -113z" /> +<glyph unicode="" d="M363 0l91 91l-235 235l-91 -91v-107h128v-128h107zM886 928q0 22 -22 22q-10 0 -17 -7l-542 -542q-7 -7 -7 -17q0 -22 22 -22q10 0 17 7l542 542q7 7 7 17zM832 1120l416 -416l-832 -832h-416v416zM1515 1024q0 -53 -37 -90l-166 -166l-416 416l166 165q36 38 90 38 q53 0 91 -38l235 -234q37 -39 37 -91z" /> +<glyph unicode="" horiz-adv-x="1024" d="M768 896q0 106 -75 181t-181 75t-181 -75t-75 -181t75 -181t181 -75t181 75t75 181zM1024 896q0 -109 -33 -179l-364 -774q-16 -33 -47.5 -52t-67.5 -19t-67.5 19t-46.5 52l-365 774q-33 70 -33 179q0 212 150 362t362 150t362 -150t150 -362z" /> +<glyph unicode="" d="M768 96v1088q-148 0 -273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1024" d="M512 384q0 36 -20 69q-1 1 -15.5 22.5t-25.5 38t-25 44t-21 50.5q-4 16 -21 16t-21 -16q-7 -23 -21 -50.5t-25 -44t-25.5 -38t-15.5 -22.5q-20 -33 -20 -69q0 -53 37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1024 512q0 -212 -150 -362t-362 -150t-362 150t-150 362 q0 145 81 275q6 9 62.5 90.5t101 151t99.5 178t83 201.5q9 30 34 47t51 17t51.5 -17t33.5 -47q28 -93 83 -201.5t99.5 -178t101 -151t62.5 -90.5q81 -127 81 -275z" /> +<glyph unicode="" horiz-adv-x="1792" d="M888 352l116 116l-152 152l-116 -116v-56h96v-96h56zM1328 1072q-16 16 -33 -1l-350 -350q-17 -17 -1 -33t33 1l350 350q17 17 1 33zM1408 478v-190q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h832 q63 0 117 -25q15 -7 18 -23q3 -17 -9 -29l-49 -49q-14 -14 -32 -8q-23 6 -45 6h-832q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832q66 0 113 47t47 113v126q0 13 9 22l64 64q15 15 35 7t20 -29zM1312 1216l288 -288l-672 -672h-288v288zM1756 1084l-92 -92 l-288 288l92 92q28 28 68 28t68 -28l152 -152q28 -28 28 -68t-28 -68z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1408 547v-259q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h255v0q13 0 22.5 -9.5t9.5 -22.5q0 -27 -26 -32q-77 -26 -133 -60q-10 -4 -16 -4h-112q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832 q66 0 113 47t47 113v214q0 19 18 29q28 13 54 37q16 16 35 8q21 -9 21 -29zM1645 1043l-384 -384q-18 -19 -45 -19q-12 0 -25 5q-39 17 -39 59v192h-160q-323 0 -438 -131q-119 -137 -74 -473q3 -23 -20 -34q-8 -2 -12 -2q-16 0 -26 13q-10 14 -21 31t-39.5 68.5t-49.5 99.5 t-38.5 114t-17.5 122q0 49 3.5 91t14 90t28 88t47 81.5t68.5 74t94.5 61.5t124.5 48.5t159.5 30.5t196.5 11h160v192q0 42 39 59q13 5 25 5q26 0 45 -19l384 -384q19 -19 19 -45t-19 -45z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1408 606v-318q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h832q63 0 117 -25q15 -7 18 -23q3 -17 -9 -29l-49 -49q-10 -10 -23 -10q-3 0 -9 2q-23 6 -45 6h-832q-66 0 -113 -47t-47 -113v-832 q0 -66 47 -113t113 -47h832q66 0 113 47t47 113v254q0 13 9 22l64 64q10 10 23 10q6 0 12 -3q20 -8 20 -29zM1639 1095l-814 -814q-24 -24 -57 -24t-57 24l-430 430q-24 24 -24 57t24 57l110 110q24 24 57 24t57 -24l263 -263l647 647q24 24 57 24t57 -24l110 -110 q24 -24 24 -57t-24 -57z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 640q0 -26 -19 -45l-256 -256q-19 -19 -45 -19t-45 19t-19 45v128h-384v-384h128q26 0 45 -19t19 -45t-19 -45l-256 -256q-19 -19 -45 -19t-45 19l-256 256q-19 19 -19 45t19 45t45 19h128v384h-384v-128q0 -26 -19 -45t-45 -19t-45 19l-256 256q-19 19 -19 45 t19 45l256 256q19 19 45 19t45 -19t19 -45v-128h384v384h-128q-26 0 -45 19t-19 45t19 45l256 256q19 19 45 19t45 -19l256 -256q19 -19 19 -45t-19 -45t-45 -19h-128v-384h384v128q0 26 19 45t45 19t45 -19l256 -256q19 -19 19 -45z" /> +<glyph unicode="" horiz-adv-x="1024" d="M979 1395q19 19 32 13t13 -32v-1472q0 -26 -13 -32t-32 13l-710 710q-9 9 -13 19v-678q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-678q4 11 13 19z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1747 1395q19 19 32 13t13 -32v-1472q0 -26 -13 -32t-32 13l-710 710q-9 9 -13 19v-710q0 -26 -13 -32t-32 13l-710 710q-9 9 -13 19v-678q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-678q4 11 13 19l710 710 q19 19 32 13t13 -32v-710q4 11 13 19z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1619 1395q19 19 32 13t13 -32v-1472q0 -26 -13 -32t-32 13l-710 710q-8 9 -13 19v-710q0 -26 -13 -32t-32 13l-710 710q-19 19 -19 45t19 45l710 710q19 19 32 13t13 -32v-710q5 11 13 19z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1384 609l-1328 -738q-23 -13 -39.5 -3t-16.5 36v1472q0 26 16.5 36t39.5 -3l1328 -738q23 -13 23 -31t-23 -31z" /> +<glyph unicode="" d="M1536 1344v-1408q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h512q26 0 45 -19t19 -45zM640 1344v-1408q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h512q26 0 45 -19t19 -45z" /> +<glyph unicode="" d="M1536 1344v-1408q0 -26 -19 -45t-45 -19h-1408q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h1408q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1664" d="M45 -115q-19 -19 -32 -13t-13 32v1472q0 26 13 32t32 -13l710 -710q8 -8 13 -19v710q0 26 13 32t32 -13l710 -710q19 -19 19 -45t-19 -45l-710 -710q-19 -19 -32 -13t-13 32v710q-5 -10 -13 -19z" /> +<glyph unicode="" horiz-adv-x="1792" d="M45 -115q-19 -19 -32 -13t-13 32v1472q0 26 13 32t32 -13l710 -710q8 -8 13 -19v710q0 26 13 32t32 -13l710 -710q8 -8 13 -19v678q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-1408q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v678q-5 -10 -13 -19l-710 -710 q-19 -19 -32 -13t-13 32v710q-5 -10 -13 -19z" /> +<glyph unicode="" horiz-adv-x="1024" d="M45 -115q-19 -19 -32 -13t-13 32v1472q0 26 13 32t32 -13l710 -710q8 -8 13 -19v678q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-1408q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v678q-5 -10 -13 -19z" /> +<glyph unicode="" horiz-adv-x="1538" d="M14 557l710 710q19 19 45 19t45 -19l710 -710q19 -19 13 -32t-32 -13h-1472q-26 0 -32 13t13 32zM1473 0h-1408q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h1408q26 0 45 -19t19 -45v-256q0 -26 -19 -45t-45 -19z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1171 1235l-531 -531l531 -531q19 -19 19 -45t-19 -45l-166 -166q-19 -19 -45 -19t-45 19l-742 742q-19 19 -19 45t19 45l742 742q19 19 45 19t45 -19l166 -166q19 -19 19 -45t-19 -45z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1107 659l-742 -742q-19 -19 -45 -19t-45 19l-166 166q-19 19 -19 45t19 45l531 531l-531 531q-19 19 -19 45t19 45l166 166q19 19 45 19t45 -19l742 -742q19 -19 19 -45t-19 -45z" /> +<glyph unicode="" d="M1216 576v128q0 26 -19 45t-45 19h-256v256q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-256h-256q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h256v-256q0 -26 19 -45t45 -19h128q26 0 45 19t19 45v256h256q26 0 45 19t19 45zM1536 640q0 -209 -103 -385.5 t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1216 576v128q0 26 -19 45t-45 19h-768q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h768q26 0 45 19t19 45zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5 t103 -385.5z" /> +<glyph unicode="" d="M1149 414q0 26 -19 45l-181 181l181 181q19 19 19 45q0 27 -19 46l-90 90q-19 19 -46 19q-26 0 -45 -19l-181 -181l-181 181q-19 19 -45 19q-27 0 -46 -19l-90 -90q-19 -19 -19 -46q0 -26 19 -45l181 -181l-181 -181q-19 -19 -19 -45q0 -27 19 -46l90 -90q19 -19 46 -19 q26 0 45 19l181 181l181 -181q19 -19 45 -19q27 0 46 19l90 90q19 19 19 46zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1284 802q0 28 -18 46l-91 90q-19 19 -45 19t-45 -19l-408 -407l-226 226q-19 19 -45 19t-45 -19l-91 -90q-18 -18 -18 -46q0 -27 18 -45l362 -362q19 -19 45 -19q27 0 46 19l543 543q18 18 18 45zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103 t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M896 160v192q0 14 -9 23t-23 9h-192q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h192q14 0 23 9t9 23zM1152 832q0 88 -55.5 163t-138.5 116t-170 41q-243 0 -371 -213q-15 -24 8 -42l132 -100q7 -6 19 -6q16 0 25 12q53 68 86 92q34 24 86 24q48 0 85.5 -26t37.5 -59 q0 -38 -20 -61t-68 -45q-63 -28 -115.5 -86.5t-52.5 -125.5v-36q0 -14 9 -23t23 -9h192q14 0 23 9t9 23q0 19 21.5 49.5t54.5 49.5q32 18 49 28.5t46 35t44.5 48t28 60.5t12.5 81zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5 t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1024 160v160q0 14 -9 23t-23 9h-96v512q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-160q0 -14 9 -23t23 -9h96v-320h-96q-14 0 -23 -9t-9 -23v-160q0 -14 9 -23t23 -9h448q14 0 23 9t9 23zM896 1056v160q0 14 -9 23t-23 9h-192q-14 0 -23 -9t-9 -23v-160q0 -14 9 -23 t23 -9h192q14 0 23 9t9 23zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1197 512h-109q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h109q-32 108 -112.5 188.5t-188.5 112.5v-109q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v109q-108 -32 -188.5 -112.5t-112.5 -188.5h109q26 0 45 -19t19 -45v-128q0 -26 -19 -45t-45 -19h-109 q32 -108 112.5 -188.5t188.5 -112.5v109q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-109q108 32 188.5 112.5t112.5 188.5zM1536 704v-128q0 -26 -19 -45t-45 -19h-143q-37 -161 -154.5 -278.5t-278.5 -154.5v-143q0 -26 -19 -45t-45 -19h-128q-26 0 -45 19t-19 45v143 q-161 37 -278.5 154.5t-154.5 278.5h-143q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h143q37 161 154.5 278.5t278.5 154.5v143q0 26 19 45t45 19h128q26 0 45 -19t19 -45v-143q161 -37 278.5 -154.5t154.5 -278.5h143q26 0 45 -19t19 -45z" /> +<glyph unicode="" d="M1097 457l-146 -146q-10 -10 -23 -10t-23 10l-137 137l-137 -137q-10 -10 -23 -10t-23 10l-146 146q-10 10 -10 23t10 23l137 137l-137 137q-10 10 -10 23t10 23l146 146q10 10 23 10t23 -10l137 -137l137 137q10 10 23 10t23 -10l146 -146q10 -10 10 -23t-10 -23 l-137 -137l137 -137q10 -10 10 -23t-10 -23zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5 t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1171 723l-422 -422q-19 -19 -45 -19t-45 19l-294 294q-19 19 -19 45t19 45l102 102q19 19 45 19t45 -19l147 -147l275 275q19 19 45 19t45 -19l102 -102q19 -19 19 -45t-19 -45zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273t73 -273t198 -198 t273 -73t273 73t198 198t73 273zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1312 643q0 161 -87 295l-754 -753q137 -89 297 -89q111 0 211.5 43.5t173.5 116.5t116 174.5t43 212.5zM313 344l755 754q-135 91 -300 91q-148 0 -273 -73t-198 -199t-73 -274q0 -162 89 -299zM1536 643q0 -157 -61 -300t-163.5 -246t-245 -164t-298.5 -61t-298.5 61 t-245 164t-163.5 246t-61 300t61 299.5t163.5 245.5t245 164t298.5 61t298.5 -61t245 -164t163.5 -245.5t61 -299.5z" /> +<glyph unicode="" d="M1536 640v-128q0 -53 -32.5 -90.5t-84.5 -37.5h-704l293 -294q38 -36 38 -90t-38 -90l-75 -76q-37 -37 -90 -37q-52 0 -91 37l-651 652q-37 37 -37 90q0 52 37 91l651 650q38 38 91 38q52 0 90 -38l75 -74q38 -38 38 -91t-38 -91l-293 -293h704q52 0 84.5 -37.5 t32.5 -90.5z" /> +<glyph unicode="" d="M1472 576q0 -54 -37 -91l-651 -651q-39 -37 -91 -37q-51 0 -90 37l-75 75q-38 38 -38 91t38 91l293 293h-704q-52 0 -84.5 37.5t-32.5 90.5v128q0 53 32.5 90.5t84.5 37.5h704l-293 294q-38 36 -38 90t38 90l75 75q38 38 90 38q53 0 91 -38l651 -651q37 -35 37 -90z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1611 565q0 -51 -37 -90l-75 -75q-38 -38 -91 -38q-54 0 -90 38l-294 293v-704q0 -52 -37.5 -84.5t-90.5 -32.5h-128q-53 0 -90.5 32.5t-37.5 84.5v704l-294 -293q-36 -38 -90 -38t-90 38l-75 75q-38 38 -38 90q0 53 38 91l651 651q35 37 90 37q54 0 91 -37l651 -651 q37 -39 37 -91z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1611 704q0 -53 -37 -90l-651 -652q-39 -37 -91 -37q-53 0 -90 37l-651 652q-38 36 -38 90q0 53 38 91l74 75q39 37 91 37q53 0 90 -37l294 -294v704q0 52 38 90t90 38h128q52 0 90 -38t38 -90v-704l294 294q37 37 90 37q52 0 91 -37l75 -75q37 -39 37 -91z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 896q0 -26 -19 -45l-512 -512q-19 -19 -45 -19t-45 19t-19 45v256h-224q-98 0 -175.5 -6t-154 -21.5t-133 -42.5t-105.5 -69.5t-80 -101t-48.5 -138.5t-17.5 -181q0 -55 5 -123q0 -6 2.5 -23.5t2.5 -26.5q0 -15 -8.5 -25t-23.5 -10q-16 0 -28 17q-7 9 -13 22 t-13.5 30t-10.5 24q-127 285 -127 451q0 199 53 333q162 403 875 403h224v256q0 26 19 45t45 19t45 -19l512 -512q19 -19 19 -45z" /> +<glyph unicode="" d="M755 480q0 -13 -10 -23l-332 -332l144 -144q19 -19 19 -45t-19 -45t-45 -19h-448q-26 0 -45 19t-19 45v448q0 26 19 45t45 19t45 -19l144 -144l332 332q10 10 23 10t23 -10l114 -114q10 -10 10 -23zM1536 1344v-448q0 -26 -19 -45t-45 -19t-45 19l-144 144l-332 -332 q-10 -10 -23 -10t-23 10l-114 114q-10 10 -10 23t10 23l332 332l-144 144q-19 19 -19 45t19 45t45 19h448q26 0 45 -19t19 -45z" /> +<glyph unicode="" d="M768 576v-448q0 -26 -19 -45t-45 -19t-45 19l-144 144l-332 -332q-10 -10 -23 -10t-23 10l-114 114q-10 10 -10 23t10 23l332 332l-144 144q-19 19 -19 45t19 45t45 19h448q26 0 45 -19t19 -45zM1523 1248q0 -13 -10 -23l-332 -332l144 -144q19 -19 19 -45t-19 -45 t-45 -19h-448q-26 0 -45 19t-19 45v448q0 26 19 45t45 19t45 -19l144 -144l332 332q10 10 23 10t23 -10l114 -114q10 -10 10 -23z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1408 800v-192q0 -40 -28 -68t-68 -28h-416v-416q0 -40 -28 -68t-68 -28h-192q-40 0 -68 28t-28 68v416h-416q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h416v416q0 40 28 68t68 28h192q40 0 68 -28t28 -68v-416h416q40 0 68 -28t28 -68z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1408 800v-192q0 -40 -28 -68t-68 -28h-1216q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h1216q40 0 68 -28t28 -68z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1482 486q46 -26 59.5 -77.5t-12.5 -97.5l-64 -110q-26 -46 -77.5 -59.5t-97.5 12.5l-266 153v-307q0 -52 -38 -90t-90 -38h-128q-52 0 -90 38t-38 90v307l-266 -153q-46 -26 -97.5 -12.5t-77.5 59.5l-64 110q-26 46 -12.5 97.5t59.5 77.5l266 154l-266 154 q-46 26 -59.5 77.5t12.5 97.5l64 110q26 46 77.5 59.5t97.5 -12.5l266 -153v307q0 52 38 90t90 38h128q52 0 90 -38t38 -90v-307l266 153q46 26 97.5 12.5t77.5 -59.5l64 -110q26 -46 12.5 -97.5t-59.5 -77.5l-266 -154z" /> +<glyph unicode="" d="M768 1408q209 0 385.5 -103t279.5 -279.5t103 -385.5t-103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103zM896 161v190q0 14 -9 23.5t-22 9.5h-192q-13 0 -23 -10t-10 -23v-190q0 -13 10 -23t23 -10h192 q13 0 22 9.5t9 23.5zM894 505l18 621q0 12 -10 18q-10 8 -24 8h-220q-14 0 -24 -8q-10 -6 -10 -18l17 -621q0 -10 10 -17.5t24 -7.5h185q14 0 23.5 7.5t10.5 17.5z" /> +<glyph unicode="" d="M928 180v56v468v192h-320v-192v-468v-56q0 -25 18 -38.5t46 -13.5h192q28 0 46 13.5t18 38.5zM472 1024h195l-126 161q-26 31 -69 31q-40 0 -68 -28t-28 -68t28 -68t68 -28zM1160 1120q0 40 -28 68t-68 28q-43 0 -69 -31l-125 -161h194q40 0 68 28t28 68zM1536 864v-320 q0 -14 -9 -23t-23 -9h-96v-416q0 -40 -28 -68t-68 -28h-1088q-40 0 -68 28t-28 68v416h-96q-14 0 -23 9t-9 23v320q0 14 9 23t23 9h440q-93 0 -158.5 65.5t-65.5 158.5t65.5 158.5t158.5 65.5q107 0 168 -77l128 -165l128 165q61 77 168 77q93 0 158.5 -65.5t65.5 -158.5 t-65.5 -158.5t-158.5 -65.5h440q14 0 23 -9t9 -23z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1280 832q0 26 -19 45t-45 19q-172 0 -318 -49.5t-259.5 -134t-235.5 -219.5q-19 -21 -19 -45q0 -26 19 -45t45 -19q24 0 45 19q27 24 74 71t67 66q137 124 268.5 176t313.5 52q26 0 45 19t19 45zM1792 1030q0 -95 -20 -193q-46 -224 -184.5 -383t-357.5 -268 q-214 -108 -438 -108q-148 0 -286 47q-15 5 -88 42t-96 37q-16 0 -39.5 -32t-45 -70t-52.5 -70t-60 -32q-30 0 -51 11t-31 24t-27 42q-2 4 -6 11t-5.5 10t-3 9.5t-1.5 13.5q0 35 31 73.5t68 65.5t68 56t31 48q0 4 -14 38t-16 44q-9 51 -9 104q0 115 43.5 220t119 184.5 t170.5 139t204 95.5q55 18 145 25.5t179.5 9t178.5 6t163.5 24t113.5 56.5l29.5 29.5t29.5 28t27 20t36.5 16t43.5 4.5q39 0 70.5 -46t47.5 -112t24 -124t8 -96z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1408 -160v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-1344q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h1344q13 0 22.5 -9.5t9.5 -22.5zM1152 896q0 -78 -24.5 -144t-64 -112.5t-87.5 -88t-96 -77.5t-87.5 -72t-64 -81.5t-24.5 -96.5q0 -96 67 -224l-4 1l1 -1 q-90 41 -160 83t-138.5 100t-113.5 122.5t-72.5 150.5t-27.5 184q0 78 24.5 144t64 112.5t87.5 88t96 77.5t87.5 72t64 81.5t24.5 96.5q0 94 -66 224l3 -1l-1 1q90 -41 160 -83t138.5 -100t113.5 -122.5t72.5 -150.5t27.5 -184z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1664 576q-152 236 -381 353q61 -104 61 -225q0 -185 -131.5 -316.5t-316.5 -131.5t-316.5 131.5t-131.5 316.5q0 121 61 225q-229 -117 -381 -353q133 -205 333.5 -326.5t434.5 -121.5t434.5 121.5t333.5 326.5zM944 960q0 20 -14 34t-34 14q-125 0 -214.5 -89.5 t-89.5 -214.5q0 -20 14 -34t34 -14t34 14t14 34q0 86 61 147t147 61q20 0 34 14t14 34zM1792 576q0 -34 -20 -69q-140 -230 -376.5 -368.5t-499.5 -138.5t-499.5 139t-376.5 368q-20 35 -20 69t20 69q140 229 376.5 368t499.5 139t499.5 -139t376.5 -368q20 -35 20 -69z" /> +<glyph unicode="" horiz-adv-x="1792" d="M555 201l78 141q-87 63 -136 159t-49 203q0 121 61 225q-229 -117 -381 -353q167 -258 427 -375zM944 960q0 20 -14 34t-34 14q-125 0 -214.5 -89.5t-89.5 -214.5q0 -20 14 -34t34 -14t34 14t14 34q0 86 61 147t147 61q20 0 34 14t14 34zM1307 1151q0 -7 -1 -9 q-105 -188 -315 -566t-316 -567l-49 -89q-10 -16 -28 -16q-12 0 -134 70q-16 10 -16 28q0 12 44 87q-143 65 -263.5 173t-208.5 245q-20 31 -20 69t20 69q153 235 380 371t496 136q89 0 180 -17l54 97q10 16 28 16q5 0 18 -6t31 -15.5t33 -18.5t31.5 -18.5t19.5 -11.5 q16 -10 16 -27zM1344 704q0 -139 -79 -253.5t-209 -164.5l280 502q8 -45 8 -84zM1792 576q0 -35 -20 -69q-39 -64 -109 -145q-150 -172 -347.5 -267t-419.5 -95l74 132q212 18 392.5 137t301.5 307q-115 179 -282 294l63 112q95 -64 182.5 -153t144.5 -184q20 -34 20 -69z " /> +<glyph unicode="" horiz-adv-x="1792" d="M1024 161v190q0 14 -9.5 23.5t-22.5 9.5h-192q-13 0 -22.5 -9.5t-9.5 -23.5v-190q0 -14 9.5 -23.5t22.5 -9.5h192q13 0 22.5 9.5t9.5 23.5zM1022 535l18 459q0 12 -10 19q-13 11 -24 11h-220q-11 0 -24 -11q-10 -7 -10 -21l17 -457q0 -10 10 -16.5t24 -6.5h185 q14 0 23.5 6.5t10.5 16.5zM1008 1469l768 -1408q35 -63 -2 -126q-17 -29 -46.5 -46t-63.5 -17h-1536q-34 0 -63.5 17t-46.5 46q-37 63 -2 126l768 1408q17 31 47 49t65 18t65 -18t47 -49z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1376 1376q44 -52 12 -148t-108 -172l-161 -161l160 -696q5 -19 -12 -33l-128 -96q-7 -6 -19 -6q-4 0 -7 1q-15 3 -21 16l-279 508l-259 -259l53 -194q5 -17 -8 -31l-96 -96q-9 -9 -23 -9h-2q-15 2 -24 13l-189 252l-252 189q-11 7 -13 23q-1 13 9 25l96 97q9 9 23 9 q6 0 8 -1l194 -53l259 259l-508 279q-14 8 -17 24q-2 16 9 27l128 128q14 13 30 8l665 -159l160 160q76 76 172 108t148 -12z" /> +<glyph unicode="" horiz-adv-x="1664" d="M128 -128h288v288h-288v-288zM480 -128h320v288h-320v-288zM128 224h288v320h-288v-320zM480 224h320v320h-320v-320zM128 608h288v288h-288v-288zM864 -128h320v288h-320v-288zM480 608h320v288h-320v-288zM1248 -128h288v288h-288v-288zM864 224h320v320h-320v-320z M512 1088v288q0 13 -9.5 22.5t-22.5 9.5h-64q-13 0 -22.5 -9.5t-9.5 -22.5v-288q0 -13 9.5 -22.5t22.5 -9.5h64q13 0 22.5 9.5t9.5 22.5zM1248 224h288v320h-288v-320zM864 608h320v288h-320v-288zM1248 608h288v288h-288v-288zM1280 1088v288q0 13 -9.5 22.5t-22.5 9.5h-64 q-13 0 -22.5 -9.5t-9.5 -22.5v-288q0 -13 9.5 -22.5t22.5 -9.5h64q13 0 22.5 9.5t9.5 22.5zM1664 1152v-1280q0 -52 -38 -90t-90 -38h-1408q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h128v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h384v96q0 66 47 113t113 47 h64q66 0 113 -47t47 -113v-96h128q52 0 90 -38t38 -90z" /> +<glyph unicode="" horiz-adv-x="1792" d="M666 1055q-60 -92 -137 -273q-22 45 -37 72.5t-40.5 63.5t-51 56.5t-63 35t-81.5 14.5h-224q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h224q250 0 410 -225zM1792 256q0 -14 -9 -23l-320 -320q-9 -9 -23 -9q-13 0 -22.5 9.5t-9.5 22.5v192q-32 0 -85 -0.5t-81 -1t-73 1 t-71 5t-64 10.5t-63 18.5t-58 28.5t-59 40t-55 53.5t-56 69.5q59 93 136 273q22 -45 37 -72.5t40.5 -63.5t51 -56.5t63 -35t81.5 -14.5h256v192q0 14 9 23t23 9q12 0 24 -10l319 -319q9 -9 9 -23zM1792 1152q0 -14 -9 -23l-320 -320q-9 -9 -23 -9q-13 0 -22.5 9.5t-9.5 22.5 v192h-256q-48 0 -87 -15t-69 -45t-51 -61.5t-45 -77.5q-32 -62 -78 -171q-29 -66 -49.5 -111t-54 -105t-64 -100t-74 -83t-90 -68.5t-106.5 -42t-128 -16.5h-224q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h224q48 0 87 15t69 45t51 61.5t45 77.5q32 62 78 171q29 66 49.5 111 t54 105t64 100t74 83t90 68.5t106.5 42t128 16.5h256v192q0 14 9 23t23 9q12 0 24 -10l319 -319q9 -9 9 -23z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 640q0 -174 -120 -321.5t-326 -233t-450 -85.5q-70 0 -145 8q-198 -175 -460 -242q-49 -14 -114 -22q-17 -2 -30.5 9t-17.5 29v1q-3 4 -0.5 12t2 10t4.5 9.5l6 9t7 8.5t8 9q7 8 31 34.5t34.5 38t31 39.5t32.5 51t27 59t26 76q-157 89 -247.5 220t-90.5 281 q0 130 71 248.5t191 204.5t286 136.5t348 50.5q244 0 450 -85.5t326 -233t120 -321.5z" /> +<glyph unicode="" d="M1536 704v-128q0 -201 -98.5 -362t-274 -251.5t-395.5 -90.5t-395.5 90.5t-274 251.5t-98.5 362v128q0 26 19 45t45 19h384q26 0 45 -19t19 -45v-128q0 -52 23.5 -90t53.5 -57t71 -30t64 -13t44 -2t44 2t64 13t71 30t53.5 57t23.5 90v128q0 26 19 45t45 19h384 q26 0 45 -19t19 -45zM512 1344v-384q0 -26 -19 -45t-45 -19h-384q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h384q26 0 45 -19t19 -45zM1536 1344v-384q0 -26 -19 -45t-45 -19h-384q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h384q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1683 205l-166 -165q-19 -19 -45 -19t-45 19l-531 531l-531 -531q-19 -19 -45 -19t-45 19l-166 165q-19 19 -19 45.5t19 45.5l742 741q19 19 45 19t45 -19l742 -741q19 -19 19 -45.5t-19 -45.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1683 728l-742 -741q-19 -19 -45 -19t-45 19l-742 741q-19 19 -19 45.5t19 45.5l166 165q19 19 45 19t45 -19l531 -531l531 531q19 19 45 19t45 -19l166 -165q19 -19 19 -45.5t-19 -45.5z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1280 32q0 -13 -9.5 -22.5t-22.5 -9.5h-960q-8 0 -13.5 2t-9 7t-5.5 8t-3 11.5t-1 11.5v13v11v160v416h-192q-26 0 -45 19t-19 45q0 24 15 41l320 384q19 22 49 22t49 -22l320 -384q15 -17 15 -41q0 -26 -19 -45t-45 -19h-192v-384h576q16 0 25 -11l160 -192q7 -11 7 -21 zM1920 448q0 -24 -15 -41l-320 -384q-20 -23 -49 -23t-49 23l-320 384q-15 17 -15 41q0 26 19 45t45 19h192v384h-576q-16 0 -25 12l-160 192q-7 9 -7 20q0 13 9.5 22.5t22.5 9.5h960q8 0 13.5 -2t9 -7t5.5 -8t3 -11.5t1 -11.5v-13v-11v-160v-416h192q26 0 45 -19t19 -45z " /> +<glyph unicode="" horiz-adv-x="1664" d="M640 0q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1536 0q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1664 1088v-512q0 -24 -16 -42.5t-41 -21.5 l-1044 -122q1 -7 4.5 -21.5t6 -26.5t2.5 -22q0 -16 -24 -64h920q26 0 45 -19t19 -45t-19 -45t-45 -19h-1024q-26 0 -45 19t-19 45q0 14 11 39.5t29.5 59.5t20.5 38l-177 823h-204q-26 0 -45 19t-19 45t19 45t45 19h256q16 0 28.5 -6.5t20 -15.5t13 -24.5t7.5 -26.5 t5.5 -29.5t4.5 -25.5h1201q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1664 928v-704q0 -92 -66 -158t-158 -66h-1216q-92 0 -158 66t-66 158v960q0 92 66 158t158 66h320q92 0 158 -66t66 -158v-32h672q92 0 158 -66t66 -158z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1879 584q0 -31 -31 -66l-336 -396q-43 -51 -120.5 -86.5t-143.5 -35.5h-1088q-34 0 -60.5 13t-26.5 43q0 31 31 66l336 396q43 51 120.5 86.5t143.5 35.5h1088q34 0 60.5 -13t26.5 -43zM1536 928v-160h-832q-94 0 -197 -47.5t-164 -119.5l-337 -396l-5 -6q0 4 -0.5 12.5 t-0.5 12.5v960q0 92 66 158t158 66h320q92 0 158 -66t66 -158v-32h544q92 0 158 -66t66 -158z" /> +<glyph unicode="" horiz-adv-x="768" d="M704 1216q0 -26 -19 -45t-45 -19h-128v-1024h128q26 0 45 -19t19 -45t-19 -45l-256 -256q-19 -19 -45 -19t-45 19l-256 256q-19 19 -19 45t19 45t45 19h128v1024h-128q-26 0 -45 19t-19 45t19 45l256 256q19 19 45 19t45 -19l256 -256q19 -19 19 -45z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 640q0 -26 -19 -45l-256 -256q-19 -19 -45 -19t-45 19t-19 45v128h-1024v-128q0 -26 -19 -45t-45 -19t-45 19l-256 256q-19 19 -19 45t19 45l256 256q19 19 45 19t45 -19t19 -45v-128h1024v128q0 26 19 45t45 19t45 -19l256 -256q19 -19 19 -45z" /> +<glyph unicode="" horiz-adv-x="2048" d="M640 640v-512h-256v512h256zM1024 1152v-1024h-256v1024h256zM2048 0v-128h-2048v1536h128v-1408h1920zM1408 896v-768h-256v768h256zM1792 1280v-1152h-256v1152h256z" /> +<glyph unicode="" d="M1280 926q-56 -25 -121 -34q68 40 93 117q-65 -38 -134 -51q-61 66 -153 66q-87 0 -148.5 -61.5t-61.5 -148.5q0 -29 5 -48q-129 7 -242 65t-192 155q-29 -50 -29 -106q0 -114 91 -175q-47 1 -100 26v-2q0 -75 50 -133.5t123 -72.5q-29 -8 -51 -8q-13 0 -39 4 q21 -63 74.5 -104t121.5 -42q-116 -90 -261 -90q-26 0 -50 3q148 -94 322 -94q112 0 210 35.5t168 95t120.5 137t75 162t24.5 168.5q0 18 -1 27q63 45 105 109zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5 t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M1536 160q0 -119 -84.5 -203.5t-203.5 -84.5h-192v608h203l30 224h-233v143q0 54 28 83t96 29l132 1v207q-96 9 -180 9q-136 0 -218 -80.5t-82 -225.5v-166h-224v-224h224v-608h-544q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960 q119 0 203.5 -84.5t84.5 -203.5v-960z" /> +<glyph unicode="" horiz-adv-x="1792" d="M928 704q0 14 -9 23t-23 9q-66 0 -113 -47t-47 -113q0 -14 9 -23t23 -9t23 9t9 23q0 40 28 68t68 28q14 0 23 9t9 23zM1152 574q0 -106 -75 -181t-181 -75t-181 75t-75 181t75 181t181 75t181 -75t75 -181zM128 0h1536v128h-1536v-128zM1280 574q0 159 -112.5 271.5 t-271.5 112.5t-271.5 -112.5t-112.5 -271.5t112.5 -271.5t271.5 -112.5t271.5 112.5t112.5 271.5zM256 1216h384v128h-384v-128zM128 1024h1536v118v138h-828l-64 -128h-644v-128zM1792 1280v-1280q0 -53 -37.5 -90.5t-90.5 -37.5h-1536q-53 0 -90.5 37.5t-37.5 90.5v1280 q0 53 37.5 90.5t90.5 37.5h1536q53 0 90.5 -37.5t37.5 -90.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M832 1024q0 80 -56 136t-136 56t-136 -56t-56 -136q0 -42 19 -83q-41 19 -83 19q-80 0 -136 -56t-56 -136t56 -136t136 -56t136 56t56 136q0 42 -19 83q41 -19 83 -19q80 0 136 56t56 136zM1683 320q0 -17 -49 -66t-66 -49q-9 0 -28.5 16t-36.5 33t-38.5 40t-24.5 26 l-96 -96l220 -220q28 -28 28 -68q0 -42 -39 -81t-81 -39q-40 0 -68 28l-671 671q-176 -131 -365 -131q-163 0 -265.5 102.5t-102.5 265.5q0 160 95 313t248 248t313 95q163 0 265.5 -102.5t102.5 -265.5q0 -189 -131 -365l355 -355l96 96q-3 3 -26 24.5t-40 38.5t-33 36.5 t-16 28.5q0 17 49 66t66 49q13 0 23 -10q6 -6 46 -44.5t82 -79.5t86.5 -86t73 -78t28.5 -41z" /> +<glyph unicode="" horiz-adv-x="1920" d="M896 640q0 106 -75 181t-181 75t-181 -75t-75 -181t75 -181t181 -75t181 75t75 181zM1664 128q0 52 -38 90t-90 38t-90 -38t-38 -90q0 -53 37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1664 1152q0 52 -38 90t-90 38t-90 -38t-38 -90q0 -53 37.5 -90.5t90.5 -37.5 t90.5 37.5t37.5 90.5zM1280 731v-185q0 -10 -7 -19.5t-16 -10.5l-155 -24q-11 -35 -32 -76q34 -48 90 -115q7 -10 7 -20q0 -12 -7 -19q-23 -30 -82.5 -89.5t-78.5 -59.5q-11 0 -21 7l-115 90q-37 -19 -77 -31q-11 -108 -23 -155q-7 -24 -30 -24h-186q-11 0 -20 7.5t-10 17.5 l-23 153q-34 10 -75 31l-118 -89q-7 -7 -20 -7q-11 0 -21 8q-144 133 -144 160q0 9 7 19q10 14 41 53t47 61q-23 44 -35 82l-152 24q-10 1 -17 9.5t-7 19.5v185q0 10 7 19.5t16 10.5l155 24q11 35 32 76q-34 48 -90 115q-7 11 -7 20q0 12 7 20q22 30 82 89t79 59q11 0 21 -7 l115 -90q34 18 77 32q11 108 23 154q7 24 30 24h186q11 0 20 -7.5t10 -17.5l23 -153q34 -10 75 -31l118 89q8 7 20 7q11 0 21 -8q144 -133 144 -160q0 -9 -7 -19q-12 -16 -42 -54t-45 -60q23 -48 34 -82l152 -23q10 -2 17 -10.5t7 -19.5zM1920 198v-140q0 -16 -149 -31 q-12 -27 -30 -52q51 -113 51 -138q0 -4 -4 -7q-122 -71 -124 -71q-8 0 -46 47t-52 68q-20 -2 -30 -2t-30 2q-14 -21 -52 -68t-46 -47q-2 0 -124 71q-4 3 -4 7q0 25 51 138q-18 25 -30 52q-149 15 -149 31v140q0 16 149 31q13 29 30 52q-51 113 -51 138q0 4 4 7q4 2 35 20 t59 34t30 16q8 0 46 -46.5t52 -67.5q20 2 30 2t30 -2q51 71 92 112l6 2q4 0 124 -70q4 -3 4 -7q0 -25 -51 -138q17 -23 30 -52q149 -15 149 -31zM1920 1222v-140q0 -16 -149 -31q-12 -27 -30 -52q51 -113 51 -138q0 -4 -4 -7q-122 -71 -124 -71q-8 0 -46 47t-52 68 q-20 -2 -30 -2t-30 2q-14 -21 -52 -68t-46 -47q-2 0 -124 71q-4 3 -4 7q0 25 51 138q-18 25 -30 52q-149 15 -149 31v140q0 16 149 31q13 29 30 52q-51 113 -51 138q0 4 4 7q4 2 35 20t59 34t30 16q8 0 46 -46.5t52 -67.5q20 2 30 2t30 -2q51 71 92 112l6 2q4 0 124 -70 q4 -3 4 -7q0 -25 -51 -138q17 -23 30 -52q149 -15 149 -31z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1408 768q0 -139 -94 -257t-256.5 -186.5t-353.5 -68.5q-86 0 -176 16q-124 -88 -278 -128q-36 -9 -86 -16h-3q-11 0 -20.5 8t-11.5 21q-1 3 -1 6.5t0.5 6.5t2 6l2.5 5t3.5 5.5t4 5t4.5 5t4 4.5q5 6 23 25t26 29.5t22.5 29t25 38.5t20.5 44q-124 72 -195 177t-71 224 q0 139 94 257t256.5 186.5t353.5 68.5t353.5 -68.5t256.5 -186.5t94 -257zM1792 512q0 -120 -71 -224.5t-195 -176.5q10 -24 20.5 -44t25 -38.5t22.5 -29t26 -29.5t23 -25q1 -1 4 -4.5t4.5 -5t4 -5t3.5 -5.5l2.5 -5t2 -6t0.5 -6.5t-1 -6.5q-3 -14 -13 -22t-22 -7 q-50 7 -86 16q-154 40 -278 128q-90 -16 -176 -16q-271 0 -472 132q58 -4 88 -4q161 0 309 45t264 129q125 92 192 212t67 254q0 77 -23 152q129 -71 204 -178t75 -230z" /> +<glyph unicode="" d="M256 192q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1408 768q0 51 -39 89.5t-89 38.5h-352q0 58 48 159.5t48 160.5q0 98 -32 145t-128 47q-26 -26 -38 -85t-30.5 -125.5t-59.5 -109.5q-22 -23 -77 -91q-4 -5 -23 -30t-31.5 -41t-34.5 -42.5 t-40 -44t-38.5 -35.5t-40 -27t-35.5 -9h-32v-640h32q13 0 31.5 -3t33 -6.5t38 -11t35 -11.5t35.5 -12.5t29 -10.5q211 -73 342 -73h121q192 0 192 167q0 26 -5 56q30 16 47.5 52.5t17.5 73.5t-18 69q53 50 53 119q0 25 -10 55.5t-25 47.5q32 1 53.5 47t21.5 81zM1536 769 q0 -89 -49 -163q9 -33 9 -69q0 -77 -38 -144q3 -21 3 -43q0 -101 -60 -178q1 -139 -85 -219.5t-227 -80.5h-36h-93q-96 0 -189.5 22.5t-216.5 65.5q-116 40 -138 40h-288q-53 0 -90.5 37.5t-37.5 90.5v640q0 53 37.5 90.5t90.5 37.5h274q36 24 137 155q58 75 107 128 q24 25 35.5 85.5t30.5 126.5t62 108q39 37 90 37q84 0 151 -32.5t102 -101.5t35 -186q0 -93 -48 -192h176q104 0 180 -76t76 -179z" /> +<glyph unicode="" d="M256 1088q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1408 512q0 35 -21.5 81t-53.5 47q15 17 25 47.5t10 55.5q0 69 -53 119q18 32 18 69t-17.5 73.5t-47.5 52.5q5 30 5 56q0 85 -49 126t-136 41h-128q-131 0 -342 -73q-5 -2 -29 -10.5 t-35.5 -12.5t-35 -11.5t-38 -11t-33 -6.5t-31.5 -3h-32v-640h32q16 0 35.5 -9t40 -27t38.5 -35.5t40 -44t34.5 -42.5t31.5 -41t23 -30q55 -68 77 -91q41 -43 59.5 -109.5t30.5 -125.5t38 -85q96 0 128 47t32 145q0 59 -48 160.5t-48 159.5h352q50 0 89 38.5t39 89.5z M1536 511q0 -103 -76 -179t-180 -76h-176q48 -99 48 -192q0 -118 -35 -186q-35 -69 -102 -101.5t-151 -32.5q-51 0 -90 37q-34 33 -54 82t-25.5 90.5t-17.5 84.5t-31 64q-48 50 -107 127q-101 131 -137 155h-274q-53 0 -90.5 37.5t-37.5 90.5v640q0 53 37.5 90.5t90.5 37.5 h288q22 0 138 40q128 44 223 66t200 22h112q140 0 226.5 -79t85.5 -216v-5q60 -77 60 -178q0 -22 -3 -43q38 -67 38 -144q0 -36 -9 -69q49 -74 49 -163z" /> +<glyph unicode="" horiz-adv-x="896" d="M832 1504v-1339l-449 -236q-22 -12 -40 -12q-21 0 -31.5 14.5t-10.5 35.5q0 6 2 20l86 500l-364 354q-25 27 -25 48q0 37 56 46l502 73l225 455q19 41 49 41z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1664 940q0 81 -21.5 143t-55 98.5t-81.5 59.5t-94 31t-98 8t-112 -25.5t-110.5 -64t-86.5 -72t-60 -61.5q-18 -22 -49 -22t-49 22q-24 28 -60 61.5t-86.5 72t-110.5 64t-112 25.5t-98 -8t-94 -31t-81.5 -59.5t-55 -98.5t-21.5 -143q0 -168 187 -355l581 -560l580 559 q188 188 188 356zM1792 940q0 -221 -229 -450l-623 -600q-18 -18 -44 -18t-44 18l-624 602q-10 8 -27.5 26t-55.5 65.5t-68 97.5t-53.5 121t-23.5 138q0 220 127 344t351 124q62 0 126.5 -21.5t120 -58t95.5 -68.5t76 -68q36 36 76 68t95.5 68.5t120 58t126.5 21.5 q224 0 351 -124t127 -344z" /> +<glyph unicode="" horiz-adv-x="1664" d="M640 96q0 -4 1 -20t0.5 -26.5t-3 -23.5t-10 -19.5t-20.5 -6.5h-320q-119 0 -203.5 84.5t-84.5 203.5v704q0 119 84.5 203.5t203.5 84.5h320q13 0 22.5 -9.5t9.5 -22.5q0 -4 1 -20t0.5 -26.5t-3 -23.5t-10 -19.5t-20.5 -6.5h-320q-66 0 -113 -47t-47 -113v-704 q0 -66 47 -113t113 -47h288h11h13t11.5 -1t11.5 -3t8 -5.5t7 -9t2 -13.5zM1568 640q0 -26 -19 -45l-544 -544q-19 -19 -45 -19t-45 19t-19 45v288h-448q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h448v288q0 26 19 45t45 19t45 -19l544 -544q19 -19 19 -45z" /> +<glyph unicode="" d="M237 122h231v694h-231v-694zM483 1030q-1 52 -36 86t-93 34t-94.5 -34t-36.5 -86q0 -51 35.5 -85.5t92.5 -34.5h1q59 0 95 34.5t36 85.5zM1068 122h231v398q0 154 -73 233t-193 79q-136 0 -209 -117h2v101h-231q3 -66 0 -694h231v388q0 38 7 56q15 35 45 59.5t74 24.5 q116 0 116 -157v-371zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1152" d="M480 672v448q0 14 -9 23t-23 9t-23 -9t-9 -23v-448q0 -14 9 -23t23 -9t23 9t9 23zM1152 320q0 -26 -19 -45t-45 -19h-429l-51 -483q-2 -12 -10.5 -20.5t-20.5 -8.5h-1q-27 0 -32 27l-76 485h-404q-26 0 -45 19t-19 45q0 123 78.5 221.5t177.5 98.5v512q-52 0 -90 38 t-38 90t38 90t90 38h640q52 0 90 -38t38 -90t-38 -90t-90 -38v-512q99 0 177.5 -98.5t78.5 -221.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1408 608v-320q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h704q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-704q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832q66 0 113 47t47 113v320 q0 14 9 23t23 9h64q14 0 23 -9t9 -23zM1792 1472v-512q0 -26 -19 -45t-45 -19t-45 19l-176 176l-652 -652q-10 -10 -23 -10t-23 10l-114 114q-10 10 -10 23t10 23l652 652l-176 176q-19 19 -19 45t19 45t45 19h512q26 0 45 -19t19 -45z" /> +<glyph unicode="" d="M1184 640q0 -26 -19 -45l-544 -544q-19 -19 -45 -19t-45 19t-19 45v288h-448q-26 0 -45 19t-19 45v384q0 26 19 45t45 19h448v288q0 26 19 45t45 19t45 -19l544 -544q19 -19 19 -45zM1536 992v-704q0 -119 -84.5 -203.5t-203.5 -84.5h-320q-13 0 -22.5 9.5t-9.5 22.5 q0 4 -1 20t-0.5 26.5t3 23.5t10 19.5t20.5 6.5h320q66 0 113 47t47 113v704q0 66 -47 113t-113 47h-288h-11h-13t-11.5 1t-11.5 3t-8 5.5t-7 9t-2 13.5q0 4 -1 20t-0.5 26.5t3 23.5t10 19.5t20.5 6.5h320q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M458 653q-74 162 -74 371h-256v-96q0 -78 94.5 -162t235.5 -113zM1536 928v96h-256q0 -209 -74 -371q141 29 235.5 113t94.5 162zM1664 1056v-128q0 -71 -41.5 -143t-112 -130t-173 -97.5t-215.5 -44.5q-42 -54 -95 -95q-38 -34 -52.5 -72.5t-14.5 -89.5q0 -54 30.5 -91 t97.5 -37q75 0 133.5 -45.5t58.5 -114.5v-64q0 -14 -9 -23t-23 -9h-832q-14 0 -23 9t-9 23v64q0 69 58.5 114.5t133.5 45.5q67 0 97.5 37t30.5 91q0 51 -14.5 89.5t-52.5 72.5q-53 41 -95 95q-113 5 -215.5 44.5t-173 97.5t-112 130t-41.5 143v128q0 40 28 68t68 28h288v96 q0 66 47 113t113 47h576q66 0 113 -47t47 -113v-96h288q40 0 68 -28t28 -68z" /> +<glyph unicode="" d="M394 184q-8 -9 -20 3q-13 11 -4 19q8 9 20 -3q12 -11 4 -19zM352 245q9 -12 0 -19q-8 -6 -17 7t0 18q9 7 17 -6zM291 305q-5 -7 -13 -2q-10 5 -7 12q3 5 13 2q10 -5 7 -12zM322 271q-6 -7 -16 3q-9 11 -2 16q6 6 16 -3q9 -11 2 -16zM451 159q-4 -12 -19 -6q-17 4 -13 15 t19 7q16 -5 13 -16zM514 154q0 -11 -16 -11q-17 -2 -17 11q0 11 16 11q17 2 17 -11zM572 164q2 -10 -14 -14t-18 8t14 15q16 2 18 -9zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-224q-16 0 -24.5 1t-19.5 5t-16 14.5t-5 27.5v239q0 97 -52 142q57 6 102.5 18t94 39 t81 66.5t53 105t20.5 150.5q0 121 -79 206q37 91 -8 204q-28 9 -81 -11t-92 -44l-38 -24q-93 26 -192 26t-192 -26q-16 11 -42.5 27t-83.5 38.5t-86 13.5q-44 -113 -7 -204q-79 -85 -79 -206q0 -85 20.5 -150t52.5 -105t80.5 -67t94 -39t102.5 -18q-40 -36 -49 -103 q-21 -10 -45 -15t-57 -5t-65.5 21.5t-55.5 62.5q-19 32 -48.5 52t-49.5 24l-20 3q-21 0 -29 -4.5t-5 -11.5t9 -14t13 -12l7 -5q22 -10 43.5 -38t31.5 -51l10 -23q13 -38 44 -61.5t67 -30t69.5 -7t55.5 3.5l23 4q0 -38 0.5 -103t0.5 -68q0 -22 -11 -33.5t-22 -13t-33 -1.5 h-224q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1280 64q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1536 64q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1664 288v-320q0 -40 -28 -68t-68 -28h-1472q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h427q21 -56 70.5 -92 t110.5 -36h256q61 0 110.5 36t70.5 92h427q40 0 68 -28t28 -68zM1339 936q-17 -40 -59 -40h-256v-448q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v448h-256q-42 0 -59 40q-17 39 14 69l448 448q18 19 45 19t45 -19l448 -448q31 -30 14 -69z" /> +<glyph unicode="" d="M1407 710q0 44 -7 113.5t-18 96.5q-12 30 -17 44t-9 36.5t-4 48.5q0 23 5 68.5t5 67.5q0 37 -10 55q-4 1 -13 1q-19 0 -58 -4.5t-59 -4.5q-60 0 -176 24t-175 24q-43 0 -94.5 -11.5t-85 -23.5t-89.5 -34q-137 -54 -202 -103q-96 -73 -159.5 -189.5t-88 -236t-24.5 -248.5 q0 -40 12.5 -120t12.5 -121q0 -23 -11 -66.5t-11 -65.5t12 -36.5t34 -14.5q24 0 72.5 11t73.5 11q57 0 169.5 -15.5t169.5 -15.5q181 0 284 36q129 45 235.5 152.5t166 245.5t59.5 275zM1535 712q0 -165 -70 -327.5t-196 -288t-281 -180.5q-124 -44 -326 -44 q-57 0 -170 14.5t-169 14.5q-24 0 -72.5 -14.5t-73.5 -14.5q-73 0 -123.5 55.5t-50.5 128.5q0 24 11 68t11 67q0 40 -12.5 120.5t-12.5 121.5q0 111 18 217.5t54.5 209.5t100.5 194t150 156q78 59 232 120q194 78 316 78q60 0 175.5 -24t173.5 -24q19 0 57 5t58 5 q81 0 118 -50.5t37 -134.5q0 -23 -5 -68t-5 -68q0 -10 1 -18.5t3 -17t4 -13.5t6.5 -16t6.5 -17q16 -40 25 -118.5t9 -136.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1408 296q0 -27 -10 -70.5t-21 -68.5q-21 -50 -122 -106q-94 -51 -186 -51q-27 0 -52.5 3.5t-57.5 12.5t-47.5 14.5t-55.5 20.5t-49 18q-98 35 -175 83q-128 79 -264.5 215.5t-215.5 264.5q-48 77 -83 175q-3 9 -18 49t-20.5 55.5t-14.5 47.5t-12.5 57.5t-3.5 52.5 q0 92 51 186q56 101 106 122q25 11 68.5 21t70.5 10q14 0 21 -3q18 -6 53 -76q11 -19 30 -54t35 -63.5t31 -53.5q3 -4 17.5 -25t21.5 -35.5t7 -28.5q0 -20 -28.5 -50t-62 -55t-62 -53t-28.5 -46q0 -9 5 -22.5t8.5 -20.5t14 -24t11.5 -19q76 -137 174 -235t235 -174 q2 -1 19 -11.5t24 -14t20.5 -8.5t22.5 -5q18 0 46 28.5t53 62t55 62t50 28.5q14 0 28.5 -7t35.5 -21.5t25 -17.5q25 -15 53.5 -31t63.5 -35t54 -30q70 -35 76 -53q3 -7 3 -21z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1120 1280h-832q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832q66 0 113 47t47 113v832q0 66 -47 113t-113 47zM1408 1120v-832q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h832 q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1152 1280h-1024v-1242l423 406l89 85l89 -85l423 -406v1242zM1164 1408q23 0 44 -9q33 -13 52.5 -41t19.5 -62v-1289q0 -34 -19.5 -62t-52.5 -41q-19 -8 -44 -8q-48 0 -83 32l-441 424l-441 -424q-36 -33 -83 -33q-23 0 -44 9q-33 13 -52.5 41t-19.5 62v1289 q0 34 19.5 62t52.5 41q21 9 44 9h1048z" /> +<glyph unicode="" d="M1280 343q0 11 -2 16q-3 8 -38.5 29.5t-88.5 49.5l-53 29q-5 3 -19 13t-25 15t-21 5q-18 0 -47 -32.5t-57 -65.5t-44 -33q-7 0 -16.5 3.5t-15.5 6.5t-17 9.5t-14 8.5q-99 55 -170.5 126.5t-126.5 170.5q-2 3 -8.5 14t-9.5 17t-6.5 15.5t-3.5 16.5q0 13 20.5 33.5t45 38.5 t45 39.5t20.5 36.5q0 10 -5 21t-15 25t-13 19q-3 6 -15 28.5t-25 45.5t-26.5 47.5t-25 40.5t-16.5 18t-16 2q-48 0 -101 -22q-46 -21 -80 -94.5t-34 -130.5q0 -16 2.5 -34t5 -30.5t9 -33t10 -29.5t12.5 -33t11 -30q60 -164 216.5 -320.5t320.5 -216.5q6 -2 30 -11t33 -12.5 t29.5 -10t33 -9t30.5 -5t34 -2.5q57 0 130.5 34t94.5 80q22 53 22 101zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1620 1128q-67 -98 -162 -167q1 -14 1 -42q0 -130 -38 -259.5t-115.5 -248.5t-184.5 -210.5t-258 -146t-323 -54.5q-271 0 -496 145q35 -4 78 -4q225 0 401 138q-105 2 -188 64.5t-114 159.5q33 -5 61 -5q43 0 85 11q-112 23 -185.5 111.5t-73.5 205.5v4q68 -38 146 -41 q-66 44 -105 115t-39 154q0 88 44 163q121 -149 294.5 -238.5t371.5 -99.5q-8 38 -8 74q0 134 94.5 228.5t228.5 94.5q140 0 236 -102q109 21 205 78q-37 -115 -142 -178q93 10 186 50z" /> +<glyph unicode="" horiz-adv-x="1024" d="M959 1524v-264h-157q-86 0 -116 -36t-30 -108v-189h293l-39 -296h-254v-759h-306v759h-255v296h255v218q0 186 104 288.5t277 102.5q147 0 228 -12z" /> +<glyph unicode="" d="M1536 640q0 -251 -146.5 -451.5t-378.5 -277.5q-27 -5 -39.5 7t-12.5 30v211q0 97 -52 142q57 6 102.5 18t94 39t81 66.5t53 105t20.5 150.5q0 121 -79 206q37 91 -8 204q-28 9 -81 -11t-92 -44l-38 -24q-93 26 -192 26t-192 -26q-16 11 -42.5 27t-83.5 38.5t-86 13.5 q-44 -113 -7 -204q-79 -85 -79 -206q0 -85 20.5 -150t52.5 -105t80.5 -67t94 -39t102.5 -18q-40 -36 -49 -103q-21 -10 -45 -15t-57 -5t-65.5 21.5t-55.5 62.5q-19 32 -48.5 52t-49.5 24l-20 3q-21 0 -29 -4.5t-5 -11.5t9 -14t13 -12l7 -5q22 -10 43.5 -38t31.5 -51l10 -23 q13 -38 44 -61.5t67 -30t69.5 -7t55.5 3.5l23 4q0 -38 0.5 -89t0.5 -54q0 -18 -13 -30t-40 -7q-232 77 -378.5 277.5t-146.5 451.5q0 209 103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1664 960v-256q0 -26 -19 -45t-45 -19h-64q-26 0 -45 19t-19 45v256q0 106 -75 181t-181 75t-181 -75t-75 -181v-192h96q40 0 68 -28t28 -68v-576q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v576q0 40 28 68t68 28h672v192q0 185 131.5 316.5t316.5 131.5 t316.5 -131.5t131.5 -316.5z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1760 1408q66 0 113 -47t47 -113v-1216q0 -66 -47 -113t-113 -47h-1600q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1600zM160 1280q-13 0 -22.5 -9.5t-9.5 -22.5v-224h1664v224q0 13 -9.5 22.5t-22.5 9.5h-1600zM1760 0q13 0 22.5 9.5t9.5 22.5v608h-1664v-608 q0 -13 9.5 -22.5t22.5 -9.5h1600zM256 128v128h256v-128h-256zM640 128v128h384v-128h-384z" /> +<glyph unicode="" horiz-adv-x="1408" d="M384 192q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM896 69q2 -28 -17 -48q-18 -21 -47 -21h-135q-25 0 -43 16.5t-20 41.5q-22 229 -184.5 391.5t-391.5 184.5q-25 2 -41.5 20t-16.5 43v135q0 29 21 47q17 17 43 17h5q160 -13 306 -80.5 t259 -181.5q114 -113 181.5 -259t80.5 -306zM1408 67q2 -27 -18 -47q-18 -20 -46 -20h-143q-26 0 -44.5 17.5t-19.5 42.5q-12 215 -101 408.5t-231.5 336t-336 231.5t-408.5 102q-25 1 -42.5 19.5t-17.5 43.5v143q0 28 20 46q18 18 44 18h3q262 -13 501.5 -120t425.5 -294 q187 -186 294 -425.5t120 -501.5z" /> +<glyph unicode="" d="M1040 320q0 -33 -23.5 -56.5t-56.5 -23.5t-56.5 23.5t-23.5 56.5t23.5 56.5t56.5 23.5t56.5 -23.5t23.5 -56.5zM1296 320q0 -33 -23.5 -56.5t-56.5 -23.5t-56.5 23.5t-23.5 56.5t23.5 56.5t56.5 23.5t56.5 -23.5t23.5 -56.5zM1408 160v320q0 13 -9.5 22.5t-22.5 9.5 h-1216q-13 0 -22.5 -9.5t-9.5 -22.5v-320q0 -13 9.5 -22.5t22.5 -9.5h1216q13 0 22.5 9.5t9.5 22.5zM178 640h1180l-157 482q-4 13 -16 21.5t-26 8.5h-782q-14 0 -26 -8.5t-16 -21.5zM1536 480v-320q0 -66 -47 -113t-113 -47h-1216q-66 0 -113 47t-47 113v320q0 25 16 75 l197 606q17 53 63 86t101 33h782q55 0 101 -33t63 -86l197 -606q16 -50 16 -75z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1664 896q53 0 90.5 -37.5t37.5 -90.5t-37.5 -90.5t-90.5 -37.5v-384q0 -52 -38 -90t-90 -38q-417 347 -812 380q-58 -19 -91 -66t-31 -100.5t40 -92.5q-20 -33 -23 -65.5t6 -58t33.5 -55t48 -50t61.5 -50.5q-29 -58 -111.5 -83t-168.5 -11.5t-132 55.5q-7 23 -29.5 87.5 t-32 94.5t-23 89t-15 101t3.5 98.5t22 110.5h-122q-66 0 -113 47t-47 113v192q0 66 47 113t113 47h480q435 0 896 384q52 0 90 -38t38 -90v-384zM1536 292v954q-394 -302 -768 -343v-270q377 -42 768 -341z" /> +<glyph unicode="" horiz-adv-x="1792" d="M912 -160q0 16 -16 16q-59 0 -101.5 42.5t-42.5 101.5q0 16 -16 16t-16 -16q0 -73 51.5 -124.5t124.5 -51.5q16 0 16 16zM246 128h1300q-266 300 -266 832q0 51 -24 105t-69 103t-121.5 80.5t-169.5 31.5t-169.5 -31.5t-121.5 -80.5t-69 -103t-24 -105q0 -532 -266 -832z M1728 128q0 -52 -38 -90t-90 -38h-448q0 -106 -75 -181t-181 -75t-181 75t-75 181h-448q-52 0 -90 38t-38 90q50 42 91 88t85 119.5t74.5 158.5t50 206t19.5 260q0 152 117 282.5t307 158.5q-8 19 -8 39q0 40 28 68t68 28t68 -28t28 -68q0 -20 -8 -39q190 -28 307 -158.5 t117 -282.5q0 -139 19.5 -260t50 -206t74.5 -158.5t85 -119.5t91 -88z" /> +<glyph unicode="" d="M1376 640l138 -135q30 -28 20 -70q-12 -41 -52 -51l-188 -48l53 -186q12 -41 -19 -70q-29 -31 -70 -19l-186 53l-48 -188q-10 -40 -51 -52q-12 -2 -19 -2q-31 0 -51 22l-135 138l-135 -138q-28 -30 -70 -20q-41 11 -51 52l-48 188l-186 -53q-41 -12 -70 19q-31 29 -19 70 l53 186l-188 48q-40 10 -52 51q-10 42 20 70l138 135l-138 135q-30 28 -20 70q12 41 52 51l188 48l-53 186q-12 41 19 70q29 31 70 19l186 -53l48 188q10 41 51 51q41 12 70 -19l135 -139l135 139q29 30 70 19q41 -10 51 -51l48 -188l186 53q41 12 70 -19q31 -29 19 -70 l-53 -186l188 -48q40 -10 52 -51q10 -42 -20 -70z" /> +<glyph unicode="" horiz-adv-x="1792" d="M256 192q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1664 768q0 51 -39 89.5t-89 38.5h-576q0 20 15 48.5t33 55t33 68t15 84.5q0 67 -44.5 97.5t-115.5 30.5q-24 0 -90 -139q-24 -44 -37 -65q-40 -64 -112 -145q-71 -81 -101 -106 q-69 -57 -140 -57h-32v-640h32q72 0 167 -32t193.5 -64t179.5 -32q189 0 189 167q0 26 -5 56q30 16 47.5 52.5t17.5 73.5t-18 69q53 50 53 119q0 25 -10 55.5t-25 47.5h331q52 0 90 38t38 90zM1792 769q0 -105 -75.5 -181t-180.5 -76h-169q-4 -62 -37 -119q3 -21 3 -43 q0 -101 -60 -178q1 -139 -85 -219.5t-227 -80.5q-133 0 -322 69q-164 59 -223 59h-288q-53 0 -90.5 37.5t-37.5 90.5v640q0 53 37.5 90.5t90.5 37.5h288q10 0 21.5 4.5t23.5 14t22.5 18t24 22.5t20.5 21.5t19 21.5t14 17q65 74 100 129q13 21 33 62t37 72t40.5 63t55 49.5 t69.5 17.5q125 0 206.5 -67t81.5 -189q0 -68 -22 -128h374q104 0 180 -76t76 -179z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1376 128h32v640h-32q-35 0 -67.5 12t-62.5 37t-50 46t-49 54q-2 3 -3.5 4.5t-4 4.5t-4.5 5q-72 81 -112 145q-14 22 -38 68q-1 3 -10.5 22.5t-18.5 36t-20 35.5t-21.5 30.5t-18.5 11.5q-71 0 -115.5 -30.5t-44.5 -97.5q0 -43 15 -84.5t33 -68t33 -55t15 -48.5h-576 q-50 0 -89 -38.5t-39 -89.5q0 -52 38 -90t90 -38h331q-15 -17 -25 -47.5t-10 -55.5q0 -69 53 -119q-18 -32 -18 -69t17.5 -73.5t47.5 -52.5q-4 -24 -4 -56q0 -85 48.5 -126t135.5 -41q84 0 183 32t194 64t167 32zM1664 192q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45 t45 -19t45 19t19 45zM1792 768v-640q0 -53 -37.5 -90.5t-90.5 -37.5h-288q-59 0 -223 -59q-190 -69 -317 -69q-142 0 -230 77.5t-87 217.5l1 5q-61 76 -61 178q0 22 3 43q-33 57 -37 119h-169q-105 0 -180.5 76t-75.5 181q0 103 76 179t180 76h374q-22 60 -22 128 q0 122 81.5 189t206.5 67q38 0 69.5 -17.5t55 -49.5t40.5 -63t37 -72t33 -62q35 -55 100 -129q2 -3 14 -17t19 -21.5t20.5 -21.5t24 -22.5t22.5 -18t23.5 -14t21.5 -4.5h288q53 0 90.5 -37.5t37.5 -90.5z" /> +<glyph unicode="" d="M1280 -64q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1408 700q0 189 -167 189q-26 0 -56 -5q-16 30 -52.5 47.5t-73.5 17.5t-69 -18q-50 53 -119 53q-25 0 -55.5 -10t-47.5 -25v331q0 52 -38 90t-90 38q-51 0 -89.5 -39t-38.5 -89v-576 q-20 0 -48.5 15t-55 33t-68 33t-84.5 15q-67 0 -97.5 -44.5t-30.5 -115.5q0 -24 139 -90q44 -24 65 -37q64 -40 145 -112q81 -71 106 -101q57 -69 57 -140v-32h640v32q0 72 32 167t64 193.5t32 179.5zM1536 705q0 -133 -69 -322q-59 -164 -59 -223v-288q0 -53 -37.5 -90.5 t-90.5 -37.5h-640q-53 0 -90.5 37.5t-37.5 90.5v288q0 10 -4.5 21.5t-14 23.5t-18 22.5t-22.5 24t-21.5 20.5t-21.5 19t-17 14q-74 65 -129 100q-21 13 -62 33t-72 37t-63 40.5t-49.5 55t-17.5 69.5q0 125 67 206.5t189 81.5q68 0 128 -22v374q0 104 76 180t179 76 q105 0 181 -75.5t76 -180.5v-169q62 -4 119 -37q21 3 43 3q101 0 178 -60q139 1 219.5 -85t80.5 -227z" /> +<glyph unicode="" d="M1408 576q0 84 -32 183t-64 194t-32 167v32h-640v-32q0 -35 -12 -67.5t-37 -62.5t-46 -50t-54 -49q-9 -8 -14 -12q-81 -72 -145 -112q-22 -14 -68 -38q-3 -1 -22.5 -10.5t-36 -18.5t-35.5 -20t-30.5 -21.5t-11.5 -18.5q0 -71 30.5 -115.5t97.5 -44.5q43 0 84.5 15t68 33 t55 33t48.5 15v-576q0 -50 38.5 -89t89.5 -39q52 0 90 38t38 90v331q46 -35 103 -35q69 0 119 53q32 -18 69 -18t73.5 17.5t52.5 47.5q24 -4 56 -4q85 0 126 48.5t41 135.5zM1280 1344q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1536 580 q0 -142 -77.5 -230t-217.5 -87l-5 1q-76 -61 -178 -61q-22 0 -43 3q-54 -30 -119 -37v-169q0 -105 -76 -180.5t-181 -75.5q-103 0 -179 76t-76 180v374q-54 -22 -128 -22q-121 0 -188.5 81.5t-67.5 206.5q0 38 17.5 69.5t49.5 55t63 40.5t72 37t62 33q55 35 129 100 q3 2 17 14t21.5 19t21.5 20.5t22.5 24t18 22.5t14 23.5t4.5 21.5v288q0 53 37.5 90.5t90.5 37.5h640q53 0 90.5 -37.5t37.5 -90.5v-288q0 -59 59 -223q69 -190 69 -317z" /> +<glyph unicode="" d="M1280 576v128q0 26 -19 45t-45 19h-502l189 189q19 19 19 45t-19 45l-91 91q-18 18 -45 18t-45 -18l-362 -362l-91 -91q-18 -18 -18 -45t18 -45l91 -91l362 -362q18 -18 45 -18t45 18l91 91q18 18 18 45t-18 45l-189 189h502q26 0 45 19t19 45zM1536 640 q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1285 640q0 27 -18 45l-91 91l-362 362q-18 18 -45 18t-45 -18l-91 -91q-18 -18 -18 -45t18 -45l189 -189h-502q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h502l-189 -189q-19 -19 -19 -45t19 -45l91 -91q18 -18 45 -18t45 18l362 362l91 91q18 18 18 45zM1536 640 q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1284 641q0 27 -18 45l-362 362l-91 91q-18 18 -45 18t-45 -18l-91 -91l-362 -362q-18 -18 -18 -45t18 -45l91 -91q18 -18 45 -18t45 18l189 189v-502q0 -26 19 -45t45 -19h128q26 0 45 19t19 45v502l189 -189q19 -19 45 -19t45 19l91 91q18 18 18 45zM1536 640 q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1284 639q0 27 -18 45l-91 91q-18 18 -45 18t-45 -18l-189 -189v502q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-502l-189 189q-19 19 -45 19t-45 -19l-91 -91q-18 -18 -18 -45t18 -45l362 -362l91 -91q18 -18 45 -18t45 18l91 91l362 362q18 18 18 45zM1536 640 q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M768 1408q209 0 385.5 -103t279.5 -279.5t103 -385.5t-103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103zM1042 887q-2 -1 -9.5 -9.5t-13.5 -9.5q2 0 4.5 5t5 11t3.5 7q6 7 22 15q14 6 52 12q34 8 51 -11 q-2 2 9.5 13t14.5 12q3 2 15 4.5t15 7.5l2 22q-12 -1 -17.5 7t-6.5 21q0 -2 -6 -8q0 7 -4.5 8t-11.5 -1t-9 -1q-10 3 -15 7.5t-8 16.5t-4 15q-2 5 -9.5 10.5t-9.5 10.5q-1 2 -2.5 5.5t-3 6.5t-4 5.5t-5.5 2.5t-7 -5t-7.5 -10t-4.5 -5q-3 2 -6 1.5t-4.5 -1t-4.5 -3t-5 -3.5 q-3 -2 -8.5 -3t-8.5 -2q15 5 -1 11q-10 4 -16 3q9 4 7.5 12t-8.5 14h5q-1 4 -8.5 8.5t-17.5 8.5t-13 6q-8 5 -34 9.5t-33 0.5q-5 -6 -4.5 -10.5t4 -14t3.5 -12.5q1 -6 -5.5 -13t-6.5 -12q0 -7 14 -15.5t10 -21.5q-3 -8 -16 -16t-16 -12q-5 -8 -1.5 -18.5t10.5 -16.5 q2 -2 1.5 -4t-3.5 -4.5t-5.5 -4t-6.5 -3.5l-3 -2q-11 -5 -20.5 6t-13.5 26q-7 25 -16 30q-23 8 -29 -1q-5 13 -41 26q-25 9 -58 4q6 1 0 15q-7 15 -19 12q3 6 4 17.5t1 13.5q3 13 12 23q1 1 7 8.5t9.5 13.5t0.5 6q35 -4 50 11q5 5 11.5 17t10.5 17q9 6 14 5.5t14.5 -5.5 t14.5 -5q14 -1 15.5 11t-7.5 20q12 -1 3 17q-5 7 -8 9q-12 4 -27 -5q-8 -4 2 -8q-1 1 -9.5 -10.5t-16.5 -17.5t-16 5q-1 1 -5.5 13.5t-9.5 13.5q-8 0 -16 -15q3 8 -11 15t-24 8q19 12 -8 27q-7 4 -20.5 5t-19.5 -4q-5 -7 -5.5 -11.5t5 -8t10.5 -5.5t11.5 -4t8.5 -3 q14 -10 8 -14q-2 -1 -8.5 -3.5t-11.5 -4.5t-6 -4q-3 -4 0 -14t-2 -14q-5 5 -9 17.5t-7 16.5q7 -9 -25 -6l-10 1q-4 0 -16 -2t-20.5 -1t-13.5 8q-4 8 0 20q1 4 4 2q-4 3 -11 9.5t-10 8.5q-46 -15 -94 -41q6 -1 12 1q5 2 13 6.5t10 5.5q34 14 42 7l5 5q14 -16 20 -25 q-7 4 -30 1q-20 -6 -22 -12q7 -12 5 -18q-4 3 -11.5 10t-14.5 11t-15 5q-16 0 -22 -1q-146 -80 -235 -222q7 -7 12 -8q4 -1 5 -9t2.5 -11t11.5 3q9 -8 3 -19q1 1 44 -27q19 -17 21 -21q3 -11 -10 -18q-1 2 -9 9t-9 4q-3 -5 0.5 -18.5t10.5 -12.5q-7 0 -9.5 -16t-2.5 -35.5 t-1 -23.5l2 -1q-3 -12 5.5 -34.5t21.5 -19.5q-13 -3 20 -43q6 -8 8 -9q3 -2 12 -7.5t15 -10t10 -10.5q4 -5 10 -22.5t14 -23.5q-2 -6 9.5 -20t10.5 -23q-1 0 -2.5 -1t-2.5 -1q3 -7 15.5 -14t15.5 -13q1 -3 2 -10t3 -11t8 -2q2 20 -24 62q-15 25 -17 29q-3 5 -5.5 15.5 t-4.5 14.5q2 0 6 -1.5t8.5 -3.5t7.5 -4t2 -3q-3 -7 2 -17.5t12 -18.5t17 -19t12 -13q6 -6 14 -19.5t0 -13.5q9 0 20 -10t17 -20q5 -8 8 -26t5 -24q2 -7 8.5 -13.5t12.5 -9.5l16 -8t13 -7q5 -2 18.5 -10.5t21.5 -11.5q10 -4 16 -4t14.5 2.5t13.5 3.5q15 2 29 -15t21 -21 q36 -19 55 -11q-2 -1 0.5 -7.5t8 -15.5t9 -14.5t5.5 -8.5q5 -6 18 -15t18 -15q6 4 7 9q-3 -8 7 -20t18 -10q14 3 14 32q-31 -15 -49 18q0 1 -2.5 5.5t-4 8.5t-2.5 8.5t0 7.5t5 3q9 0 10 3.5t-2 12.5t-4 13q-1 8 -11 20t-12 15q-5 -9 -16 -8t-16 9q0 -1 -1.5 -5.5t-1.5 -6.5 q-13 0 -15 1q1 3 2.5 17.5t3.5 22.5q1 4 5.5 12t7.5 14.5t4 12.5t-4.5 9.5t-17.5 2.5q-19 -1 -26 -20q-1 -3 -3 -10.5t-5 -11.5t-9 -7q-7 -3 -24 -2t-24 5q-13 8 -22.5 29t-9.5 37q0 10 2.5 26.5t3 25t-5.5 24.5q3 2 9 9.5t10 10.5q2 1 4.5 1.5t4.5 0t4 1.5t3 6q-1 1 -4 3 q-3 3 -4 3q7 -3 28.5 1.5t27.5 -1.5q15 -11 22 2q0 1 -2.5 9.5t-0.5 13.5q5 -27 29 -9q3 -3 15.5 -5t17.5 -5q3 -2 7 -5.5t5.5 -4.5t5 0.5t8.5 6.5q10 -14 12 -24q11 -40 19 -44q7 -3 11 -2t4.5 9.5t0 14t-1.5 12.5l-1 8v18l-1 8q-15 3 -18.5 12t1.5 18.5t15 18.5q1 1 8 3.5 t15.5 6.5t12.5 8q21 19 15 35q7 0 11 9q-1 0 -5 3t-7.5 5t-4.5 2q9 5 2 16q5 3 7.5 11t7.5 10q9 -12 21 -2q7 8 1 16q5 7 20.5 10.5t18.5 9.5q7 -2 8 2t1 12t3 12q4 5 15 9t13 5l17 11q3 4 0 4q18 -2 31 11q10 11 -6 20q3 6 -3 9.5t-15 5.5q3 1 11.5 0.5t10.5 1.5 q15 10 -7 16q-17 5 -43 -12zM879 10q206 36 351 189q-3 3 -12.5 4.5t-12.5 3.5q-18 7 -24 8q1 7 -2.5 13t-8 9t-12.5 8t-11 7q-2 2 -7 6t-7 5.5t-7.5 4.5t-8.5 2t-10 -1l-3 -1q-3 -1 -5.5 -2.5t-5.5 -3t-4 -3t0 -2.5q-21 17 -36 22q-5 1 -11 5.5t-10.5 7t-10 1.5t-11.5 -7 q-5 -5 -6 -15t-2 -13q-7 5 0 17.5t2 18.5q-3 6 -10.5 4.5t-12 -4.5t-11.5 -8.5t-9 -6.5t-8.5 -5.5t-8.5 -7.5q-3 -4 -6 -12t-5 -11q-2 4 -11.5 6.5t-9.5 5.5q2 -10 4 -35t5 -38q7 -31 -12 -48q-27 -25 -29 -40q-4 -22 12 -26q0 -7 -8 -20.5t-7 -21.5q0 -6 2 -16z" /> +<glyph unicode="" horiz-adv-x="1664" d="M384 64q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1028 484l-682 -682q-37 -37 -90 -37q-52 0 -91 37l-106 108q-38 36 -38 90q0 53 38 91l681 681q39 -98 114.5 -173.5t173.5 -114.5zM1662 919q0 -39 -23 -106q-47 -134 -164.5 -217.5 t-258.5 -83.5q-185 0 -316.5 131.5t-131.5 316.5t131.5 316.5t316.5 131.5q58 0 121.5 -16.5t107.5 -46.5q16 -11 16 -28t-16 -28l-293 -169v-224l193 -107q5 3 79 48.5t135.5 81t70.5 35.5q15 0 23.5 -10t8.5 -25z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1024 128h640v128h-640v-128zM640 640h1024v128h-1024v-128zM1280 1152h384v128h-384v-128zM1792 320v-256q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 832v-256q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19 t-19 45v256q0 26 19 45t45 19h1664q26 0 45 -19t19 -45zM1792 1344v-256q0 -26 -19 -45t-45 -19h-1664q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h1664q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1403 1241q17 -41 -14 -70l-493 -493v-742q0 -42 -39 -59q-13 -5 -25 -5q-27 0 -45 19l-256 256q-19 19 -19 45v486l-493 493q-31 29 -14 70q17 39 59 39h1280q42 0 59 -39z" /> +<glyph unicode="" horiz-adv-x="1792" d="M640 1280h512v128h-512v-128zM1792 640v-480q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v480h672v-160q0 -26 19 -45t45 -19h320q26 0 45 19t19 45v160h672zM1024 640v-128h-256v128h256zM1792 1120v-384h-1792v384q0 66 47 113t113 47h352v160q0 40 28 68 t68 28h576q40 0 68 -28t28 -68v-160h352q66 0 113 -47t47 -113z" /> +<glyph unicode="" d="M1283 995l-355 -355l355 -355l144 144q29 31 70 14q39 -17 39 -59v-448q0 -26 -19 -45t-45 -19h-448q-42 0 -59 40q-17 39 14 69l144 144l-355 355l-355 -355l144 -144q31 -30 14 -69q-17 -40 -59 -40h-448q-26 0 -45 19t-19 45v448q0 42 40 59q39 17 69 -14l144 -144 l355 355l-355 355l-144 -144q-19 -19 -45 -19q-12 0 -24 5q-40 17 -40 59v448q0 26 19 45t45 19h448q42 0 59 -40q17 -39 -14 -69l-144 -144l355 -355l355 355l-144 144q-31 30 -14 69q17 40 59 40h448q26 0 45 -19t19 -45v-448q0 -42 -39 -59q-13 -5 -25 -5q-26 0 -45 19z " /> +<glyph unicode="" horiz-adv-x="1920" d="M593 640q-162 -5 -265 -128h-134q-82 0 -138 40.5t-56 118.5q0 353 124 353q6 0 43.5 -21t97.5 -42.5t119 -21.5q67 0 133 23q-5 -37 -5 -66q0 -139 81 -256zM1664 3q0 -120 -73 -189.5t-194 -69.5h-874q-121 0 -194 69.5t-73 189.5q0 53 3.5 103.5t14 109t26.5 108.5 t43 97.5t62 81t85.5 53.5t111.5 20q10 0 43 -21.5t73 -48t107 -48t135 -21.5t135 21.5t107 48t73 48t43 21.5q61 0 111.5 -20t85.5 -53.5t62 -81t43 -97.5t26.5 -108.5t14 -109t3.5 -103.5zM640 1280q0 -106 -75 -181t-181 -75t-181 75t-75 181t75 181t181 75t181 -75 t75 -181zM1344 896q0 -159 -112.5 -271.5t-271.5 -112.5t-271.5 112.5t-112.5 271.5t112.5 271.5t271.5 112.5t271.5 -112.5t112.5 -271.5zM1920 671q0 -78 -56 -118.5t-138 -40.5h-134q-103 123 -265 128q81 117 81 256q0 29 -5 66q66 -23 133 -23q59 0 119 21.5t97.5 42.5 t43.5 21q124 0 124 -353zM1792 1280q0 -106 -75 -181t-181 -75t-181 75t-75 181t75 181t181 75t181 -75t75 -181z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1456 320q0 40 -28 68l-208 208q-28 28 -68 28q-42 0 -72 -32q3 -3 19 -18.5t21.5 -21.5t15 -19t13 -25.5t3.5 -27.5q0 -40 -28 -68t-68 -28q-15 0 -27.5 3.5t-25.5 13t-19 15t-21.5 21.5t-18.5 19q-33 -31 -33 -73q0 -40 28 -68l206 -207q27 -27 68 -27q40 0 68 26 l147 146q28 28 28 67zM753 1025q0 40 -28 68l-206 207q-28 28 -68 28q-39 0 -68 -27l-147 -146q-28 -28 -28 -67q0 -40 28 -68l208 -208q27 -27 68 -27q42 0 72 31q-3 3 -19 18.5t-21.5 21.5t-15 19t-13 25.5t-3.5 27.5q0 40 28 68t68 28q15 0 27.5 -3.5t25.5 -13t19 -15 t21.5 -21.5t18.5 -19q33 31 33 73zM1648 320q0 -120 -85 -203l-147 -146q-83 -83 -203 -83q-121 0 -204 85l-206 207q-83 83 -83 203q0 123 88 209l-88 88q-86 -88 -208 -88q-120 0 -204 84l-208 208q-84 84 -84 204t85 203l147 146q83 83 203 83q121 0 204 -85l206 -207 q83 -83 83 -203q0 -123 -88 -209l88 -88q86 88 208 88q120 0 204 -84l208 -208q84 -84 84 -204z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1920 384q0 -159 -112.5 -271.5t-271.5 -112.5h-1088q-185 0 -316.5 131.5t-131.5 316.5q0 132 71 241.5t187 163.5q-2 28 -2 43q0 212 150 362t362 150q158 0 286.5 -88t187.5 -230q70 62 166 62q106 0 181 -75t75 -181q0 -75 -41 -138q129 -30 213 -134.5t84 -239.5z " /> +<glyph unicode="" horiz-adv-x="1664" d="M1527 88q56 -89 21.5 -152.5t-140.5 -63.5h-1152q-106 0 -140.5 63.5t21.5 152.5l503 793v399h-64q-26 0 -45 19t-19 45t19 45t45 19h512q26 0 45 -19t19 -45t-19 -45t-45 -19h-64v-399zM748 813l-272 -429h712l-272 429l-20 31v37v399h-128v-399v-37z" /> +<glyph unicode="" horiz-adv-x="1792" d="M960 640q26 0 45 -19t19 -45t-19 -45t-45 -19t-45 19t-19 45t19 45t45 19zM1260 576l507 -398q28 -20 25 -56q-5 -35 -35 -51l-128 -64q-13 -7 -29 -7q-17 0 -31 8l-690 387l-110 -66q-8 -4 -12 -5q14 -49 10 -97q-7 -77 -56 -147.5t-132 -123.5q-132 -84 -277 -84 q-136 0 -222 78q-90 84 -79 207q7 76 56 147t131 124q132 84 278 84q83 0 151 -31q9 13 22 22l122 73l-122 73q-13 9 -22 22q-68 -31 -151 -31q-146 0 -278 84q-82 53 -131 124t-56 147q-5 59 15.5 113t63.5 93q85 79 222 79q145 0 277 -84q83 -52 132 -123t56 -148 q4 -48 -10 -97q4 -1 12 -5l110 -66l690 387q14 8 31 8q16 0 29 -7l128 -64q30 -16 35 -51q3 -36 -25 -56zM579 836q46 42 21 108t-106 117q-92 59 -192 59q-74 0 -113 -36q-46 -42 -21 -108t106 -117q92 -59 192 -59q74 0 113 36zM494 91q81 51 106 117t-21 108 q-39 36 -113 36q-100 0 -192 -59q-81 -51 -106 -117t21 -108q39 -36 113 -36q100 0 192 59zM672 704l96 -58v11q0 36 33 56l14 8l-79 47l-26 -26q-3 -3 -10 -11t-12 -12q-2 -2 -4 -3.5t-3 -2.5zM896 480l96 -32l736 576l-128 64l-768 -431v-113l-160 -96l9 -8q2 -2 7 -6 q4 -4 11 -12t11 -12l26 -26zM1600 64l128 64l-520 408l-177 -138q-2 -3 -13 -7z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1696 1152q40 0 68 -28t28 -68v-1216q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v288h-544q-40 0 -68 28t-28 68v672q0 40 20 88t48 76l408 408q28 28 76 48t88 20h416q40 0 68 -28t28 -68v-328q68 40 128 40h416zM1152 939l-299 -299h299v299zM512 1323l-299 -299 h299v299zM708 676l316 316v416h-384v-416q0 -40 -28 -68t-68 -28h-416v-640h512v256q0 40 20 88t48 76zM1664 -128v1152h-384v-416q0 -40 -28 -68t-68 -28h-416v-640h896z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1404 151q0 -117 -79 -196t-196 -79q-135 0 -235 100l-777 776q-113 115 -113 271q0 159 110 270t269 111q158 0 273 -113l605 -606q10 -10 10 -22q0 -16 -30.5 -46.5t-46.5 -30.5q-13 0 -23 10l-606 607q-79 77 -181 77q-106 0 -179 -75t-73 -181q0 -105 76 -181 l776 -777q63 -63 145 -63q64 0 106 42t42 106q0 82 -63 145l-581 581q-26 24 -60 24q-29 0 -48 -19t-19 -48q0 -32 25 -59l410 -410q10 -10 10 -22q0 -16 -31 -47t-47 -31q-12 0 -22 10l-410 410q-63 61 -63 149q0 82 57 139t139 57q88 0 149 -63l581 -581q100 -98 100 -235 z" /> +<glyph unicode="" d="M384 0h768v384h-768v-384zM1280 0h128v896q0 14 -10 38.5t-20 34.5l-281 281q-10 10 -34 20t-39 10v-416q0 -40 -28 -68t-68 -28h-576q-40 0 -68 28t-28 68v416h-128v-1280h128v416q0 40 28 68t68 28h832q40 0 68 -28t28 -68v-416zM896 928v320q0 13 -9.5 22.5t-22.5 9.5 h-192q-13 0 -22.5 -9.5t-9.5 -22.5v-320q0 -13 9.5 -22.5t22.5 -9.5h192q13 0 22.5 9.5t9.5 22.5zM1536 896v-928q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1344q0 40 28 68t68 28h928q40 0 88 -20t76 -48l280 -280q28 -28 48 -76t20 -88z" /> +<glyph unicode="" d="M1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M1536 192v-128q0 -26 -19 -45t-45 -19h-1408q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1408q26 0 45 -19t19 -45zM1536 704v-128q0 -26 -19 -45t-45 -19h-1408q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1408q26 0 45 -19t19 -45zM1536 1216v-128q0 -26 -19 -45 t-45 -19h-1408q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h1408q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1792" d="M384 128q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM384 640q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM1792 224v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1216q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5 t22.5 9.5h1216q13 0 22.5 -9.5t9.5 -22.5zM384 1152q0 -80 -56 -136t-136 -56t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM1792 736v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1216q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1216q13 0 22.5 -9.5t9.5 -22.5z M1792 1248v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1216q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1216q13 0 22.5 -9.5t9.5 -22.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M381 -84q0 -80 -54.5 -126t-135.5 -46q-106 0 -172 66l57 88q49 -45 106 -45q29 0 50.5 14.5t21.5 42.5q0 64 -105 56l-26 56q8 10 32.5 43.5t42.5 54t37 38.5v1q-16 0 -48.5 -1t-48.5 -1v-53h-106v152h333v-88l-95 -115q51 -12 81 -49t30 -88zM383 543v-159h-362 q-6 36 -6 54q0 51 23.5 93t56.5 68t66 47.5t56.5 43.5t23.5 45q0 25 -14.5 38.5t-39.5 13.5q-46 0 -81 -58l-85 59q24 51 71.5 79.5t105.5 28.5q73 0 123 -41.5t50 -112.5q0 -50 -34 -91.5t-75 -64.5t-75.5 -50.5t-35.5 -52.5h127v60h105zM1792 224v-192q0 -13 -9.5 -22.5 t-22.5 -9.5h-1216q-13 0 -22.5 9.5t-9.5 22.5v192q0 14 9 23t23 9h1216q13 0 22.5 -9.5t9.5 -22.5zM384 1123v-99h-335v99h107q0 41 0.5 122t0.5 121v12h-2q-8 -17 -50 -54l-71 76l136 127h106v-404h108zM1792 736v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1216q-13 0 -22.5 9.5 t-9.5 22.5v192q0 14 9 23t23 9h1216q13 0 22.5 -9.5t9.5 -22.5zM1792 1248v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1216q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1216q13 0 22.5 -9.5t9.5 -22.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1760 640q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-1728q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h1728zM483 704q-28 35 -51 80q-48 97 -48 188q0 181 134 309q133 127 393 127q50 0 167 -19q66 -12 177 -48q10 -38 21 -118q14 -123 14 -183q0 -18 -5 -45l-12 -3l-84 6 l-14 2q-50 149 -103 205q-88 91 -210 91q-114 0 -182 -59q-67 -58 -67 -146q0 -73 66 -140t279 -129q69 -20 173 -66q58 -28 95 -52h-743zM990 448h411q7 -39 7 -92q0 -111 -41 -212q-23 -55 -71 -104q-37 -35 -109 -81q-80 -48 -153 -66q-80 -21 -203 -21q-114 0 -195 23 l-140 40q-57 16 -72 28q-8 8 -8 22v13q0 108 -2 156q-1 30 0 68l2 37v44l102 2q15 -34 30 -71t22.5 -56t12.5 -27q35 -57 80 -94q43 -36 105 -57q59 -22 132 -22q64 0 139 27q77 26 122 86q47 61 47 129q0 84 -81 157q-34 29 -137 71z" /> +<glyph unicode="" d="M48 1313q-37 2 -45 4l-3 88q13 1 40 1q60 0 112 -4q132 -7 166 -7q86 0 168 3q116 4 146 5q56 0 86 2l-1 -14l2 -64v-9q-60 -9 -124 -9q-60 0 -79 -25q-13 -14 -13 -132q0 -13 0.5 -32.5t0.5 -25.5l1 -229l14 -280q6 -124 51 -202q35 -59 96 -92q88 -47 177 -47 q104 0 191 28q56 18 99 51q48 36 65 64q36 56 53 114q21 73 21 229q0 79 -3.5 128t-11 122.5t-13.5 159.5l-4 59q-5 67 -24 88q-34 35 -77 34l-100 -2l-14 3l2 86h84l205 -10q76 -3 196 10l18 -2q6 -38 6 -51q0 -7 -4 -31q-45 -12 -84 -13q-73 -11 -79 -17q-15 -15 -15 -41 q0 -7 1.5 -27t1.5 -31q8 -19 22 -396q6 -195 -15 -304q-15 -76 -41 -122q-38 -65 -112 -123q-75 -57 -182 -89q-109 -33 -255 -33q-167 0 -284 46q-119 47 -179 122q-61 76 -83 195q-16 80 -16 237v333q0 188 -17 213q-25 36 -147 39zM1536 -96v64q0 14 -9 23t-23 9h-1472 q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h1472q14 0 23 9t9 23z" /> +<glyph unicode="" horiz-adv-x="1664" d="M512 160v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM512 544v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1024 160v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23 v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM512 928v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1024 544v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1536 160v192 q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1024 928v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1536 544v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192 q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1536 928v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM1664 1248v-1088q0 -66 -47 -113t-113 -47h-1344q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h1344q66 0 113 -47t47 -113 z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1190 955l293 293l-107 107l-293 -293zM1637 1248q0 -27 -18 -45l-1286 -1286q-18 -18 -45 -18t-45 18l-198 198q-18 18 -18 45t18 45l1286 1286q18 18 45 18t45 -18l198 -198q18 -18 18 -45zM286 1438l98 -30l-98 -30l-30 -98l-30 98l-98 30l98 30l30 98zM636 1276 l196 -60l-196 -60l-60 -196l-60 196l-196 60l196 60l60 196zM1566 798l98 -30l-98 -30l-30 -98l-30 98l-98 30l98 30l30 98zM926 1438l98 -30l-98 -30l-30 -98l-30 98l-98 30l98 30l30 98z" /> +<glyph unicode="" horiz-adv-x="1792" d="M640 128q0 52 -38 90t-90 38t-90 -38t-38 -90t38 -90t90 -38t90 38t38 90zM256 640h384v256h-158q-13 0 -22 -9l-195 -195q-9 -9 -9 -22v-30zM1536 128q0 52 -38 90t-90 38t-90 -38t-38 -90t38 -90t90 -38t90 38t38 90zM1792 1216v-1024q0 -15 -4 -26.5t-13.5 -18.5 t-16.5 -11.5t-23.5 -6t-22.5 -2t-25.5 0t-22.5 0.5q0 -106 -75 -181t-181 -75t-181 75t-75 181h-384q0 -106 -75 -181t-181 -75t-181 75t-75 181h-64q-3 0 -22.5 -0.5t-25.5 0t-22.5 2t-23.5 6t-16.5 11.5t-13.5 18.5t-4 26.5q0 26 19 45t45 19v320q0 8 -0.5 35t0 38 t2.5 34.5t6.5 37t14 30.5t22.5 30l198 198q19 19 50.5 32t58.5 13h160v192q0 26 19 45t45 19h1024q26 0 45 -19t19 -45z" /> +<glyph unicode="" d="M1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103q-111 0 -218 32q59 93 78 164q9 34 54 211q20 -39 73 -67.5t114 -28.5q121 0 216 68.5t147 188.5t52 270q0 114 -59.5 214t-172.5 163t-255 63q-105 0 -196 -29t-154.5 -77t-109 -110.5t-67 -129.5t-21.5 -134 q0 -104 40 -183t117 -111q30 -12 38 20q2 7 8 31t8 30q6 23 -11 43q-51 61 -51 151q0 151 104.5 259.5t273.5 108.5q151 0 235.5 -82t84.5 -213q0 -170 -68.5 -289t-175.5 -119q-61 0 -98 43.5t-23 104.5q8 35 26.5 93.5t30 103t11.5 75.5q0 50 -27 83t-77 33 q-62 0 -105 -57t-43 -142q0 -73 25 -122l-99 -418q-17 -70 -13 -177q-206 91 -333 281t-127 423q0 209 103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1248 1408q119 0 203.5 -84.5t84.5 -203.5v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-725q85 122 108 210q9 34 53 209q21 -39 73.5 -67t112.5 -28q181 0 295.5 147.5t114.5 373.5q0 84 -35 162.5t-96.5 139t-152.5 97t-197 36.5q-104 0 -194.5 -28.5t-153 -76.5 t-107.5 -109.5t-66.5 -128t-21.5 -132.5q0 -102 39.5 -180t116.5 -110q13 -5 23.5 0t14.5 19q10 44 15 61q6 23 -11 42q-50 62 -50 150q0 150 103.5 256.5t270.5 106.5q149 0 232.5 -81t83.5 -210q0 -168 -67.5 -286t-173.5 -118q-60 0 -97 43.5t-23 103.5q8 34 26.5 92.5 t29.5 102t11 74.5q0 49 -26.5 81.5t-75.5 32.5q-61 0 -103.5 -56.5t-42.5 -139.5q0 -72 24 -121l-98 -414q-24 -100 -7 -254h-183q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960z" /> +<glyph unicode="" d="M829 318q0 -76 -58.5 -112.5t-139.5 -36.5q-41 0 -80.5 9.5t-75.5 28.5t-58 53t-22 78q0 46 25 80t65.5 51.5t82 25t84.5 7.5q20 0 31 -2q2 -1 23 -16.5t26 -19t23 -18t24.5 -22t19 -22.5t17 -26t9 -26.5t4.5 -31.5zM755 863q0 -60 -33 -99.5t-92 -39.5q-53 0 -93 42.5 t-57.5 96.5t-17.5 106q0 61 32 104t92 43q53 0 93.5 -45t58 -101t17.5 -107zM861 1120l88 64h-265q-85 0 -161 -32t-127.5 -98t-51.5 -153q0 -93 64.5 -154.5t158.5 -61.5q22 0 43 3q-13 -29 -13 -54q0 -44 40 -94q-175 -12 -257 -63q-47 -29 -75.5 -73t-28.5 -95 q0 -43 18.5 -77.5t48.5 -56.5t69 -37t77.5 -21t76.5 -6q60 0 120.5 15.5t113.5 46t86 82.5t33 117q0 49 -20 89.5t-49 66.5t-58 47.5t-49 44t-20 44.5t15.5 42.5t37.5 39.5t44 42t37.5 59.5t15.5 82.5q0 60 -22.5 99.5t-72.5 90.5h83zM1152 672h128v64h-128v128h-64v-128 h-128v-64h128v-160h64v160zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M735 740q0 -36 32 -70.5t77.5 -68t90.5 -73.5t77 -104t32 -142q0 -90 -48 -173q-72 -122 -211 -179.5t-298 -57.5q-132 0 -246.5 41.5t-171.5 137.5q-37 60 -37 131q0 81 44.5 150t118.5 115q131 82 404 100q-32 42 -47.5 74t-15.5 73q0 36 21 85q-46 -4 -68 -4 q-148 0 -249.5 96.5t-101.5 244.5q0 82 36 159t99 131q77 66 182.5 98t217.5 32h418l-138 -88h-131q74 -63 112 -133t38 -160q0 -72 -24.5 -129.5t-59 -93t-69.5 -65t-59.5 -61.5t-24.5 -66zM589 836q38 0 78 16.5t66 43.5q53 57 53 159q0 58 -17 125t-48.5 129.5 t-84.5 103.5t-117 41q-42 0 -82.5 -19.5t-65.5 -52.5q-47 -59 -47 -160q0 -46 10 -97.5t31.5 -103t52 -92.5t75 -67t96.5 -26zM591 -37q58 0 111.5 13t99 39t73 73t27.5 109q0 25 -7 49t-14.5 42t-27 41.5t-29.5 35t-38.5 34.5t-36.5 29t-41.5 30t-36.5 26q-16 2 -48 2 q-53 0 -105 -7t-107.5 -25t-97 -46t-68.5 -74.5t-27 -105.5q0 -70 35 -123.5t91.5 -83t119 -44t127.5 -14.5zM1401 839h213v-108h-213v-219h-105v219h-212v108h212v217h105v-217z" /> +<glyph unicode="" horiz-adv-x="1920" d="M768 384h384v96h-128v448h-114l-148 -137l77 -80q42 37 55 57h2v-288h-128v-96zM1280 640q0 -70 -21 -142t-59.5 -134t-101.5 -101t-138 -39t-138 39t-101.5 101t-59.5 134t-21 142t21 142t59.5 134t101.5 101t138 39t138 -39t101.5 -101t59.5 -134t21 -142zM1792 384 v512q-106 0 -181 75t-75 181h-1152q0 -106 -75 -181t-181 -75v-512q106 0 181 -75t75 -181h1152q0 106 75 181t181 75zM1920 1216v-1152q0 -26 -19 -45t-45 -19h-1792q-26 0 -45 19t-19 45v1152q0 26 19 45t45 19h1792q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1024" d="M1024 832q0 -26 -19 -45l-448 -448q-19 -19 -45 -19t-45 19l-448 448q-19 19 -19 45t19 45t45 19h896q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1024" d="M1024 320q0 -26 -19 -45t-45 -19h-896q-26 0 -45 19t-19 45t19 45l448 448q19 19 45 19t45 -19l448 -448q19 -19 19 -45z" /> +<glyph unicode="" horiz-adv-x="640" d="M640 1088v-896q0 -26 -19 -45t-45 -19t-45 19l-448 448q-19 19 -19 45t19 45l448 448q19 19 45 19t45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="640" d="M576 640q0 -26 -19 -45l-448 -448q-19 -19 -45 -19t-45 19t-19 45v896q0 26 19 45t45 19t45 -19l448 -448q19 -19 19 -45z" /> +<glyph unicode="" horiz-adv-x="1664" d="M160 0h608v1152h-640v-1120q0 -13 9.5 -22.5t22.5 -9.5zM1536 32v1120h-640v-1152h608q13 0 22.5 9.5t9.5 22.5zM1664 1248v-1216q0 -66 -47 -113t-113 -47h-1344q-66 0 -113 47t-47 113v1216q0 66 47 113t113 47h1344q66 0 113 -47t47 -113z" /> +<glyph unicode="" horiz-adv-x="1024" d="M1024 448q0 -26 -19 -45l-448 -448q-19 -19 -45 -19t-45 19l-448 448q-19 19 -19 45t19 45t45 19h896q26 0 45 -19t19 -45zM1024 832q0 -26 -19 -45t-45 -19h-896q-26 0 -45 19t-19 45t19 45l448 448q19 19 45 19t45 -19l448 -448q19 -19 19 -45z" /> +<glyph unicode="" horiz-adv-x="1024" d="M1024 448q0 -26 -19 -45l-448 -448q-19 -19 -45 -19t-45 19l-448 448q-19 19 -19 45t19 45t45 19h896q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1024" d="M1024 832q0 -26 -19 -45t-45 -19h-896q-26 0 -45 19t-19 45t19 45l448 448q19 19 45 19t45 -19l448 -448q19 -19 19 -45z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 826v-794q0 -66 -47 -113t-113 -47h-1472q-66 0 -113 47t-47 113v794q44 -49 101 -87q362 -246 497 -345q57 -42 92.5 -65.5t94.5 -48t110 -24.5h1h1q51 0 110 24.5t94.5 48t92.5 65.5q170 123 498 345q57 39 100 87zM1792 1120q0 -79 -49 -151t-122 -123 q-376 -261 -468 -325q-10 -7 -42.5 -30.5t-54 -38t-52 -32.5t-57.5 -27t-50 -9h-1h-1q-23 0 -50 9t-57.5 27t-52 32.5t-54 38t-42.5 30.5q-91 64 -262 182.5t-205 142.5q-62 42 -117 115.5t-55 136.5q0 78 41.5 130t118.5 52h1472q65 0 112.5 -47t47.5 -113z" /> +<glyph unicode="" d="M349 911v-991h-330v991h330zM370 1217q1 -73 -50.5 -122t-135.5 -49h-2q-82 0 -132 49t-50 122q0 74 51.5 122.5t134.5 48.5t133 -48.5t51 -122.5zM1536 488v-568h-329v530q0 105 -40.5 164.5t-126.5 59.5q-63 0 -105.5 -34.5t-63.5 -85.5q-11 -30 -11 -81v-553h-329 q2 399 2 647t-1 296l-1 48h329v-144h-2q20 32 41 56t56.5 52t87 43.5t114.5 15.5q171 0 275 -113.5t104 -332.5z" /> +<glyph unicode="" d="M1536 640q0 -156 -61 -298t-164 -245t-245 -164t-298 -61q-172 0 -327 72.5t-264 204.5q-7 10 -6.5 22.5t8.5 20.5l137 138q10 9 25 9q16 -2 23 -12q73 -95 179 -147t225 -52q104 0 198.5 40.5t163.5 109.5t109.5 163.5t40.5 198.5t-40.5 198.5t-109.5 163.5 t-163.5 109.5t-198.5 40.5q-98 0 -188 -35.5t-160 -101.5l137 -138q31 -30 14 -69q-17 -40 -59 -40h-448q-26 0 -45 19t-19 45v448q0 42 40 59q39 17 69 -14l130 -129q107 101 244.5 156.5t284.5 55.5q156 0 298 -61t245 -164t164 -245t61 -298z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1771 0q0 -53 -37 -90l-107 -108q-39 -37 -91 -37q-53 0 -90 37l-363 364q-38 36 -38 90q0 53 43 96l-256 256l-126 -126q-14 -14 -34 -14t-34 14q2 -2 12.5 -12t12.5 -13t10 -11.5t10 -13.5t6 -13.5t5.5 -16.5t1.5 -18q0 -38 -28 -68q-3 -3 -16.5 -18t-19 -20.5 t-18.5 -16.5t-22 -15.5t-22 -9t-26 -4.5q-40 0 -68 28l-408 408q-28 28 -28 68q0 13 4.5 26t9 22t15.5 22t16.5 18.5t20.5 19t18 16.5q30 28 68 28q10 0 18 -1.5t16.5 -5.5t13.5 -6t13.5 -10t11.5 -10t13 -12.5t12 -12.5q-14 14 -14 34t14 34l348 348q14 14 34 14t34 -14 q-2 2 -12.5 12t-12.5 13t-10 11.5t-10 13.5t-6 13.5t-5.5 16.5t-1.5 18q0 38 28 68q3 3 16.5 18t19 20.5t18.5 16.5t22 15.5t22 9t26 4.5q40 0 68 -28l408 -408q28 -28 28 -68q0 -13 -4.5 -26t-9 -22t-15.5 -22t-16.5 -18.5t-20.5 -19t-18 -16.5q-30 -28 -68 -28 q-10 0 -18 1.5t-16.5 5.5t-13.5 6t-13.5 10t-11.5 10t-13 12.5t-12 12.5q14 -14 14 -34t-14 -34l-126 -126l256 -256q43 43 96 43q52 0 91 -37l363 -363q37 -39 37 -91z" /> +<glyph unicode="" horiz-adv-x="1792" d="M384 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM576 832q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1004 351l101 382q6 26 -7.5 48.5t-38.5 29.5 t-48 -6.5t-30 -39.5l-101 -382q-60 -5 -107 -43.5t-63 -98.5q-20 -77 20 -146t117 -89t146 20t89 117q16 60 -6 117t-72 91zM1664 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1024 1024q0 53 -37.5 90.5 t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1472 832q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1792 384q0 -261 -141 -483q-19 -29 -54 -29h-1402q-35 0 -54 29 q-141 221 -141 483q0 182 71 348t191 286t286 191t348 71t348 -71t286 -191t191 -286t71 -348z" /> +<glyph unicode="" horiz-adv-x="1792" d="M896 1152q-204 0 -381.5 -69.5t-282 -187.5t-104.5 -255q0 -112 71.5 -213.5t201.5 -175.5l87 -50l-27 -96q-24 -91 -70 -172q152 63 275 171l43 38l57 -6q69 -8 130 -8q204 0 381.5 69.5t282 187.5t104.5 255t-104.5 255t-282 187.5t-381.5 69.5zM1792 640 q0 -174 -120 -321.5t-326 -233t-450 -85.5q-70 0 -145 8q-198 -175 -460 -242q-49 -14 -114 -22h-5q-15 0 -27 10.5t-16 27.5v1q-3 4 -0.5 12t2 10t4.5 9.5l6 9t7 8.5t8 9q7 8 31 34.5t34.5 38t31 39.5t32.5 51t27 59t26 76q-157 89 -247.5 220t-90.5 281q0 174 120 321.5 t326 233t450 85.5t450 -85.5t326 -233t120 -321.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M704 1152q-153 0 -286 -52t-211.5 -141t-78.5 -191q0 -82 53 -158t149 -132l97 -56l-35 -84q34 20 62 39l44 31l53 -10q78 -14 153 -14q153 0 286 52t211.5 141t78.5 191t-78.5 191t-211.5 141t-286 52zM704 1280q191 0 353.5 -68.5t256.5 -186.5t94 -257t-94 -257 t-256.5 -186.5t-353.5 -68.5q-86 0 -176 16q-124 -88 -278 -128q-36 -9 -86 -16h-3q-11 0 -20.5 8t-11.5 21q-1 3 -1 6.5t0.5 6.5t2 6l2.5 5t3.5 5.5t4 5t4.5 5t4 4.5q5 6 23 25t26 29.5t22.5 29t25 38.5t20.5 44q-124 72 -195 177t-71 224q0 139 94 257t256.5 186.5 t353.5 68.5zM1526 111q10 -24 20.5 -44t25 -38.5t22.5 -29t26 -29.5t23 -25q1 -1 4 -4.5t4.5 -5t4 -5t3.5 -5.5l2.5 -5t2 -6t0.5 -6.5t-1 -6.5q-3 -14 -13 -22t-22 -7q-50 7 -86 16q-154 40 -278 128q-90 -16 -176 -16q-271 0 -472 132q58 -4 88 -4q161 0 309 45t264 129 q125 92 192 212t67 254q0 77 -23 152q129 -71 204 -178t75 -230q0 -120 -71 -224.5t-195 -176.5z" /> +<glyph unicode="" horiz-adv-x="896" d="M885 970q18 -20 7 -44l-540 -1157q-13 -25 -42 -25q-4 0 -14 2q-17 5 -25.5 19t-4.5 30l197 808l-406 -101q-4 -1 -12 -1q-18 0 -31 11q-18 15 -13 39l201 825q4 14 16 23t28 9h328q19 0 32 -12.5t13 -29.5q0 -8 -5 -18l-171 -463l396 98q8 2 12 2q19 0 34 -15z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 288v-320q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h96v192h-512v-192h96q40 0 68 -28t28 -68v-320q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h96v192h-512v-192h96q40 0 68 -28t28 -68v-320 q0 -40 -28 -68t-68 -28h-320q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h96v192q0 52 38 90t90 38h512v192h-96q-40 0 -68 28t-28 68v320q0 40 28 68t68 28h320q40 0 68 -28t28 -68v-320q0 -40 -28 -68t-68 -28h-96v-192h512q52 0 90 -38t38 -90v-192h96q40 0 68 -28t28 -68 z" /> +<glyph unicode="" horiz-adv-x="1664" d="M896 708v-580q0 -104 -76 -180t-180 -76t-180 76t-76 180q0 26 19 45t45 19t45 -19t19 -45q0 -50 39 -89t89 -39t89 39t39 89v580q33 11 64 11t64 -11zM1664 681q0 -13 -9.5 -22.5t-22.5 -9.5q-11 0 -23 10q-49 46 -93 69t-102 23q-68 0 -128 -37t-103 -97 q-7 -10 -17.5 -28t-14.5 -24q-11 -17 -28 -17q-18 0 -29 17q-4 6 -14.5 24t-17.5 28q-43 60 -102.5 97t-127.5 37t-127.5 -37t-102.5 -97q-7 -10 -17.5 -28t-14.5 -24q-11 -17 -29 -17q-17 0 -28 17q-4 6 -14.5 24t-17.5 28q-43 60 -103 97t-128 37q-58 0 -102 -23t-93 -69 q-12 -10 -23 -10q-13 0 -22.5 9.5t-9.5 22.5q0 5 1 7q45 183 172.5 319.5t298 204.5t360.5 68q140 0 274.5 -40t246.5 -113.5t194.5 -187t115.5 -251.5q1 -2 1 -7zM896 1408v-98q-42 2 -64 2t-64 -2v98q0 26 19 45t45 19t45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1792" d="M768 -128h896v640h-416q-40 0 -68 28t-28 68v416h-384v-1152zM1024 1312v64q0 13 -9.5 22.5t-22.5 9.5h-704q-13 0 -22.5 -9.5t-9.5 -22.5v-64q0 -13 9.5 -22.5t22.5 -9.5h704q13 0 22.5 9.5t9.5 22.5zM1280 640h299l-299 299v-299zM1792 512v-672q0 -40 -28 -68t-68 -28 h-960q-40 0 -68 28t-28 68v160h-544q-40 0 -68 28t-28 68v1344q0 40 28 68t68 28h1088q40 0 68 -28t28 -68v-328q21 -13 36 -28l408 -408q28 -28 48 -76t20 -88z" /> +<glyph unicode="" horiz-adv-x="1024" d="M736 960q0 -13 -9.5 -22.5t-22.5 -9.5t-22.5 9.5t-9.5 22.5q0 46 -54 71t-106 25q-13 0 -22.5 9.5t-9.5 22.5t9.5 22.5t22.5 9.5q50 0 99.5 -16t87 -54t37.5 -90zM896 960q0 72 -34.5 134t-90 101.5t-123 62t-136.5 22.5t-136.5 -22.5t-123 -62t-90 -101.5t-34.5 -134 q0 -101 68 -180q10 -11 30.5 -33t30.5 -33q128 -153 141 -298h228q13 145 141 298q10 11 30.5 33t30.5 33q68 79 68 180zM1024 960q0 -155 -103 -268q-45 -49 -74.5 -87t-59.5 -95.5t-34 -107.5q47 -28 47 -82q0 -37 -25 -64q25 -27 25 -64q0 -52 -45 -81q13 -23 13 -47 q0 -46 -31.5 -71t-77.5 -25q-20 -44 -60 -70t-87 -26t-87 26t-60 70q-46 0 -77.5 25t-31.5 71q0 24 13 47q-45 29 -45 81q0 37 25 64q-25 27 -25 64q0 54 47 82q-4 50 -34 107.5t-59.5 95.5t-74.5 87q-103 113 -103 268q0 99 44.5 184.5t117 142t164 89t186.5 32.5 t186.5 -32.5t164 -89t117 -142t44.5 -184.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 352v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-1376v-192q0 -13 -9.5 -22.5t-22.5 -9.5q-12 0 -24 10l-319 320q-9 9 -9 22q0 14 9 23l320 320q9 9 23 9q13 0 22.5 -9.5t9.5 -22.5v-192h1376q13 0 22.5 -9.5t9.5 -22.5zM1792 896q0 -14 -9 -23l-320 -320q-9 -9 -23 -9 q-13 0 -22.5 9.5t-9.5 22.5v192h-1376q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h1376v192q0 14 9 23t23 9q12 0 24 -10l319 -319q9 -9 9 -23z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1280 608q0 14 -9 23t-23 9h-224v352q0 13 -9.5 22.5t-22.5 9.5h-192q-13 0 -22.5 -9.5t-9.5 -22.5v-352h-224q-13 0 -22.5 -9.5t-9.5 -22.5q0 -14 9 -23l352 -352q9 -9 23 -9t23 9l351 351q10 12 10 24zM1920 384q0 -159 -112.5 -271.5t-271.5 -112.5h-1088 q-185 0 -316.5 131.5t-131.5 316.5q0 130 70 240t188 165q-2 30 -2 43q0 212 150 362t362 150q156 0 285.5 -87t188.5 -231q71 62 166 62q106 0 181 -75t75 -181q0 -76 -41 -138q130 -31 213.5 -135.5t83.5 -238.5z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1280 672q0 14 -9 23l-352 352q-9 9 -23 9t-23 -9l-351 -351q-10 -12 -10 -24q0 -14 9 -23t23 -9h224v-352q0 -13 9.5 -22.5t22.5 -9.5h192q13 0 22.5 9.5t9.5 22.5v352h224q13 0 22.5 9.5t9.5 22.5zM1920 384q0 -159 -112.5 -271.5t-271.5 -112.5h-1088 q-185 0 -316.5 131.5t-131.5 316.5q0 130 70 240t188 165q-2 30 -2 43q0 212 150 362t362 150q156 0 285.5 -87t188.5 -231q71 62 166 62q106 0 181 -75t75 -181q0 -76 -41 -138q130 -31 213.5 -135.5t83.5 -238.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M384 192q0 -26 -19 -45t-45 -19t-45 19t-19 45t19 45t45 19t45 -19t19 -45zM1408 131q0 -121 -73 -190t-194 -69h-874q-121 0 -194 69t-73 190q0 68 5.5 131t24 138t47.5 132.5t81 103t120 60.5q-22 -52 -22 -120v-203q-58 -20 -93 -70t-35 -111q0 -80 56 -136t136 -56 t136 56t56 136q0 61 -35.5 111t-92.5 70v203q0 62 25 93q132 -104 295 -104t295 104q25 -31 25 -93v-64q-106 0 -181 -75t-75 -181v-89q-32 -29 -32 -71q0 -40 28 -68t68 -28t68 28t28 68q0 42 -32 71v89q0 52 38 90t90 38t90 -38t38 -90v-89q-32 -29 -32 -71q0 -40 28 -68 t68 -28t68 28t28 68q0 42 -32 71v89q0 68 -34.5 127.5t-93.5 93.5q0 10 0.5 42.5t0 48t-2.5 41.5t-7 47t-13 40q68 -15 120 -60.5t81 -103t47.5 -132.5t24 -138t5.5 -131zM1088 1024q0 -159 -112.5 -271.5t-271.5 -112.5t-271.5 112.5t-112.5 271.5t112.5 271.5t271.5 112.5 t271.5 -112.5t112.5 -271.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1280 832q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1408 832q0 -62 -35.5 -111t-92.5 -70v-395q0 -159 -131.5 -271.5t-316.5 -112.5t-316.5 112.5t-131.5 271.5v132q-164 20 -274 128t-110 252v512q0 26 19 45t45 19q6 0 16 -2q17 30 47 48 t65 18q53 0 90.5 -37.5t37.5 -90.5t-37.5 -90.5t-90.5 -37.5q-33 0 -64 18v-402q0 -106 94 -181t226 -75t226 75t94 181v402q-31 -18 -64 -18q-53 0 -90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5q35 0 65 -18t47 -48q10 2 16 2q26 0 45 -19t19 -45v-512q0 -144 -110 -252 t-274 -128v-132q0 -106 94 -181t226 -75t226 75t94 181v395q-57 21 -92.5 70t-35.5 111q0 80 56 136t136 56t136 -56t56 -136z" /> +<glyph unicode="" horiz-adv-x="1792" d="M640 1152h512v128h-512v-128zM288 1152v-1280h-64q-92 0 -158 66t-66 158v832q0 92 66 158t158 66h64zM1408 1152v-1280h-1024v1280h128v160q0 40 28 68t68 28h576q40 0 68 -28t28 -68v-160h128zM1792 928v-832q0 -92 -66 -158t-158 -66h-64v1280h64q92 0 158 -66 t66 -158z" /> +<glyph unicode="" horiz-adv-x="1792" d="M912 -160q0 16 -16 16q-59 0 -101.5 42.5t-42.5 101.5q0 16 -16 16t-16 -16q0 -73 51.5 -124.5t124.5 -51.5q16 0 16 16zM1728 128q0 -52 -38 -90t-90 -38h-448q0 -106 -75 -181t-181 -75t-181 75t-75 181h-448q-52 0 -90 38t-38 90q50 42 91 88t85 119.5t74.5 158.5 t50 206t19.5 260q0 152 117 282.5t307 158.5q-8 19 -8 39q0 40 28 68t68 28t68 -28t28 -68q0 -20 -8 -39q190 -28 307 -158.5t117 -282.5q0 -139 19.5 -260t50 -206t74.5 -158.5t85 -119.5t91 -88z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1664 896q0 80 -56 136t-136 56h-64v-384h64q80 0 136 56t56 136zM0 128h1792q0 -106 -75 -181t-181 -75h-1280q-106 0 -181 75t-75 181zM1856 896q0 -159 -112.5 -271.5t-271.5 -112.5h-64v-32q0 -92 -66 -158t-158 -66h-704q-92 0 -158 66t-66 158v736q0 26 19 45 t45 19h1152q159 0 271.5 -112.5t112.5 -271.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M640 1472v-640q0 -61 -35.5 -111t-92.5 -70v-779q0 -52 -38 -90t-90 -38h-128q-52 0 -90 38t-38 90v779q-57 20 -92.5 70t-35.5 111v640q0 26 19 45t45 19t45 -19t19 -45v-416q0 -26 19 -45t45 -19t45 19t19 45v416q0 26 19 45t45 19t45 -19t19 -45v-416q0 -26 19 -45 t45 -19t45 19t19 45v416q0 26 19 45t45 19t45 -19t19 -45zM1408 1472v-1600q0 -52 -38 -90t-90 -38h-128q-52 0 -90 38t-38 90v512h-224q-13 0 -22.5 9.5t-9.5 22.5v800q0 132 94 226t226 94h256q26 0 45 -19t19 -45z" /> +<glyph unicode="" d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z M384 736q0 14 9 23t23 9h704q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-704q-14 0 -23 9t-9 23v64zM1120 512q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-704q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h704zM1120 256q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-704 q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h704z" /> +<glyph unicode="" horiz-adv-x="1408" d="M384 224v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M640 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M1152 224v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM896 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M640 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 992v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M1152 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM896 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M640 992v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 1248v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M1152 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM896 992v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M640 1248v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM1152 992v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M896 1248v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM1152 1248v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M896 -128h384v1536h-1152v-1536h384v224q0 13 9.5 22.5t22.5 9.5h320q13 0 22.5 -9.5t9.5 -22.5v-224zM1408 1472v-1664q0 -26 -19 -45t-45 -19h-1280q-26 0 -45 19t-19 45v1664q0 26 19 45t45 19h1280q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1408" d="M384 224v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M640 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM384 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M1152 224v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM896 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M640 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM1152 480v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M896 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5zM1152 736v-64q0 -13 -9.5 -22.5t-22.5 -9.5h-64q-13 0 -22.5 9.5t-9.5 22.5v64q0 13 9.5 22.5t22.5 9.5h64q13 0 22.5 -9.5t9.5 -22.5z M896 -128h384v1152h-256v-32q0 -40 -28 -68t-68 -28h-448q-40 0 -68 28t-28 68v32h-256v-1152h384v224q0 13 9.5 22.5t22.5 9.5h320q13 0 22.5 -9.5t9.5 -22.5v-224zM896 1056v320q0 13 -9.5 22.5t-22.5 9.5h-64q-13 0 -22.5 -9.5t-9.5 -22.5v-96h-128v96q0 13 -9.5 22.5 t-22.5 9.5h-64q-13 0 -22.5 -9.5t-9.5 -22.5v-320q0 -13 9.5 -22.5t22.5 -9.5h64q13 0 22.5 9.5t9.5 22.5v96h128v-96q0 -13 9.5 -22.5t22.5 -9.5h64q13 0 22.5 9.5t9.5 22.5zM1408 1088v-1280q0 -26 -19 -45t-45 -19h-1280q-26 0 -45 19t-19 45v1280q0 26 19 45t45 19h320 v288q0 40 28 68t68 28h448q40 0 68 -28t28 -68v-288h320q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1920" d="M640 128q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM256 640h384v256h-158q-14 -2 -22 -9l-195 -195q-7 -12 -9 -22v-30zM1536 128q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5 t90.5 37.5t37.5 90.5zM1664 800v192q0 14 -9 23t-23 9h-224v224q0 14 -9 23t-23 9h-192q-14 0 -23 -9t-9 -23v-224h-224q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h224v-224q0 -14 9 -23t23 -9h192q14 0 23 9t9 23v224h224q14 0 23 9t9 23zM1920 1344v-1152 q0 -26 -19 -45t-45 -19h-192q0 -106 -75 -181t-181 -75t-181 75t-75 181h-384q0 -106 -75 -181t-181 -75t-181 75t-75 181h-128q-26 0 -45 19t-19 45t19 45t45 19v416q0 26 13 58t32 51l198 198q19 19 51 32t58 13h160v320q0 26 19 45t45 19h1152q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1280 416v192q0 14 -9 23t-23 9h-224v224q0 14 -9 23t-23 9h-192q-14 0 -23 -9t-9 -23v-224h-224q-14 0 -23 -9t-9 -23v-192q0 -14 9 -23t23 -9h224v-224q0 -14 9 -23t23 -9h192q14 0 23 9t9 23v224h224q14 0 23 9t9 23zM640 1152h512v128h-512v-128zM256 1152v-1280h-32 q-92 0 -158 66t-66 158v832q0 92 66 158t158 66h32zM1440 1152v-1280h-1088v1280h160v160q0 40 28 68t68 28h576q40 0 68 -28t28 -68v-160h160zM1792 928v-832q0 -92 -66 -158t-158 -66h-32v1280h32q92 0 158 -66t66 -158z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1920 576q-1 -32 -288 -96l-352 -32l-224 -64h-64l-293 -352h69q26 0 45 -4.5t19 -11.5t-19 -11.5t-45 -4.5h-96h-160h-64v32h64v416h-160l-192 -224h-96l-32 32v192h32v32h128v8l-192 24v128l192 24v8h-128v32h-32v192l32 32h96l192 -224h160v416h-64v32h64h160h96 q26 0 45 -4.5t19 -11.5t-19 -11.5t-45 -4.5h-69l293 -352h64l224 -64l352 -32q261 -58 287 -93z" /> +<glyph unicode="" horiz-adv-x="1664" d="M640 640v384h-256v-256q0 -53 37.5 -90.5t90.5 -37.5h128zM1664 192v-192h-1152v192l128 192h-128q-159 0 -271.5 112.5t-112.5 271.5v320l-64 64l32 128h480l32 128h960l32 -192l-64 -32v-800z" /> +<glyph unicode="" d="M1280 192v896q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-320h-512v320q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-896q0 -26 19 -45t45 -19h128q26 0 45 19t19 45v320h512v-320q0 -26 19 -45t45 -19h128q26 0 45 19t19 45zM1536 1120v-960 q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M1280 576v128q0 26 -19 45t-45 19h-320v320q0 26 -19 45t-45 19h-128q-26 0 -45 -19t-19 -45v-320h-320q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h320v-320q0 -26 19 -45t45 -19h128q26 0 45 19t19 45v320h320q26 0 45 19t19 45zM1536 1120v-960 q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1024" d="M627 160q0 -13 -10 -23l-50 -50q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l50 -50q10 -10 10 -23t-10 -23l-393 -393l393 -393q10 -10 10 -23zM1011 160q0 -13 -10 -23l-50 -50q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23 t10 23l466 466q10 10 23 10t23 -10l50 -50q10 -10 10 -23t-10 -23l-393 -393l393 -393q10 -10 10 -23z" /> +<glyph unicode="" horiz-adv-x="1024" d="M595 576q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l393 393l-393 393q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l466 -466q10 -10 10 -23zM979 576q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23 l393 393l-393 393q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l466 -466q10 -10 10 -23z" /> +<glyph unicode="" horiz-adv-x="1152" d="M1075 224q0 -13 -10 -23l-50 -50q-10 -10 -23 -10t-23 10l-393 393l-393 -393q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l466 -466q10 -10 10 -23zM1075 608q0 -13 -10 -23l-50 -50q-10 -10 -23 -10t-23 10l-393 393l-393 -393 q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l466 -466q10 -10 10 -23z" /> +<glyph unicode="" horiz-adv-x="1152" d="M1075 672q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l393 -393l393 393q10 10 23 10t23 -10l50 -50q10 -10 10 -23zM1075 1056q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23 t10 23l50 50q10 10 23 10t23 -10l393 -393l393 393q10 10 23 10t23 -10l50 -50q10 -10 10 -23z" /> +<glyph unicode="" horiz-adv-x="640" d="M627 992q0 -13 -10 -23l-393 -393l393 -393q10 -10 10 -23t-10 -23l-50 -50q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l50 -50q10 -10 10 -23z" /> +<glyph unicode="" horiz-adv-x="640" d="M595 576q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l393 393l-393 393q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l466 -466q10 -10 10 -23z" /> +<glyph unicode="" horiz-adv-x="1152" d="M1075 352q0 -13 -10 -23l-50 -50q-10 -10 -23 -10t-23 10l-393 393l-393 -393q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l466 -466q10 -10 10 -23z" /> +<glyph unicode="" horiz-adv-x="1152" d="M1075 800q0 -13 -10 -23l-466 -466q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l393 -393l393 393q10 10 23 10t23 -10l50 -50q10 -10 10 -23z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1792 544v832q0 13 -9.5 22.5t-22.5 9.5h-1600q-13 0 -22.5 -9.5t-9.5 -22.5v-832q0 -13 9.5 -22.5t22.5 -9.5h1600q13 0 22.5 9.5t9.5 22.5zM1920 1376v-1088q0 -66 -47 -113t-113 -47h-544q0 -37 16 -77.5t32 -71t16 -43.5q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19 t-19 45q0 14 16 44t32 70t16 78h-544q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h1600q66 0 113 -47t47 -113z" /> +<glyph unicode="" horiz-adv-x="1920" d="M416 256q-66 0 -113 47t-47 113v704q0 66 47 113t113 47h1088q66 0 113 -47t47 -113v-704q0 -66 -47 -113t-113 -47h-1088zM384 1120v-704q0 -13 9.5 -22.5t22.5 -9.5h1088q13 0 22.5 9.5t9.5 22.5v704q0 13 -9.5 22.5t-22.5 9.5h-1088q-13 0 -22.5 -9.5t-9.5 -22.5z M1760 192h160v-96q0 -40 -47 -68t-113 -28h-1600q-66 0 -113 28t-47 68v96h160h1600zM1040 96q16 0 16 16t-16 16h-160q-16 0 -16 -16t16 -16h160z" /> +<glyph unicode="" horiz-adv-x="1152" d="M640 128q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1024 288v960q0 13 -9.5 22.5t-22.5 9.5h-832q-13 0 -22.5 -9.5t-9.5 -22.5v-960q0 -13 9.5 -22.5t22.5 -9.5h832q13 0 22.5 9.5t9.5 22.5zM1152 1248v-1088q0 -66 -47 -113t-113 -47h-832 q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h832q66 0 113 -47t47 -113z" /> +<glyph unicode="" horiz-adv-x="768" d="M464 128q0 33 -23.5 56.5t-56.5 23.5t-56.5 -23.5t-23.5 -56.5t23.5 -56.5t56.5 -23.5t56.5 23.5t23.5 56.5zM672 288v704q0 13 -9.5 22.5t-22.5 9.5h-512q-13 0 -22.5 -9.5t-9.5 -22.5v-704q0 -13 9.5 -22.5t22.5 -9.5h512q13 0 22.5 9.5t9.5 22.5zM480 1136 q0 16 -16 16h-160q-16 0 -16 -16t16 -16h160q16 0 16 16zM768 1152v-1024q0 -52 -38 -90t-90 -38h-512q-52 0 -90 38t-38 90v1024q0 52 38 90t90 38h512q52 0 90 -38t38 -90z" /> +<glyph unicode="" d="M768 1184q-148 0 -273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273t-73 273t-198 198t-273 73zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103 t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M768 576v-384q0 -80 -56 -136t-136 -56h-384q-80 0 -136 56t-56 136v704q0 104 40.5 198.5t109.5 163.5t163.5 109.5t198.5 40.5h64q26 0 45 -19t19 -45v-128q0 -26 -19 -45t-45 -19h-64q-106 0 -181 -75t-75 -181v-32q0 -40 28 -68t68 -28h224q80 0 136 -56t56 -136z M1664 576v-384q0 -80 -56 -136t-136 -56h-384q-80 0 -136 56t-56 136v704q0 104 40.5 198.5t109.5 163.5t163.5 109.5t198.5 40.5h64q26 0 45 -19t19 -45v-128q0 -26 -19 -45t-45 -19h-64q-106 0 -181 -75t-75 -181v-32q0 -40 28 -68t68 -28h224q80 0 136 -56t56 -136z" /> +<glyph unicode="" horiz-adv-x="1664" d="M768 1216v-704q0 -104 -40.5 -198.5t-109.5 -163.5t-163.5 -109.5t-198.5 -40.5h-64q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h64q106 0 181 75t75 181v32q0 40 -28 68t-68 28h-224q-80 0 -136 56t-56 136v384q0 80 56 136t136 56h384q80 0 136 -56t56 -136zM1664 1216 v-704q0 -104 -40.5 -198.5t-109.5 -163.5t-163.5 -109.5t-198.5 -40.5h-64q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h64q106 0 181 75t75 181v32q0 40 -28 68t-68 28h-224q-80 0 -136 56t-56 136v384q0 80 56 136t136 56h384q80 0 136 -56t56 -136z" /> +<glyph unicode="" horiz-adv-x="1568" d="M496 192q0 -60 -42.5 -102t-101.5 -42q-60 0 -102 42t-42 102t42 102t102 42q59 0 101.5 -42t42.5 -102zM928 0q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM320 640q0 -66 -47 -113t-113 -47t-113 47t-47 113 t47 113t113 47t113 -47t47 -113zM1360 192q0 -46 -33 -79t-79 -33t-79 33t-33 79t33 79t79 33t79 -33t33 -79zM528 1088q0 -73 -51.5 -124.5t-124.5 -51.5t-124.5 51.5t-51.5 124.5t51.5 124.5t124.5 51.5t124.5 -51.5t51.5 -124.5zM992 1280q0 -80 -56 -136t-136 -56 t-136 56t-56 136t56 136t136 56t136 -56t56 -136zM1536 640q0 -40 -28 -68t-68 -28t-68 28t-28 68t28 68t68 28t68 -28t28 -68zM1328 1088q0 -33 -23.5 -56.5t-56.5 -23.5t-56.5 23.5t-23.5 56.5t23.5 56.5t56.5 23.5t56.5 -23.5t23.5 -56.5z" /> +<glyph unicode="" d="M1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 416q0 -166 -127 -451q-3 -7 -10.5 -24t-13.5 -30t-13 -22q-12 -17 -28 -17q-15 0 -23.5 10t-8.5 25q0 9 2.5 26.5t2.5 23.5q5 68 5 123q0 101 -17.5 181t-48.5 138.5t-80 101t-105.5 69.5t-133 42.5t-154 21.5t-175.5 6h-224v-256q0 -26 -19 -45t-45 -19t-45 19 l-512 512q-19 19 -19 45t19 45l512 512q19 19 45 19t45 -19t19 -45v-256h224q713 0 875 -403q53 -134 53 -333z" /> +<glyph unicode="" horiz-adv-x="1664" d="M640 320q0 -40 -12.5 -82t-43 -76t-72.5 -34t-72.5 34t-43 76t-12.5 82t12.5 82t43 76t72.5 34t72.5 -34t43 -76t12.5 -82zM1280 320q0 -40 -12.5 -82t-43 -76t-72.5 -34t-72.5 34t-43 76t-12.5 82t12.5 82t43 76t72.5 34t72.5 -34t43 -76t12.5 -82zM1440 320 q0 120 -69 204t-187 84q-41 0 -195 -21q-71 -11 -157 -11t-157 11q-152 21 -195 21q-118 0 -187 -84t-69 -204q0 -88 32 -153.5t81 -103t122 -60t140 -29.5t149 -7h168q82 0 149 7t140 29.5t122 60t81 103t32 153.5zM1664 496q0 -207 -61 -331q-38 -77 -105.5 -133t-141 -86 t-170 -47.5t-171.5 -22t-167 -4.5q-78 0 -142 3t-147.5 12.5t-152.5 30t-137 51.5t-121 81t-86 115q-62 123 -62 331q0 237 136 396q-27 82 -27 170q0 116 51 218q108 0 190 -39.5t189 -123.5q147 35 309 35q148 0 280 -32q105 82 187 121t189 39q51 -102 51 -218 q0 -87 -27 -168q136 -160 136 -398z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1536 224v704q0 40 -28 68t-68 28h-704q-40 0 -68 28t-28 68v64q0 40 -28 68t-68 28h-320q-40 0 -68 -28t-28 -68v-960q0 -40 28 -68t68 -28h1216q40 0 68 28t28 68zM1664 928v-704q0 -92 -66 -158t-158 -66h-1216q-92 0 -158 66t-66 158v960q0 92 66 158t158 66h320 q92 0 158 -66t66 -158v-32h672q92 0 158 -66t66 -158z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1781 605q0 35 -53 35h-1088q-40 0 -85.5 -21.5t-71.5 -52.5l-294 -363q-18 -24 -18 -40q0 -35 53 -35h1088q40 0 86 22t71 53l294 363q18 22 18 39zM640 768h768v160q0 40 -28 68t-68 28h-576q-40 0 -68 28t-28 68v64q0 40 -28 68t-68 28h-320q-40 0 -68 -28t-28 -68 v-853l256 315q44 53 116 87.5t140 34.5zM1909 605q0 -62 -46 -120l-295 -363q-43 -53 -116 -87.5t-140 -34.5h-1088q-92 0 -158 66t-66 158v960q0 92 66 158t158 66h320q92 0 158 -66t66 -158v-32h544q92 0 158 -66t66 -158v-160h192q54 0 99 -24.5t67 -70.5q15 -32 15 -68z " /> +<glyph unicode="" horiz-adv-x="1792" /> +<glyph unicode="" horiz-adv-x="1792" /> +<glyph unicode="" d="M1134 461q-37 -121 -138 -195t-228 -74t-228 74t-138 195q-8 25 4 48.5t38 31.5q25 8 48.5 -4t31.5 -38q25 -80 92.5 -129.5t151.5 -49.5t151.5 49.5t92.5 129.5q8 26 32 38t49 4t37 -31.5t4 -48.5zM640 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5 t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1152 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1408 640q0 130 -51 248.5t-136.5 204t-204 136.5t-248.5 51t-248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5 t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1134 307q8 -25 -4 -48.5t-37 -31.5t-49 4t-32 38q-25 80 -92.5 129.5t-151.5 49.5t-151.5 -49.5t-92.5 -129.5q-8 -26 -31.5 -38t-48.5 -4q-26 8 -38 31.5t-4 48.5q37 121 138 195t228 74t228 -74t138 -195zM640 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5 t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1152 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1408 640q0 130 -51 248.5t-136.5 204t-204 136.5t-248.5 51t-248.5 -51t-204 -136.5t-136.5 -204 t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1152 448q0 -26 -19 -45t-45 -19h-640q-26 0 -45 19t-19 45t19 45t45 19h640q26 0 45 -19t19 -45zM640 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1152 896q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5 t-37.5 90.5t37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1408 640q0 130 -51 248.5t-136.5 204t-204 136.5t-248.5 51t-248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5zM1536 640 q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1920" d="M832 448v128q0 14 -9 23t-23 9h-192v192q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-192h-192q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h192v-192q0 -14 9 -23t23 -9h128q14 0 23 9t9 23v192h192q14 0 23 9t9 23zM1408 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5 t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1664 640q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1920 512q0 -212 -150 -362t-362 -150q-192 0 -338 128h-220q-146 -128 -338 -128q-212 0 -362 150 t-150 362t150 362t362 150h896q212 0 362 -150t150 -362z" /> +<glyph unicode="" horiz-adv-x="1920" d="M384 368v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM512 624v-96q0 -16 -16 -16h-224q-16 0 -16 16v96q0 16 16 16h224q16 0 16 -16zM384 880v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1408 368v-96q0 -16 -16 -16 h-864q-16 0 -16 16v96q0 16 16 16h864q16 0 16 -16zM768 624v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM640 880v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1024 624v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16 h96q16 0 16 -16zM896 880v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1280 624v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1664 368v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1152 880v-96 q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1408 880v-96q0 -16 -16 -16h-96q-16 0 -16 16v96q0 16 16 16h96q16 0 16 -16zM1664 880v-352q0 -16 -16 -16h-224q-16 0 -16 16v96q0 16 16 16h112v240q0 16 16 16h96q16 0 16 -16zM1792 128v896h-1664v-896 h1664zM1920 1024v-896q0 -53 -37.5 -90.5t-90.5 -37.5h-1664q-53 0 -90.5 37.5t-37.5 90.5v896q0 53 37.5 90.5t90.5 37.5h1664q53 0 90.5 -37.5t37.5 -90.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1664 491v616q-169 -91 -306 -91q-82 0 -145 32q-100 49 -184 76.5t-178 27.5q-173 0 -403 -127v-599q245 113 433 113q55 0 103.5 -7.5t98 -26t77 -31t82.5 -39.5l28 -14q44 -22 101 -22q120 0 293 92zM320 1280q0 -35 -17.5 -64t-46.5 -46v-1266q0 -14 -9 -23t-23 -9 h-64q-14 0 -23 9t-9 23v1266q-29 17 -46.5 46t-17.5 64q0 53 37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1792 1216v-763q0 -39 -35 -57q-10 -5 -17 -9q-218 -116 -369 -116q-88 0 -158 35l-28 14q-64 33 -99 48t-91 29t-114 14q-102 0 -235.5 -44t-228.5 -102 q-15 -9 -33 -9q-16 0 -32 8q-32 19 -32 56v742q0 35 31 55q35 21 78.5 42.5t114 52t152.5 49.5t155 19q112 0 209 -31t209 -86q38 -19 89 -19q122 0 310 112q22 12 31 17q31 16 62 -2q31 -20 31 -55z" /> +<glyph unicode="" horiz-adv-x="1792" d="M832 536v192q-181 -16 -384 -117v-185q205 96 384 110zM832 954v197q-172 -8 -384 -126v-189q215 111 384 118zM1664 491v184q-235 -116 -384 -71v224q-20 6 -39 15q-5 3 -33 17t-34.5 17t-31.5 15t-34.5 15.5t-32.5 13t-36 12.5t-35 8.5t-39.5 7.5t-39.5 4t-44 2 q-23 0 -49 -3v-222h19q102 0 192.5 -29t197.5 -82q19 -9 39 -15v-188q42 -17 91 -17q120 0 293 92zM1664 918v189q-169 -91 -306 -91q-45 0 -78 8v-196q148 -42 384 90zM320 1280q0 -35 -17.5 -64t-46.5 -46v-1266q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v1266 q-29 17 -46.5 46t-17.5 64q0 53 37.5 90.5t90.5 37.5t90.5 -37.5t37.5 -90.5zM1792 1216v-763q0 -39 -35 -57q-10 -5 -17 -9q-218 -116 -369 -116q-88 0 -158 35l-28 14q-64 33 -99 48t-91 29t-114 14q-102 0 -235.5 -44t-228.5 -102q-15 -9 -33 -9q-16 0 -32 8 q-32 19 -32 56v742q0 35 31 55q35 21 78.5 42.5t114 52t152.5 49.5t155 19q112 0 209 -31t209 -86q38 -19 89 -19q122 0 310 112q22 12 31 17q31 16 62 -2q31 -20 31 -55z" /> +<glyph unicode="" horiz-adv-x="1664" d="M585 553l-466 -466q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l393 393l-393 393q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l466 -466q10 -10 10 -23t-10 -23zM1664 96v-64q0 -14 -9 -23t-23 -9h-960q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h960q14 0 23 -9 t9 -23z" /> +<glyph unicode="" horiz-adv-x="1920" d="M617 137l-50 -50q-10 -10 -23 -10t-23 10l-466 466q-10 10 -10 23t10 23l466 466q10 10 23 10t23 -10l50 -50q10 -10 10 -23t-10 -23l-393 -393l393 -393q10 -10 10 -23t-10 -23zM1208 1204l-373 -1291q-4 -13 -15.5 -19.5t-23.5 -2.5l-62 17q-13 4 -19.5 15.5t-2.5 24.5 l373 1291q4 13 15.5 19.5t23.5 2.5l62 -17q13 -4 19.5 -15.5t2.5 -24.5zM1865 553l-466 -466q-10 -10 -23 -10t-23 10l-50 50q-10 10 -10 23t10 23l393 393l-393 393q-10 10 -10 23t10 23l50 50q10 10 23 10t23 -10l466 -466q10 -10 10 -23t-10 -23z" /> +<glyph unicode="" horiz-adv-x="1792" d="M640 454v-70q0 -42 -39 -59q-13 -5 -25 -5q-27 0 -45 19l-512 512q-19 19 -19 45t19 45l512 512q29 31 70 14q39 -17 39 -59v-69l-397 -398q-19 -19 -19 -45t19 -45zM1792 416q0 -58 -17 -133.5t-38.5 -138t-48 -125t-40.5 -90.5l-20 -40q-8 -17 -28 -17q-6 0 -9 1 q-25 8 -23 34q43 400 -106 565q-64 71 -170.5 110.5t-267.5 52.5v-251q0 -42 -39 -59q-13 -5 -25 -5q-27 0 -45 19l-512 512q-19 19 -19 45t19 45l512 512q29 31 70 14q39 -17 39 -59v-262q411 -28 599 -221q169 -173 169 -509z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1186 579l257 250l-356 52l-66 10l-30 60l-159 322v-963l59 -31l318 -168l-60 355l-12 66zM1638 841l-363 -354l86 -500q5 -33 -6 -51.5t-34 -18.5q-17 0 -40 12l-449 236l-449 -236q-23 -12 -40 -12q-23 0 -34 18.5t-6 51.5l86 500l-364 354q-32 32 -23 59.5t54 34.5 l502 73l225 455q20 41 49 41q28 0 49 -41l225 -455l502 -73q45 -7 54 -34.5t-24 -59.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1401 1187l-640 -1280q-17 -35 -57 -35q-5 0 -15 2q-22 5 -35.5 22.5t-13.5 39.5v576h-576q-22 0 -39.5 13.5t-22.5 35.5t4 42t29 30l1280 640q13 7 29 7q27 0 45 -19q15 -14 18.5 -34.5t-6.5 -39.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M557 256h595v595zM512 301l595 595h-595v-595zM1664 224v-192q0 -14 -9 -23t-23 -9h-224v-224q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v224h-864q-14 0 -23 9t-9 23v864h-224q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h224v224q0 14 9 23t23 9h192q14 0 23 -9t9 -23 v-224h851l246 247q10 9 23 9t23 -9q9 -10 9 -23t-9 -23l-247 -246v-851h224q14 0 23 -9t9 -23z" /> +<glyph unicode="" horiz-adv-x="1024" d="M288 64q0 40 -28 68t-68 28t-68 -28t-28 -68t28 -68t68 -28t68 28t28 68zM288 1216q0 40 -28 68t-68 28t-68 -28t-28 -68t28 -68t68 -28t68 28t28 68zM928 1088q0 40 -28 68t-68 28t-68 -28t-28 -68t28 -68t68 -28t68 28t28 68zM1024 1088q0 -52 -26 -96.5t-70 -69.5 q-2 -287 -226 -414q-68 -38 -203 -81q-128 -40 -169.5 -71t-41.5 -100v-26q44 -25 70 -69.5t26 -96.5q0 -80 -56 -136t-136 -56t-136 56t-56 136q0 52 26 96.5t70 69.5v820q-44 25 -70 69.5t-26 96.5q0 80 56 136t136 56t136 -56t56 -136q0 -52 -26 -96.5t-70 -69.5v-497 q54 26 154 57q55 17 87.5 29.5t70.5 31t59 39.5t40.5 51t28 69.5t8.5 91.5q-44 25 -70 69.5t-26 96.5q0 80 56 136t136 56t136 -56t56 -136z" /> +<glyph unicode="" horiz-adv-x="1664" d="M439 265l-256 -256q-10 -9 -23 -9q-12 0 -23 9q-9 10 -9 23t9 23l256 256q10 9 23 9t23 -9q9 -10 9 -23t-9 -23zM608 224v-320q0 -14 -9 -23t-23 -9t-23 9t-9 23v320q0 14 9 23t23 9t23 -9t9 -23zM384 448q0 -14 -9 -23t-23 -9h-320q-14 0 -23 9t-9 23t9 23t23 9h320 q14 0 23 -9t9 -23zM1648 320q0 -120 -85 -203l-147 -146q-83 -83 -203 -83q-121 0 -204 85l-334 335q-21 21 -42 56l239 18l273 -274q27 -27 68 -27.5t68 26.5l147 146q28 28 28 67q0 40 -28 68l-274 275l18 239q35 -21 56 -42l336 -336q84 -86 84 -204zM1031 1044l-239 -18 l-273 274q-28 28 -68 28q-39 0 -68 -27l-147 -146q-28 -28 -28 -67q0 -40 28 -68l274 -274l-18 -240q-35 21 -56 42l-336 336q-84 86 -84 204q0 120 85 203l147 146q83 83 203 83q121 0 204 -85l334 -335q21 -21 42 -56zM1664 960q0 -14 -9 -23t-23 -9h-320q-14 0 -23 9 t-9 23t9 23t23 9h320q14 0 23 -9t9 -23zM1120 1504v-320q0 -14 -9 -23t-23 -9t-23 9t-9 23v320q0 14 9 23t23 9t23 -9t9 -23zM1527 1353l-256 -256q-11 -9 -23 -9t-23 9q-9 10 -9 23t9 23l256 256q10 9 23 9t23 -9q9 -10 9 -23t-9 -23z" /> +<glyph unicode="" horiz-adv-x="1024" d="M704 280v-240q0 -16 -12 -28t-28 -12h-240q-16 0 -28 12t-12 28v240q0 16 12 28t28 12h240q16 0 28 -12t12 -28zM1020 880q0 -54 -15.5 -101t-35 -76.5t-55 -59.5t-57.5 -43.5t-61 -35.5q-41 -23 -68.5 -65t-27.5 -67q0 -17 -12 -32.5t-28 -15.5h-240q-15 0 -25.5 18.5 t-10.5 37.5v45q0 83 65 156.5t143 108.5q59 27 84 56t25 76q0 42 -46.5 74t-107.5 32q-65 0 -108 -29q-35 -25 -107 -115q-13 -16 -31 -16q-12 0 -25 8l-164 125q-13 10 -15.5 25t5.5 28q160 266 464 266q80 0 161 -31t146 -83t106 -127.5t41 -158.5z" /> +<glyph unicode="" horiz-adv-x="640" d="M640 192v-128q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h64v384h-64q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h384q26 0 45 -19t19 -45v-576h64q26 0 45 -19t19 -45zM512 1344v-192q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v192 q0 26 19 45t45 19h256q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="640" d="M512 288v-224q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v224q0 26 19 45t45 19h256q26 0 45 -19t19 -45zM542 1344l-28 -768q-1 -26 -20.5 -45t-45.5 -19h-256q-26 0 -45.5 19t-20.5 45l-28 768q-1 26 17.5 45t44.5 19h320q26 0 44.5 -19t17.5 -45z" /> +<glyph unicode="" d="M897 167v-167h-248l-159 252l-24 42q-8 9 -11 21h-3l-9 -21q-10 -20 -25 -44l-155 -250h-258v167h128l197 291l-185 272h-137v168h276l139 -228q2 -4 23 -42q8 -9 11 -21h3q3 9 11 21l25 42l140 228h257v-168h-125l-184 -267l204 -296h109zM1534 846v-206h-514l-3 27 q-4 28 -4 46q0 64 26 117t65 86.5t84 65t84 54.5t65 54t26 64q0 38 -29.5 62.5t-70.5 24.5q-51 0 -97 -39q-14 -11 -36 -38l-105 92q26 37 63 66q83 65 188 65q110 0 178 -59.5t68 -158.5q0 -56 -24.5 -103t-62 -76.5t-81.5 -58.5t-82 -50.5t-65.5 -51.5t-30.5 -63h232v80 h126z" /> +<glyph unicode="" d="M897 167v-167h-248l-159 252l-24 42q-8 9 -11 21h-3l-9 -21q-10 -20 -25 -44l-155 -250h-258v167h128l197 291l-185 272h-137v168h276l139 -228q2 -4 23 -42q8 -9 11 -21h3q3 9 11 21l25 42l140 228h257v-168h-125l-184 -267l204 -296h109zM1536 -50v-206h-514l-4 27 q-3 45 -3 46q0 64 26 117t65 86.5t84 65t84 54.5t65 54t26 64q0 38 -29.5 62.5t-70.5 24.5q-51 0 -97 -39q-14 -11 -36 -38l-105 92q26 37 63 66q80 65 188 65q110 0 178 -59.5t68 -158.5q0 -66 -34.5 -118.5t-84 -86t-99.5 -62.5t-87 -63t-41 -73h232v80h126z" /> +<glyph unicode="" horiz-adv-x="1920" d="M896 128l336 384h-768l-336 -384h768zM1909 1205q15 -34 9.5 -71.5t-30.5 -65.5l-896 -1024q-38 -44 -96 -44h-768q-38 0 -69.5 20.5t-47.5 54.5q-15 34 -9.5 71.5t30.5 65.5l896 1024q38 44 96 44h768q38 0 69.5 -20.5t47.5 -54.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1664 438q0 -81 -44.5 -135t-123.5 -54q-41 0 -77.5 17.5t-59 38t-56.5 38t-71 17.5q-110 0 -110 -124q0 -39 16 -115t15 -115v-5q-22 0 -33 -1q-34 -3 -97.5 -11.5t-115.5 -13.5t-98 -5q-61 0 -103 26.5t-42 83.5q0 37 17.5 71t38 56.5t38 59t17.5 77.5q0 79 -54 123.5 t-135 44.5q-84 0 -143 -45.5t-59 -127.5q0 -43 15 -83t33.5 -64.5t33.5 -53t15 -50.5q0 -45 -46 -89q-37 -35 -117 -35q-95 0 -245 24q-9 2 -27.5 4t-27.5 4l-13 2q-1 0 -3 1q-2 0 -2 1v1024q2 -1 17.5 -3.5t34 -5t21.5 -3.5q150 -24 245 -24q80 0 117 35q46 44 46 89 q0 22 -15 50.5t-33.5 53t-33.5 64.5t-15 83q0 82 59 127.5t144 45.5q80 0 134 -44.5t54 -123.5q0 -41 -17.5 -77.5t-38 -59t-38 -56.5t-17.5 -71q0 -57 42 -83.5t103 -26.5q64 0 180 15t163 17v-2q-1 -2 -3.5 -17.5t-5 -34t-3.5 -21.5q-24 -150 -24 -245q0 -80 35 -117 q44 -46 89 -46q22 0 50.5 15t53 33.5t64.5 33.5t83 15q82 0 127.5 -59t45.5 -143z" /> +<glyph unicode="" horiz-adv-x="1152" d="M1152 832v-128q0 -221 -147.5 -384.5t-364.5 -187.5v-132h256q26 0 45 -19t19 -45t-19 -45t-45 -19h-640q-26 0 -45 19t-19 45t19 45t45 19h256v132q-217 24 -364.5 187.5t-147.5 384.5v128q0 26 19 45t45 19t45 -19t19 -45v-128q0 -185 131.5 -316.5t316.5 -131.5 t316.5 131.5t131.5 316.5v128q0 26 19 45t45 19t45 -19t19 -45zM896 1216v-512q0 -132 -94 -226t-226 -94t-226 94t-94 226v512q0 132 94 226t226 94t226 -94t94 -226z" /> +<glyph unicode="" horiz-adv-x="1408" d="M271 591l-101 -101q-42 103 -42 214v128q0 26 19 45t45 19t45 -19t19 -45v-128q0 -53 15 -113zM1385 1193l-361 -361v-128q0 -132 -94 -226t-226 -94q-55 0 -109 19l-96 -96q97 -51 205 -51q185 0 316.5 131.5t131.5 316.5v128q0 26 19 45t45 19t45 -19t19 -45v-128 q0 -221 -147.5 -384.5t-364.5 -187.5v-132h256q26 0 45 -19t19 -45t-19 -45t-45 -19h-640q-26 0 -45 19t-19 45t19 45t45 19h256v132q-125 13 -235 81l-254 -254q-10 -10 -23 -10t-23 10l-82 82q-10 10 -10 23t10 23l1234 1234q10 10 23 10t23 -10l82 -82q10 -10 10 -23 t-10 -23zM1005 1325l-621 -621v512q0 132 94 226t226 94q102 0 184.5 -59t116.5 -152z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1088 576v640h-448v-1137q119 63 213 137q235 184 235 360zM1280 1344v-768q0 -86 -33.5 -170.5t-83 -150t-118 -127.5t-126.5 -103t-121 -77.5t-89.5 -49.5t-42.5 -20q-12 -6 -26 -6t-26 6q-16 7 -42.5 20t-89.5 49.5t-121 77.5t-126.5 103t-118 127.5t-83 150 t-33.5 170.5v768q0 26 19 45t45 19h1152q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1664" d="M128 -128h1408v1024h-1408v-1024zM512 1088v288q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-288q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1280 1088v288q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-288q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1664 1152v-1280 q0 -52 -38 -90t-90 -38h-1408q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h128v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h384v96q0 66 47 113t113 47h64q66 0 113 -47t47 -113v-96h128q52 0 90 -38t38 -90z" /> +<glyph unicode="" horiz-adv-x="1408" d="M512 1344q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1408 1376v-320q0 -16 -12 -25q-8 -7 -20 -7q-4 0 -7 1l-448 96q-11 2 -18 11t-7 20h-256v-102q111 -23 183.5 -111t72.5 -203v-800q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v800 q0 106 62.5 190.5t161.5 114.5v111h-32q-59 0 -115 -23.5t-91.5 -53t-66 -66.5t-40.5 -53.5t-14 -24.5q-17 -35 -57 -35q-16 0 -29 7q-23 12 -31.5 37t3.5 49q5 10 14.5 26t37.5 53.5t60.5 70t85 67t108.5 52.5q-25 42 -25 86q0 66 47 113t113 47t113 -47t47 -113 q0 -33 -14 -64h302q0 11 7 20t18 11l448 96q3 1 7 1q12 0 20 -7q12 -9 12 -25z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1440 1088q0 40 -28 68t-68 28t-68 -28t-28 -68t28 -68t68 -28t68 28t28 68zM1664 1376q0 -249 -75.5 -430.5t-253.5 -360.5q-81 -80 -195 -176l-20 -379q-2 -16 -16 -26l-384 -224q-7 -4 -16 -4q-12 0 -23 9l-64 64q-13 14 -8 32l85 276l-281 281l-276 -85q-3 -1 -9 -1 q-14 0 -23 9l-64 64q-17 19 -5 39l224 384q10 14 26 16l379 20q96 114 176 195q188 187 358 258t431 71q14 0 24 -9.5t10 -22.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1745 763l-164 -763h-334l178 832q13 56 -15 88q-27 33 -83 33h-169l-204 -953h-334l204 953h-286l-204 -953h-334l204 953l-153 327h1276q101 0 189.5 -40.5t147.5 -113.5q60 -73 81 -168.5t0 -194.5z" /> +<glyph unicode="" d="M909 141l102 102q19 19 19 45t-19 45l-307 307l307 307q19 19 19 45t-19 45l-102 102q-19 19 -45 19t-45 -19l-454 -454q-19 -19 -19 -45t19 -45l454 -454q19 -19 45 -19t45 19zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5 t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M717 141l454 454q19 19 19 45t-19 45l-454 454q-19 19 -45 19t-45 -19l-102 -102q-19 -19 -19 -45t19 -45l307 -307l-307 -307q-19 -19 -19 -45t19 -45l102 -102q19 -19 45 -19t45 19zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5 t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1165 397l102 102q19 19 19 45t-19 45l-454 454q-19 19 -45 19t-45 -19l-454 -454q-19 -19 -19 -45t19 -45l102 -102q19 -19 45 -19t45 19l307 307l307 -307q19 -19 45 -19t45 19zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5 t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M813 237l454 454q19 19 19 45t-19 45l-102 102q-19 19 -45 19t-45 -19l-307 -307l-307 307q-19 19 -45 19t-45 -19l-102 -102q-19 -19 -19 -45t19 -45l454 -454q19 -19 45 -19t45 19zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5 t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1130 939l16 175h-884l47 -534h612l-22 -228l-197 -53l-196 53l-13 140h-175l22 -278l362 -100h4v1l359 99l50 544h-644l-15 181h674zM0 1408h1408l-128 -1438l-578 -162l-574 162z" /> +<glyph unicode="" horiz-adv-x="1792" d="M275 1408h1505l-266 -1333l-804 -267l-698 267l71 356h297l-29 -147l422 -161l486 161l68 339h-1208l58 297h1209l38 191h-1208z" /> +<glyph unicode="" horiz-adv-x="1792" d="M960 1280q0 26 -19 45t-45 19t-45 -19t-19 -45t19 -45t45 -19t45 19t19 45zM1792 352v-352q0 -22 -20 -30q-8 -2 -12 -2q-13 0 -23 9l-93 93q-119 -143 -318.5 -226.5t-429.5 -83.5t-429.5 83.5t-318.5 226.5l-93 -93q-9 -9 -23 -9q-4 0 -12 2q-20 8 -20 30v352 q0 14 9 23t23 9h352q22 0 30 -20q8 -19 -7 -35l-100 -100q67 -91 189.5 -153.5t271.5 -82.5v647h-192q-26 0 -45 19t-19 45v128q0 26 19 45t45 19h192v163q-58 34 -93 92.5t-35 128.5q0 106 75 181t181 75t181 -75t75 -181q0 -70 -35 -128.5t-93 -92.5v-163h192q26 0 45 -19 t19 -45v-128q0 -26 -19 -45t-45 -19h-192v-647q149 20 271.5 82.5t189.5 153.5l-100 100q-15 16 -7 35q8 20 30 20h352q14 0 23 -9t9 -23z" /> +<glyph unicode="" horiz-adv-x="1152" d="M1056 768q40 0 68 -28t28 -68v-576q0 -40 -28 -68t-68 -28h-960q-40 0 -68 28t-28 68v576q0 40 28 68t68 28h32v320q0 185 131.5 316.5t316.5 131.5t316.5 -131.5t131.5 -316.5q0 -26 -19 -45t-45 -19h-64q-26 0 -45 19t-19 45q0 106 -75 181t-181 75t-181 -75t-75 -181 v-320h736z" /> +<glyph unicode="" d="M1024 640q0 -106 -75 -181t-181 -75t-181 75t-75 181t75 181t181 75t181 -75t75 -181zM1152 640q0 159 -112.5 271.5t-271.5 112.5t-271.5 -112.5t-112.5 -271.5t112.5 -271.5t271.5 -112.5t271.5 112.5t112.5 271.5zM1280 640q0 -212 -150 -362t-362 -150t-362 150 t-150 362t150 362t362 150t362 -150t150 -362zM1408 640q0 130 -51 248.5t-136.5 204t-204 136.5t-248.5 51t-248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5zM1536 640 q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M384 800v-192q0 -40 -28 -68t-68 -28h-192q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68zM896 800v-192q0 -40 -28 -68t-68 -28h-192q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68zM1408 800v-192q0 -40 -28 -68t-68 -28h-192 q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68z" /> +<glyph unicode="" horiz-adv-x="384" d="M384 288v-192q0 -40 -28 -68t-68 -28h-192q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68zM384 800v-192q0 -40 -28 -68t-68 -28h-192q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68zM384 1312v-192q0 -40 -28 -68t-68 -28h-192 q-40 0 -68 28t-28 68v192q0 40 28 68t68 28h192q40 0 68 -28t28 -68z" /> +<glyph unicode="" d="M512 256q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM863 162q-13 232 -177 396t-396 177q-14 1 -24 -9t-10 -23v-128q0 -13 8.5 -22t21.5 -10q154 -11 264 -121t121 -264q1 -13 10 -21.5t22 -8.5h128q13 0 23 10 t9 24zM1247 161q-5 154 -56 297.5t-139.5 260t-205 205t-260 139.5t-297.5 56q-14 1 -23 -9q-10 -10 -10 -23v-128q0 -13 9 -22t22 -10q204 -7 378 -111.5t278.5 -278.5t111.5 -378q1 -13 10 -22t22 -9h128q13 0 23 10q11 9 9 23zM1536 1120v-960q0 -119 -84.5 -203.5 t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M768 1408q209 0 385.5 -103t279.5 -279.5t103 -385.5t-103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103zM1152 585q32 18 32 55t-32 55l-544 320q-31 19 -64 1q-32 -19 -32 -56v-640q0 -37 32 -56 q16 -8 32 -8q17 0 32 9z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1024 1084l316 -316l-572 -572l-316 316zM813 105l618 618q19 19 19 45t-19 45l-362 362q-18 18 -45 18t-45 -18l-618 -618q-19 -19 -19 -45t19 -45l362 -362q18 -18 45 -18t45 18zM1702 742l-907 -908q-37 -37 -90.5 -37t-90.5 37l-126 126q56 56 56 136t-56 136 t-136 56t-136 -56l-125 126q-37 37 -37 90.5t37 90.5l907 906q37 37 90.5 37t90.5 -37l125 -125q-56 -56 -56 -136t56 -136t136 -56t136 56l126 -125q37 -37 37 -90.5t-37 -90.5z" /> +<glyph unicode="" d="M1280 576v128q0 26 -19 45t-45 19h-896q-26 0 -45 -19t-19 -45v-128q0 -26 19 -45t45 -19h896q26 0 45 19t19 45zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5 t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1152 736v-64q0 -14 -9 -23t-23 -9h-832q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h832q14 0 23 -9t9 -23zM1280 288v832q0 66 -47 113t-113 47h-832q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832q66 0 113 47t47 113zM1408 1120v-832q0 -119 -84.5 -203.5 t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h832q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1024" d="M1018 933q-18 -37 -58 -37h-192v-864q0 -14 -9 -23t-23 -9h-704q-21 0 -29 18q-8 20 4 35l160 192q9 11 25 11h320v640h-192q-40 0 -58 37q-17 37 9 68l320 384q18 22 49 22t49 -22l320 -384q27 -32 9 -68z" /> +<glyph unicode="" horiz-adv-x="1024" d="M32 1280h704q13 0 22.5 -9.5t9.5 -23.5v-863h192q40 0 58 -37t-9 -69l-320 -384q-18 -22 -49 -22t-49 22l-320 384q-26 31 -9 69q18 37 58 37h192v640h-320q-14 0 -25 11l-160 192q-13 14 -4 34q9 19 29 19z" /> +<glyph unicode="" d="M685 237l614 614q19 19 19 45t-19 45l-102 102q-19 19 -45 19t-45 -19l-467 -467l-211 211q-19 19 -45 19t-45 -19l-102 -102q-19 -19 -19 -45t19 -45l358 -358q19 -19 45 -19t45 19zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5 t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M404 428l152 -152l-52 -52h-56v96h-96v56zM818 818q14 -13 -3 -30l-291 -291q-17 -17 -30 -3q-14 13 3 30l291 291q17 17 30 3zM544 128l544 544l-288 288l-544 -544v-288h288zM1152 736l92 92q28 28 28 68t-28 68l-152 152q-28 28 -68 28t-68 -28l-92 -92zM1536 1120 v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M1280 608v480q0 26 -19 45t-45 19h-480q-42 0 -59 -39q-17 -41 14 -70l144 -144l-534 -534q-19 -19 -19 -45t19 -45l102 -102q19 -19 45 -19t45 19l534 534l144 -144q18 -19 45 -19q12 0 25 5q39 17 39 59zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960 q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M1005 435l352 352q19 19 19 45t-19 45l-352 352q-30 31 -69 14q-40 -17 -40 -59v-160q-119 0 -216 -19.5t-162.5 -51t-114 -79t-76.5 -95.5t-44.5 -109t-21.5 -111.5t-5 -110.5q0 -181 167 -404q10 -12 25 -12q7 0 13 3q22 9 19 33q-44 354 62 473q46 52 130 75.5 t224 23.5v-160q0 -42 40 -59q12 -5 24 -5q26 0 45 19zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M640 448l256 128l-256 128v-256zM1024 1039v-542l-512 -256v542zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103 t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1145 861q18 -35 -5 -66l-320 -448q-19 -27 -52 -27t-52 27l-320 448q-23 31 -5 66q17 35 57 35h640q40 0 57 -35zM1280 160v960q0 13 -9.5 22.5t-22.5 9.5h-960q-13 0 -22.5 -9.5t-9.5 -22.5v-960q0 -13 9.5 -22.5t22.5 -9.5h960q13 0 22.5 9.5t9.5 22.5zM1536 1120 v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M1145 419q-17 -35 -57 -35h-640q-40 0 -57 35q-18 35 5 66l320 448q19 27 52 27t52 -27l320 -448q23 -31 5 -66zM1280 160v960q0 13 -9.5 22.5t-22.5 9.5h-960q-13 0 -22.5 -9.5t-9.5 -22.5v-960q0 -13 9.5 -22.5t22.5 -9.5h960q13 0 22.5 9.5t9.5 22.5zM1536 1120v-960 q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M1088 640q0 -33 -27 -52l-448 -320q-31 -23 -66 -5q-35 17 -35 57v640q0 40 35 57q35 18 66 -5l448 -320q27 -19 27 -52zM1280 160v960q0 14 -9 23t-23 9h-960q-14 0 -23 -9t-9 -23v-960q0 -14 9 -23t23 -9h960q14 0 23 9t9 23zM1536 1120v-960q0 -119 -84.5 -203.5 t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1024" d="M976 229l35 -159q3 -12 -3 -22.5t-17 -14.5l-5 -1q-4 -2 -10.5 -3.5t-16 -4.5t-21.5 -5.5t-25.5 -5t-30 -5t-33.5 -4.5t-36.5 -3t-38.5 -1q-234 0 -409 130.5t-238 351.5h-95q-13 0 -22.5 9.5t-9.5 22.5v113q0 13 9.5 22.5t22.5 9.5h66q-2 57 1 105h-67q-14 0 -23 9 t-9 23v114q0 14 9 23t23 9h98q67 210 243.5 338t400.5 128q102 0 194 -23q11 -3 20 -15q6 -11 3 -24l-43 -159q-3 -13 -14 -19.5t-24 -2.5l-4 1q-4 1 -11.5 2.5l-17.5 3.5t-22.5 3.5t-26 3t-29 2.5t-29.5 1q-126 0 -226 -64t-150 -176h468q16 0 25 -12q10 -12 7 -26 l-24 -114q-5 -26 -32 -26h-488q-3 -37 0 -105h459q15 0 25 -12q9 -12 6 -27l-24 -112q-2 -11 -11 -18.5t-20 -7.5h-387q48 -117 149.5 -185.5t228.5 -68.5q18 0 36 1.5t33.5 3.5t29.5 4.5t24.5 5t18.5 4.5l12 3l5 2q13 5 26 -2q12 -7 15 -21z" /> +<glyph unicode="" horiz-adv-x="1024" d="M1020 399v-367q0 -14 -9 -23t-23 -9h-956q-14 0 -23 9t-9 23v150q0 13 9.5 22.5t22.5 9.5h97v383h-95q-14 0 -23 9.5t-9 22.5v131q0 14 9 23t23 9h95v223q0 171 123.5 282t314.5 111q185 0 335 -125q9 -8 10 -20.5t-7 -22.5l-103 -127q-9 -11 -22 -12q-13 -2 -23 7 q-5 5 -26 19t-69 32t-93 18q-85 0 -137 -47t-52 -123v-215h305q13 0 22.5 -9t9.5 -23v-131q0 -13 -9.5 -22.5t-22.5 -9.5h-305v-379h414v181q0 13 9 22.5t23 9.5h162q14 0 23 -9.5t9 -22.5z" /> +<glyph unicode="" horiz-adv-x="1024" d="M978 351q0 -153 -99.5 -263.5t-258.5 -136.5v-175q0 -14 -9 -23t-23 -9h-135q-13 0 -22.5 9.5t-9.5 22.5v175q-66 9 -127.5 31t-101.5 44.5t-74 48t-46.5 37.5t-17.5 18q-17 21 -2 41l103 135q7 10 23 12q15 2 24 -9l2 -2q113 -99 243 -125q37 -8 74 -8q81 0 142.5 43 t61.5 122q0 28 -15 53t-33.5 42t-58.5 37.5t-66 32t-80 32.5q-39 16 -61.5 25t-61.5 26.5t-62.5 31t-56.5 35.5t-53.5 42.5t-43.5 49t-35.5 58t-21 66.5t-8.5 78q0 138 98 242t255 134v180q0 13 9.5 22.5t22.5 9.5h135q14 0 23 -9t9 -23v-176q57 -6 110.5 -23t87 -33.5 t63.5 -37.5t39 -29t15 -14q17 -18 5 -38l-81 -146q-8 -15 -23 -16q-14 -3 -27 7q-3 3 -14.5 12t-39 26.5t-58.5 32t-74.5 26t-85.5 11.5q-95 0 -155 -43t-60 -111q0 -26 8.5 -48t29.5 -41.5t39.5 -33t56 -31t60.5 -27t70 -27.5q53 -20 81 -31.5t76 -35t75.5 -42.5t62 -50 t53 -63.5t31.5 -76.5t13 -94z" /> +<glyph unicode="" horiz-adv-x="898" d="M898 1066v-102q0 -14 -9 -23t-23 -9h-168q-23 -144 -129 -234t-276 -110q167 -178 459 -536q14 -16 4 -34q-8 -18 -29 -18h-195q-16 0 -25 12q-306 367 -498 571q-9 9 -9 22v127q0 13 9.5 22.5t22.5 9.5h112q132 0 212.5 43t102.5 125h-427q-14 0 -23 9t-9 23v102 q0 14 9 23t23 9h413q-57 113 -268 113h-145q-13 0 -22.5 9.5t-9.5 22.5v133q0 14 9 23t23 9h832q14 0 23 -9t9 -23v-102q0 -14 -9 -23t-23 -9h-233q47 -61 64 -144h171q14 0 23 -9t9 -23z" /> +<glyph unicode="" horiz-adv-x="1027" d="M603 0h-172q-13 0 -22.5 9t-9.5 23v330h-288q-13 0 -22.5 9t-9.5 23v103q0 13 9.5 22.5t22.5 9.5h288v85h-288q-13 0 -22.5 9t-9.5 23v104q0 13 9.5 22.5t22.5 9.5h214l-321 578q-8 16 0 32q10 16 28 16h194q19 0 29 -18l215 -425q19 -38 56 -125q10 24 30.5 68t27.5 61 l191 420q8 19 29 19h191q17 0 27 -16q9 -14 1 -31l-313 -579h215q13 0 22.5 -9.5t9.5 -22.5v-104q0 -14 -9.5 -23t-22.5 -9h-290v-85h290q13 0 22.5 -9.5t9.5 -22.5v-103q0 -14 -9.5 -23t-22.5 -9h-290v-330q0 -13 -9.5 -22.5t-22.5 -9.5z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1043 971q0 100 -65 162t-171 62h-320v-448h320q106 0 171 62t65 162zM1280 971q0 -193 -126.5 -315t-326.5 -122h-340v-118h505q14 0 23 -9t9 -23v-128q0 -14 -9 -23t-23 -9h-505v-192q0 -14 -9.5 -23t-22.5 -9h-167q-14 0 -23 9t-9 23v192h-224q-14 0 -23 9t-9 23v128 q0 14 9 23t23 9h224v118h-224q-14 0 -23 9t-9 23v149q0 13 9 22.5t23 9.5h224v629q0 14 9 23t23 9h539q200 0 326.5 -122t126.5 -315z" /> +<glyph unicode="" horiz-adv-x="1792" d="M514 341l81 299h-159l75 -300q1 -1 1 -3t1 -3q0 1 0.5 3.5t0.5 3.5zM630 768l35 128h-292l32 -128h225zM822 768h139l-35 128h-70zM1271 340l78 300h-162l81 -299q0 -1 0.5 -3.5t1.5 -3.5q0 1 0.5 3t0.5 3zM1382 768l33 128h-297l34 -128h230zM1792 736v-64q0 -14 -9 -23 t-23 -9h-213l-164 -616q-7 -24 -31 -24h-159q-24 0 -31 24l-166 616h-209l-167 -616q-7 -24 -31 -24h-159q-11 0 -19.5 7t-10.5 17l-160 616h-208q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h175l-33 128h-142q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h109l-89 344q-5 15 5 28 q10 12 26 12h137q26 0 31 -24l90 -360h359l97 360q7 24 31 24h126q24 0 31 -24l98 -360h365l93 360q5 24 31 24h137q16 0 26 -12q10 -13 5 -28l-91 -344h111q14 0 23 -9t9 -23v-64q0 -14 -9 -23t-23 -9h-145l-34 -128h179q14 0 23 -9t9 -23z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1167 896q18 -182 -131 -258q117 -28 175 -103t45 -214q-7 -71 -32.5 -125t-64.5 -89t-97 -58.5t-121.5 -34.5t-145.5 -15v-255h-154v251q-80 0 -122 1v-252h-154v255q-18 0 -54 0.5t-55 0.5h-200l31 183h111q50 0 58 51v402h16q-6 1 -16 1v287q-13 68 -89 68h-111v164 l212 -1q64 0 97 1v252h154v-247q82 2 122 2v245h154v-252q79 -7 140 -22.5t113 -45t82.5 -78t36.5 -114.5zM952 351q0 36 -15 64t-37 46t-57.5 30.5t-65.5 18.5t-74 9t-69 3t-64.5 -1t-47.5 -1v-338q8 0 37 -0.5t48 -0.5t53 1.5t58.5 4t57 8.5t55.5 14t47.5 21t39.5 30 t24.5 40t9.5 51zM881 827q0 33 -12.5 58.5t-30.5 42t-48 28t-55 16.5t-61.5 8t-58 2.5t-54 -1t-39.5 -0.5v-307q5 0 34.5 -0.5t46.5 0t50 2t55 5.5t51.5 11t48.5 18.5t37 27t27 38.5t9 51z" /> +<glyph unicode="" d="M1024 1024v472q22 -14 36 -28l408 -408q14 -14 28 -36h-472zM896 992q0 -40 28 -68t68 -28h544v-1056q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h800v-544z" /> +<glyph unicode="" d="M1468 1060q14 -14 28 -36h-472v472q22 -14 36 -28zM992 896h544v-1056q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h800v-544q0 -40 28 -68t68 -28zM1152 160v64q0 14 -9 23t-23 9h-704q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h704 q14 0 23 9t9 23zM1152 416v64q0 14 -9 23t-23 9h-704q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h704q14 0 23 9t9 23zM1152 672v64q0 14 -9 23t-23 9h-704q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h704q14 0 23 9t9 23z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1191 1128h177l-72 218l-12 47q-2 16 -2 20h-4l-3 -20q0 -1 -3.5 -18t-7.5 -29zM736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192q14 0 23 -9t9 -23zM1572 -23 v-233h-584v90l369 529q12 18 21 27l11 9v3q-2 0 -6.5 -0.5t-7.5 -0.5q-12 -3 -30 -3h-232v-115h-120v229h567v-89l-369 -530q-6 -8 -21 -26l-11 -11v-2l14 2q9 2 30 2h248v119h121zM1661 874v-106h-288v106h75l-47 144h-243l-47 -144h75v-106h-287v106h70l230 662h162 l230 -662h70z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1191 104h177l-72 218l-12 47q-2 16 -2 20h-4l-3 -20q0 -1 -3.5 -18t-7.5 -29zM736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192q14 0 23 -9t9 -23zM1661 -150 v-106h-288v106h75l-47 144h-243l-47 -144h75v-106h-287v106h70l230 662h162l230 -662h70zM1572 1001v-233h-584v90l369 529q12 18 21 27l11 9v3q-2 0 -6.5 -0.5t-7.5 -0.5q-12 -3 -30 -3h-232v-115h-120v229h567v-89l-369 -530q-6 -8 -21 -26l-11 -10v-3l14 3q9 1 30 1h248 v119h121z" /> +<glyph unicode="" horiz-adv-x="1792" d="M736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192q14 0 23 -9t9 -23zM1792 -32v-192q0 -14 -9 -23t-23 -9h-832q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h832 q14 0 23 -9t9 -23zM1600 480v-192q0 -14 -9 -23t-23 -9h-640q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h640q14 0 23 -9t9 -23zM1408 992v-192q0 -14 -9 -23t-23 -9h-448q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h448q14 0 23 -9t9 -23zM1216 1504v-192q0 -14 -9 -23t-23 -9h-256 q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h256q14 0 23 -9t9 -23z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1216 -32v-192q0 -14 -9 -23t-23 -9h-256q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h256q14 0 23 -9t9 -23zM736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192 q14 0 23 -9t9 -23zM1408 480v-192q0 -14 -9 -23t-23 -9h-448q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h448q14 0 23 -9t9 -23zM1600 992v-192q0 -14 -9 -23t-23 -9h-640q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h640q14 0 23 -9t9 -23zM1792 1504v-192q0 -14 -9 -23t-23 -9h-832 q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h832q14 0 23 -9t9 -23z" /> +<glyph unicode="" d="M1346 223q0 63 -44 116t-103 53q-52 0 -83 -37t-31 -94t36.5 -95t104.5 -38q50 0 85 27t35 68zM736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192q14 0 23 -9t9 -23 zM1486 165q0 -62 -13 -121.5t-41 -114t-68 -95.5t-98.5 -65.5t-127.5 -24.5q-62 0 -108 16q-24 8 -42 15l39 113q15 -7 31 -11q37 -13 75 -13q84 0 134.5 58.5t66.5 145.5h-2q-21 -23 -61.5 -37t-84.5 -14q-106 0 -173 71.5t-67 172.5q0 105 72 178t181 73q123 0 205 -94.5 t82 -252.5zM1456 882v-114h-469v114h167v432q0 7 0.5 19t0.5 17v16h-2l-7 -12q-8 -13 -26 -31l-62 -58l-82 86l192 185h123v-654h165z" /> +<glyph unicode="" d="M1346 1247q0 63 -44 116t-103 53q-52 0 -83 -37t-31 -94t36.5 -95t104.5 -38q50 0 85 27t35 68zM736 96q0 -12 -10 -24l-319 -319q-10 -9 -23 -9q-12 0 -23 9l-320 320q-15 16 -7 35q8 20 30 20h192v1376q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1376h192q14 0 23 -9 t9 -23zM1456 -142v-114h-469v114h167v432q0 7 0.5 19t0.5 17v16h-2l-7 -12q-8 -13 -26 -31l-62 -58l-82 86l192 185h123v-654h165zM1486 1189q0 -62 -13 -121.5t-41 -114t-68 -95.5t-98.5 -65.5t-127.5 -24.5q-62 0 -108 16q-24 8 -42 15l39 113q15 -7 31 -11q37 -13 75 -13 q84 0 134.5 58.5t66.5 145.5h-2q-21 -23 -61.5 -37t-84.5 -14q-106 0 -173 71.5t-67 172.5q0 105 72 178t181 73q123 0 205 -94.5t82 -252.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M256 192q0 26 -19 45t-45 19q-27 0 -45.5 -19t-18.5 -45q0 -27 18.5 -45.5t45.5 -18.5q26 0 45 18.5t19 45.5zM416 704v-640q0 -26 -19 -45t-45 -19h-288q-26 0 -45 19t-19 45v640q0 26 19 45t45 19h288q26 0 45 -19t19 -45zM1600 704q0 -86 -55 -149q15 -44 15 -76 q3 -76 -43 -137q17 -56 0 -117q-15 -57 -54 -94q9 -112 -49 -181q-64 -76 -197 -78h-36h-76h-17q-66 0 -144 15.5t-121.5 29t-120.5 39.5q-123 43 -158 44q-26 1 -45 19.5t-19 44.5v641q0 25 18 43.5t43 20.5q24 2 76 59t101 121q68 87 101 120q18 18 31 48t17.5 48.5 t13.5 60.5q7 39 12.5 61t19.5 52t34 50q19 19 45 19q46 0 82.5 -10.5t60 -26t40 -40.5t24 -45t12 -50t5 -45t0.5 -39q0 -38 -9.5 -76t-19 -60t-27.5 -56q-3 -6 -10 -18t-11 -22t-8 -24h277q78 0 135 -57t57 -135z" /> +<glyph unicode="" horiz-adv-x="1664" d="M256 960q0 -26 -19 -45t-45 -19q-27 0 -45.5 19t-18.5 45q0 27 18.5 45.5t45.5 18.5q26 0 45 -18.5t19 -45.5zM416 448v640q0 26 -19 45t-45 19h-288q-26 0 -45 -19t-19 -45v-640q0 -26 19 -45t45 -19h288q26 0 45 19t19 45zM1545 597q55 -61 55 -149q-1 -78 -57.5 -135 t-134.5 -57h-277q4 -14 8 -24t11 -22t10 -18q18 -37 27 -57t19 -58.5t10 -76.5q0 -24 -0.5 -39t-5 -45t-12 -50t-24 -45t-40 -40.5t-60 -26t-82.5 -10.5q-26 0 -45 19q-20 20 -34 50t-19.5 52t-12.5 61q-9 42 -13.5 60.5t-17.5 48.5t-31 48q-33 33 -101 120q-49 64 -101 121 t-76 59q-25 2 -43 20.5t-18 43.5v641q0 26 19 44.5t45 19.5q35 1 158 44q77 26 120.5 39.5t121.5 29t144 15.5h17h76h36q133 -2 197 -78q58 -69 49 -181q39 -37 54 -94q17 -61 0 -117q46 -61 43 -137q0 -32 -15 -76z" /> +<glyph unicode="" d="M919 233v157q0 50 -29 50q-17 0 -33 -16v-224q16 -16 33 -16q29 0 29 49zM1103 355h66v34q0 51 -33 51t-33 -51v-34zM532 621v-70h-80v-423h-74v423h-78v70h232zM733 495v-367h-67v40q-39 -45 -76 -45q-33 0 -42 28q-6 16 -6 54v290h66v-270q0 -24 1 -26q1 -15 15 -15 q20 0 42 31v280h67zM985 384v-146q0 -52 -7 -73q-12 -42 -53 -42q-35 0 -68 41v-36h-67v493h67v-161q32 40 68 40q41 0 53 -42q7 -21 7 -74zM1236 255v-9q0 -29 -2 -43q-3 -22 -15 -40q-27 -40 -80 -40q-52 0 -81 38q-21 27 -21 86v129q0 59 20 86q29 38 80 38t78 -38 q21 -28 21 -86v-76h-133v-65q0 -51 34 -51q24 0 30 26q0 1 0.5 7t0.5 16.5v21.5h68zM785 1079v-156q0 -51 -32 -51t-32 51v156q0 52 32 52t32 -52zM1318 366q0 177 -19 260q-10 44 -43 73.5t-76 34.5q-136 15 -412 15q-275 0 -411 -15q-44 -5 -76.5 -34.5t-42.5 -73.5 q-20 -87 -20 -260q0 -176 20 -260q10 -43 42.5 -73t75.5 -35q137 -15 412 -15t412 15q43 5 75.5 35t42.5 73q20 84 20 260zM563 1017l90 296h-75l-51 -195l-53 195h-78l24 -69t23 -69q35 -103 46 -158v-201h74v201zM852 936v130q0 58 -21 87q-29 38 -78 38q-51 0 -78 -38 q-21 -29 -21 -87v-130q0 -58 21 -87q27 -38 78 -38q49 0 78 38q21 27 21 87zM1033 816h67v370h-67v-283q-22 -31 -42 -31q-15 0 -16 16q-1 2 -1 26v272h-67v-293q0 -37 6 -55q11 -27 43 -27q36 0 77 45v-40zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960 q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M971 292v-211q0 -67 -39 -67q-23 0 -45 22v301q22 22 45 22q39 0 39 -67zM1309 291v-46h-90v46q0 68 45 68t45 -68zM343 509h107v94h-312v-94h105v-569h100v569zM631 -60h89v494h-89v-378q-30 -42 -57 -42q-18 0 -21 21q-1 3 -1 35v364h-89v-391q0 -49 8 -73 q12 -37 58 -37q48 0 102 61v-54zM1060 88v197q0 73 -9 99q-17 56 -71 56q-50 0 -93 -54v217h-89v-663h89v48q45 -55 93 -55q54 0 71 55q9 27 9 100zM1398 98v13h-91q0 -51 -2 -61q-7 -36 -40 -36q-46 0 -46 69v87h179v103q0 79 -27 116q-39 51 -106 51q-68 0 -107 -51 q-28 -37 -28 -116v-173q0 -79 29 -116q39 -51 108 -51q72 0 108 53q18 27 21 54q2 9 2 58zM790 1011v210q0 69 -43 69t-43 -69v-210q0 -70 43 -70t43 70zM1509 260q0 -234 -26 -350q-14 -59 -58 -99t-102 -46q-184 -21 -555 -21t-555 21q-58 6 -102.5 46t-57.5 99 q-26 112 -26 350q0 234 26 350q14 59 58 99t103 47q183 20 554 20t555 -20q58 -7 102.5 -47t57.5 -99q26 -112 26 -350zM511 1536h102l-121 -399v-271h-100v271q-14 74 -61 212q-37 103 -65 187h106l71 -263zM881 1203v-175q0 -81 -28 -118q-37 -51 -106 -51q-67 0 -105 51 q-28 38 -28 118v175q0 80 28 117q38 51 105 51q69 0 106 -51q28 -37 28 -117zM1216 1365v-499h-91v55q-53 -62 -103 -62q-46 0 -59 37q-8 24 -8 75v394h91v-367q0 -33 1 -35q3 -22 21 -22q27 0 57 43v381h91z" /> +<glyph unicode="" horiz-adv-x="1408" d="M597 869q-10 -18 -257 -456q-27 -46 -65 -46h-239q-21 0 -31 17t0 36l253 448q1 0 0 1l-161 279q-12 22 -1 37q9 15 32 15h239q40 0 66 -45zM1403 1511q11 -16 0 -37l-528 -934v-1l336 -615q11 -20 1 -37q-10 -15 -32 -15h-239q-42 0 -66 45l-339 622q18 32 531 942 q25 45 64 45h241q22 0 31 -15z" /> +<glyph unicode="" d="M685 771q0 1 -126 222q-21 34 -52 34h-184q-18 0 -26 -11q-7 -12 1 -29l125 -216v-1l-196 -346q-9 -14 0 -28q8 -13 24 -13h185q31 0 50 36zM1309 1268q-7 12 -24 12h-187q-30 0 -49 -35l-411 -729q1 -2 262 -481q20 -35 52 -35h184q18 0 25 12q8 13 -1 28l-260 476v1 l409 723q8 16 0 28zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1280 640q0 37 -30 54l-512 320q-31 20 -65 2q-33 -18 -33 -56v-640q0 -38 33 -56q16 -8 31 -8q20 0 34 10l512 320q30 17 30 54zM1792 640q0 -96 -1 -150t-8.5 -136.5t-22.5 -147.5q-16 -73 -69 -123t-124 -58q-222 -25 -671 -25t-671 25q-71 8 -124.5 58t-69.5 123 q-14 65 -21.5 147.5t-8.5 136.5t-1 150t1 150t8.5 136.5t22.5 147.5q16 73 69 123t124 58q222 25 671 25t671 -25q71 -8 124.5 -58t69.5 -123q14 -65 21.5 -147.5t8.5 -136.5t1 -150z" /> +<glyph unicode="" horiz-adv-x="1792" d="M402 829l494 -305l-342 -285l-490 319zM1388 274v-108l-490 -293v-1l-1 1l-1 -1v1l-489 293v108l147 -96l342 284v2l1 -1l1 1v-2l343 -284zM554 1418l342 -285l-494 -304l-338 270zM1390 829l338 -271l-489 -319l-343 285zM1239 1418l489 -319l-338 -270l-494 304z" /> +<glyph unicode="" horiz-adv-x="1408" d="M928 135v-151l-707 -1v151zM1169 481v-701l-1 -35v-1h-1132l-35 1h-1v736h121v-618h928v618h120zM241 393l704 -65l-13 -150l-705 65zM309 709l683 -183l-39 -146l-683 183zM472 1058l609 -360l-77 -130l-609 360zM832 1389l398 -585l-124 -85l-399 584zM1285 1536 l121 -697l-149 -26l-121 697z" /> +<glyph unicode="" d="M1362 110v648h-135q20 -63 20 -131q0 -126 -64 -232.5t-174 -168.5t-240 -62q-197 0 -337 135.5t-140 327.5q0 68 20 131h-141v-648q0 -26 17.5 -43.5t43.5 -17.5h1069q25 0 43 17.5t18 43.5zM1078 643q0 124 -90.5 211.5t-218.5 87.5q-127 0 -217.5 -87.5t-90.5 -211.5 t90.5 -211.5t217.5 -87.5q128 0 218.5 87.5t90.5 211.5zM1362 1003v165q0 28 -20 48.5t-49 20.5h-174q-29 0 -49 -20.5t-20 -48.5v-165q0 -29 20 -49t49 -20h174q29 0 49 20t20 49zM1536 1211v-1142q0 -81 -58 -139t-139 -58h-1142q-81 0 -139 58t-58 139v1142q0 81 58 139 t139 58h1142q81 0 139 -58t58 -139z" /> +<glyph unicode="" d="M1248 1408q119 0 203.5 -84.5t84.5 -203.5v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960zM698 640q0 88 -62 150t-150 62t-150 -62t-62 -150t62 -150t150 -62t150 62t62 150zM1262 640q0 88 -62 150 t-150 62t-150 -62t-62 -150t62 -150t150 -62t150 62t62 150z" /> +<glyph unicode="" d="M768 914l201 -306h-402zM1133 384h94l-459 691l-459 -691h94l104 160h522zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M815 677q8 -63 -50.5 -101t-111.5 -6q-39 17 -53.5 58t-0.5 82t52 58q36 18 72.5 12t64 -35.5t27.5 -67.5zM926 698q-14 107 -113 164t-197 13q-63 -28 -100.5 -88.5t-34.5 -129.5q4 -91 77.5 -155t165.5 -56q91 8 152 84t50 168zM1165 1240q-20 27 -56 44.5t-58 22 t-71 12.5q-291 47 -566 -2q-43 -7 -66 -12t-55 -22t-50 -43q30 -28 76 -45.5t73.5 -22t87.5 -11.5q228 -29 448 -1q63 8 89.5 12t72.5 21.5t75 46.5zM1222 205q-8 -26 -15.5 -76.5t-14 -84t-28.5 -70t-58 -56.5q-86 -48 -189.5 -71.5t-202 -22t-201.5 18.5q-46 8 -81.5 18 t-76.5 27t-73 43.5t-52 61.5q-25 96 -57 292l6 16l18 9q223 -148 506.5 -148t507.5 148q21 -6 24 -23t-5 -45t-8 -37zM1403 1166q-26 -167 -111 -655q-5 -30 -27 -56t-43.5 -40t-54.5 -31q-252 -126 -610 -88q-248 27 -394 139q-15 12 -25.5 26.5t-17 35t-9 34t-6 39.5 t-5.5 35q-9 50 -26.5 150t-28 161.5t-23.5 147.5t-22 158q3 26 17.5 48.5t31.5 37.5t45 30t46 22.5t48 18.5q125 46 313 64q379 37 676 -50q155 -46 215 -122q16 -20 16.5 -51t-5.5 -54z" /> +<glyph unicode="" d="M848 666q0 43 -41 66t-77 1q-43 -20 -42.5 -72.5t43.5 -70.5q39 -23 81 4t36 72zM928 682q8 -66 -36 -121t-110 -61t-119 40t-56 113q-2 49 25.5 93t72.5 64q70 31 141.5 -10t81.5 -118zM1100 1073q-20 -21 -53.5 -34t-53 -16t-63.5 -8q-155 -20 -324 0q-44 6 -63 9.5 t-52.5 16t-54.5 32.5q13 19 36 31t40 15.5t47 8.5q198 35 408 1q33 -5 51 -8.5t43 -16t39 -31.5zM1142 327q0 7 5.5 26.5t3 32t-17.5 16.5q-161 -106 -365 -106t-366 106l-12 -6l-5 -12q26 -154 41 -210q47 -81 204 -108q249 -46 428 53q34 19 49 51.5t22.5 85.5t12.5 71z M1272 1020q9 53 -8 75q-43 55 -155 88q-216 63 -487 36q-132 -12 -226 -46q-38 -15 -59.5 -25t-47 -34t-29.5 -54q8 -68 19 -138t29 -171t24 -137q1 -5 5 -31t7 -36t12 -27t22 -28q105 -80 284 -100q259 -28 440 63q24 13 39.5 23t31 29t19.5 40q48 267 80 473zM1536 1120 v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1024" d="M944 207l80 -237q-23 -35 -111 -66t-177 -32q-104 -2 -190.5 26t-142.5 74t-95 106t-55.5 120t-16.5 118v544h-168v215q72 26 129 69.5t91 90t58 102t34 99t15 88.5q1 5 4.5 8.5t7.5 3.5h244v-424h333v-252h-334v-518q0 -30 6.5 -56t22.5 -52.5t49.5 -41.5t81.5 -14 q78 2 134 29z" /> +<glyph unicode="" d="M1136 75l-62 183q-44 -22 -103 -22q-36 -1 -62 10.5t-38.5 31.5t-17.5 40.5t-5 43.5v398h257v194h-256v326h-188q-8 0 -9 -10q-5 -44 -17.5 -87t-39 -95t-77 -95t-118.5 -68v-165h130v-418q0 -57 21.5 -115t65 -111t121 -85.5t176.5 -30.5q69 1 136.5 25t85.5 50z M1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="768" d="M765 237q8 -19 -5 -35l-350 -384q-10 -10 -23 -10q-14 0 -24 10l-355 384q-13 16 -5 35q9 19 29 19h224v1248q0 14 9 23t23 9h192q14 0 23 -9t9 -23v-1248h224q21 0 29 -19z" /> +<glyph unicode="" horiz-adv-x="768" d="M765 1043q-9 -19 -29 -19h-224v-1248q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v1248h-224q-21 0 -29 19t5 35l350 384q10 10 23 10q14 0 24 -10l355 -384q13 -16 5 -35z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 736v-192q0 -14 -9 -23t-23 -9h-1248v-224q0 -21 -19 -29t-35 5l-384 350q-10 10 -10 23q0 14 10 24l384 354q16 14 35 6q19 -9 19 -29v-224h1248q14 0 23 -9t9 -23z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1728 643q0 -14 -10 -24l-384 -354q-16 -14 -35 -6q-19 9 -19 29v224h-1248q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h1248v224q0 21 19 29t35 -5l384 -350q10 -10 10 -23z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1393 321q-39 -125 -123 -250q-129 -196 -257 -196q-49 0 -140 32q-86 32 -151 32q-61 0 -142 -33q-81 -34 -132 -34q-152 0 -301 259q-147 261 -147 503q0 228 113 374q112 144 284 144q72 0 177 -30q104 -30 138 -30q45 0 143 34q102 34 173 34q119 0 213 -65 q52 -36 104 -100q-79 -67 -114 -118q-65 -94 -65 -207q0 -124 69 -223t158 -126zM1017 1494q0 -61 -29 -136q-30 -75 -93 -138q-54 -54 -108 -72q-37 -11 -104 -17q3 149 78 257q74 107 250 148q1 -3 2.5 -11t2.5 -11q0 -4 0.5 -10t0.5 -10z" /> +<glyph unicode="" horiz-adv-x="1664" d="M682 530v-651l-682 94v557h682zM682 1273v-659h-682v565zM1664 530v-786l-907 125v661h907zM1664 1408v-794h-907v669z" /> +<glyph unicode="" horiz-adv-x="1408" d="M493 1053q16 0 27.5 11.5t11.5 27.5t-11.5 27.5t-27.5 11.5t-27 -11.5t-11 -27.5t11 -27.5t27 -11.5zM915 1053q16 0 27 11.5t11 27.5t-11 27.5t-27 11.5t-27.5 -11.5t-11.5 -27.5t11.5 -27.5t27.5 -11.5zM103 869q42 0 72 -30t30 -72v-430q0 -43 -29.5 -73t-72.5 -30 t-73 30t-30 73v430q0 42 30 72t73 30zM1163 850v-666q0 -46 -32 -78t-77 -32h-75v-227q0 -43 -30 -73t-73 -30t-73 30t-30 73v227h-138v-227q0 -43 -30 -73t-73 -30q-42 0 -72 30t-30 73l-1 227h-74q-46 0 -78 32t-32 78v666h918zM931 1255q107 -55 171 -153.5t64 -215.5 h-925q0 117 64 215.5t172 153.5l-71 131q-7 13 5 20q13 6 20 -6l72 -132q95 42 201 42t201 -42l72 132q7 12 20 6q12 -7 5 -20zM1408 767v-430q0 -43 -30 -73t-73 -30q-42 0 -72 30t-30 73v430q0 43 30 72.5t72 29.5q43 0 73 -29.5t30 -72.5z" /> +<glyph unicode="" d="M663 1125q-11 -1 -15.5 -10.5t-8.5 -9.5q-5 -1 -5 5q0 12 19 15h10zM750 1111q-4 -1 -11.5 6.5t-17.5 4.5q24 11 32 -2q3 -6 -3 -9zM399 684q-4 1 -6 -3t-4.5 -12.5t-5.5 -13.5t-10 -13q-7 -10 -1 -12q4 -1 12.5 7t12.5 18q1 3 2 7t2 6t1.5 4.5t0.5 4v3t-1 2.5t-3 2z M1254 325q0 18 -55 42q4 15 7.5 27.5t5 26t3 21.5t0.5 22.5t-1 19.5t-3.5 22t-4 20.5t-5 25t-5.5 26.5q-10 48 -47 103t-72 75q24 -20 57 -83q87 -162 54 -278q-11 -40 -50 -42q-31 -4 -38.5 18.5t-8 83.5t-11.5 107q-9 39 -19.5 69t-19.5 45.5t-15.5 24.5t-13 15t-7.5 7 q-14 62 -31 103t-29.5 56t-23.5 33t-15 40q-4 21 6 53.5t4.5 49.5t-44.5 25q-15 3 -44.5 18t-35.5 16q-8 1 -11 26t8 51t36 27q37 3 51 -30t4 -58q-11 -19 -2 -26.5t30 -0.5q13 4 13 36v37q-5 30 -13.5 50t-21 30.5t-23.5 15t-27 7.5q-107 -8 -89 -134q0 -15 -1 -15 q-9 9 -29.5 10.5t-33 -0.5t-15.5 5q1 57 -16 90t-45 34q-27 1 -41.5 -27.5t-16.5 -59.5q-1 -15 3.5 -37t13 -37.5t15.5 -13.5q10 3 16 14q4 9 -7 8q-7 0 -15.5 14.5t-9.5 33.5q-1 22 9 37t34 14q17 0 27 -21t9.5 -39t-1.5 -22q-22 -15 -31 -29q-8 -12 -27.5 -23.5 t-20.5 -12.5q-13 -14 -15.5 -27t7.5 -18q14 -8 25 -19.5t16 -19t18.5 -13t35.5 -6.5q47 -2 102 15q2 1 23 7t34.5 10.5t29.5 13t21 17.5q9 14 20 8q5 -3 6.5 -8.5t-3 -12t-16.5 -9.5q-20 -6 -56.5 -21.5t-45.5 -19.5q-44 -19 -70 -23q-25 -5 -79 2q-10 2 -9 -2t17 -19 q25 -23 67 -22q17 1 36 7t36 14t33.5 17.5t30 17t24.5 12t17.5 2.5t8.5 -11q0 -2 -1 -4.5t-4 -5t-6 -4.5t-8.5 -5t-9 -4.5t-10 -5t-9.5 -4.5q-28 -14 -67.5 -44t-66.5 -43t-49 -1q-21 11 -63 73q-22 31 -25 22q-1 -3 -1 -10q0 -25 -15 -56.5t-29.5 -55.5t-21 -58t11.5 -63 q-23 -6 -62.5 -90t-47.5 -141q-2 -18 -1.5 -69t-5.5 -59q-8 -24 -29 -3q-32 31 -36 94q-2 28 4 56q4 19 -1 18l-4 -5q-36 -65 10 -166q5 -12 25 -28t24 -20q20 -23 104 -90.5t93 -76.5q16 -15 17.5 -38t-14 -43t-45.5 -23q8 -15 29 -44.5t28 -54t7 -70.5q46 24 7 92 q-4 8 -10.5 16t-9.5 12t-2 6q3 5 13 9.5t20 -2.5q46 -52 166 -36q133 15 177 87q23 38 34 30q12 -6 10 -52q-1 -25 -23 -92q-9 -23 -6 -37.5t24 -15.5q3 19 14.5 77t13.5 90q2 21 -6.5 73.5t-7.5 97t23 70.5q15 18 51 18q1 37 34.5 53t72.5 10.5t60 -22.5zM626 1152 q3 17 -2.5 30t-11.5 15q-9 2 -9 -7q2 -5 5 -6q10 0 7 -15q-3 -20 8 -20q3 0 3 3zM1045 955q-2 8 -6.5 11.5t-13 5t-14.5 5.5q-5 3 -9.5 8t-7 8t-5.5 6.5t-4 4t-4 -1.5q-14 -16 7 -43.5t39 -31.5q9 -1 14.5 8t3.5 20zM867 1168q0 11 -5 19.5t-11 12.5t-9 3q-14 -1 -7 -7l4 -2 q14 -4 18 -31q0 -3 8 2zM921 1401q0 2 -2.5 5t-9 7t-9.5 6q-15 15 -24 15q-9 -1 -11.5 -7.5t-1 -13t-0.5 -12.5q-1 -4 -6 -10.5t-6 -9t3 -8.5q4 -3 8 0t11 9t15 9q1 1 9 1t15 2t9 7zM1486 60q20 -12 31 -24.5t12 -24t-2.5 -22.5t-15.5 -22t-23.5 -19.5t-30 -18.5 t-31.5 -16.5t-32 -15.5t-27 -13q-38 -19 -85.5 -56t-75.5 -64q-17 -16 -68 -19.5t-89 14.5q-18 9 -29.5 23.5t-16.5 25.5t-22 19.5t-47 9.5q-44 1 -130 1q-19 0 -57 -1.5t-58 -2.5q-44 -1 -79.5 -15t-53.5 -30t-43.5 -28.5t-53.5 -11.5q-29 1 -111 31t-146 43q-19 4 -51 9.5 t-50 9t-39.5 9.5t-33.5 14.5t-17 19.5q-10 23 7 66.5t18 54.5q1 16 -4 40t-10 42.5t-4.5 36.5t10.5 27q14 12 57 14t60 12q30 18 42 35t12 51q21 -73 -32 -106q-32 -20 -83 -15q-34 3 -43 -10q-13 -15 5 -57q2 -6 8 -18t8.5 -18t4.5 -17t1 -22q0 -15 -17 -49t-14 -48 q3 -17 37 -26q20 -6 84.5 -18.5t99.5 -20.5q24 -6 74 -22t82.5 -23t55.5 -4q43 6 64.5 28t23 48t-7.5 58.5t-19 52t-20 36.5q-121 190 -169 242q-68 74 -113 40q-11 -9 -15 15q-3 16 -2 38q1 29 10 52t24 47t22 42q8 21 26.5 72t29.5 78t30 61t39 54q110 143 124 195 q-12 112 -16 310q-2 90 24 151.5t106 104.5q39 21 104 21q53 1 106 -13.5t89 -41.5q57 -42 91.5 -121.5t29.5 -147.5q-5 -95 30 -214q34 -113 133 -218q55 -59 99.5 -163t59.5 -191q8 -49 5 -84.5t-12 -55.5t-20 -22q-10 -2 -23.5 -19t-27 -35.5t-40.5 -33.5t-61 -14 q-18 1 -31.5 5t-22.5 13.5t-13.5 15.5t-11.5 20.5t-9 19.5q-22 37 -41 30t-28 -49t7 -97q20 -70 1 -195q-10 -65 18 -100.5t73 -33t85 35.5q59 49 89.5 66.5t103.5 42.5q53 18 77 36.5t18.5 34.5t-25 28.5t-51.5 23.5q-33 11 -49.5 48t-15 72.5t15.5 47.5q1 -31 8 -56.5 t14.5 -40.5t20.5 -28.5t21 -19t21.5 -13t16.5 -9.5z" /> +<glyph unicode="" d="M1024 36q-42 241 -140 498h-2l-2 -1q-16 -6 -43 -16.5t-101 -49t-137 -82t-131 -114.5t-103 -148l-15 11q184 -150 418 -150q132 0 256 52zM839 643q-21 49 -53 111q-311 -93 -673 -93q-1 -7 -1 -21q0 -124 44 -236.5t124 -201.5q50 89 123.5 166.5t142.5 124.5t130.5 81 t99.5 48l37 13q4 1 13 3.5t13 4.5zM732 855q-120 213 -244 378q-138 -65 -234 -186t-128 -272q302 0 606 80zM1416 536q-210 60 -409 29q87 -239 128 -469q111 75 185 189.5t96 250.5zM611 1277q-1 0 -2 -1q1 1 2 1zM1201 1132q-185 164 -433 164q-76 0 -155 -19 q131 -170 246 -382q69 26 130 60.5t96.5 61.5t65.5 57t37.5 40.5zM1424 647q-3 232 -149 410l-1 -1q-9 -12 -19 -24.5t-43.5 -44.5t-71 -60.5t-100 -65t-131.5 -64.5q25 -53 44 -95q2 -6 6.5 -17.5t7.5 -16.5q36 5 74.5 7t73.5 2t69 -1.5t64 -4t56.5 -5.5t48 -6.5t36.5 -6 t25 -4.5zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1173 473q0 50 -19.5 91.5t-48.5 68.5t-73 49t-82.5 34t-87.5 23l-104 24q-30 7 -44 10.5t-35 11.5t-30 16t-16.5 21t-7.5 30q0 77 144 77q43 0 77 -12t54 -28.5t38 -33.5t40 -29t48 -12q47 0 75.5 32t28.5 77q0 55 -56 99.5t-142 67.5t-182 23q-68 0 -132 -15.5 t-119.5 -47t-89 -87t-33.5 -128.5q0 -61 19 -106.5t56 -75.5t80 -48.5t103 -32.5l146 -36q90 -22 112 -36q32 -20 32 -60q0 -39 -40 -64.5t-105 -25.5q-51 0 -91.5 16t-65 38.5t-45.5 45t-46 38.5t-54 16q-50 0 -75.5 -30t-25.5 -75q0 -92 122 -157.5t291 -65.5 q73 0 140 18.5t122.5 53.5t88.5 93.5t33 131.5zM1536 256q0 -159 -112.5 -271.5t-271.5 -112.5q-130 0 -234 80q-77 -16 -150 -16q-143 0 -273.5 55.5t-225 150t-150 225t-55.5 273.5q0 73 16 150q-80 104 -80 234q0 159 112.5 271.5t271.5 112.5q130 0 234 -80 q77 16 150 16q143 0 273.5 -55.5t225 -150t150 -225t55.5 -273.5q0 -73 -16 -150q80 -104 80 -234z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1000 1102l37 194q5 23 -9 40t-35 17h-712q-23 0 -38.5 -17t-15.5 -37v-1101q0 -7 6 -1l291 352q23 26 38 33.5t48 7.5h239q22 0 37 14.5t18 29.5q24 130 37 191q4 21 -11.5 40t-36.5 19h-294q-29 0 -48 19t-19 48v42q0 29 19 47.5t48 18.5h346q18 0 35 13.5t20 29.5z M1227 1324q-15 -73 -53.5 -266.5t-69.5 -350t-35 -173.5q-6 -22 -9 -32.5t-14 -32.5t-24.5 -33t-38.5 -21t-58 -10h-271q-13 0 -22 -10q-8 -9 -426 -494q-22 -25 -58.5 -28.5t-48.5 5.5q-55 22 -55 98v1410q0 55 38 102.5t120 47.5h888q95 0 127 -53t10 -159zM1227 1324 l-158 -790q4 17 35 173.5t69.5 350t53.5 266.5z" /> +<glyph unicode="" d="M704 192v1024q0 14 -9 23t-23 9h-480q-14 0 -23 -9t-9 -23v-1024q0 -14 9 -23t23 -9h480q14 0 23 9t9 23zM1376 576v640q0 14 -9 23t-23 9h-480q-14 0 -23 -9t-9 -23v-640q0 -14 9 -23t23 -9h480q14 0 23 9t9 23zM1536 1344v-1408q0 -26 -19 -45t-45 -19h-1408 q-26 0 -45 19t-19 45v1408q0 26 19 45t45 19h1408q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1280 480q0 -40 -28 -68t-68 -28q-51 0 -80 43l-227 341h-45v-132l247 -411q9 -15 9 -33q0 -26 -19 -45t-45 -19h-192v-272q0 -46 -33 -79t-79 -33h-160q-46 0 -79 33t-33 79v272h-192q-26 0 -45 19t-19 45q0 18 9 33l247 411v132h-45l-227 -341q-29 -43 -80 -43 q-40 0 -68 28t-28 68q0 29 16 53l256 384q73 107 176 107h384q103 0 176 -107l256 -384q16 -24 16 -53zM864 1280q0 -93 -65.5 -158.5t-158.5 -65.5t-158.5 65.5t-65.5 158.5t65.5 158.5t158.5 65.5t158.5 -65.5t65.5 -158.5z" /> +<glyph unicode="" horiz-adv-x="1024" d="M1024 832v-416q0 -40 -28 -68t-68 -28t-68 28t-28 68v352h-64v-912q0 -46 -33 -79t-79 -33t-79 33t-33 79v464h-64v-464q0 -46 -33 -79t-79 -33t-79 33t-33 79v912h-64v-352q0 -40 -28 -68t-68 -28t-68 28t-28 68v416q0 80 56 136t136 56h640q80 0 136 -56t56 -136z M736 1280q0 -93 -65.5 -158.5t-158.5 -65.5t-158.5 65.5t-65.5 158.5t65.5 158.5t158.5 65.5t158.5 -65.5t65.5 -158.5z" /> +<glyph unicode="" d="M773 234l350 473q16 22 24.5 59t-6 85t-61.5 79q-40 26 -83 25.5t-73.5 -17.5t-54.5 -45q-36 -40 -96 -40q-59 0 -95 40q-24 28 -54.5 45t-73.5 17.5t-84 -25.5q-46 -31 -60.5 -79t-6 -85t24.5 -59zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103 t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1472 640q0 117 -45.5 223.5t-123 184t-184 123t-223.5 45.5t-223.5 -45.5t-184 -123t-123 -184t-45.5 -223.5t45.5 -223.5t123 -184t184 -123t223.5 -45.5t223.5 45.5t184 123t123 184t45.5 223.5zM1748 363q-4 -15 -20 -20l-292 -96v-306q0 -16 -13 -26q-15 -10 -29 -4 l-292 94l-180 -248q-10 -13 -26 -13t-26 13l-180 248l-292 -94q-14 -6 -29 4q-13 10 -13 26v306l-292 96q-16 5 -20 20q-5 17 4 29l180 248l-180 248q-9 13 -4 29q4 15 20 20l292 96v306q0 16 13 26q15 10 29 4l292 -94l180 248q9 12 26 12t26 -12l180 -248l292 94 q14 6 29 -4q13 -10 13 -26v-306l292 -96q16 -5 20 -20q5 -16 -4 -29l-180 -248l180 -248q9 -12 4 -29z" /> +<glyph unicode="" d="M1262 233q-54 -9 -110 -9q-182 0 -337 90t-245 245t-90 337q0 192 104 357q-201 -60 -328.5 -229t-127.5 -384q0 -130 51 -248.5t136.5 -204t204 -136.5t248.5 -51q144 0 273.5 61.5t220.5 171.5zM1465 318q-94 -203 -283.5 -324.5t-413.5 -121.5q-156 0 -298 61 t-245 164t-164 245t-61 298q0 153 57.5 292.5t156 241.5t235.5 164.5t290 68.5q44 2 61 -39q18 -41 -15 -72q-86 -78 -131.5 -181.5t-45.5 -218.5q0 -148 73 -273t198 -198t273 -73q118 0 228 51q41 18 72 -13q14 -14 17.5 -34t-4.5 -38z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1088 704q0 26 -19 45t-45 19h-256q-26 0 -45 -19t-19 -45t19 -45t45 -19h256q26 0 45 19t19 45zM1664 896v-960q0 -26 -19 -45t-45 -19h-1408q-26 0 -45 19t-19 45v960q0 26 19 45t45 19h1408q26 0 45 -19t19 -45zM1728 1344v-256q0 -26 -19 -45t-45 -19h-1536 q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h1536q26 0 45 -19t19 -45z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1632 576q0 -26 -19 -45t-45 -19h-224q0 -171 -67 -290l208 -209q19 -19 19 -45t-19 -45q-18 -19 -45 -19t-45 19l-198 197q-5 -5 -15 -13t-42 -28.5t-65 -36.5t-82 -29t-97 -13v896h-128v-896q-51 0 -101.5 13.5t-87 33t-66 39t-43.5 32.5l-15 14l-183 -207 q-20 -21 -48 -21q-24 0 -43 16q-19 18 -20.5 44.5t15.5 46.5l202 227q-58 114 -58 274h-224q-26 0 -45 19t-19 45t19 45t45 19h224v294l-173 173q-19 19 -19 45t19 45t45 19t45 -19l173 -173h844l173 173q19 19 45 19t45 -19t19 -45t-19 -45l-173 -173v-294h224q26 0 45 -19 t19 -45zM1152 1152h-640q0 133 93.5 226.5t226.5 93.5t226.5 -93.5t93.5 -226.5z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1917 1016q23 -64 -150 -294q-24 -32 -65 -85q-78 -100 -90 -131q-17 -41 14 -81q17 -21 81 -82h1l1 -1l1 -1l2 -2q141 -131 191 -221q3 -5 6.5 -12.5t7 -26.5t-0.5 -34t-25 -27.5t-59 -12.5l-256 -4q-24 -5 -56 5t-52 22l-20 12q-30 21 -70 64t-68.5 77.5t-61 58 t-56.5 15.5q-3 -1 -8 -3.5t-17 -14.5t-21.5 -29.5t-17 -52t-6.5 -77.5q0 -15 -3.5 -27.5t-7.5 -18.5l-4 -5q-18 -19 -53 -22h-115q-71 -4 -146 16.5t-131.5 53t-103 66t-70.5 57.5l-25 24q-10 10 -27.5 30t-71.5 91t-106 151t-122.5 211t-130.5 272q-6 16 -6 27t3 16l4 6 q15 19 57 19l274 2q12 -2 23 -6.5t16 -8.5l5 -3q16 -11 24 -32q20 -50 46 -103.5t41 -81.5l16 -29q29 -60 56 -104t48.5 -68.5t41.5 -38.5t34 -14t27 5q2 1 5 5t12 22t13.5 47t9.5 81t0 125q-2 40 -9 73t-14 46l-6 12q-25 34 -85 43q-13 2 5 24q17 19 38 30q53 26 239 24 q82 -1 135 -13q20 -5 33.5 -13.5t20.5 -24t10.5 -32t3.5 -45.5t-1 -55t-2.5 -70.5t-1.5 -82.5q0 -11 -1 -42t-0.5 -48t3.5 -40.5t11.5 -39t22.5 -24.5q8 -2 17 -4t26 11t38 34.5t52 67t68 107.5q60 104 107 225q4 10 10 17.5t11 10.5l4 3l5 2.5t13 3t20 0.5l288 2 q39 5 64 -2.5t31 -16.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M675 252q21 34 11 69t-45 50q-34 14 -73 1t-60 -46q-22 -34 -13 -68.5t43 -50.5t74.5 -2.5t62.5 47.5zM769 373q8 13 3.5 26.5t-17.5 18.5q-14 5 -28.5 -0.5t-21.5 -18.5q-17 -31 13 -45q14 -5 29 0.5t22 18.5zM943 266q-45 -102 -158 -150t-224 -12 q-107 34 -147.5 126.5t6.5 187.5q47 93 151.5 139t210.5 19q111 -29 158.5 -119.5t2.5 -190.5zM1255 426q-9 96 -89 170t-208.5 109t-274.5 21q-223 -23 -369.5 -141.5t-132.5 -264.5q9 -96 89 -170t208.5 -109t274.5 -21q223 23 369.5 141.5t132.5 264.5zM1563 422 q0 -68 -37 -139.5t-109 -137t-168.5 -117.5t-226 -83t-270.5 -31t-275 33.5t-240.5 93t-171.5 151t-65 199.5q0 115 69.5 245t197.5 258q169 169 341.5 236t246.5 -7q65 -64 20 -209q-4 -14 -1 -20t10 -7t14.5 0.5t13.5 3.5l6 2q139 59 246 59t153 -61q45 -63 0 -178 q-2 -13 -4.5 -20t4.5 -12.5t12 -7.5t17 -6q57 -18 103 -47t80 -81.5t34 -116.5zM1489 1046q42 -47 54.5 -108.5t-6.5 -117.5q-8 -23 -29.5 -34t-44.5 -4q-23 8 -34 29.5t-4 44.5q20 63 -24 111t-107 35q-24 -5 -45 8t-25 37q-5 24 8 44.5t37 25.5q60 13 119 -5.5t101 -65.5z M1670 1209q87 -96 112.5 -222.5t-13.5 -241.5q-9 -27 -34 -40t-52 -4t-40 34t-5 52q28 82 10 172t-80 158q-62 69 -148 95.5t-173 8.5q-28 -6 -52 9.5t-30 43.5t9.5 51.5t43.5 29.5q123 26 244 -11.5t208 -134.5z" /> +<glyph unicode="" d="M1133 -34q-171 -94 -368 -94q-196 0 -367 94q138 87 235.5 211t131.5 268q35 -144 132.5 -268t235.5 -211zM638 1394v-485q0 -252 -126.5 -459.5t-330.5 -306.5q-181 215 -181 495q0 187 83.5 349.5t229.5 269.5t325 137zM1536 638q0 -280 -181 -495 q-204 99 -330.5 306.5t-126.5 459.5v485q179 -30 325 -137t229.5 -269.5t83.5 -349.5z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1402 433q-32 -80 -76 -138t-91 -88.5t-99 -46.5t-101.5 -14.5t-96.5 8.5t-86.5 22t-69.5 27.5t-46 22.5l-17 10q-113 -228 -289.5 -359.5t-384.5 -132.5q-19 0 -32 13t-13 32t13 31.5t32 12.5q173 1 322.5 107.5t251.5 294.5q-36 -14 -72 -23t-83 -13t-91 2.5t-93 28.5 t-92 59t-84.5 100t-74.5 146q114 47 214 57t167.5 -7.5t124.5 -56.5t88.5 -77t56.5 -82q53 131 79 291q-7 -1 -18 -2.5t-46.5 -2.5t-69.5 0.5t-81.5 10t-88.5 23t-84 42.5t-75 65t-54.5 94.5t-28.5 127.5q70 28 133.5 36.5t112.5 -1t92 -30t73.5 -50t56 -61t42 -63t27.5 -56 t16 -39.5l4 -16q12 122 12 195q-8 6 -21.5 16t-49 44.5t-63.5 71.5t-54 93t-33 112.5t12 127t70 138.5q73 -25 127.5 -61.5t84.5 -76.5t48 -85t20.5 -89t-0.5 -85.5t-13 -76.5t-19 -62t-17 -42l-7 -15q1 -5 1 -50.5t-1 -71.5q3 7 10 18.5t30.5 43t50.5 58t71 55.5t91.5 44.5 t112 14.5t132.5 -24q-2 -78 -21.5 -141.5t-50 -104.5t-69.5 -71.5t-81.5 -45.5t-84.5 -24t-80 -9.5t-67.5 1t-46.5 4.5l-17 3q-23 -147 -73 -283q6 7 18 18.5t49.5 41t77.5 52.5t99.5 42t117.5 20t129 -23.5t137 -77.5z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1259 283v-66q0 -85 -57.5 -144.5t-138.5 -59.5h-57l-260 -269v269h-529q-81 0 -138.5 59.5t-57.5 144.5v66h1238zM1259 609v-255h-1238v255h1238zM1259 937v-255h-1238v255h1238zM1259 1077v-67h-1238v67q0 84 57.5 143.5t138.5 59.5h846q81 0 138.5 -59.5t57.5 -143.5z " /> +<glyph unicode="" d="M1152 640q0 -14 -9 -23l-320 -320q-9 -9 -23 -9q-13 0 -22.5 9.5t-9.5 22.5v192h-352q-13 0 -22.5 9.5t-9.5 22.5v192q0 13 9.5 22.5t22.5 9.5h352v192q0 14 9 23t23 9q12 0 24 -10l319 -319q9 -9 9 -23zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198 t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1152 736v-192q0 -13 -9.5 -22.5t-22.5 -9.5h-352v-192q0 -14 -9 -23t-23 -9q-12 0 -24 10l-319 319q-9 9 -9 23t9 23l320 320q9 9 23 9q13 0 22.5 -9.5t9.5 -22.5v-192h352q13 0 22.5 -9.5t9.5 -22.5zM1312 640q0 148 -73 273t-198 198t-273 73t-273 -73t-198 -198 t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1024 960v-640q0 -26 -19 -45t-45 -19q-20 0 -37 12l-448 320q-27 19 -27 52t27 52l448 320q17 12 37 12q26 0 45 -19t19 -45zM1280 160v960q0 13 -9.5 22.5t-22.5 9.5h-960q-13 0 -22.5 -9.5t-9.5 -22.5v-960q0 -13 9.5 -22.5t22.5 -9.5h960q13 0 22.5 9.5t9.5 22.5z M1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" d="M1024 640q0 -106 -75 -181t-181 -75t-181 75t-75 181t75 181t181 75t181 -75t75 -181zM768 1184q-148 0 -273 -73t-198 -198t-73 -273t73 -273t198 -198t273 -73t273 73t198 198t73 273t-73 273t-198 198t-273 73zM1536 640q0 -209 -103 -385.5t-279.5 -279.5 t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1023 349l102 -204q-58 -179 -210 -290t-339 -111q-156 0 -288.5 77.5t-210 210t-77.5 288.5q0 181 104.5 330t274.5 211l17 -131q-122 -54 -195 -165.5t-73 -244.5q0 -185 131.5 -316.5t316.5 -131.5q126 0 232.5 65t165 175.5t49.5 236.5zM1571 249l58 -114l-256 -128 q-13 -7 -29 -7q-40 0 -57 35l-239 477h-472q-24 0 -42.5 16.5t-21.5 40.5l-96 779q-2 16 6 42q14 51 57 82.5t97 31.5q66 0 113 -47t47 -113q0 -69 -52 -117.5t-120 -41.5l37 -289h423v-128h-407l16 -128h455q40 0 57 -35l228 -455z" /> +<glyph unicode="" d="M1254 899q16 85 -21 132q-52 65 -187 45q-17 -3 -41 -12.5t-57.5 -30.5t-64.5 -48.5t-59.5 -70t-44.5 -91.5q80 7 113.5 -16t26.5 -99q-5 -52 -52 -143q-43 -78 -71 -99q-44 -32 -87 14q-23 24 -37.5 64.5t-19 73t-10 84t-8.5 71.5q-23 129 -34 164q-12 37 -35.5 69 t-50.5 40q-57 16 -127 -25q-54 -32 -136.5 -106t-122.5 -102v-7q16 -8 25.5 -26t21.5 -20q21 -3 54.5 8.5t58 10.5t41.5 -30q11 -18 18.5 -38.5t15 -48t12.5 -40.5q17 -46 53 -187q36 -146 57 -197q42 -99 103 -125q43 -12 85 -1.5t76 31.5q131 77 250 237 q104 139 172.5 292.5t82.5 226.5zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1152" d="M1152 704q0 -191 -94.5 -353t-256.5 -256.5t-353 -94.5h-160q-14 0 -23 9t-9 23v611l-215 -66q-3 -1 -9 -1q-10 0 -19 6q-13 10 -13 26v128q0 23 23 31l233 71v93l-215 -66q-3 -1 -9 -1q-10 0 -19 6q-13 10 -13 26v128q0 23 23 31l233 71v250q0 14 9 23t23 9h160 q14 0 23 -9t9 -23v-181l375 116q15 5 28 -5t13 -26v-128q0 -23 -23 -31l-393 -121v-93l375 116q15 5 28 -5t13 -26v-128q0 -23 -23 -31l-393 -121v-487q188 13 318 151t130 328q0 14 9 23t23 9h160q14 0 23 -9t9 -23z" /> +<glyph unicode="" horiz-adv-x="1408" d="M1152 736v-64q0 -14 -9 -23t-23 -9h-352v-352q0 -14 -9 -23t-23 -9h-64q-14 0 -23 9t-9 23v352h-352q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h352v352q0 14 9 23t23 9h64q14 0 23 -9t9 -23v-352h352q14 0 23 -9t9 -23zM1280 288v832q0 66 -47 113t-113 47h-832 q-66 0 -113 -47t-47 -113v-832q0 -66 47 -113t113 -47h832q66 0 113 47t47 113zM1408 1120v-832q0 -119 -84.5 -203.5t-203.5 -84.5h-832q-119 0 -203.5 84.5t-84.5 203.5v832q0 119 84.5 203.5t203.5 84.5h832q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="2176" d="M620 416q-110 -64 -268 -64h-128v64h-64q-13 0 -22.5 23.5t-9.5 56.5q0 24 7 49q-58 2 -96.5 10.5t-38.5 20.5t38.5 20.5t96.5 10.5q-7 25 -7 49q0 33 9.5 56.5t22.5 23.5h64v64h128q158 0 268 -64h1113q42 -7 106.5 -18t80.5 -14q89 -15 150 -40.5t83.5 -47.5t22.5 -40 t-22.5 -40t-83.5 -47.5t-150 -40.5q-16 -3 -80.5 -14t-106.5 -18h-1113zM1739 668q53 -36 53 -92t-53 -92l81 -30q68 48 68 122t-68 122zM625 400h1015q-217 -38 -456 -80q-57 0 -113 -24t-83 -48l-28 -24l-288 -288q-26 -26 -70.5 -45t-89.5 -19h-96l-93 464h29 q157 0 273 64zM352 816h-29l93 464h96q46 0 90 -19t70 -45l288 -288q4 -4 11 -10.5t30.5 -23t48.5 -29t61.5 -23t72.5 -10.5l456 -80h-1015q-116 64 -273 64z" /> +<glyph unicode="" horiz-adv-x="1664" d="M1519 760q62 0 103.5 -40.5t41.5 -101.5q0 -97 -93 -130l-172 -59l56 -167q7 -21 7 -47q0 -59 -42 -102t-101 -43q-47 0 -85.5 27t-53.5 72l-55 165l-310 -106l55 -164q8 -24 8 -47q0 -59 -42 -102t-102 -43q-47 0 -85 27t-53 72l-55 163l-153 -53q-29 -9 -50 -9 q-61 0 -101.5 40t-40.5 101q0 47 27.5 85t71.5 53l156 53l-105 313l-156 -54q-26 -8 -48 -8q-60 0 -101 40.5t-41 100.5q0 47 27.5 85t71.5 53l157 53l-53 159q-8 24 -8 47q0 60 42 102.5t102 42.5q47 0 85 -27t53 -72l54 -160l310 105l-54 160q-8 24 -8 47q0 59 42.5 102 t101.5 43q47 0 85.5 -27.5t53.5 -71.5l53 -161l162 55q21 6 43 6q60 0 102.5 -39.5t42.5 -98.5q0 -45 -30 -81.5t-74 -51.5l-157 -54l105 -316l164 56q24 8 46 8zM725 498l310 105l-105 315l-310 -107z" /> +<glyph unicode="" d="M1248 1408q119 0 203.5 -84.5t84.5 -203.5v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960zM1280 352v436q-31 -35 -64 -55q-34 -22 -132.5 -85t-151.5 -99q-98 -69 -164 -69v0v0q-66 0 -164 69 q-46 32 -141.5 92.5t-142.5 92.5q-12 8 -33 27t-31 27v-436q0 -40 28 -68t68 -28h832q40 0 68 28t28 68zM1280 925q0 41 -27.5 70t-68.5 29h-832q-40 0 -68 -28t-28 -68q0 -37 30.5 -76.5t67.5 -64.5q47 -32 137.5 -89t129.5 -83q3 -2 17 -11.5t21 -14t21 -13t23.5 -13 t21.5 -9.5t22.5 -7.5t20.5 -2.5t20.5 2.5t22.5 7.5t21.5 9.5t23.5 13t21 13t21 14t17 11.5l267 174q35 23 66.5 62.5t31.5 73.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M127 640q0 163 67 313l367 -1005q-196 95 -315 281t-119 411zM1415 679q0 -19 -2.5 -38.5t-10 -49.5t-11.5 -44t-17.5 -59t-17.5 -58l-76 -256l-278 826q46 3 88 8q19 2 26 18.5t-2.5 31t-28.5 13.5l-205 -10q-75 1 -202 10q-12 1 -20.5 -5t-11.5 -15t-1.5 -18.5t9 -16.5 t19.5 -8l80 -8l120 -328l-168 -504l-280 832q46 3 88 8q19 2 26 18.5t-2.5 31t-28.5 13.5l-205 -10q-7 0 -23 0.5t-26 0.5q105 160 274.5 253.5t367.5 93.5q147 0 280.5 -53t238.5 -149h-10q-55 0 -92 -40.5t-37 -95.5q0 -12 2 -24t4 -21.5t8 -23t9 -21t12 -22.5t12.5 -21 t14.5 -24t14 -23q63 -107 63 -212zM909 573l237 -647q1 -6 5 -11q-126 -44 -255 -44q-112 0 -217 32zM1570 1009q95 -174 95 -369q0 -209 -104 -385.5t-279 -278.5l235 678q59 169 59 276q0 42 -6 79zM896 1536q182 0 348 -71t286 -191t191 -286t71 -348t-71 -348t-191 -286 t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71zM896 -215q173 0 331.5 68t273 182.5t182.5 273t68 331.5t-68 331.5t-182.5 273t-273 182.5t-331.5 68t-331.5 -68t-273 -182.5t-182.5 -273t-68 -331.5t68 -331.5t182.5 -273 t273 -182.5t331.5 -68z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1086 1536v-1536l-272 -128q-228 20 -414 102t-293 208.5t-107 272.5q0 140 100.5 263.5t275 205.5t391.5 108v-172q-217 -38 -356.5 -150t-139.5 -255q0 -152 154.5 -267t388.5 -145v1360zM1755 954l37 -390l-525 114l147 83q-119 70 -280 99v172q277 -33 481 -157z" /> +<glyph unicode="" horiz-adv-x="2048" d="M960 1536l960 -384v-128h-128q0 -26 -20.5 -45t-48.5 -19h-1526q-28 0 -48.5 19t-20.5 45h-128v128zM256 896h256v-768h128v768h256v-768h128v768h256v-768h128v768h256v-768h59q28 0 48.5 -19t20.5 -45v-64h-1664v64q0 26 20.5 45t48.5 19h59v768zM1851 -64 q28 0 48.5 -19t20.5 -45v-128h-1920v128q0 26 20.5 45t48.5 19h1782z" /> +<glyph unicode="" horiz-adv-x="2304" d="M1774 700l18 -316q4 -69 -82 -128t-235 -93.5t-323 -34.5t-323 34.5t-235 93.5t-82 128l18 316l574 -181q22 -7 48 -7t48 7zM2304 1024q0 -23 -22 -31l-1120 -352q-4 -1 -10 -1t-10 1l-652 206q-43 -34 -71 -111.5t-34 -178.5q63 -36 63 -109q0 -69 -58 -107l58 -433 q2 -14 -8 -25q-9 -11 -24 -11h-192q-15 0 -24 11q-10 11 -8 25l58 433q-58 38 -58 107q0 73 65 111q11 207 98 330l-333 104q-22 8 -22 31t22 31l1120 352q4 1 10 1t10 -1l1120 -352q22 -8 22 -31z" /> +<glyph unicode="" d="M859 579l13 -707q-62 11 -105 11q-41 0 -105 -11l13 707q-40 69 -168.5 295.5t-216.5 374.5t-181 287q58 -15 108 -15q43 0 111 15q63 -111 133.5 -229.5t167 -276.5t138.5 -227q37 61 109.5 177.5t117.5 190t105 176t107 189.5q54 -14 107 -14q56 0 114 14v0 q-28 -39 -60 -88.5t-49.5 -78.5t-56.5 -96t-49 -84q-146 -248 -353 -610z" /> +<glyph unicode="" horiz-adv-x="1280" d="M981 197q0 25 -7 49t-14.5 42t-27 41.5t-29.5 35t-38.5 34.5t-36.5 29t-41.5 30t-36.5 26q-16 2 -49 2q-53 0 -104.5 -7t-107 -25t-97 -46t-68.5 -74.5t-27 -105.5q0 -56 23.5 -102t61 -75.5t87 -50t100 -29t101.5 -8.5q58 0 111.5 13t99 39t73 73t27.5 109zM864 1055 q0 59 -17 125.5t-48 129t-84 103.5t-117 41q-42 0 -82.5 -19.5t-66.5 -52.5q-46 -59 -46 -160q0 -46 10 -97.5t31.5 -103t52 -92.5t75 -67t96.5 -26q37 0 77.5 16.5t65.5 43.5q53 56 53 159zM752 1536h417l-137 -88h-132q75 -63 113 -133t38 -160q0 -72 -24.5 -129.5 t-59.5 -93t-69.5 -65t-59 -61.5t-24.5 -66q0 -36 32 -70.5t77 -68t90.5 -73.5t77.5 -104t32 -142q0 -91 -49 -173q-71 -122 -209.5 -179.5t-298.5 -57.5q-132 0 -246.5 41.5t-172.5 137.5q-36 59 -36 131q0 81 44.5 150t118.5 115q131 82 404 100q-32 41 -47.5 73.5 t-15.5 73.5q0 40 21 85q-46 -4 -68 -4q-148 0 -249.5 96.5t-101.5 244.5q0 82 36 159t99 131q76 66 182 98t218 32z" /> +<glyph unicode="" horiz-adv-x="1984" d="M831 572q0 -56 -40.5 -96t-96.5 -40q-57 0 -98 40t-41 96q0 57 41.5 98t97.5 41t96.5 -41t40.5 -98zM1292 711q56 0 96.5 -41t40.5 -98q0 -56 -40.5 -96t-96.5 -40q-57 0 -98 40t-41 96q0 57 41.5 98t97.5 41zM1984 722q0 -62 -31 -114t-83 -82q5 -33 5 -61 q0 -121 -68.5 -230.5t-197.5 -193.5q-125 -82 -285.5 -125.5t-335.5 -43.5q-176 0 -336.5 43.5t-284.5 125.5q-129 84 -197.5 193t-68.5 231q0 29 5 66q-48 31 -77 81.5t-29 109.5q0 94 66 160t160 66q83 0 148 -55q248 158 592 164l134 423q4 14 17.5 21.5t28.5 4.5 l347 -82q22 50 68.5 81t102.5 31q77 0 131.5 -54.5t54.5 -131.5t-54.5 -132t-131.5 -55q-76 0 -130.5 54t-55.5 131l-315 74l-116 -366q327 -14 560 -166q64 58 151 58q94 0 160 -66t66 -160zM1664 1459q-45 0 -77 -32t-32 -77t32 -77t77 -32t77 32t32 77t-32 77t-77 32z M77 722q0 -67 51 -111q49 131 180 235q-36 25 -82 25q-62 0 -105.5 -43.5t-43.5 -105.5zM1567 105q112 73 171.5 166t59.5 194t-59.5 193.5t-171.5 165.5q-116 75 -265.5 115.5t-313.5 40.5t-313.5 -40.5t-265.5 -115.5q-112 -73 -171.5 -165.5t-59.5 -193.5t59.5 -194 t171.5 -166q116 -75 265.5 -115.5t313.5 -40.5t313.5 40.5t265.5 115.5zM1850 605q57 46 57 117q0 62 -43.5 105.5t-105.5 43.5q-49 0 -86 -28q131 -105 178 -238zM1258 237q11 11 27 11t27 -11t11 -27.5t-11 -27.5q-99 -99 -319 -99h-2q-220 0 -319 99q-11 11 -11 27.5 t11 27.5t27 11t27 -11q77 -77 265 -77h2q188 0 265 77z" /> +<glyph unicode="" d="M950 393q7 7 17.5 7t17.5 -7t7 -18t-7 -18q-65 -64 -208 -64h-1h-1q-143 0 -207 64q-8 7 -8 18t8 18q7 7 17.5 7t17.5 -7q49 -51 172 -51h1h1q122 0 173 51zM671 613q0 -37 -26 -64t-63 -27t-63 27t-26 64t26 63t63 26t63 -26t26 -63zM1214 1049q-29 0 -50 21t-21 50 q0 30 21 51t50 21q30 0 51 -21t21 -51q0 -29 -21 -50t-51 -21zM1216 1408q132 0 226 -94t94 -227v-894q0 -133 -94 -227t-226 -94h-896q-132 0 -226 94t-94 227v894q0 133 94 227t226 94h896zM1321 596q35 14 57 45.5t22 70.5q0 51 -36 87.5t-87 36.5q-60 0 -98 -48 q-151 107 -375 115l83 265l206 -49q1 -50 36.5 -85t84.5 -35q50 0 86 35.5t36 85.5t-36 86t-86 36q-36 0 -66 -20.5t-45 -53.5l-227 54q-9 2 -17.5 -2.5t-11.5 -14.5l-95 -302q-224 -4 -381 -113q-36 43 -93 43q-51 0 -87 -36.5t-36 -87.5q0 -37 19.5 -67.5t52.5 -45.5 q-7 -25 -7 -54q0 -98 74 -181.5t201.5 -132t278.5 -48.5q150 0 277.5 48.5t201.5 132t74 181.5q0 27 -6 54zM971 702q37 0 63 -26t26 -63t-26 -64t-63 -27t-63 27t-26 64t26 63t63 26z" /> +<glyph unicode="" d="M866 697l90 27v62q0 79 -58 135t-138 56t-138 -55.5t-58 -134.5v-283q0 -20 -14 -33.5t-33 -13.5t-32.5 13.5t-13.5 33.5v120h-151v-122q0 -82 57.5 -139t139.5 -57q81 0 138.5 56.5t57.5 136.5v280q0 19 13.5 33t33.5 14q19 0 32.5 -14t13.5 -33v-54zM1199 502v122h-150 v-126q0 -20 -13.5 -33.5t-33.5 -13.5q-19 0 -32.5 14t-13.5 33v123l-90 -26l-60 28v-123q0 -80 58 -137t139 -57t138.5 57t57.5 139zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103 t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1920" d="M1062 824v118q0 42 -30 72t-72 30t-72 -30t-30 -72v-612q0 -175 -126 -299t-303 -124q-178 0 -303.5 125.5t-125.5 303.5v266h328v-262q0 -43 30 -72.5t72 -29.5t72 29.5t30 72.5v620q0 171 126.5 292t301.5 121q176 0 302 -122t126 -294v-136l-195 -58zM1592 602h328 v-266q0 -178 -125.5 -303.5t-303.5 -125.5q-177 0 -303 124.5t-126 300.5v268l131 -61l195 58v-270q0 -42 30 -71.5t72 -29.5t72 29.5t30 71.5v275z" /> +<glyph unicode="" d="M1472 160v480h-704v704h-480q-93 0 -158.5 -65.5t-65.5 -158.5v-480h704v-704h480q93 0 158.5 65.5t65.5 158.5zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5 t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="2048" d="M328 1254h204v-983h-532v697h328v286zM328 435v369h-123v-369h123zM614 968v-697h205v697h-205zM614 1254v-204h205v204h-205zM901 968h533v-942h-533v163h328v82h-328v697zM1229 435v369h-123v-369h123zM1516 968h532v-942h-532v163h327v82h-327v697zM1843 435v369h-123 v-369h123z" /> +<glyph unicode="" d="M1046 516q0 -64 -38 -109t-91 -45q-43 0 -70 15v277q28 17 70 17q53 0 91 -45.5t38 -109.5zM703 944q0 -64 -38 -109.5t-91 -45.5q-43 0 -70 15v277q28 17 70 17q53 0 91 -45t38 -109zM1265 513q0 134 -88 229t-213 95q-20 0 -39 -3q-23 -78 -78 -136q-87 -95 -211 -101 v-636l211 41v206q51 -19 117 -19q125 0 213 95t88 229zM922 940q0 134 -88.5 229t-213.5 95q-74 0 -141 -36h-186v-840l211 41v206q55 -19 116 -19q125 0 213.5 95t88.5 229zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960 q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="2038" d="M1222 607q75 3 143.5 -20.5t118 -58.5t101 -94.5t84 -108t75.5 -120.5q33 -56 78.5 -109t75.5 -80.5t99 -88.5q-48 -30 -108.5 -57.5t-138.5 -59t-114 -47.5q-44 37 -74 115t-43.5 164.5t-33 180.5t-42.5 168.5t-72.5 123t-122.5 48.5l-10 -2l-6 -4q4 -5 13 -14 q6 -5 28 -23.5t25.5 -22t19 -18t18 -20.5t11.5 -21t10.5 -27.5t4.5 -31t4 -40.5l1 -33q1 -26 -2.5 -57.5t-7.5 -52t-12.5 -58.5t-11.5 -53q-35 1 -101 -9.5t-98 -10.5q-39 0 -72 10q-2 16 -2 47q0 74 3 96q2 13 31.5 41.5t57 59t26.5 51.5q-24 2 -43 -24 q-36 -53 -111.5 -99.5t-136.5 -46.5q-25 0 -75.5 63t-106.5 139.5t-84 96.5q-6 4 -27 30q-482 -112 -513 -112q-16 0 -28 11t-12 27q0 15 8.5 26.5t22.5 14.5l486 106q-8 14 -8 25t5.5 17.5t16 11.5t20 7t23 4.5t18.5 4.5q4 1 15.5 7.5t17.5 6.5q15 0 28 -16t20 -33 q163 37 172 37q17 0 29.5 -11t12.5 -28q0 -15 -8.5 -26t-23.5 -14l-182 -40l-1 -16q-1 -26 81.5 -117.5t104.5 -91.5q47 0 119 80t72 129q0 36 -23.5 53t-51 18.5t-51 11.5t-23.5 34q0 16 10 34l-68 19q43 44 43 117q0 26 -5 58q82 16 144 16q44 0 71.5 -1.5t48.5 -8.5 t31 -13.5t20.5 -24.5t15.5 -33.5t17 -47.5t24 -60l50 25q-3 -40 -23 -60t-42.5 -21t-40 -6.5t-16.5 -20.5zM1282 842q-5 5 -13.5 15.5t-12 14.5t-10.5 11.5t-10 10.5l-8 8t-8.5 7.5t-8 5t-8.5 4.5q-7 3 -14.5 5t-20.5 2.5t-22 0.5h-32.5h-37.5q-126 0 -217 -43 q16 30 36 46.5t54 29.5t65.5 36t46 36.5t50 55t43.5 50.5q12 -9 28 -31.5t32 -36.5t38 -13l12 1v-76l22 -1q247 95 371 190q28 21 50 39t42.5 37.5t33 31t29.5 34t24 31t24.5 37t23 38t27 47.5t29.5 53l7 9q-2 -53 -43 -139q-79 -165 -205 -264t-306 -142q-14 -3 -42 -7.5 t-50 -9.5t-39 -14q3 -19 24.5 -46t21.5 -34q0 -11 -26 -30zM1061 -79q39 26 131.5 47.5t146.5 21.5q9 0 22.5 -15.5t28 -42.5t26 -50t24 -51t14.5 -33q-121 -45 -244 -45q-61 0 -125 11zM822 568l48 12l109 -177l-73 -48zM1323 51q3 -15 3 -16q0 -7 -17.5 -14.5t-46 -13 t-54 -9.5t-53.5 -7.5t-32 -4.5l-7 43q21 2 60.5 8.5t72 10t60.5 3.5h14zM866 679l-96 -20l-6 17q10 1 32.5 7t34.5 6q19 0 35 -10zM1061 45h31l10 -83l-41 -12v95zM1950 1535v1v-1zM1950 1535l-1 -5l-2 -2l1 3zM1950 1535l1 1z" /> +<glyph unicode="" d="M1167 -50q-5 19 -24 5q-30 -22 -87 -39t-131 -17q-129 0 -193 49q-5 4 -13 4q-11 0 -26 -12q-7 -6 -7.5 -16t7.5 -20q34 -32 87.5 -46t102.5 -12.5t99 4.5q41 4 84.5 20.5t65 30t28.5 20.5q12 12 7 29zM1128 65q-19 47 -39 61q-23 15 -76 15q-47 0 -71 -10 q-29 -12 -78 -56q-26 -24 -12 -44q9 -8 17.5 -4.5t31.5 23.5q3 2 10.5 8.5t10.5 8.5t10 7t11.5 7t12.5 5t15 4.5t16.5 2.5t20.5 1q27 0 44.5 -7.5t23 -14.5t13.5 -22q10 -17 12.5 -20t12.5 1q23 12 14 34zM1483 346q0 22 -5 44.5t-16.5 45t-34 36.5t-52.5 14 q-33 0 -97 -41.5t-129 -83.5t-101 -42q-27 -1 -63.5 19t-76 49t-83.5 58t-100 49t-111 19q-115 -1 -197 -78.5t-84 -178.5q-2 -112 74 -164q29 -20 62.5 -28.5t103.5 -8.5q57 0 132 32.5t134 71t120 70.5t93 31q26 -1 65 -31.5t71.5 -67t68 -67.5t55.5 -32q35 -3 58.5 14 t55.5 63q28 41 42.5 101t14.5 106zM1536 506q0 -164 -62 -304.5t-166 -236t-242.5 -149.5t-290.5 -54t-293 57.5t-247.5 157t-170.5 241.5t-64 302q0 89 19.5 172.5t49 145.5t70.5 118.5t78.5 94t78.5 69.5t64.5 46.5t42.5 24.5q14 8 51 26.5t54.5 28.5t48 30t60.5 44 q36 28 58 72.5t30 125.5q129 -155 186 -193q44 -29 130 -68t129 -66q21 -13 39 -25t60.5 -46.5t76 -70.5t75 -95t69 -122t47 -148.5t19.5 -177.5z" /> +<glyph unicode="" d="M1070 463l-160 -160l-151 -152l-30 -30q-65 -64 -151.5 -87t-171.5 -2q-16 -70 -72 -115t-129 -45q-85 0 -145 60.5t-60 145.5q0 72 44.5 128t113.5 72q-22 86 1 173t88 152l12 12l151 -152l-11 -11q-37 -37 -37 -89t37 -90q37 -37 89 -37t89 37l30 30l151 152l161 160z M729 1145l12 -12l-152 -152l-12 12q-37 37 -89 37t-89 -37t-37 -89.5t37 -89.5l29 -29l152 -152l160 -160l-151 -152l-161 160l-151 152l-30 30q-68 67 -90 159.5t5 179.5q-70 15 -115 71t-45 129q0 85 60 145.5t145 60.5q76 0 133.5 -49t69.5 -123q84 20 169.5 -3.5 t149.5 -87.5zM1536 78q0 -85 -60 -145.5t-145 -60.5q-74 0 -131 47t-71 118q-86 -28 -179.5 -6t-161.5 90l-11 12l151 152l12 -12q37 -37 89 -37t89 37t37 89t-37 89l-30 30l-152 152l-160 160l152 152l160 -160l152 -152l29 -30q64 -64 87.5 -150.5t2.5 -171.5 q76 -11 126.5 -68.5t50.5 -134.5zM1534 1202q0 -77 -51 -135t-127 -69q26 -85 3 -176.5t-90 -158.5l-12 -12l-151 152l12 12q37 37 37 89t-37 89t-89 37t-89 -37l-30 -30l-152 -152l-160 -160l-152 152l161 160l152 152l29 30q67 67 159 89.5t178 -3.5q11 75 68.5 126 t135.5 51q85 0 145 -60.5t60 -145.5z" /> +<glyph unicode="" d="M654 458q-1 -3 -12.5 0.5t-31.5 11.5l-20 9q-44 20 -87 49q-7 5 -41 31.5t-38 28.5q-67 -103 -134 -181q-81 -95 -105 -110q-4 -2 -19.5 -4t-18.5 0q6 4 82 92q21 24 85.5 115t78.5 118q17 30 51 98.5t36 77.5q-8 1 -110 -33q-8 -2 -27.5 -7.5t-34.5 -9.5t-17 -5 q-2 -2 -2 -10.5t-1 -9.5q-5 -10 -31 -15q-23 -7 -47 0q-18 4 -28 21q-4 6 -5 23q6 2 24.5 5t29.5 6q58 16 105 32q100 35 102 35q10 2 43 19.5t44 21.5q9 3 21.5 8t14.5 5.5t6 -0.5q2 -12 -1 -33q0 -2 -12.5 -27t-26.5 -53.5t-17 -33.5q-25 -50 -77 -131l64 -28 q12 -6 74.5 -32t67.5 -28q4 -1 10.5 -25.5t4.5 -30.5zM449 944q3 -15 -4 -28q-12 -23 -50 -38q-30 -12 -60 -12q-26 3 -49 26q-14 15 -18 41l1 3q3 -3 19.5 -5t26.5 0t58 16q36 12 55 14q17 0 21 -17zM1147 815l63 -227l-139 42zM39 15l694 232v1032l-694 -233v-1031z M1280 332l102 -31l-181 657l-100 31l-216 -536l102 -31l45 110l211 -65zM777 1294l573 -184v380zM1088 -29l158 -13l-54 -160l-40 66q-130 -83 -276 -108q-58 -12 -91 -12h-84q-79 0 -199.5 39t-183.5 85q-8 7 -8 16q0 8 5 13.5t13 5.5q4 0 18 -7.5t30.5 -16.5t20.5 -11 q73 -37 159.5 -61.5t157.5 -24.5q95 0 167 14.5t157 50.5q15 7 30.5 15.5t34 19t28.5 16.5zM1536 1050v-1079l-774 246q-14 -6 -375 -127.5t-368 -121.5q-13 0 -18 13q0 1 -1 3v1078q3 9 4 10q5 6 20 11q106 35 149 50v384l558 -198q2 0 160.5 55t316 108.5t161.5 53.5 q20 0 20 -21v-418z" /> +<glyph unicode="" horiz-adv-x="1792" d="M288 1152q66 0 113 -47t47 -113v-1088q0 -66 -47 -113t-113 -47h-128q-66 0 -113 47t-47 113v1088q0 66 47 113t113 47h128zM1664 989q58 -34 93 -93t35 -128v-768q0 -106 -75 -181t-181 -75h-864q-66 0 -113 47t-47 113v1536q0 40 28 68t68 28h672q40 0 88 -20t76 -48 l152 -152q28 -28 48 -76t20 -88v-163zM928 0v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM928 256v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM928 512v128q0 14 -9 23 t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1184 0v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1184 256v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128 q14 0 23 9t9 23zM1184 512v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1440 0v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1440 256v128q0 14 -9 23t-23 9h-128 q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1440 512v128q0 14 -9 23t-23 9h-128q-14 0 -23 -9t-9 -23v-128q0 -14 9 -23t23 -9h128q14 0 23 9t9 23zM1536 896v256h-160q-40 0 -68 28t-28 68v160h-640v-512h896z" /> +<glyph unicode="" d="M1344 1536q26 0 45 -19t19 -45v-1664q0 -26 -19 -45t-45 -19h-1280q-26 0 -45 19t-19 45v1664q0 26 19 45t45 19h1280zM512 1248v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23zM512 992v-64q0 -14 9 -23t23 -9h64q14 0 23 9 t9 23v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23zM512 736v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23zM512 480v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23zM384 160v64 q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM384 416v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM384 672v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64 q14 0 23 9t9 23zM384 928v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM384 1184v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM896 -96v192q0 14 -9 23t-23 9h-320q-14 0 -23 -9 t-9 -23v-192q0 -14 9 -23t23 -9h320q14 0 23 9t9 23zM896 416v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM896 672v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM896 928v64 q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM896 1184v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1152 160v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64 q14 0 23 9t9 23zM1152 416v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1152 672v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1152 928v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9 t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1152 1184v64q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-64q0 -14 9 -23t23 -9h64q14 0 23 9t9 23z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1188 988l-292 -292v-824q0 -46 -33 -79t-79 -33t-79 33t-33 79v384h-64v-384q0 -46 -33 -79t-79 -33t-79 33t-33 79v824l-292 292q-28 28 -28 68t28 68t68 28t68 -28l228 -228h368l228 228q28 28 68 28t68 -28t28 -68t-28 -68zM864 1152q0 -93 -65.5 -158.5 t-158.5 -65.5t-158.5 65.5t-65.5 158.5t65.5 158.5t158.5 65.5t158.5 -65.5t65.5 -158.5z" /> +<glyph unicode="" horiz-adv-x="1664" d="M780 1064q0 -60 -19 -113.5t-63 -92.5t-105 -39q-76 0 -138 57.5t-92 135.5t-30 151q0 60 19 113.5t63 92.5t105 39q77 0 138.5 -57.5t91.5 -135t30 -151.5zM438 581q0 -80 -42 -139t-119 -59q-76 0 -141.5 55.5t-100.5 133.5t-35 152q0 80 42 139.5t119 59.5 q76 0 141.5 -55.5t100.5 -134t35 -152.5zM832 608q118 0 255 -97.5t229 -237t92 -254.5q0 -46 -17 -76.5t-48.5 -45t-64.5 -20t-76 -5.5q-68 0 -187.5 45t-182.5 45q-66 0 -192.5 -44.5t-200.5 -44.5q-183 0 -183 146q0 86 56 191.5t139.5 192.5t187.5 146t193 59zM1071 819 q-61 0 -105 39t-63 92.5t-19 113.5q0 74 30 151.5t91.5 135t138.5 57.5q61 0 105 -39t63 -92.5t19 -113.5q0 -73 -30 -151t-92 -135.5t-138 -57.5zM1503 923q77 0 119 -59.5t42 -139.5q0 -74 -35 -152t-100.5 -133.5t-141.5 -55.5q-77 0 -119 59t-42 139q0 74 35 152.5 t100.5 134t141.5 55.5z" /> +<glyph unicode="" horiz-adv-x="768" d="M704 1008q0 -145 -57 -243.5t-152 -135.5l45 -821q2 -26 -16 -45t-44 -19h-192q-26 0 -44 19t-16 45l45 821q-95 37 -152 135.5t-57 243.5q0 128 42.5 249.5t117.5 200t160 78.5t160 -78.5t117.5 -200t42.5 -249.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M896 -93l640 349v636l-640 -233v-752zM832 772l698 254l-698 254l-698 -254zM1664 1024v-768q0 -35 -18 -65t-49 -47l-704 -384q-28 -16 -61 -16t-61 16l-704 384q-31 17 -49 47t-18 65v768q0 40 23 73t61 47l704 256q22 8 44 8t44 -8l704 -256q38 -14 61 -47t23 -73z " /> +<glyph unicode="" horiz-adv-x="2304" d="M640 -96l384 192v314l-384 -164v-342zM576 358l404 173l-404 173l-404 -173zM1664 -96l384 192v314l-384 -164v-342zM1600 358l404 173l-404 173l-404 -173zM1152 651l384 165v266l-384 -164v-267zM1088 1030l441 189l-441 189l-441 -189zM2176 512v-416q0 -36 -19 -67 t-52 -47l-448 -224q-25 -14 -57 -14t-57 14l-448 224q-5 2 -7 4q-2 -2 -7 -4l-448 -224q-25 -14 -57 -14t-57 14l-448 224q-33 16 -52 47t-19 67v416q0 38 21.5 70t56.5 48l434 186v400q0 38 21.5 70t56.5 48l448 192q23 10 50 10t50 -10l448 -192q35 -16 56.5 -48t21.5 -70 v-400l434 -186q36 -16 57 -48t21 -70z" /> +<glyph unicode="" horiz-adv-x="2048" d="M1848 1197h-511v-124h511v124zM1596 771q-90 0 -146 -52.5t-62 -142.5h408q-18 195 -200 195zM1612 186q63 0 122 32t76 87h221q-100 -307 -427 -307q-214 0 -340.5 132t-126.5 347q0 208 130.5 345.5t336.5 137.5q138 0 240.5 -68t153 -179t50.5 -248q0 -17 -2 -47h-658 q0 -111 57.5 -171.5t166.5 -60.5zM277 236h296q205 0 205 167q0 180 -199 180h-302v-347zM277 773h281q78 0 123.5 36.5t45.5 113.5q0 144 -190 144h-260v-294zM0 1282h594q87 0 155 -14t126.5 -47.5t90 -96.5t31.5 -154q0 -181 -172 -263q114 -32 172 -115t58 -204 q0 -75 -24.5 -136.5t-66 -103.5t-98.5 -71t-121 -42t-134 -13h-611v1260z" /> +<glyph unicode="" d="M1248 1408q119 0 203.5 -84.5t84.5 -203.5v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960zM499 1041h-371v-787h382q117 0 197 57.5t80 170.5q0 158 -143 200q107 52 107 164q0 57 -19.5 96.5 t-56.5 60.5t-79 29.5t-97 8.5zM477 723h-176v184h163q119 0 119 -90q0 -94 -106 -94zM486 388h-185v217h189q124 0 124 -113q0 -104 -128 -104zM1136 356q-68 0 -104 38t-36 107h411q1 10 1 30q0 132 -74.5 220.5t-203.5 88.5q-128 0 -210 -86t-82 -216q0 -135 79 -217 t213 -82q205 0 267 191h-138q-11 -34 -47.5 -54t-75.5 -20zM1126 722q113 0 124 -122h-254q4 56 39 89t91 33zM964 988h319v-77h-319v77z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1582 954q0 -101 -71.5 -172.5t-172.5 -71.5t-172.5 71.5t-71.5 172.5t71.5 172.5t172.5 71.5t172.5 -71.5t71.5 -172.5zM812 212q0 104 -73 177t-177 73q-27 0 -54 -6l104 -42q77 -31 109.5 -106.5t1.5 -151.5q-31 -77 -107 -109t-152 -1q-21 8 -62 24.5t-61 24.5 q32 -60 91 -96.5t130 -36.5q104 0 177 73t73 177zM1642 953q0 126 -89.5 215.5t-215.5 89.5q-127 0 -216.5 -89.5t-89.5 -215.5q0 -127 89.5 -216t216.5 -89q126 0 215.5 89t89.5 216zM1792 953q0 -189 -133.5 -322t-321.5 -133l-437 -319q-12 -129 -109 -218t-229 -89 q-121 0 -214 76t-118 192l-230 92v429l389 -157q79 48 173 48q13 0 35 -2l284 407q2 187 135.5 319t320.5 132q188 0 321.5 -133.5t133.5 -321.5z" /> +<glyph unicode="" d="M1242 889q0 80 -57 136.5t-137 56.5t-136.5 -57t-56.5 -136q0 -80 56.5 -136.5t136.5 -56.5t137 56.5t57 136.5zM632 301q0 -83 -58 -140.5t-140 -57.5q-56 0 -103 29t-72 77q52 -20 98 -40q60 -24 120 1.5t85 86.5q24 60 -1.5 120t-86.5 84l-82 33q22 5 42 5 q82 0 140 -57.5t58 -140.5zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v153l172 -69q20 -92 93.5 -152t168.5 -60q104 0 181 70t87 173l345 252q150 0 255.5 105.5t105.5 254.5q0 150 -105.5 255.5t-255.5 105.5 q-148 0 -253 -104.5t-107 -252.5l-225 -322q-9 1 -28 1q-75 0 -137 -37l-297 119v468q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5zM1289 887q0 -100 -71 -170.5t-171 -70.5t-170.5 70.5t-70.5 170.5t70.5 171t170.5 71q101 0 171.5 -70.5t70.5 -171.5z " /> +<glyph unicode="" horiz-adv-x="1792" d="M836 367l-15 -368l-2 -22l-420 29q-36 3 -67 31.5t-47 65.5q-11 27 -14.5 55t4 65t12 55t21.5 64t19 53q78 -12 509 -28zM449 953l180 -379l-147 92q-63 -72 -111.5 -144.5t-72.5 -125t-39.5 -94.5t-18.5 -63l-4 -21l-190 357q-17 26 -18 56t6 47l8 18q35 63 114 188 l-140 86zM1680 436l-188 -359q-12 -29 -36.5 -46.5t-43.5 -20.5l-18 -4q-71 -7 -219 -12l8 -164l-230 367l211 362l7 -173q170 -16 283 -5t170 33zM895 1360q-47 -63 -265 -435l-317 187l-19 12l225 356q20 31 60 45t80 10q24 -2 48.5 -12t42 -21t41.5 -33t36 -34.5 t36 -39.5t32 -35zM1550 1053l212 -363q18 -37 12.5 -76t-27.5 -74q-13 -20 -33 -37t-38 -28t-48.5 -22t-47 -16t-51.5 -14t-46 -12q-34 72 -265 436l313 195zM1407 1279l142 83l-220 -373l-419 20l151 86q-34 89 -75 166t-75.5 123.5t-64.5 80t-47 46.5l-17 13l405 -1 q31 3 58 -10.5t39 -28.5l11 -15q39 -61 112 -190z" /> +<glyph unicode="" horiz-adv-x="2048" d="M480 448q0 66 -47 113t-113 47t-113 -47t-47 -113t47 -113t113 -47t113 47t47 113zM516 768h1016l-89 357q-2 8 -14 17.5t-21 9.5h-768q-9 0 -21 -9.5t-14 -17.5zM1888 448q0 66 -47 113t-113 47t-113 -47t-47 -113t47 -113t113 -47t113 47t47 113zM2048 544v-384 q0 -14 -9 -23t-23 -9h-96v-128q0 -80 -56 -136t-136 -56t-136 56t-56 136v128h-1024v-128q0 -80 -56 -136t-136 -56t-136 56t-56 136v128h-96q-14 0 -23 9t-9 23v384q0 93 65.5 158.5t158.5 65.5h28l105 419q23 94 104 157.5t179 63.5h768q98 0 179 -63.5t104 -157.5 l105 -419h28q93 0 158.5 -65.5t65.5 -158.5z" /> +<glyph unicode="" horiz-adv-x="2048" d="M1824 640q93 0 158.5 -65.5t65.5 -158.5v-384q0 -14 -9 -23t-23 -9h-96v-64q0 -80 -56 -136t-136 -56t-136 56t-56 136v64h-1024v-64q0 -80 -56 -136t-136 -56t-136 56t-56 136v64h-96q-14 0 -23 9t-9 23v384q0 93 65.5 158.5t158.5 65.5h28l105 419q23 94 104 157.5 t179 63.5h128v224q0 14 9 23t23 9h448q14 0 23 -9t9 -23v-224h128q98 0 179 -63.5t104 -157.5l105 -419h28zM320 160q66 0 113 47t47 113t-47 113t-113 47t-113 -47t-47 -113t47 -113t113 -47zM516 640h1016l-89 357q-2 8 -14 17.5t-21 9.5h-768q-9 0 -21 -9.5t-14 -17.5z M1728 160q66 0 113 47t47 113t-47 113t-113 47t-113 -47t-47 -113t47 -113t113 -47z" /> +<glyph unicode="" d="M1504 64q0 -26 -19 -45t-45 -19h-462q1 -17 6 -87.5t5 -108.5q0 -25 -18 -42.5t-43 -17.5h-320q-25 0 -43 17.5t-18 42.5q0 38 5 108.5t6 87.5h-462q-26 0 -45 19t-19 45t19 45l402 403h-229q-26 0 -45 19t-19 45t19 45l402 403h-197q-26 0 -45 19t-19 45t19 45l384 384 q19 19 45 19t45 -19l384 -384q19 -19 19 -45t-19 -45t-45 -19h-197l402 -403q19 -19 19 -45t-19 -45t-45 -19h-229l402 -403q19 -19 19 -45z" /> +<glyph unicode="" d="M1127 326q0 32 -30 51q-193 115 -447 115q-133 0 -287 -34q-42 -9 -42 -52q0 -20 13.5 -34.5t35.5 -14.5q5 0 37 8q132 27 243 27q226 0 397 -103q19 -11 33 -11q19 0 33 13.5t14 34.5zM1223 541q0 40 -35 61q-237 141 -548 141q-153 0 -303 -42q-48 -13 -48 -64 q0 -25 17.5 -42.5t42.5 -17.5q7 0 37 8q122 33 251 33q279 0 488 -124q24 -13 38 -13q25 0 42.5 17.5t17.5 42.5zM1331 789q0 47 -40 70q-126 73 -293 110.5t-343 37.5q-204 0 -364 -47q-23 -7 -38.5 -25.5t-15.5 -48.5q0 -31 20.5 -52t51.5 -21q11 0 40 8q133 37 307 37 q159 0 309.5 -34t253.5 -95q21 -12 40 -12q29 0 50.5 20.5t21.5 51.5zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M1397 1408q58 0 98.5 -40.5t40.5 -98.5v-1258q0 -58 -40.5 -98.5t-98.5 -40.5h-1258q-58 0 -98.5 40.5t-40.5 98.5v1258q0 58 40.5 98.5t98.5 40.5h1258zM1465 11v1258q0 28 -20 48t-48 20h-1258q-28 0 -48 -20t-20 -48v-1258q0 -28 20 -48t48 -20h1258q28 0 48 20t20 48 zM694 749l188 -387l533 145v-496q0 -7 -5.5 -12.5t-12.5 -5.5h-1258q-7 0 -12.5 5.5t-5.5 12.5v141l711 195l-212 439q4 1 12 2.5t12 1.5q170 32 303.5 21.5t221 -46t143.5 -94.5q27 -28 -25 -42q-64 -16 -256 -62l-97 198q-111 7 -240 -16zM1397 1287q7 0 12.5 -5.5 t5.5 -12.5v-428q-85 30 -188 52q-294 64 -645 12l-18 -3l-65 134h-233l85 -190q-132 -51 -230 -137v560q0 7 5.5 12.5t12.5 5.5h1258zM286 387q-14 -3 -26 4.5t-14 21.5q-24 203 166 305l129 -270z" /> +<glyph unicode="" horiz-adv-x="2304" d="M784 164l16 241l-16 523q-1 10 -7.5 17t-16.5 7q-9 0 -16 -7t-7 -17l-14 -523l14 -241q1 -10 7.5 -16.5t15.5 -6.5q22 0 24 23zM1080 193l11 211l-12 586q0 16 -13 24q-8 5 -16 5t-16 -5q-13 -8 -13 -24l-1 -6l-10 -579q0 -1 11 -236v-1q0 -10 6 -17q9 -11 23 -11 q11 0 20 9q9 7 9 20zM35 533l20 -128l-20 -126q-2 -9 -9 -9t-9 9l-17 126l17 128q2 9 9 9t9 -9zM121 612l26 -207l-26 -203q-2 -9 -10 -9q-9 0 -9 10l-23 202l23 207q0 9 9 9q8 0 10 -9zM401 159zM213 650l25 -245l-25 -237q0 -11 -11 -11q-10 0 -12 11l-21 237l21 245 q2 12 12 12q11 0 11 -12zM307 657l23 -252l-23 -244q-2 -13 -14 -13q-13 0 -13 13l-21 244l21 252q0 13 13 13q12 0 14 -13zM401 639l21 -234l-21 -246q-2 -16 -16 -16q-6 0 -10.5 4.5t-4.5 11.5l-20 246l20 234q0 6 4.5 10.5t10.5 4.5q14 0 16 -15zM784 164zM495 785 l21 -380l-21 -246q0 -7 -5 -12.5t-12 -5.5q-16 0 -18 18l-18 246l18 380q2 18 18 18q7 0 12 -5.5t5 -12.5zM589 871l19 -468l-19 -244q0 -8 -5.5 -13.5t-13.5 -5.5q-18 0 -20 19l-16 244l16 468q2 19 20 19q8 0 13.5 -5.5t5.5 -13.5zM687 911l18 -506l-18 -242 q-2 -21 -22 -21q-19 0 -21 21l-16 242l16 506q0 9 6.5 15.5t14.5 6.5q9 0 15 -6.5t7 -15.5zM1079 169v0v0zM881 915l15 -510l-15 -239q0 -10 -7.5 -17.5t-17.5 -7.5t-17 7t-8 18l-14 239l14 510q0 11 7.5 18t17.5 7t17.5 -7t7.5 -18zM980 896l14 -492l-14 -236q0 -11 -8 -19 t-19 -8t-19 8t-9 19l-12 236l12 492q1 12 9 20t19 8t18.5 -8t8.5 -20zM1192 404l-14 -231v0q0 -13 -9 -22t-22 -9t-22 9t-10 22l-6 114l-6 117l12 636v3q2 15 12 24q9 7 20 7q8 0 15 -5q14 -8 16 -26zM2304 423q0 -117 -83 -199.5t-200 -82.5h-786q-13 2 -22 11t-9 22v899 q0 23 28 33q85 34 181 34q195 0 338 -131.5t160 -323.5q53 22 110 22q117 0 200 -83t83 -201z" /> +<glyph unicode="" d="M768 768q237 0 443 43t325 127v-170q0 -69 -103 -128t-280 -93.5t-385 -34.5t-385 34.5t-280 93.5t-103 128v170q119 -84 325 -127t443 -43zM768 0q237 0 443 43t325 127v-170q0 -69 -103 -128t-280 -93.5t-385 -34.5t-385 34.5t-280 93.5t-103 128v170q119 -84 325 -127 t443 -43zM768 384q237 0 443 43t325 127v-170q0 -69 -103 -128t-280 -93.5t-385 -34.5t-385 34.5t-280 93.5t-103 128v170q119 -84 325 -127t443 -43zM768 1536q208 0 385 -34.5t280 -93.5t103 -128v-128q0 -69 -103 -128t-280 -93.5t-385 -34.5t-385 34.5t-280 93.5 t-103 128v128q0 69 103 128t280 93.5t385 34.5z" /> +<glyph unicode="" d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z M894 465q33 -26 84 -56q59 7 117 7q147 0 177 -49q16 -22 2 -52q0 -1 -1 -2l-2 -2v-1q-6 -38 -71 -38q-48 0 -115 20t-130 53q-221 -24 -392 -83q-153 -262 -242 -262q-15 0 -28 7l-24 12q-1 1 -6 5q-10 10 -6 36q9 40 56 91.5t132 96.5q14 9 23 -6q2 -2 2 -4q52 85 107 197 q68 136 104 262q-24 82 -30.5 159.5t6.5 127.5q11 40 42 40h21h1q23 0 35 -15q18 -21 9 -68q-2 -6 -4 -8q1 -3 1 -8v-30q-2 -123 -14 -192q55 -164 146 -238zM318 54q52 24 137 158q-51 -40 -87.5 -84t-49.5 -74zM716 974q-15 -42 -2 -132q1 7 7 44q0 3 7 43q1 4 4 8 q-1 1 -1 2t-0.5 1.5t-0.5 1.5q-1 22 -13 36q0 -1 -1 -2v-2zM592 313q135 54 284 81q-2 1 -13 9.5t-16 13.5q-76 67 -127 176q-27 -86 -83 -197q-30 -56 -45 -83zM1238 329q-24 24 -140 24q76 -28 124 -28q14 0 18 1q0 1 -2 3z" /> +<glyph unicode="" d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z M233 768v-107h70l164 -661h159l128 485q7 20 10 46q2 16 2 24h4l3 -24q1 -3 3.5 -20t5.5 -26l128 -485h159l164 661h70v107h-300v-107h90l-99 -438q-5 -20 -7 -46l-2 -21h-4l-3 21q-1 5 -4 21t-5 25l-144 545h-114l-144 -545q-2 -9 -4.5 -24.5t-3.5 -21.5l-4 -21h-4l-2 21 q-2 26 -7 46l-99 438h90v107h-300z" /> +<glyph unicode="" d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z M429 106v-106h281v106h-75l103 161q5 7 10 16.5t7.5 13.5t3.5 4h2q1 -4 5 -10q2 -4 4.5 -7.5t6 -8t6.5 -8.5l107 -161h-76v-106h291v106h-68l-192 273l195 282h67v107h-279v-107h74l-103 -159q-4 -7 -10 -16.5t-9 -13.5l-2 -3h-2q-1 4 -5 10q-6 11 -17 23l-106 159h76v107 h-290v-107h68l189 -272l-194 -283h-68z" /> +<glyph unicode="" d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z M416 106v-106h327v106h-93v167h137q76 0 118 15q67 23 106.5 87t39.5 146q0 81 -37 141t-100 87q-48 19 -130 19h-368v-107h92v-555h-92zM769 386h-119v268h120q52 0 83 -18q56 -33 56 -115q0 -89 -62 -120q-31 -15 -78 -15z" /> +<glyph unicode="" d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z M1280 320v-320h-1024v192l192 192l128 -128l384 384zM448 512q-80 0 -136 56t-56 136t56 136t136 56t136 -56t56 -136t-56 -136t-136 -56z" /> +<glyph unicode="" d="M640 1152v128h-128v-128h128zM768 1024v128h-128v-128h128zM640 896v128h-128v-128h128zM768 768v128h-128v-128h128zM1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400 v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-128v-128h-128v128h-512v-1536h1280zM781 593l107 -349q8 -27 8 -52q0 -83 -72.5 -137.5t-183.5 -54.5t-183.5 54.5t-72.5 137.5q0 25 8 52q21 63 120 396v128h128v-128h79 q22 0 39 -13t23 -34zM640 128q53 0 90.5 19t37.5 45t-37.5 45t-90.5 19t-90.5 -19t-37.5 -45t37.5 -45t90.5 -19z" /> +<glyph unicode="" d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z M620 686q20 -8 20 -30v-544q0 -22 -20 -30q-8 -2 -12 -2q-12 0 -23 9l-166 167h-131q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h131l166 167q16 15 35 7zM1037 -3q31 0 50 24q129 159 129 363t-129 363q-16 21 -43 24t-47 -14q-21 -17 -23.5 -43.5t14.5 -47.5 q100 -123 100 -282t-100 -282q-17 -21 -14.5 -47.5t23.5 -42.5q18 -15 40 -15zM826 145q27 0 47 20q87 93 87 219t-87 219q-18 19 -45 20t-46 -17t-20 -44.5t18 -46.5q52 -57 52 -131t-52 -131q-19 -20 -18 -46.5t20 -44.5q20 -17 44 -17z" /> +<glyph unicode="" d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z M768 768q52 0 90 -38t38 -90v-384q0 -52 -38 -90t-90 -38h-384q-52 0 -90 38t-38 90v384q0 52 38 90t90 38h384zM1260 766q20 -8 20 -30v-576q0 -22 -20 -30q-8 -2 -12 -2q-14 0 -23 9l-265 266v90l265 266q9 9 23 9q4 0 12 -2z" /> +<glyph unicode="" d="M1468 1156q28 -28 48 -76t20 -88v-1152q0 -40 -28 -68t-68 -28h-1344q-40 0 -68 28t-28 68v1600q0 40 28 68t68 28h896q40 0 88 -20t76 -48zM1024 1400v-376h376q-10 29 -22 41l-313 313q-12 12 -41 22zM1408 -128v1024h-416q-40 0 -68 28t-28 68v416h-768v-1536h1280z M480 768q8 11 21 12.5t24 -6.5l51 -38q11 -8 12.5 -21t-6.5 -24l-182 -243l182 -243q8 -11 6.5 -24t-12.5 -21l-51 -38q-11 -8 -24 -6.5t-21 12.5l-226 301q-14 19 0 38zM1282 467q14 -19 0 -38l-226 -301q-8 -11 -21 -12.5t-24 6.5l-51 38q-11 8 -12.5 21t6.5 24l182 243 l-182 243q-8 11 -6.5 24t12.5 21l51 38q11 8 24 6.5t21 -12.5zM662 6q-13 2 -20.5 13t-5.5 24l138 831q2 13 13 20.5t24 5.5l63 -10q13 -2 20.5 -13t5.5 -24l-138 -831q-2 -13 -13 -20.5t-24 -5.5z" /> +<glyph unicode="" d="M1497 709v-198q-101 -23 -198 -23q-65 -136 -165.5 -271t-181.5 -215.5t-128 -106.5q-80 -45 -162 3q-28 17 -60.5 43.5t-85 83.5t-102.5 128.5t-107.5 184t-105.5 244t-91.5 314.5t-70.5 390h283q26 -218 70 -398.5t104.5 -317t121.5 -235.5t140 -195q169 169 287 406 q-142 72 -223 220t-81 333q0 192 104 314.5t284 122.5q178 0 273 -105.5t95 -297.5q0 -159 -58 -286q-7 -1 -19.5 -3t-46 -2t-63 6t-62 25.5t-50.5 51.5q31 103 31 184q0 87 -29 132t-79 45q-53 0 -85 -49.5t-32 -140.5q0 -186 105 -293.5t267 -107.5q62 0 121 14z" /> +<glyph unicode="" horiz-adv-x="1792" d="M216 367l603 -402v359l-334 223zM154 511l193 129l-193 129v-258zM973 -35l603 402l-269 180l-334 -223v-359zM896 458l272 182l-272 182l-272 -182zM485 733l334 223v359l-603 -402zM1445 640l193 -129v258zM1307 733l269 180l-603 402v-359zM1792 913v-546 q0 -41 -34 -64l-819 -546q-21 -13 -43 -13t-43 13l-819 546q-34 23 -34 64v546q0 41 34 64l819 546q21 13 43 13t43 -13l819 -546q34 -23 34 -64z" /> +<glyph unicode="" horiz-adv-x="2048" d="M1800 764q111 -46 179.5 -145.5t68.5 -221.5q0 -164 -118 -280.5t-285 -116.5q-4 0 -11.5 0.5t-10.5 0.5h-1209h-1h-2h-5q-170 10 -288 125.5t-118 280.5q0 110 55 203t147 147q-12 39 -12 82q0 115 82 196t199 81q95 0 172 -58q75 154 222.5 248t326.5 94 q166 0 306 -80.5t221.5 -218.5t81.5 -301q0 -6 -0.5 -18t-0.5 -18zM468 498q0 -122 84 -193t208 -71q137 0 240 99q-16 20 -47.5 56.5t-43.5 50.5q-67 -65 -144 -65q-55 0 -93.5 33.5t-38.5 87.5q0 53 38.5 87t91.5 34q44 0 84.5 -21t73 -55t65 -75t69 -82t77 -75t97 -55 t121.5 -21q121 0 204.5 71.5t83.5 190.5q0 121 -84 192t-207 71q-143 0 -241 -97q14 -16 29.5 -34t34.5 -40t29 -34q66 64 142 64q52 0 92 -33t40 -84q0 -57 -37 -91.5t-94 -34.5q-43 0 -82.5 21t-72 55t-65.5 75t-69.5 82t-77.5 75t-96.5 55t-118.5 21q-122 0 -207 -70.5 t-85 -189.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M896 1536q182 0 348 -71t286 -191t191 -286t71 -348t-71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71zM896 1408q-190 0 -361 -90l194 -194q82 28 167 28t167 -28l194 194q-171 90 -361 90zM218 279l194 194 q-28 82 -28 167t28 167l-194 194q-90 -171 -90 -361t90 -361zM896 -128q190 0 361 90l-194 194q-82 -28 -167 -28t-167 28l-194 -194q171 -90 361 -90zM896 256q159 0 271.5 112.5t112.5 271.5t-112.5 271.5t-271.5 112.5t-271.5 -112.5t-112.5 -271.5t112.5 -271.5 t271.5 -112.5zM1380 473l194 -194q90 171 90 361t-90 361l-194 -194q28 -82 28 -167t-28 -167z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 640q0 -182 -71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348q0 222 101 414.5t276.5 317t390.5 155.5v-260q-221 -45 -366.5 -221t-145.5 -406q0 -130 51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5 q0 230 -145.5 406t-366.5 221v260q215 -31 390.5 -155.5t276.5 -317t101 -414.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M19 662q8 217 116 406t305 318h5q0 -1 -1 -3q-8 -8 -28 -33.5t-52 -76.5t-60 -110.5t-44.5 -135.5t-14 -150.5t39 -157.5t108.5 -154q50 -50 102 -69.5t90.5 -11.5t69.5 23.5t47 32.5l16 16q39 51 53 116.5t6.5 122.5t-21 107t-26.5 80l-14 29q-10 25 -30.5 49.5t-43 41 t-43.5 29.5t-35 19l-13 6l104 115q39 -17 78 -52t59 -61l19 -27q1 48 -18.5 103.5t-40.5 87.5l-20 31l161 183l160 -181q-33 -46 -52.5 -102.5t-22.5 -90.5l-4 -33q22 37 61.5 72.5t67.5 52.5l28 17l103 -115q-44 -14 -85 -50t-60 -65l-19 -29q-31 -56 -48 -133.5t-7 -170 t57 -156.5q33 -45 77.5 -60.5t85 -5.5t76 26.5t57.5 33.5l21 16q60 53 96.5 115t48.5 121.5t10 121.5t-18 118t-37 107.5t-45.5 93t-45 72t-34.5 47.5l-13 17q-14 13 -7 13l10 -3q40 -29 62.5 -46t62 -50t64 -58t58.5 -65t55.5 -77t45.5 -88t38 -103t23.5 -117t10.5 -136 q3 -259 -108 -465t-312 -321t-456 -115q-185 0 -351 74t-283.5 198t-184 293t-60.5 353z" /> +<glyph unicode="" horiz-adv-x="1792" d="M874 -102v-66q-208 6 -385 109.5t-283 275.5l58 34q29 -49 73 -99l65 57q148 -168 368 -212l-17 -86q65 -12 121 -13zM276 428l-83 -28q22 -60 49 -112l-57 -33q-98 180 -98 385t98 385l57 -33q-30 -56 -49 -112l82 -28q-35 -100 -35 -212q0 -109 36 -212zM1528 251 l58 -34q-106 -172 -283 -275.5t-385 -109.5v66q56 1 121 13l-17 86q220 44 368 212l65 -57q44 50 73 99zM1377 805l-233 -80q14 -42 14 -85t-14 -85l232 -80q-31 -92 -98 -169l-185 162q-57 -67 -147 -85l48 -241q-52 -10 -98 -10t-98 10l48 241q-90 18 -147 85l-185 -162 q-67 77 -98 169l232 80q-14 42 -14 85t14 85l-233 80q33 93 99 169l185 -162q59 68 147 86l-48 240q44 10 98 10t98 -10l-48 -240q88 -18 147 -86l185 162q66 -76 99 -169zM874 1448v-66q-65 -2 -121 -13l17 -86q-220 -42 -368 -211l-65 56q-38 -42 -73 -98l-57 33 q106 172 282 275.5t385 109.5zM1705 640q0 -205 -98 -385l-57 33q27 52 49 112l-83 28q36 103 36 212q0 112 -35 212l82 28q-19 56 -49 112l57 33q98 -180 98 -385zM1585 1063l-57 -33q-35 56 -73 98l-65 -56q-148 169 -368 211l17 86q-56 11 -121 13v66q209 -6 385 -109.5 t282 -275.5zM1748 640q0 173 -67.5 331t-181.5 272t-272 181.5t-331 67.5t-331 -67.5t-272 -181.5t-181.5 -272t-67.5 -331t67.5 -331t181.5 -272t272 -181.5t331 -67.5t331 67.5t272 181.5t181.5 272t67.5 331zM1792 640q0 -182 -71 -348t-191 -286t-286 -191t-348 -71 t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71t348 -71t286 -191t191 -286t71 -348z" /> +<glyph unicode="" d="M582 228q0 -66 -93 -66q-107 0 -107 63q0 64 98 64q102 0 102 -61zM546 694q0 -85 -74 -85q-77 0 -77 84q0 90 77 90q36 0 55 -25.5t19 -63.5zM712 769v125q-78 -29 -135 -29q-50 29 -110 29q-86 0 -145 -57t-59 -143q0 -50 29.5 -102t73.5 -67v-3q-38 -17 -38 -85 q0 -53 41 -77v-3q-113 -37 -113 -139q0 -45 20 -78.5t54 -51t72 -25.5t81 -8q224 0 224 188q0 67 -48 99t-126 46q-27 5 -51.5 20.5t-24.5 39.5q0 44 49 52q77 15 122 70t45 134q0 24 -10 52q37 9 49 13zM771 350h137q-2 27 -2 82v387q0 46 2 69h-137q3 -23 3 -71v-392 q0 -50 -3 -75zM1280 366v121q-30 -21 -68 -21q-53 0 -53 82v225h52q9 0 26.5 -1t26.5 -1v117h-105q0 82 3 102h-140q4 -24 4 -55v-47h-60v-117q36 3 37 3q3 0 11 -0.5t12 -0.5v-2h-2v-217q0 -37 2.5 -64t11.5 -56.5t24.5 -48.5t43.5 -31t66 -12q64 0 108 24zM924 1072 q0 36 -24 63.5t-60 27.5t-60.5 -27t-24.5 -64q0 -36 25 -62.5t60 -26.5t59.5 27t24.5 62zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M595 22q0 100 -165 100q-158 0 -158 -104q0 -101 172 -101q151 0 151 105zM536 777q0 61 -30 102t-89 41q-124 0 -124 -145q0 -135 124 -135q119 0 119 137zM805 1101v-202q-36 -12 -79 -22q16 -43 16 -84q0 -127 -73 -216.5t-197 -112.5q-40 -8 -59.5 -27t-19.5 -58 q0 -31 22.5 -51.5t58 -32t78.5 -22t86 -25.5t78.5 -37.5t58 -64t22.5 -98.5q0 -304 -363 -304q-69 0 -130 12.5t-116 41t-87.5 82t-32.5 127.5q0 165 182 225v4q-67 41 -67 126q0 109 63 137v4q-72 24 -119.5 108.5t-47.5 165.5q0 139 95 231.5t235 92.5q96 0 178 -47 q98 0 218 47zM1123 220h-222q4 45 4 134v609q0 94 -4 128h222q-4 -33 -4 -124v-613q0 -89 4 -134zM1724 442v-196q-71 -39 -174 -39q-62 0 -107 20t-70 50t-39.5 78t-18.5 92t-4 103v351h2v4q-7 0 -19 1t-18 1q-21 0 -59 -6v190h96v76q0 54 -6 89h227q-6 -41 -6 -165h171 v-190q-15 0 -43.5 2t-42.5 2h-85v-365q0 -131 87 -131q61 0 109 33zM1148 1389q0 -58 -39 -101.5t-96 -43.5q-58 0 -98 43.5t-40 101.5q0 59 39.5 103t98.5 44q58 0 96.5 -44.5t38.5 -102.5z" /> +<glyph unicode="" d="M825 547l343 588h-150q-21 -39 -63.5 -118.5t-68 -128.5t-59.5 -118.5t-60 -128.5h-3q-21 48 -44.5 97t-52 105.5t-46.5 92t-54 104.5t-49 95h-150l323 -589v-435h134v436zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960 q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1280" d="M842 964q0 -80 -57 -136.5t-136 -56.5q-60 0 -111 35q-62 -67 -115 -146q-247 -371 -202 -859q1 -22 -12.5 -38.5t-34.5 -18.5h-5q-20 0 -35 13.5t-17 33.5q-14 126 -3.5 247.5t29.5 217t54 186t69 155.5t74 125q61 90 132 165q-16 35 -16 77q0 80 56.5 136.5t136.5 56.5 t136.5 -56.5t56.5 -136.5zM1223 953q0 -158 -78 -292t-212.5 -212t-292.5 -78q-64 0 -131 14q-21 5 -32.5 23.5t-6.5 39.5q5 20 23 31.5t39 7.5q51 -13 108 -13q97 0 186 38t153 102t102 153t38 186t-38 186t-102 153t-153 102t-186 38t-186 -38t-153 -102t-102 -153 t-38 -186q0 -114 52 -218q10 -20 3.5 -40t-25.5 -30t-39.5 -3t-30.5 26q-64 123 -64 265q0 119 46.5 227t124.5 186t186 124t226 46q158 0 292.5 -78t212.5 -212.5t78 -292.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M270 730q-8 19 -8 52q0 20 11 49t24 45q-1 22 7.5 53t22.5 43q0 139 92.5 288.5t217.5 209.5q139 66 324 66q133 0 266 -55q49 -21 90 -48t71 -56t55 -68t42 -74t32.5 -84.5t25.5 -89.5t22 -98l1 -5q55 -83 55 -150q0 -14 -9 -40t-9 -38q0 -1 1.5 -3.5t3.5 -5t2 -3.5 q77 -114 120.5 -214.5t43.5 -208.5q0 -43 -19.5 -100t-55.5 -57q-9 0 -19.5 7.5t-19 17.5t-19 26t-16 26.5t-13.5 26t-9 17.5q-1 1 -3 1l-5 -4q-59 -154 -132 -223q20 -20 61.5 -38.5t69 -41.5t35.5 -65q-2 -4 -4 -16t-7 -18q-64 -97 -302 -97q-53 0 -110.5 9t-98 20 t-104.5 30q-15 5 -23 7q-14 4 -46 4.5t-40 1.5q-41 -45 -127.5 -65t-168.5 -20q-35 0 -69 1.5t-93 9t-101 20.5t-74.5 40t-32.5 64q0 40 10 59.5t41 48.5q11 2 40.5 13t49.5 12q4 0 14 2q2 2 2 4l-2 3q-48 11 -108 105.5t-73 156.5l-5 3q-4 0 -12 -20q-18 -41 -54.5 -74.5 t-77.5 -37.5h-1q-4 0 -6 4.5t-5 5.5q-23 54 -23 100q0 275 252 466z" /> +<glyph unicode="" horiz-adv-x="2048" d="M580 1075q0 41 -25 66t-66 25q-43 0 -76 -25.5t-33 -65.5q0 -39 33 -64.5t76 -25.5q41 0 66 24.5t25 65.5zM1323 568q0 28 -25.5 50t-65.5 22q-27 0 -49.5 -22.5t-22.5 -49.5q0 -28 22.5 -50.5t49.5 -22.5q40 0 65.5 22t25.5 51zM1087 1075q0 41 -24.5 66t-65.5 25 q-43 0 -76 -25.5t-33 -65.5q0 -39 33 -64.5t76 -25.5q41 0 65.5 24.5t24.5 65.5zM1722 568q0 28 -26 50t-65 22q-27 0 -49.5 -22.5t-22.5 -49.5q0 -28 22.5 -50.5t49.5 -22.5q39 0 65 22t26 51zM1456 965q-31 4 -70 4q-169 0 -311 -77t-223.5 -208.5t-81.5 -287.5 q0 -78 23 -152q-35 -3 -68 -3q-26 0 -50 1.5t-55 6.5t-44.5 7t-54.5 10.5t-50 10.5l-253 -127l72 218q-290 203 -290 490q0 169 97.5 311t264 223.5t363.5 81.5q176 0 332.5 -66t262 -182.5t136.5 -260.5zM2048 404q0 -117 -68.5 -223.5t-185.5 -193.5l55 -181l-199 109 q-150 -37 -218 -37q-169 0 -311 70.5t-223.5 191.5t-81.5 264t81.5 264t223.5 191.5t311 70.5q161 0 303 -70.5t227.5 -192t85.5 -263.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1764 1525q33 -24 27 -64l-256 -1536q-5 -29 -32 -45q-14 -8 -31 -8q-11 0 -24 5l-453 185l-242 -295q-18 -23 -49 -23q-13 0 -22 4q-19 7 -30.5 23.5t-11.5 36.5v349l864 1059l-1069 -925l-395 162q-37 14 -40 55q-2 40 32 59l1664 960q15 9 32 9q20 0 36 -11z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1764 1525q33 -24 27 -64l-256 -1536q-5 -29 -32 -45q-14 -8 -31 -8q-11 0 -24 5l-527 215l-298 -327q-18 -21 -47 -21q-14 0 -23 4q-19 7 -30 23.5t-11 36.5v452l-472 193q-37 14 -40 55q-3 39 32 59l1664 960q35 21 68 -2zM1422 26l221 1323l-1434 -827l336 -137 l863 639l-478 -797z" /> +<glyph unicode="" d="M1536 640q0 -156 -61 -298t-164 -245t-245 -164t-298 -61q-172 0 -327 72.5t-264 204.5q-7 10 -6.5 22.5t8.5 20.5l137 138q10 9 25 9q16 -2 23 -12q73 -95 179 -147t225 -52q104 0 198.5 40.5t163.5 109.5t109.5 163.5t40.5 198.5t-40.5 198.5t-109.5 163.5 t-163.5 109.5t-198.5 40.5q-98 0 -188 -35.5t-160 -101.5l137 -138q31 -30 14 -69q-17 -40 -59 -40h-448q-26 0 -45 19t-19 45v448q0 42 40 59q39 17 69 -14l130 -129q107 101 244.5 156.5t284.5 55.5q156 0 298 -61t245 -164t164 -245t61 -298zM896 928v-448q0 -14 -9 -23 t-23 -9h-320q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h224v352q0 14 9 23t23 9h64q14 0 23 -9t9 -23z" /> +<glyph unicode="" d="M768 1280q-130 0 -248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5t-51 248.5t-136.5 204t-204 136.5t-248.5 51zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103 t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1682 -128q-44 0 -132.5 3.5t-133.5 3.5q-44 0 -132 -3.5t-132 -3.5q-24 0 -37 20.5t-13 45.5q0 31 17 46t39 17t51 7t45 15q33 21 33 140l-1 391q0 21 -1 31q-13 4 -50 4h-675q-38 0 -51 -4q-1 -10 -1 -31l-1 -371q0 -142 37 -164q16 -10 48 -13t57 -3.5t45 -15 t20 -45.5q0 -26 -12.5 -48t-36.5 -22q-47 0 -139.5 3.5t-138.5 3.5q-43 0 -128 -3.5t-127 -3.5q-23 0 -35.5 21t-12.5 45q0 30 15.5 45t36 17.5t47.5 7.5t42 15q33 23 33 143l-1 57v813q0 3 0.5 26t0 36.5t-1.5 38.5t-3.5 42t-6.5 36.5t-11 31.5t-16 18q-15 10 -45 12t-53 2 t-41 14t-18 45q0 26 12 48t36 22q46 0 138.5 -3.5t138.5 -3.5q42 0 126.5 3.5t126.5 3.5q25 0 37.5 -22t12.5 -48q0 -30 -17 -43.5t-38.5 -14.5t-49.5 -4t-43 -13q-35 -21 -35 -160l1 -320q0 -21 1 -32q13 -3 39 -3h699q25 0 38 3q1 11 1 32l1 320q0 139 -35 160 q-18 11 -58.5 12.5t-66 13t-25.5 49.5q0 26 12.5 48t37.5 22q44 0 132 -3.5t132 -3.5q43 0 129 3.5t129 3.5q25 0 37.5 -22t12.5 -48q0 -30 -17.5 -44t-40 -14.5t-51.5 -3t-44 -12.5q-35 -23 -35 -161l1 -943q0 -119 34 -140q16 -10 46 -13.5t53.5 -4.5t41.5 -15.5t18 -44.5 q0 -26 -12 -48t-36 -22z" /> +<glyph unicode="" horiz-adv-x="1280" d="M1278 1347v-73q0 -29 -18.5 -61t-42.5 -32q-50 0 -54 -1q-26 -6 -32 -31q-3 -11 -3 -64v-1152q0 -25 -18 -43t-43 -18h-108q-25 0 -43 18t-18 43v1218h-143v-1218q0 -25 -17.5 -43t-43.5 -18h-108q-26 0 -43.5 18t-17.5 43v496q-147 12 -245 59q-126 58 -192 179 q-64 117 -64 259q0 166 88 286q88 118 209 159q111 37 417 37h479q25 0 43 -18t18 -43z" /> +<glyph unicode="" d="M352 128v-128h-352v128h352zM704 256q26 0 45 -19t19 -45v-256q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h256zM864 640v-128h-864v128h864zM224 1152v-128h-224v128h224zM1536 128v-128h-736v128h736zM576 1280q26 0 45 -19t19 -45v-256 q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h256zM1216 768q26 0 45 -19t19 -45v-256q0 -26 -19 -45t-45 -19h-256q-26 0 -45 19t-19 45v256q0 26 19 45t45 19h256zM1536 640v-128h-224v128h224zM1536 1152v-128h-864v128h864z" /> +<glyph unicode="" d="M1216 512q133 0 226.5 -93.5t93.5 -226.5t-93.5 -226.5t-226.5 -93.5t-226.5 93.5t-93.5 226.5q0 12 2 34l-360 180q-92 -86 -218 -86q-133 0 -226.5 93.5t-93.5 226.5t93.5 226.5t226.5 93.5q126 0 218 -86l360 180q-2 22 -2 34q0 133 93.5 226.5t226.5 93.5 t226.5 -93.5t93.5 -226.5t-93.5 -226.5t-226.5 -93.5q-126 0 -218 86l-360 -180q2 -22 2 -34t-2 -34l360 -180q92 86 218 86z" /> +<glyph unicode="" d="M1280 341q0 88 -62.5 151t-150.5 63q-84 0 -145 -58l-241 120q2 16 2 23t-2 23l241 120q61 -58 145 -58q88 0 150.5 63t62.5 151t-62.5 150.5t-150.5 62.5t-151 -62.5t-63 -150.5q0 -7 2 -23l-241 -120q-62 57 -145 57q-88 0 -150.5 -62.5t-62.5 -150.5t62.5 -150.5 t150.5 -62.5q83 0 145 57l241 -120q-2 -16 -2 -23q0 -88 63 -150.5t151 -62.5t150.5 62.5t62.5 150.5zM1536 1120v-960q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M571 947q-10 25 -34 35t-49 0q-108 -44 -191 -127t-127 -191q-10 -25 0 -49t35 -34q13 -5 24 -5q42 0 60 40q34 84 98.5 148.5t148.5 98.5q25 11 35 35t0 49zM1513 1303l46 -46l-244 -243l68 -68q19 -19 19 -45.5t-19 -45.5l-64 -64q89 -161 89 -343q0 -143 -55.5 -273.5 t-150 -225t-225 -150t-273.5 -55.5t-273.5 55.5t-225 150t-150 225t-55.5 273.5t55.5 273.5t150 225t225 150t273.5 55.5q182 0 343 -89l64 64q19 19 45.5 19t45.5 -19l68 -68zM1521 1359q-10 -10 -22 -10q-13 0 -23 10l-91 90q-9 10 -9 23t9 23q10 9 23 9t23 -9l90 -91 q10 -9 10 -22.5t-10 -22.5zM1751 1129q-11 -9 -23 -9t-23 9l-90 91q-10 9 -10 22.5t10 22.5q9 10 22.5 10t22.5 -10l91 -90q9 -10 9 -23t-9 -23zM1792 1312q0 -14 -9 -23t-23 -9h-96q-14 0 -23 9t-9 23t9 23t23 9h96q14 0 23 -9t9 -23zM1600 1504v-96q0 -14 -9 -23t-23 -9 t-23 9t-9 23v96q0 14 9 23t23 9t23 -9t9 -23zM1751 1449l-91 -90q-10 -10 -22 -10q-13 0 -23 10q-10 9 -10 22.5t10 22.5l90 91q10 9 23 9t23 -9q9 -10 9 -23t-9 -23z" /> +<glyph unicode="" horiz-adv-x="1792" d="M609 720l287 208l287 -208l-109 -336h-355zM896 1536q182 0 348 -71t286 -191t191 -286t71 -348t-71 -348t-191 -286t-286 -191t-348 -71t-348 71t-286 191t-191 286t-71 348t71 348t191 286t286 191t348 71zM1515 186q149 203 149 454v3l-102 -89l-240 224l63 323 l134 -12q-150 206 -389 282l53 -124l-287 -159l-287 159l53 124q-239 -76 -389 -282l135 12l62 -323l-240 -224l-102 89v-3q0 -251 149 -454l30 132l326 -40l139 -298l-116 -69q117 -39 240 -39t240 39l-116 69l139 298l326 40z" /> +<glyph unicode="" horiz-adv-x="1792" d="M448 224v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM256 608v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM832 224v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23 v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM640 608v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM66 768q-28 0 -47 19t-19 46v129h514v-129q0 -27 -19 -46t-46 -19h-383zM1216 224v-192q0 -14 -9 -23t-23 -9h-192 q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1024 608v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1600 224v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23 zM1408 608v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1792 1016v-13h-514v10q0 104 -382 102q-382 -1 -382 -102v-10h-514v13q0 17 8.5 43t34 64t65.5 75.5t110.5 76t160 67.5t224 47.5t293.5 18.5t293 -18.5t224 -47.5 t160.5 -67.5t110.5 -76t65.5 -75.5t34 -64t8.5 -43zM1792 608v-192q0 -14 -9 -23t-23 -9h-192q-14 0 -23 9t-9 23v192q0 14 9 23t23 9h192q14 0 23 -9t9 -23zM1792 962v-129q0 -27 -19 -46t-46 -19h-384q-27 0 -46 19t-19 46v129h514z" /> +<glyph unicode="" horiz-adv-x="1792" d="M704 1216v-768q0 -26 -19 -45t-45 -19v-576q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v512l249 873q7 23 31 23h424zM1024 1216v-704h-256v704h256zM1792 320v-512q0 -26 -19 -45t-45 -19h-512q-26 0 -45 19t-19 45v576q-26 0 -45 19t-19 45v768h424q24 0 31 -23z M736 1504v-224h-352v224q0 14 9 23t23 9h288q14 0 23 -9t9 -23zM1408 1504v-224h-352v224q0 14 9 23t23 9h288q14 0 23 -9t9 -23z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1755 1083q37 -37 37 -90t-37 -91l-401 -400l150 -150l-160 -160q-163 -163 -389.5 -186.5t-411.5 100.5l-362 -362h-181v181l362 362q-124 185 -100.5 411.5t186.5 389.5l160 160l150 -150l400 401q38 37 91 37t90 -37t37 -90.5t-37 -90.5l-400 -401l234 -234l401 400 q38 37 91 37t90 -37z" /> +<glyph unicode="" horiz-adv-x="1792" d="M873 796q0 -83 -63.5 -142.5t-152.5 -59.5t-152.5 59.5t-63.5 142.5q0 84 63.5 143t152.5 59t152.5 -59t63.5 -143zM1375 796q0 -83 -63 -142.5t-153 -59.5q-89 0 -152.5 59.5t-63.5 142.5q0 84 63.5 143t152.5 59q90 0 153 -59t63 -143zM1600 616v667q0 87 -32 123.5 t-111 36.5h-1112q-83 0 -112.5 -34t-29.5 -126v-673q43 -23 88.5 -40t81 -28t81 -18.5t71 -11t70 -4t58.5 -0.5t56.5 2t44.5 2q68 1 95 -27q6 -6 10 -9q26 -25 61 -51q7 91 118 87q5 0 36.5 -1.5t43 -2t45.5 -1t53 1t54.5 4.5t61 8.5t62 13.5t67 19.5t67.5 27t72 34.5z M1763 621q-121 -149 -372 -252q84 -285 -23 -465q-66 -113 -183 -148q-104 -32 -182 15q-86 51 -82 164l-1 326v1q-8 2 -24.5 6t-23.5 5l-1 -338q4 -114 -83 -164q-79 -47 -183 -15q-117 36 -182 150q-105 180 -22 463q-251 103 -372 252q-25 37 -4 63t60 -1q3 -2 11 -7 t11 -8v694q0 72 47 123t114 51h1257q67 0 114 -51t47 -123v-694l21 15q39 27 60 1t-4 -63z" /> +<glyph unicode="" horiz-adv-x="1792" d="M896 1102v-434h-145v434h145zM1294 1102v-434h-145v434h145zM1294 342l253 254v795h-1194v-1049h326v-217l217 217h398zM1692 1536v-1013l-434 -434h-326l-217 -217h-217v217h-398v1158l109 289h1483z" /> +<glyph unicode="" d="M773 217v-127q-1 -292 -6 -305q-12 -32 -51 -40q-54 -9 -181.5 38t-162.5 89q-13 15 -17 36q-1 12 4 26q4 10 34 47t181 216q1 0 60 70q15 19 39.5 24.5t49.5 -3.5q24 -10 37.5 -29t12.5 -42zM624 468q-3 -55 -52 -70l-120 -39q-275 -88 -292 -88q-35 2 -54 36 q-12 25 -17 75q-8 76 1 166.5t30 124.5t56 32q13 0 202 -77q70 -29 115 -47l84 -34q23 -9 35.5 -30.5t11.5 -48.5zM1450 171q-7 -54 -91.5 -161t-135.5 -127q-37 -14 -63 7q-14 10 -184 287l-47 77q-14 21 -11.5 46t19.5 46q35 43 83 26q1 -1 119 -40q203 -66 242 -79.5 t47 -20.5q28 -22 22 -61zM778 803q5 -102 -54 -122q-58 -17 -114 71l-378 598q-8 35 19 62q41 43 207.5 89.5t224.5 31.5q40 -10 49 -45q3 -18 22 -305.5t24 -379.5zM1440 695q3 -39 -26 -59q-15 -10 -329 -86q-67 -15 -91 -23l1 2q-23 -6 -46 4t-37 32q-30 47 0 87 q1 1 75 102q125 171 150 204t34 39q28 19 65 2q48 -23 123 -133.5t81 -167.5v-3z" /> +<glyph unicode="" horiz-adv-x="2048" d="M1024 1024h-384v-384h384v384zM1152 384v-128h-640v128h640zM1152 1152v-640h-640v640h640zM1792 384v-128h-512v128h512zM1792 640v-128h-512v128h512zM1792 896v-128h-512v128h512zM1792 1152v-128h-512v128h512zM256 192v960h-128v-960q0 -26 19 -45t45 -19t45 19 t19 45zM1920 192v1088h-1536v-1088q0 -33 -11 -64h1483q26 0 45 19t19 45zM2048 1408v-1216q0 -80 -56 -136t-136 -56h-1664q-80 0 -136 56t-56 136v1088h256v128h1792z" /> +<glyph unicode="" horiz-adv-x="2048" d="M1024 13q-20 0 -93 73.5t-73 93.5q0 32 62.5 54t103.5 22t103.5 -22t62.5 -54q0 -20 -73 -93.5t-93 -73.5zM1294 284q-2 0 -40 25t-101.5 50t-128.5 25t-128.5 -25t-101 -50t-40.5 -25q-18 0 -93.5 75t-75.5 93q0 13 10 23q78 77 196 121t233 44t233 -44t196 -121 q10 -10 10 -23q0 -18 -75.5 -93t-93.5 -75zM1567 556q-11 0 -23 8q-136 105 -252 154.5t-268 49.5q-85 0 -170.5 -22t-149 -53t-113.5 -62t-79 -53t-31 -22q-17 0 -92 75t-75 93q0 12 10 22q132 132 320 205t380 73t380 -73t320 -205q10 -10 10 -22q0 -18 -75 -93t-92 -75z M1838 827q-11 0 -22 9q-179 157 -371.5 236.5t-420.5 79.5t-420.5 -79.5t-371.5 -236.5q-11 -9 -22 -9q-17 0 -92.5 75t-75.5 93q0 13 10 23q187 186 445 288t527 102t527 -102t445 -288q10 -10 10 -23q0 -18 -75.5 -93t-92.5 -75z" /> +<glyph unicode="" horiz-adv-x="1792" d="M384 0q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM768 0q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM384 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5 t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1152 0q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM768 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5 t37.5 90.5zM384 768q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1152 384q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM768 768q0 53 -37.5 90.5t-90.5 37.5 t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1536 0v384q0 52 -38 90t-90 38t-90 -38t-38 -90v-384q0 -52 38 -90t90 -38t90 38t38 90zM1152 768q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5z M1536 1088v256q0 26 -19 45t-45 19h-1280q-26 0 -45 -19t-19 -45v-256q0 -26 19 -45t45 -19h1280q26 0 45 19t19 45zM1536 768q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1664 1408v-1536q0 -52 -38 -90t-90 -38 h-1408q-52 0 -90 38t-38 90v1536q0 52 38 90t90 38h1408q52 0 90 -38t38 -90z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1112 1090q0 159 -237 159h-70q-32 0 -59.5 -21.5t-34.5 -52.5l-63 -276q-2 -5 -2 -16q0 -24 17 -39.5t41 -15.5h53q69 0 128.5 13t112.5 41t83.5 81.5t30.5 126.5zM1716 938q0 -265 -220 -428q-219 -161 -612 -161h-61q-32 0 -59 -21.5t-34 -52.5l-73 -316 q-8 -36 -40.5 -61.5t-69.5 -25.5h-213q-31 0 -53 20t-22 51q0 10 13 65h151q34 0 64 23.5t38 56.5l73 316q8 33 37.5 57t63.5 24h61q390 0 607 160t217 421q0 129 -51 207q183 -92 183 -335zM1533 1123q0 -264 -221 -428q-218 -161 -612 -161h-60q-32 0 -59.5 -22t-34.5 -53 l-73 -315q-8 -36 -40 -61.5t-69 -25.5h-214q-31 0 -52.5 19.5t-21.5 51.5q0 8 2 20l300 1301q8 36 40.5 61.5t69.5 25.5h444q68 0 125 -4t120.5 -15t113.5 -30t96.5 -50.5t77.5 -74t49.5 -103.5t18.5 -136z" /> +<glyph unicode="" horiz-adv-x="1792" d="M602 949q19 -61 31 -123.5t17 -141.5t-14 -159t-62 -145q-21 81 -67 157t-95.5 127t-99 90.5t-78.5 57.5t-33 19q-62 34 -81.5 100t14.5 128t101 81.5t129 -14.5q138 -83 238 -177zM927 1236q11 -25 20.5 -46t36.5 -100.5t42.5 -150.5t25.5 -179.5t0 -205.5t-47.5 -209.5 t-105.5 -208.5q-51 -72 -138 -72q-54 0 -98 31q-57 40 -69 109t28 127q60 85 81 195t13 199.5t-32 180.5t-39 128t-22 52q-31 63 -8.5 129.5t85.5 97.5q34 17 75 17q47 0 88.5 -25t63.5 -69zM1248 567q-17 -160 -72 -311q-17 131 -63 246q25 174 -5 361q-27 178 -94 342 q114 -90 212 -211q9 -37 15 -80q26 -179 7 -347zM1520 1440q9 -17 23.5 -49.5t43.5 -117.5t50.5 -178t34 -227.5t5 -269t-47 -300t-112.5 -323.5q-22 -48 -66 -75.5t-95 -27.5q-39 0 -74 16q-67 31 -92.5 100t4.5 136q58 126 90 257.5t37.5 239.5t-3.5 213.5t-26.5 180.5 t-38.5 138.5t-32.5 90t-15.5 32.5q-34 65 -11.5 135.5t87.5 104.5q37 20 81 20q49 0 91.5 -25.5t66.5 -70.5z" /> +<glyph unicode="" horiz-adv-x="2304" d="M1975 546h-138q14 37 66 179l3 9q4 10 10 26t9 26l12 -55zM531 611l-58 295q-11 54 -75 54h-268l-2 -13q311 -79 403 -336zM710 960l-162 -438l-17 89q-26 70 -85 129.5t-131 88.5l135 -510h175l261 641h-176zM849 318h166l104 642h-166zM1617 944q-69 27 -149 27 q-123 0 -201 -59t-79 -153q-1 -102 145 -174q48 -23 67 -41t19 -39q0 -30 -30 -46t-69 -16q-86 0 -156 33l-22 11l-23 -144q74 -34 185 -34q130 -1 208.5 59t80.5 160q0 106 -140 174q-49 25 -71 42t-22 38q0 22 24.5 38.5t70.5 16.5q70 1 124 -24l15 -8zM2042 960h-128 q-65 0 -87 -54l-246 -588h174l35 96h212q5 -22 20 -96h154zM2304 1280v-1280q0 -52 -38 -90t-90 -38h-2048q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h2048q52 0 90 -38t38 -90z" /> +<glyph unicode="" horiz-adv-x="2304" d="M671 603h-13q-47 0 -47 -32q0 -22 20 -22q17 0 28 15t12 39zM1066 639h62v3q1 4 0.5 6.5t-1 7t-2 8t-4.5 6.5t-7.5 5t-11.5 2q-28 0 -36 -38zM1606 603h-12q-48 0 -48 -32q0 -22 20 -22q17 0 28 15t12 39zM1925 629q0 41 -30 41q-19 0 -31 -20t-12 -51q0 -42 28 -42 q20 0 32.5 20t12.5 52zM480 770h87l-44 -262h-56l32 201l-71 -201h-39l-4 200l-34 -200h-53l44 262h81l2 -163zM733 663q0 -6 -4 -42q-16 -101 -17 -113h-47l1 22q-20 -26 -58 -26q-23 0 -37.5 16t-14.5 42q0 39 26 60.5t73 21.5q14 0 23 -1q0 3 0.5 5.5t1 4.5t0.5 3 q0 20 -36 20q-29 0 -59 -10q0 4 7 48q38 11 67 11q74 0 74 -62zM889 721l-8 -49q-22 3 -41 3q-27 0 -27 -17q0 -8 4.5 -12t21.5 -11q40 -19 40 -60q0 -72 -87 -71q-34 0 -58 6q0 2 7 49q29 -8 51 -8q32 0 32 19q0 7 -4.5 11.5t-21.5 12.5q-43 20 -43 59q0 72 84 72 q30 0 50 -4zM977 721h28l-7 -52h-29q-2 -17 -6.5 -40.5t-7 -38.5t-2.5 -18q0 -16 19 -16q8 0 16 2l-8 -47q-21 -7 -40 -7q-43 0 -45 47q0 12 8 56q3 20 25 146h55zM1180 648q0 -23 -7 -52h-111q-3 -22 10 -33t38 -11q30 0 58 14l-9 -54q-30 -8 -57 -8q-95 0 -95 95 q0 55 27.5 90.5t69.5 35.5q35 0 55.5 -21t20.5 -56zM1319 722q-13 -23 -22 -62q-22 2 -31 -24t-25 -128h-56l3 14q22 130 29 199h51l-3 -33q14 21 25.5 29.5t28.5 4.5zM1506 763l-9 -57q-28 14 -50 14q-31 0 -51 -27.5t-20 -70.5q0 -30 13.5 -47t38.5 -17q21 0 48 13 l-10 -59q-28 -8 -50 -8q-45 0 -71.5 30.5t-26.5 82.5q0 70 35.5 114.5t91.5 44.5q26 0 61 -13zM1668 663q0 -18 -4 -42q-13 -79 -17 -113h-46l1 22q-20 -26 -59 -26q-23 0 -37 16t-14 42q0 39 25.5 60.5t72.5 21.5q15 0 23 -1q2 7 2 13q0 20 -36 20q-29 0 -59 -10q0 4 8 48 q38 11 67 11q73 0 73 -62zM1809 722q-14 -24 -21 -62q-23 2 -31.5 -23t-25.5 -129h-56l3 14q19 104 29 199h52q0 -11 -4 -33q15 21 26.5 29.5t27.5 4.5zM1950 770h56l-43 -262h-53l3 19q-23 -23 -52 -23q-31 0 -49.5 24t-18.5 64q0 53 27.5 92t64.5 39q31 0 53 -29z M2061 640q0 148 -72.5 273t-198 198t-273.5 73q-181 0 -328 -110q127 -116 171 -284h-50q-44 150 -158 253q-114 -103 -158 -253h-50q44 168 171 284q-147 110 -328 110q-148 0 -273.5 -73t-198 -198t-72.5 -273t72.5 -273t198 -198t273.5 -73q181 0 328 110 q-120 111 -165 264h50q46 -138 152 -233q106 95 152 233h50q-45 -153 -165 -264q147 -110 328 -110q148 0 273.5 73t198 198t72.5 273zM2304 1280v-1280q0 -52 -38 -90t-90 -38h-2048q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h2048q52 0 90 -38t38 -90z" /> +<glyph unicode="" horiz-adv-x="2304" d="M313 759q0 -51 -36 -84q-29 -26 -89 -26h-17v220h17q61 0 89 -27q36 -31 36 -83zM2089 824q0 -52 -64 -52h-19v101h20q63 0 63 -49zM380 759q0 74 -50 120.5t-129 46.5h-95v-333h95q74 0 119 38q60 51 60 128zM410 593h65v333h-65v-333zM730 694q0 40 -20.5 62t-75.5 42 q-29 10 -39.5 19t-10.5 23q0 16 13.5 26.5t34.5 10.5q29 0 53 -27l34 44q-41 37 -98 37q-44 0 -74 -27.5t-30 -67.5q0 -35 18 -55.5t64 -36.5q37 -13 45 -19q19 -12 19 -34q0 -20 -14 -33.5t-36 -13.5q-48 0 -71 44l-42 -40q44 -64 115 -64q51 0 83 30.5t32 79.5zM1008 604 v77q-37 -37 -78 -37q-49 0 -80.5 32.5t-31.5 82.5q0 48 31.5 81.5t77.5 33.5q43 0 81 -38v77q-40 20 -80 20q-74 0 -125.5 -50.5t-51.5 -123.5t51 -123.5t125 -50.5q42 0 81 19zM2240 0v527q-65 -40 -144.5 -84t-237.5 -117t-329.5 -137.5t-417.5 -134.5t-504 -118h1569 q26 0 45 19t19 45zM1389 757q0 75 -53 128t-128 53t-128 -53t-53 -128t53 -128t128 -53t128 53t53 128zM1541 584l144 342h-71l-90 -224l-89 224h-71l142 -342h35zM1714 593h184v56h-119v90h115v56h-115v74h119v57h-184v-333zM2105 593h80l-105 140q76 16 76 94q0 47 -31 73 t-87 26h-97v-333h65v133h9zM2304 1274v-1268q0 -56 -38.5 -95t-93.5 -39h-2040q-55 0 -93.5 39t-38.5 95v1268q0 56 38.5 95t93.5 39h2040q55 0 93.5 -39t38.5 -95z" /> +<glyph unicode="" horiz-adv-x="2304" d="M119 854h89l-45 108zM740 328l74 79l-70 79h-163v-49h142v-55h-142v-54h159zM898 406l99 -110v217zM1186 453q0 33 -40 33h-84v-69h83q41 0 41 36zM1475 457q0 29 -42 29h-82v-61h81q43 0 43 32zM1197 923q0 29 -42 29h-82v-60h81q43 0 43 31zM1656 854h89l-44 108z M699 1009v-271h-66v212l-94 -212h-57l-94 212v-212h-132l-25 60h-135l-25 -60h-70l116 271h96l110 -257v257h106l85 -184l77 184h108zM1255 453q0 -20 -5.5 -35t-14 -25t-22.5 -16.5t-26 -10t-31.5 -4.5t-31.5 -1t-32.5 0.5t-29.5 0.5v-91h-126l-80 90l-83 -90h-256v271h260 l80 -89l82 89h207q109 0 109 -89zM964 794v-56h-217v271h217v-57h-152v-49h148v-55h-148v-54h152zM2304 235v-229q0 -55 -38.5 -94.5t-93.5 -39.5h-2040q-55 0 -93.5 39.5t-38.5 94.5v678h111l25 61h55l25 -61h218v46l19 -46h113l20 47v-47h541v99l10 1q10 0 10 -14v-86h279 v23q23 -12 55 -18t52.5 -6.5t63 0.5t51.5 1l25 61h56l25 -61h227v58l34 -58h182v378h-180v-44l-25 44h-185v-44l-23 44h-249q-69 0 -109 -22v22h-172v-22q-24 22 -73 22h-628l-43 -97l-43 97h-198v-44l-22 44h-169l-78 -179v391q0 55 38.5 94.5t93.5 39.5h2040 q55 0 93.5 -39.5t38.5 -94.5v-678h-120q-51 0 -81 -22v22h-177q-55 0 -78 -22v22h-316v-22q-31 22 -87 22h-209v-22q-23 22 -91 22h-234l-54 -58l-50 58h-349v-378h343l55 59l52 -59h211v89h21q59 0 90 13v-102h174v99h8q8 0 10 -2t2 -10v-87h529q57 0 88 24v-24h168 q60 0 95 17zM1546 469q0 -23 -12 -43t-34 -29q25 -9 34 -26t9 -46v-54h-65v45q0 33 -12 43.5t-46 10.5h-69v-99h-65v271h154q48 0 77 -15t29 -58zM1269 936q0 -24 -12.5 -44t-33.5 -29q26 -9 34.5 -25.5t8.5 -46.5v-53h-65q0 9 0.5 26.5t0 25t-3 18.5t-8.5 16t-17.5 8.5 t-29.5 3.5h-70v-98h-64v271l153 -1q49 0 78 -14.5t29 -57.5zM1798 327v-56h-216v271h216v-56h-151v-49h148v-55h-148v-54zM1372 1009v-271h-66v271h66zM2065 357q0 -86 -102 -86h-126v58h126q34 0 34 25q0 16 -17 21t-41.5 5t-49.5 3.5t-42 22.5t-17 55q0 39 26 60t66 21 h130v-57h-119q-36 0 -36 -25q0 -16 17.5 -20.5t42 -4t49 -2.5t42 -21.5t17.5 -54.5zM2304 407v-101q-24 -35 -88 -35h-125v58h125q33 0 33 25q0 13 -12.5 19t-31 5.5t-40 2t-40 8t-31 24t-12.5 48.5q0 39 26.5 60t66.5 21h129v-57h-118q-36 0 -36 -25q0 -20 29 -22t68.5 -5 t56.5 -26zM2139 1008v-270h-92l-122 203v-203h-132l-26 60h-134l-25 -60h-75q-129 0 -129 133q0 138 133 138h63v-59q-7 0 -28 1t-28.5 0.5t-23 -2t-21.5 -6.5t-14.5 -13.5t-11.5 -23t-3 -33.5q0 -38 13.5 -58t49.5 -20h29l92 213h97l109 -256v256h99l114 -188v188h66z" /> +<glyph unicode="" horiz-adv-x="2304" d="M322 689h-15q-19 0 -19 18q0 28 19 85q5 15 15 19.5t28 4.5q77 0 77 -49q0 -41 -30.5 -59.5t-74.5 -18.5zM664 528q-47 0 -47 29q0 62 123 62l3 -3q-5 -88 -79 -88zM1438 687h-15q-19 0 -19 19q0 28 19 85q5 15 14.5 19t28.5 4q77 0 77 -49q0 -41 -30.5 -59.5 t-74.5 -18.5zM1780 527q-47 0 -47 30q0 62 123 62l3 -3q-5 -89 -79 -89zM373 894h-128q-8 0 -14.5 -4t-8.5 -7.5t-7 -12.5q-3 -7 -45 -190t-42 -192q0 -7 5.5 -12.5t13.5 -5.5h62q25 0 32.5 34.5l15 69t32.5 34.5q47 0 87.5 7.5t80.5 24.5t63.5 52.5t23.5 84.5 q0 36 -14.5 61t-41 36.5t-53.5 15.5t-62 4zM719 798q-38 0 -74 -6q-2 0 -8.5 -1t-9 -1.5l-7.5 -1.5t-7.5 -2t-6.5 -3t-6.5 -4t-5 -5t-4.5 -7t-4 -9q-9 -29 -9 -39t9 -10q5 0 21.5 5t19.5 6q30 8 58 8q74 0 74 -36q0 -11 -10 -14q-8 -2 -18 -3t-21.5 -1.5t-17.5 -1.5 q-38 -4 -64.5 -10t-56.5 -19.5t-45.5 -39t-15.5 -62.5q0 -38 26 -59.5t64 -21.5q24 0 45.5 6.5t33 13t38.5 23.5q-3 -7 -3 -15t5.5 -13.5t12.5 -5.5h56q1 1 7 3.5t7.5 3.5t5 3.5t5 5.5t2.5 8l45 194q4 13 4 30q0 81 -145 81zM1247 793h-74q-22 0 -39 -23q-5 -7 -29.5 -51 t-46.5 -81.5t-26 -38.5l-5 4q0 77 -27 166q-1 5 -3.5 8.5t-6 6.5t-6.5 5t-8.5 3t-8.5 1.5t-9.5 1t-9 0.5h-10h-8.5q-38 0 -38 -21l1 -5q5 -53 25 -151t25 -143q2 -16 2 -24q0 -19 -30.5 -61.5t-30.5 -58.5q0 -13 40 -13q61 0 76 25l245 415q10 20 10 26q0 9 -8 9zM1489 892 h-129q-18 0 -29 -23q-6 -13 -46.5 -191.5t-40.5 -190.5q0 -20 43 -20h7.5h9h9t9.5 1t8.5 2t8.5 3t6.5 4.5t5.5 6t3 8.5l21 91q2 10 10.5 17t19.5 7q47 0 87.5 7t80.5 24.5t63.5 52.5t23.5 84q0 36 -14.5 61t-41 36.5t-53.5 15.5t-62 4zM1835 798q-26 0 -74 -6 q-38 -6 -48 -16q-7 -8 -11 -19q-8 -24 -8 -39q0 -10 8 -10q1 0 41 12q30 8 58 8q74 0 74 -36q0 -12 -10 -14q-4 -1 -57 -7q-38 -4 -64.5 -10t-56.5 -19.5t-45.5 -39t-15.5 -62.5t26 -58.5t64 -21.5q24 0 45 6t34 13t38 24q-3 -15 -3 -16q0 -5 2 -8.5t6.5 -5.5t8 -3.5 t10.5 -2t9.5 -0.5h9.5h8q42 0 48 25l45 194q3 15 3 31q0 81 -145 81zM2157 889h-55q-25 0 -33 -40q-10 -44 -36.5 -167t-42.5 -190v-5q0 -16 16 -18h1h57q10 0 18.5 6.5t10.5 16.5l83 374h-1l1 5q0 7 -5.5 12.5t-13.5 5.5zM2304 1280v-1280q0 -52 -38 -90t-90 -38h-2048 q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h2048q52 0 90 -38t38 -90z" /> +<glyph unicode="" horiz-adv-x="2304" d="M1597 633q0 -69 -21 -106q-19 -35 -52 -35q-23 0 -41 9v224q29 30 57 30q57 0 57 -122zM2035 669h-110q6 98 56 98q51 0 54 -98zM476 534q0 59 -33 91.5t-101 57.5q-36 13 -52 24t-16 25q0 26 38 26q58 0 124 -33l18 112q-67 32 -149 32q-77 0 -123 -38q-48 -39 -48 -109 q0 -58 32.5 -90.5t99.5 -56.5q39 -14 54.5 -25.5t15.5 -27.5q0 -31 -48 -31q-29 0 -70 12.5t-72 30.5l-18 -113q72 -41 168 -41q81 0 129 37q51 41 51 117zM771 749l19 111h-96v135l-129 -21l-18 -114l-46 -8l-17 -103h62v-219q0 -84 44 -120q38 -30 111 -30q32 0 79 11v118 q-32 -7 -44 -7q-42 0 -42 50v197h77zM1087 724v139q-15 3 -28 3q-32 0 -55.5 -16t-33.5 -46l-10 56h-131v-471h150v306q26 31 82 31q16 0 26 -2zM1124 389h150v471h-150v-471zM1746 638q0 122 -45 179q-40 52 -111 52q-64 0 -117 -56l-8 47h-132v-645l150 25v151 q36 -11 68 -11q83 0 134 56q61 65 61 202zM1278 986q0 33 -23 56t-56 23t-56 -23t-23 -56t23 -56.5t56 -23.5t56 23.5t23 56.5zM2176 629q0 113 -48 176q-50 64 -144 64q-96 0 -151.5 -66t-55.5 -180q0 -128 63 -188q55 -55 161 -55q101 0 160 40l-16 103q-57 -31 -128 -31 q-43 0 -63 19q-23 19 -28 66h248q2 14 2 52zM2304 1280v-1280q0 -52 -38 -90t-90 -38h-2048q-52 0 -90 38t-38 90v1280q0 52 38 90t90 38h2048q52 0 90 -38t38 -90z" /> +<glyph unicode="" horiz-adv-x="2048" d="M1558 684q61 -356 298 -556q0 -52 -38 -90t-90 -38h-448q0 -106 -75 -181t-181 -75t-180.5 74.5t-75.5 180.5zM1024 -176q16 0 16 16t-16 16q-59 0 -101.5 42.5t-42.5 101.5q0 16 -16 16t-16 -16q0 -73 51.5 -124.5t124.5 -51.5zM2026 1424q8 -10 7.5 -23.5t-10.5 -22.5 l-1872 -1622q-10 -8 -23.5 -7t-21.5 11l-84 96q-8 10 -7.5 23.5t10.5 21.5l186 161q-19 32 -19 66q50 42 91 88t85 119.5t74.5 158.5t50 206t19.5 260q0 152 117 282.5t307 158.5q-8 19 -8 39q0 40 28 68t68 28t68 -28t28 -68q0 -20 -8 -39q124 -18 219 -82.5t148 -157.5 l418 363q10 8 23.5 7t21.5 -11z" /> +<glyph unicode="" horiz-adv-x="2048" d="M1040 -160q0 16 -16 16q-59 0 -101.5 42.5t-42.5 101.5q0 16 -16 16t-16 -16q0 -73 51.5 -124.5t124.5 -51.5q16 0 16 16zM503 315l877 760q-42 88 -132.5 146.5t-223.5 58.5q-93 0 -169.5 -31.5t-121.5 -80.5t-69 -103t-24 -105q0 -384 -137 -645zM1856 128 q0 -52 -38 -90t-90 -38h-448q0 -106 -75 -181t-181 -75t-180.5 74.5t-75.5 180.5l149 129h757q-166 187 -227 459l111 97q61 -356 298 -556zM1942 1520l84 -96q8 -10 7.5 -23.5t-10.5 -22.5l-1872 -1622q-10 -8 -23.5 -7t-21.5 11l-84 96q-8 10 -7.5 23.5t10.5 21.5l186 161 q-19 32 -19 66q50 42 91 88t85 119.5t74.5 158.5t50 206t19.5 260q0 152 117 282.5t307 158.5q-8 19 -8 39q0 40 28 68t68 28t68 -28t28 -68q0 -20 -8 -39q124 -18 219 -82.5t148 -157.5l418 363q10 8 23.5 7t21.5 -11z" /> +<glyph unicode="" horiz-adv-x="1408" d="M512 160v704q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-704q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM768 160v704q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-704q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM1024 160v704q0 14 -9 23t-23 9h-64q-14 0 -23 -9t-9 -23v-704 q0 -14 9 -23t23 -9h64q14 0 23 9t9 23zM480 1152h448l-48 117q-7 9 -17 11h-317q-10 -2 -17 -11zM1408 1120v-64q0 -14 -9 -23t-23 -9h-96v-948q0 -83 -47 -143.5t-113 -60.5h-832q-66 0 -113 58.5t-47 141.5v952h-96q-14 0 -23 9t-9 23v64q0 14 9 23t23 9h309l70 167 q15 37 54 63t79 26h320q40 0 79 -26t54 -63l70 -167h309q14 0 23 -9t9 -23z" /> +<glyph unicode="" d="M1150 462v-109q0 -50 -36.5 -89t-94 -60.5t-118 -32.5t-117.5 -11q-205 0 -342.5 139t-137.5 346q0 203 136 339t339 136q34 0 75.5 -4.5t93 -18t92.5 -34t69 -56.5t28 -81v-109q0 -16 -16 -16h-118q-16 0 -16 16v70q0 43 -65.5 67.5t-137.5 24.5q-140 0 -228.5 -91.5 t-88.5 -237.5q0 -151 91.5 -249.5t233.5 -98.5q68 0 138 24t70 66v70q0 7 4.5 11.5t10.5 4.5h119q6 0 11 -4.5t5 -11.5zM768 1280q-130 0 -248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51t248.5 51t204 136.5t136.5 204t51 248.5 t-51 248.5t-136.5 204t-204 136.5t-248.5 51zM1536 640q0 -209 -103 -385.5t-279.5 -279.5t-385.5 -103t-385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103t385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" d="M972 761q0 108 -53.5 169t-147.5 61q-63 0 -124 -30.5t-110 -84.5t-79.5 -137t-30.5 -180q0 -112 53.5 -173t150.5 -61q96 0 176 66.5t122.5 166t42.5 203.5zM1536 640q0 -111 -37 -197t-98.5 -135t-131.5 -74.5t-145 -27.5q-6 0 -15.5 -0.5t-16.5 -0.5q-95 0 -142 53 q-28 33 -33 83q-52 -66 -131.5 -110t-173.5 -44q-161 0 -249.5 95.5t-88.5 269.5q0 157 66 290t179 210.5t246 77.5q87 0 155 -35.5t106 -99.5l2 19l11 56q1 6 5.5 12t9.5 6h118q5 0 13 -11q5 -5 3 -16l-120 -614q-5 -24 -5 -48q0 -39 12.5 -52t44.5 -13q28 1 57 5.5t73 24 t77 50t57 89.5t24 137q0 292 -174 466t-466 174q-130 0 -248.5 -51t-204 -136.5t-136.5 -204t-51 -248.5t51 -248.5t136.5 -204t204 -136.5t248.5 -51q228 0 405 144q11 9 24 8t21 -12l41 -49q8 -12 7 -24q-2 -13 -12 -22q-102 -83 -227.5 -128t-258.5 -45q-156 0 -298 61 t-245 164t-164 245t-61 298t61 298t164 245t245 164t298 61q344 0 556 -212t212 -556z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1698 1442q94 -94 94 -226.5t-94 -225.5l-225 -223l104 -104q10 -10 10 -23t-10 -23l-210 -210q-10 -10 -23 -10t-23 10l-105 105l-603 -603q-37 -37 -90 -37h-203l-256 -128l-64 64l128 256v203q0 53 37 90l603 603l-105 105q-10 10 -10 23t10 23l210 210q10 10 23 10 t23 -10l104 -104l223 225q93 94 225.5 94t226.5 -94zM512 64l576 576l-192 192l-576 -576v-192h192z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1615 1536q70 0 122.5 -46.5t52.5 -116.5q0 -63 -45 -151q-332 -629 -465 -752q-97 -91 -218 -91q-126 0 -216.5 92.5t-90.5 219.5q0 128 92 212l638 579q59 54 130 54zM706 502q39 -76 106.5 -130t150.5 -76l1 -71q4 -213 -129.5 -347t-348.5 -134q-123 0 -218 46.5 t-152.5 127.5t-86.5 183t-29 220q7 -5 41 -30t62 -44.5t59 -36.5t46 -17q41 0 55 37q25 66 57.5 112.5t69.5 76t88 47.5t103 25.5t125 10.5z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1792 128v-384h-1792v384q45 0 85 14t59 27.5t47 37.5q30 27 51.5 38t56.5 11t55.5 -11t52.5 -38q29 -25 47 -38t58 -27t86 -14q45 0 85 14.5t58 27t48 37.5q21 19 32.5 27t31 15t43.5 7q35 0 56.5 -11t51.5 -38q28 -24 47 -37.5t59 -27.5t85 -14t85 14t59 27.5t47 37.5 q30 27 51.5 38t56.5 11q34 0 55.5 -11t51.5 -38q28 -24 47 -37.5t59 -27.5t85 -14zM1792 448v-192q-35 0 -55.5 11t-52.5 38q-29 25 -47 38t-58 27t-85 14q-46 0 -86 -14t-58 -27t-47 -38q-22 -19 -33 -27t-31 -15t-44 -7q-35 0 -56.5 11t-51.5 38q-29 25 -47 38t-58 27 t-86 14q-45 0 -85 -14.5t-58 -27t-48 -37.5q-21 -19 -32.5 -27t-31 -15t-43.5 -7q-35 0 -56.5 11t-51.5 38q-28 24 -47 37.5t-59 27.5t-85 14q-46 0 -86 -14t-58 -27t-47 -38q-30 -27 -51.5 -38t-56.5 -11v192q0 80 56 136t136 56h64v448h256v-448h256v448h256v-448h256v448 h256v-448h64q80 0 136 -56t56 -136zM512 1312q0 -77 -36 -118.5t-92 -41.5q-53 0 -90.5 37.5t-37.5 90.5q0 29 9.5 51t23.5 34t31 28t31 31.5t23.5 44.5t9.5 67q38 0 83 -74t45 -150zM1024 1312q0 -77 -36 -118.5t-92 -41.5q-53 0 -90.5 37.5t-37.5 90.5q0 29 9.5 51 t23.5 34t31 28t31 31.5t23.5 44.5t9.5 67q38 0 83 -74t45 -150zM1536 1312q0 -77 -36 -118.5t-92 -41.5q-53 0 -90.5 37.5t-37.5 90.5q0 29 9.5 51t23.5 34t31 28t31 31.5t23.5 44.5t9.5 67q38 0 83 -74t45 -150z" /> +<glyph unicode="" horiz-adv-x="2048" d="M2048 0v-128h-2048v1536h128v-1408h1920zM1664 1024l256 -896h-1664v576l448 576l576 -576z" /> +<glyph unicode="" horiz-adv-x="1792" d="M768 646l546 -546q-106 -108 -247.5 -168t-298.5 -60q-209 0 -385.5 103t-279.5 279.5t-103 385.5t103 385.5t279.5 279.5t385.5 103v-762zM955 640h773q0 -157 -60 -298.5t-168 -247.5zM1664 768h-768v768q209 0 385.5 -103t279.5 -279.5t103 -385.5z" /> +<glyph unicode="" horiz-adv-x="2048" d="M2048 0v-128h-2048v1536h128v-1408h1920zM1920 1248v-435q0 -21 -19.5 -29.5t-35.5 7.5l-121 121l-633 -633q-10 -10 -23 -10t-23 10l-233 233l-416 -416l-192 192l585 585q10 10 23 10t23 -10l233 -233l464 464l-121 121q-16 16 -7.5 35.5t29.5 19.5h435q14 0 23 -9 t9 -23z" /> +<glyph unicode="" horiz-adv-x="1792" d="M1292 832q0 -6 10 -41q10 -29 25 -49.5t41 -34t44 -20t55 -16.5q325 -91 325 -332q0 -146 -105.5 -242.5t-254.5 -96.5q-59 0 -111.5 18.5t-91.5 45.5t-77 74.5t-63 87.5t-53.5 103.5t-43.5 103t-39.5 106.5t-35.5 95q-32 81 -61.5 133.5t-73.5 96.5t-104 64t-142 20 q-96 0 -183 -55.5t-138 -144.5t-51 -185q0 -160 106.5 -279.5t263.5 -119.5q177 0 258 95q56 63 83 116l84 -152q-15 -34 -44 -70l1 -1q-131 -152 -388 -152q-147 0 -269.5 79t-190.5 207.5t-68 274.5q0 105 43.5 206t116 176.5t172 121.5t204.5 46q87 0 159 -19t123.5 -50 t95 -80t72.5 -99t58.5 -117t50.5 -124.5t50 -130.5t55 -127q96 -200 233 -200q81 0 138.5 48.5t57.5 128.5q0 42 -19 72t-50.5 46t-72.5 31.5t-84.5 27t-87.5 34t-81 52t-65 82t-39 122.5q-3 16 -3 33q0 110 87.5 192t198.5 78q78 -3 120.5 -14.5t90.5 -53.5h-1 q12 -11 23 -24.5t26 -36t19 -27.5l-129 -99q-26 49 -54 70v1q-23 21 -97 21q-49 0 -84 -33t-35 -83z" /> +<glyph unicode="" d="M1432 484q0 173 -234 239q-35 10 -53 16.5t-38 25t-29 46.5q0 2 -2 8.5t-3 12t-1 7.5q0 36 24.5 59.5t60.5 23.5q54 0 71 -15h-1q20 -15 39 -51l93 71q-39 54 -49 64q-33 29 -67.5 39t-85.5 10q-80 0 -142 -57.5t-62 -137.5q0 -7 2 -23q16 -96 64.5 -140t148.5 -73 q29 -8 49 -15.5t45 -21.5t38.5 -34.5t13.5 -46.5v-5q1 -58 -40.5 -93t-100.5 -35q-97 0 -167 144q-23 47 -51.5 121.5t-48 125.5t-54 110.5t-74 95.5t-103.5 60.5t-147 24.5q-101 0 -192 -56t-144 -148t-50 -192v-1q4 -108 50.5 -199t133.5 -147.5t196 -56.5q186 0 279 110 q20 27 31 51l-60 109q-42 -80 -99 -116t-146 -36q-115 0 -191 87t-76 204q0 105 82 189t186 84q112 0 170 -53.5t104 -172.5q8 -21 25.5 -68.5t28.5 -76.5t31.5 -74.5t38.5 -74t45.5 -62.5t55.5 -53.5t66 -33t80 -13.5q107 0 183 69.5t76 174.5zM1536 1120v-960 q0 -119 -84.5 -203.5t-203.5 -84.5h-960q-119 0 -203.5 84.5t-84.5 203.5v960q0 119 84.5 203.5t203.5 84.5h960q119 0 203.5 -84.5t84.5 -203.5z" /> +<glyph unicode="" horiz-adv-x="2048" d="M1152 640q0 104 -40.5 198.5t-109.5 163.5t-163.5 109.5t-198.5 40.5t-198.5 -40.5t-163.5 -109.5t-109.5 -163.5t-40.5 -198.5t40.5 -198.5t109.5 -163.5t163.5 -109.5t198.5 -40.5t198.5 40.5t163.5 109.5t109.5 163.5t40.5 198.5zM1920 640q0 104 -40.5 198.5 t-109.5 163.5t-163.5 109.5t-198.5 40.5h-386q119 -90 188.5 -224t69.5 -288t-69.5 -288t-188.5 -224h386q104 0 198.5 40.5t163.5 109.5t109.5 163.5t40.5 198.5zM2048 640q0 -130 -51 -248.5t-136.5 -204t-204 -136.5t-248.5 -51h-768q-130 0 -248.5 51t-204 136.5 t-136.5 204t-51 248.5t51 248.5t136.5 204t204 136.5t248.5 51h768q130 0 248.5 -51t204 -136.5t136.5 -204t51 -248.5z" /> +<glyph unicode="" horiz-adv-x="2048" d="M0 640q0 130 51 248.5t136.5 204t204 136.5t248.5 51h768q130 0 248.5 -51t204 -136.5t136.5 -204t51 -248.5t-51 -248.5t-136.5 -204t-204 -136.5t-248.5 -51h-768q-130 0 -248.5 51t-204 136.5t-136.5 204t-51 248.5zM1408 128q104 0 198.5 40.5t163.5 109.5 t109.5 163.5t40.5 198.5t-40.5 198.5t-109.5 163.5t-163.5 109.5t-198.5 40.5t-198.5 -40.5t-163.5 -109.5t-109.5 -163.5t-40.5 -198.5t40.5 -198.5t109.5 -163.5t163.5 -109.5t198.5 -40.5z" /> +<glyph unicode="" horiz-adv-x="2304" d="M762 384h-314q-40 0 -57.5 35t6.5 67l188 251q-65 31 -137 31q-132 0 -226 -94t-94 -226t94 -226t226 -94q115 0 203 72.5t111 183.5zM576 512h186q-18 85 -75 148zM1056 512l288 384h-480l-99 -132q105 -103 126 -252h165zM2176 448q0 132 -94 226t-226 94 q-60 0 -121 -24l174 -260q15 -23 10 -49t-27 -40q-15 -11 -36 -11q-35 0 -53 29l-174 260q-93 -95 -93 -225q0 -132 94 -226t226 -94t226 94t94 226zM2304 448q0 -185 -131.5 -316.5t-316.5 -131.5t-316.5 131.5t-131.5 316.5q0 97 39.5 183.5t109.5 149.5l-65 98l-353 -469 q-18 -26 -51 -26h-197q-23 -164 -149 -274t-294 -110q-185 0 -316.5 131.5t-131.5 316.5t131.5 316.5t316.5 131.5q114 0 215 -55l137 183h-224q-26 0 -45 19t-19 45t19 45t45 19h384v-128h435l-85 128h-222q-26 0 -45 19t-19 45t19 45t45 19h256q33 0 53 -28l267 -400 q91 44 192 44q185 0 316.5 -131.5t131.5 -316.5z" /> +<glyph unicode="" d="M384 320q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1408 320q0 53 -37.5 90.5t-90.5 37.5t-90.5 -37.5t-37.5 -90.5t37.5 -90.5t90.5 -37.5t90.5 37.5t37.5 90.5zM1362 716l-72 384q-5 23 -22.5 37.5t-40.5 14.5 h-918q-23 0 -40.5 -14.5t-22.5 -37.5l-72 -384q-5 -30 14 -53t49 -23h1062q30 0 49 23t14 53zM1136 1328q0 20 -14 34t-34 14h-640q-20 0 -34 -14t-14 -34t14 -34t34 -14h640q20 0 34 14t14 34zM1536 603v-603h-128v-128q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5 t-37.5 90.5v128h-768v-128q0 -53 -37.5 -90.5t-90.5 -37.5t-90.5 37.5t-37.5 90.5v128h-128v603q0 112 25 223l103 454q9 78 97.5 137t230 89t312.5 30t312.5 -30t230 -89t97.5 -137l105 -454q23 -102 23 -223z" /> +<glyph unicode="" horiz-adv-x="2048" d="M1463 704q0 -35 -25 -60.5t-61 -25.5h-702q-36 0 -61 25.5t-25 60.5t25 60.5t61 25.5h702q36 0 61 -25.5t25 -60.5zM1677 704q0 86 -23 170h-982q-36 0 -61 25t-25 60q0 36 25 61t61 25h908q-88 143 -235 227t-320 84q-177 0 -327.5 -87.5t-238 -237.5t-87.5 -327 q0 -86 23 -170h982q36 0 61 -25t25 -60q0 -36 -25 -61t-61 -25h-908q88 -143 235.5 -227t320.5 -84q132 0 253 51.5t208 139t139 208t52 253.5zM2048 959q0 -35 -25 -60t-61 -25h-131q17 -85 17 -170q0 -167 -65.5 -319.5t-175.5 -263t-262.5 -176t-319.5 -65.5 q-246 0 -448.5 133t-301.5 350h-189q-36 0 -61 25t-25 61q0 35 25 60t61 25h132q-17 85 -17 170q0 167 65.5 319.5t175.5 263t262.5 176t320.5 65.5q245 0 447.5 -133t301.5 -350h188q36 0 61 -25t25 -61z" /> +<glyph unicode="" horiz-adv-x="1280" d="M953 1158l-114 -328l117 -21q165 451 165 518q0 56 -38 56q-57 0 -130 -225zM654 471l33 -88q37 42 71 67l-33 5.5t-38.5 7t-32.5 8.5zM362 1367q0 -98 159 -521q18 10 49 10q15 0 75 -5l-121 351q-75 220 -123 220q-19 0 -29 -17.5t-10 -37.5zM283 608q0 -36 51.5 -119 t117.5 -153t100 -70q14 0 25.5 13t11.5 27q0 24 -32 102q-13 32 -32 72t-47.5 89t-61.5 81t-62 32q-20 0 -45.5 -27t-25.5 -47zM125 273q0 -41 25 -104q59 -145 183.5 -227t281.5 -82q227 0 382 170q152 169 152 427q0 43 -1 67t-11.5 62t-30.5 56q-56 49 -211.5 75.5 t-270.5 26.5q-37 0 -49 -11q-12 -5 -12 -35q0 -34 21.5 -60t55.5 -40t77.5 -23.5t87.5 -11.5t85 -4t70 0h23q24 0 40 -19q15 -19 19 -55q-28 -28 -96 -54q-61 -22 -93 -46q-64 -46 -108.5 -114t-44.5 -137q0 -31 18.5 -88.5t18.5 -87.5l-3 -12q-4 -12 -4 -14 q-137 10 -146 216q-8 -2 -41 -2q2 -7 2 -21q0 -53 -40.5 -89.5t-94.5 -36.5q-82 0 -166.5 78t-84.5 159q0 34 33 67q52 -64 60 -76q77 -104 133 -104q12 0 26.5 8.5t14.5 20.5q0 34 -87.5 145t-116.5 111q-43 0 -70 -44.5t-27 -90.5zM11 264q0 101 42.5 163t136.5 88 q-28 74 -28 104q0 62 61 123t122 61q29 0 70 -15q-163 462 -163 567q0 80 41 130.5t119 50.5q131 0 325 -581q6 -17 8 -23q6 16 29 79.5t43.5 118.5t54 127.5t64.5 123t70.5 86.5t76.5 36q71 0 112 -49t41 -122q0 -108 -159 -550q61 -15 100.5 -46t58.5 -78t26 -93.5 t7 -110.5q0 -150 -47 -280t-132 -225t-211 -150t-278 -55q-111 0 -223 42q-149 57 -258 191.5t-109 286.5z" /> +<glyph unicode="" horiz-adv-x="2048" d="M785 528h207q-14 -158 -98.5 -248.5t-214.5 -90.5q-162 0 -254.5 116t-92.5 316q0 194 93 311.5t233 117.5q148 0 232 -87t97 -247h-203q-5 64 -35.5 99t-81.5 35q-57 0 -88.5 -60.5t-31.5 -177.5q0 -48 5 -84t18 -69.5t40 -51.5t66 -18q95 0 109 139zM1497 528h206 q-14 -158 -98 -248.5t-214 -90.5q-162 0 -254.5 116t-92.5 316q0 194 93 311.5t233 117.5q148 0 232 -87t97 -247h-204q-4 64 -35 99t-81 35q-57 0 -88.5 -60.5t-31.5 -177.5q0 -48 5 -84t18 -69.5t39.5 -51.5t65.5 -18q49 0 76.5 38t33.5 101zM1856 647q0 207 -15.5 307 t-60.5 161q-6 8 -13.5 14t-21.5 15t-16 11q-86 63 -697 63q-625 0 -710 -63q-5 -4 -17.5 -11.5t-21 -14t-14.5 -14.5q-45 -60 -60 -159.5t-15 -308.5q0 -208 15 -307.5t60 -160.5q6 -8 15 -15t20.5 -14t17.5 -12q44 -33 239.5 -49t470.5 -16q610 0 697 65q5 4 17 11t20.5 14 t13.5 16q46 60 61 159t15 309zM2048 1408v-1536h-2048v1536h2048z" /> +<glyph unicode="" d="M992 912v-496q0 -14 -9 -23t-23 -9h-160q-14 0 -23 9t-9 23v496q0 112 -80 192t-192 80h-272v-1152q0 -14 -9 -23t-23 -9h-160q-14 0 -23 9t-9 23v1344q0 14 9 23t23 9h464q135 0 249 -66.5t180.5 -180.5t66.5 -249zM1376 1376v-880q0 -135 -66.5 -249t-180.5 -180.5 t-249 -66.5h-464q-14 0 -23 9t-9 23v960q0 14 9 23t23 9h160q14 0 23 -9t9 -23v-768h272q112 0 192 80t80 192v880q0 14 9 23t23 9h160q14 0 23 -9t9 -23z" /> +<glyph unicode="" d="M1311 694v-114q0 -24 -13.5 -38t-37.5 -14h-202q-24 0 -38 14t-14 38v114q0 24 14 38t38 14h202q24 0 37.5 -14t13.5 -38zM821 464v250q0 53 -32.5 85.5t-85.5 32.5h-133q-68 0 -96 -52q-28 52 -96 52h-130q-53 0 -85.5 -32.5t-32.5 -85.5v-250q0 -22 21 -22h55 q22 0 22 22v230q0 24 13.5 38t38.5 14h94q24 0 38 -14t14 -38v-230q0 -22 21 -22h54q22 0 22 22v230q0 24 14 38t38 14h97q24 0 37.5 -14t13.5 -38v-230q0 -22 22 -22h55q21 0 21 22zM1410 560v154q0 53 -33 85.5t-86 32.5h-264q-53 0 -86 -32.5t-33 -85.5v-410 q0 -21 22 -21h55q21 0 21 21v180q31 -42 94 -42h191q53 0 86 32.5t33 85.5zM1536 1176v-1072q0 -96 -68 -164t-164 -68h-1072q-96 0 -164 68t-68 164v1072q0 96 68 164t164 68h1072q96 0 164 -68t68 -164z" /> +<glyph unicode="" horiz-adv-x="1792" /> +<glyph unicode="" horiz-adv-x="1792" /> +<glyph unicode="" horiz-adv-x="1792" /> +</font> +</defs></svg>
\ No newline at end of file diff --git a/python/vespa/docs/fonts/fontawesome-webfont.ttf b/python/vespa/docs/fonts/fontawesome-webfont.ttf Binary files differnew file mode 100644 index 00000000000..96a3639cdde --- /dev/null +++ b/python/vespa/docs/fonts/fontawesome-webfont.ttf diff --git a/python/vespa/docs/fonts/fontawesome-webfont.woff b/python/vespa/docs/fonts/fontawesome-webfont.woff Binary files differnew file mode 100644 index 00000000000..628b6a52a87 --- /dev/null +++ b/python/vespa/docs/fonts/fontawesome-webfont.woff diff --git a/python/vespa/docs/fonts/glyphicons-halflings-regular.eot b/python/vespa/docs/fonts/glyphicons-halflings-regular.eot Binary files differnew file mode 100644 index 00000000000..b93a4953fff --- /dev/null +++ b/python/vespa/docs/fonts/glyphicons-halflings-regular.eot diff --git a/python/vespa/docs/fonts/glyphicons-halflings-regular.svg b/python/vespa/docs/fonts/glyphicons-halflings-regular.svg new file mode 100644 index 00000000000..94fb5490a2e --- /dev/null +++ b/python/vespa/docs/fonts/glyphicons-halflings-regular.svg @@ -0,0 +1,288 @@ +<?xml version="1.0" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd" > +<svg xmlns="http://www.w3.org/2000/svg"> +<metadata></metadata> +<defs> +<font id="glyphicons_halflingsregular" horiz-adv-x="1200" > +<font-face units-per-em="1200" ascent="960" descent="-240" /> +<missing-glyph horiz-adv-x="500" /> +<glyph horiz-adv-x="0" /> +<glyph horiz-adv-x="400" /> +<glyph unicode=" " /> +<glyph unicode="*" d="M600 1100q15 0 34 -1.5t30 -3.5l11 -1q10 -2 17.5 -10.5t7.5 -18.5v-224l158 158q7 7 18 8t19 -6l106 -106q7 -8 6 -19t-8 -18l-158 -158h224q10 0 18.5 -7.5t10.5 -17.5q6 -41 6 -75q0 -15 -1.5 -34t-3.5 -30l-1 -11q-2 -10 -10.5 -17.5t-18.5 -7.5h-224l158 -158 q7 -7 8 -18t-6 -19l-106 -106q-8 -7 -19 -6t-18 8l-158 158v-224q0 -10 -7.5 -18.5t-17.5 -10.5q-41 -6 -75 -6q-15 0 -34 1.5t-30 3.5l-11 1q-10 2 -17.5 10.5t-7.5 18.5v224l-158 -158q-7 -7 -18 -8t-19 6l-106 106q-7 8 -6 19t8 18l158 158h-224q-10 0 -18.5 7.5 t-10.5 17.5q-6 41 -6 75q0 15 1.5 34t3.5 30l1 11q2 10 10.5 17.5t18.5 7.5h224l-158 158q-7 7 -8 18t6 19l106 106q8 7 19 6t18 -8l158 -158v224q0 10 7.5 18.5t17.5 10.5q41 6 75 6z" /> +<glyph unicode="+" d="M450 1100h200q21 0 35.5 -14.5t14.5 -35.5v-350h350q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-350v-350q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v350h-350q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5 h350v350q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode=" " /> +<glyph unicode="¥" d="M825 1100h250q10 0 12.5 -5t-5.5 -13l-364 -364q-6 -6 -11 -18h268q10 0 13 -6t-3 -14l-120 -160q-6 -8 -18 -14t-22 -6h-125v-100h275q10 0 13 -6t-3 -14l-120 -160q-6 -8 -18 -14t-22 -6h-125v-174q0 -11 -7.5 -18.5t-18.5 -7.5h-148q-11 0 -18.5 7.5t-7.5 18.5v174 h-275q-10 0 -13 6t3 14l120 160q6 8 18 14t22 6h125v100h-275q-10 0 -13 6t3 14l120 160q6 8 18 14t22 6h118q-5 12 -11 18l-364 364q-8 8 -5.5 13t12.5 5h250q25 0 43 -18l164 -164q8 -8 18 -8t18 8l164 164q18 18 43 18z" /> +<glyph unicode=" " horiz-adv-x="650" /> +<glyph unicode=" " horiz-adv-x="1300" /> +<glyph unicode=" " horiz-adv-x="650" /> +<glyph unicode=" " horiz-adv-x="1300" /> +<glyph unicode=" " horiz-adv-x="433" /> +<glyph unicode=" " horiz-adv-x="325" /> +<glyph unicode=" " horiz-adv-x="216" /> +<glyph unicode=" " horiz-adv-x="216" /> +<glyph unicode=" " horiz-adv-x="162" /> +<glyph unicode=" " horiz-adv-x="260" /> +<glyph unicode=" " horiz-adv-x="72" /> +<glyph unicode=" " horiz-adv-x="260" /> +<glyph unicode=" " horiz-adv-x="325" /> +<glyph unicode="€" d="M744 1198q242 0 354 -189q60 -104 66 -209h-181q0 45 -17.5 82.5t-43.5 61.5t-58 40.5t-60.5 24t-51.5 7.5q-19 0 -40.5 -5.5t-49.5 -20.5t-53 -38t-49 -62.5t-39 -89.5h379l-100 -100h-300q-6 -50 -6 -100h406l-100 -100h-300q9 -74 33 -132t52.5 -91t61.5 -54.5t59 -29 t47 -7.5q22 0 50.5 7.5t60.5 24.5t58 41t43.5 61t17.5 80h174q-30 -171 -128 -278q-107 -117 -274 -117q-206 0 -324 158q-36 48 -69 133t-45 204h-217l100 100h112q1 47 6 100h-218l100 100h134q20 87 51 153.5t62 103.5q117 141 297 141z" /> +<glyph unicode="₽" d="M428 1200h350q67 0 120 -13t86 -31t57 -49.5t35 -56.5t17 -64.5t6.5 -60.5t0.5 -57v-16.5v-16.5q0 -36 -0.5 -57t-6.5 -61t-17 -65t-35 -57t-57 -50.5t-86 -31.5t-120 -13h-178l-2 -100h288q10 0 13 -6t-3 -14l-120 -160q-6 -8 -18 -14t-22 -6h-138v-175q0 -11 -5.5 -18 t-15.5 -7h-149q-10 0 -17.5 7.5t-7.5 17.5v175h-267q-10 0 -13 6t3 14l120 160q6 8 18 14t22 6h117v100h-267q-10 0 -13 6t3 14l120 160q6 8 18 14t22 6h117v475q0 10 7.5 17.5t17.5 7.5zM600 1000v-300h203q64 0 86.5 33t22.5 119q0 84 -22.5 116t-86.5 32h-203z" /> +<glyph unicode="−" d="M250 700h800q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-800q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="⌛" d="M1000 1200v-150q0 -21 -14.5 -35.5t-35.5 -14.5h-50v-100q0 -91 -49.5 -165.5t-130.5 -109.5q81 -35 130.5 -109.5t49.5 -165.5v-150h50q21 0 35.5 -14.5t14.5 -35.5v-150h-800v150q0 21 14.5 35.5t35.5 14.5h50v150q0 91 49.5 165.5t130.5 109.5q-81 35 -130.5 109.5 t-49.5 165.5v100h-50q-21 0 -35.5 14.5t-14.5 35.5v150h800zM400 1000v-100q0 -60 32.5 -109.5t87.5 -73.5q28 -12 44 -37t16 -55t-16 -55t-44 -37q-55 -24 -87.5 -73.5t-32.5 -109.5v-150h400v150q0 60 -32.5 109.5t-87.5 73.5q-28 12 -44 37t-16 55t16 55t44 37 q55 24 87.5 73.5t32.5 109.5v100h-400z" /> +<glyph unicode="◼" horiz-adv-x="500" d="M0 0z" /> +<glyph unicode="☁" d="M503 1089q110 0 200.5 -59.5t134.5 -156.5q44 14 90 14q120 0 205 -86.5t85 -206.5q0 -121 -85 -207.5t-205 -86.5h-750q-79 0 -135.5 57t-56.5 137q0 69 42.5 122.5t108.5 67.5q-2 12 -2 37q0 153 108 260.5t260 107.5z" /> +<glyph unicode="⛺" d="M774 1193.5q16 -9.5 20.5 -27t-5.5 -33.5l-136 -187l467 -746h30q20 0 35 -18.5t15 -39.5v-42h-1200v42q0 21 15 39.5t35 18.5h30l468 746l-135 183q-10 16 -5.5 34t20.5 28t34 5.5t28 -20.5l111 -148l112 150q9 16 27 20.5t34 -5zM600 200h377l-182 112l-195 534v-646z " /> +<glyph unicode="✉" d="M25 1100h1150q10 0 12.5 -5t-5.5 -13l-564 -567q-8 -8 -18 -8t-18 8l-564 567q-8 8 -5.5 13t12.5 5zM18 882l264 -264q8 -8 8 -18t-8 -18l-264 -264q-8 -8 -13 -5.5t-5 12.5v550q0 10 5 12.5t13 -5.5zM918 618l264 264q8 8 13 5.5t5 -12.5v-550q0 -10 -5 -12.5t-13 5.5 l-264 264q-8 8 -8 18t8 18zM818 482l364 -364q8 -8 5.5 -13t-12.5 -5h-1150q-10 0 -12.5 5t5.5 13l364 364q8 8 18 8t18 -8l164 -164q8 -8 18 -8t18 8l164 164q8 8 18 8t18 -8z" /> +<glyph unicode="✏" d="M1011 1210q19 0 33 -13l153 -153q13 -14 13 -33t-13 -33l-99 -92l-214 214l95 96q13 14 32 14zM1013 800l-615 -614l-214 214l614 614zM317 96l-333 -112l110 335z" /> +<glyph unicode="" d="M700 650v-550h250q21 0 35.5 -14.5t14.5 -35.5v-50h-800v50q0 21 14.5 35.5t35.5 14.5h250v550l-500 550h1200z" /> +<glyph unicode="" d="M368 1017l645 163q39 15 63 0t24 -49v-831q0 -55 -41.5 -95.5t-111.5 -63.5q-79 -25 -147 -4.5t-86 75t25.5 111.5t122.5 82q72 24 138 8v521l-600 -155v-606q0 -42 -44 -90t-109 -69q-79 -26 -147 -5.5t-86 75.5t25.5 111.5t122.5 82.5q72 24 138 7v639q0 38 14.5 59 t53.5 34z" /> +<glyph unicode="" d="M500 1191q100 0 191 -39t156.5 -104.5t104.5 -156.5t39 -191l-1 -2l1 -5q0 -141 -78 -262l275 -274q23 -26 22.5 -44.5t-22.5 -42.5l-59 -58q-26 -20 -46.5 -20t-39.5 20l-275 274q-119 -77 -261 -77l-5 1l-2 -1q-100 0 -191 39t-156.5 104.5t-104.5 156.5t-39 191 t39 191t104.5 156.5t156.5 104.5t191 39zM500 1022q-88 0 -162 -43t-117 -117t-43 -162t43 -162t117 -117t162 -43t162 43t117 117t43 162t-43 162t-117 117t-162 43z" /> +<glyph unicode="" d="M649 949q48 68 109.5 104t121.5 38.5t118.5 -20t102.5 -64t71 -100.5t27 -123q0 -57 -33.5 -117.5t-94 -124.5t-126.5 -127.5t-150 -152.5t-146 -174q-62 85 -145.5 174t-150 152.5t-126.5 127.5t-93.5 124.5t-33.5 117.5q0 64 28 123t73 100.5t104 64t119 20 t120.5 -38.5t104.5 -104z" /> +<glyph unicode="" d="M407 800l131 353q7 19 17.5 19t17.5 -19l129 -353h421q21 0 24 -8.5t-14 -20.5l-342 -249l130 -401q7 -20 -0.5 -25.5t-24.5 6.5l-343 246l-342 -247q-17 -12 -24.5 -6.5t-0.5 25.5l130 400l-347 251q-17 12 -14 20.5t23 8.5h429z" /> +<glyph unicode="" d="M407 800l131 353q7 19 17.5 19t17.5 -19l129 -353h421q21 0 24 -8.5t-14 -20.5l-342 -249l130 -401q7 -20 -0.5 -25.5t-24.5 6.5l-343 246l-342 -247q-17 -12 -24.5 -6.5t-0.5 25.5l130 400l-347 251q-17 12 -14 20.5t23 8.5h429zM477 700h-240l197 -142l-74 -226 l193 139l195 -140l-74 229l192 140h-234l-78 211z" /> +<glyph unicode="" d="M600 1200q124 0 212 -88t88 -212v-250q0 -46 -31 -98t-69 -52v-75q0 -10 6 -21.5t15 -17.5l358 -230q9 -5 15 -16.5t6 -21.5v-93q0 -10 -7.5 -17.5t-17.5 -7.5h-1150q-10 0 -17.5 7.5t-7.5 17.5v93q0 10 6 21.5t15 16.5l358 230q9 6 15 17.5t6 21.5v75q-38 0 -69 52 t-31 98v250q0 124 88 212t212 88z" /> +<glyph unicode="" d="M25 1100h1150q10 0 17.5 -7.5t7.5 -17.5v-1050q0 -10 -7.5 -17.5t-17.5 -7.5h-1150q-10 0 -17.5 7.5t-7.5 17.5v1050q0 10 7.5 17.5t17.5 7.5zM100 1000v-100h100v100h-100zM875 1000h-550q-10 0 -17.5 -7.5t-7.5 -17.5v-350q0 -10 7.5 -17.5t17.5 -7.5h550 q10 0 17.5 7.5t7.5 17.5v350q0 10 -7.5 17.5t-17.5 7.5zM1000 1000v-100h100v100h-100zM100 800v-100h100v100h-100zM1000 800v-100h100v100h-100zM100 600v-100h100v100h-100zM1000 600v-100h100v100h-100zM875 500h-550q-10 0 -17.5 -7.5t-7.5 -17.5v-350q0 -10 7.5 -17.5 t17.5 -7.5h550q10 0 17.5 7.5t7.5 17.5v350q0 10 -7.5 17.5t-17.5 7.5zM100 400v-100h100v100h-100zM1000 400v-100h100v100h-100zM100 200v-100h100v100h-100zM1000 200v-100h100v100h-100z" /> +<glyph unicode="" d="M50 1100h400q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-400q-21 0 -35.5 14.5t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5zM650 1100h400q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-400q-21 0 -35.5 14.5t-14.5 35.5v400 q0 21 14.5 35.5t35.5 14.5zM50 500h400q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-400q-21 0 -35.5 14.5t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5zM650 500h400q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-400 q-21 0 -35.5 14.5t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M50 1100h200q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5zM450 1100h200q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v200 q0 21 14.5 35.5t35.5 14.5zM850 1100h200q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5zM50 700h200q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-200 q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5zM450 700h200q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5zM850 700h200q21 0 35.5 -14.5t14.5 -35.5v-200 q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5zM50 300h200q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5zM450 300h200 q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5zM850 300h200q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5 t35.5 14.5z" /> +<glyph unicode="" d="M50 1100h200q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5zM450 1100h700q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-700q-21 0 -35.5 14.5t-14.5 35.5v200 q0 21 14.5 35.5t35.5 14.5zM50 700h200q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5zM450 700h700q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-700 q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5zM50 300h200q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5zM450 300h700q21 0 35.5 -14.5t14.5 -35.5v-200 q0 -21 -14.5 -35.5t-35.5 -14.5h-700q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M465 477l571 571q8 8 18 8t17 -8l177 -177q8 -7 8 -17t-8 -18l-783 -784q-7 -8 -17.5 -8t-17.5 8l-384 384q-8 8 -8 18t8 17l177 177q7 8 17 8t18 -8l171 -171q7 -7 18 -7t18 7z" /> +<glyph unicode="" d="M904 1083l178 -179q8 -8 8 -18.5t-8 -17.5l-267 -268l267 -268q8 -7 8 -17.5t-8 -18.5l-178 -178q-8 -8 -18.5 -8t-17.5 8l-268 267l-268 -267q-7 -8 -17.5 -8t-18.5 8l-178 178q-8 8 -8 18.5t8 17.5l267 268l-267 268q-8 7 -8 17.5t8 18.5l178 178q8 8 18.5 8t17.5 -8 l268 -267l268 268q7 7 17.5 7t18.5 -7z" /> +<glyph unicode="" d="M507 1177q98 0 187.5 -38.5t154.5 -103.5t103.5 -154.5t38.5 -187.5q0 -141 -78 -262l300 -299q8 -8 8 -18.5t-8 -18.5l-109 -108q-7 -8 -17.5 -8t-18.5 8l-300 299q-119 -77 -261 -77q-98 0 -188 38.5t-154.5 103t-103 154.5t-38.5 188t38.5 187.5t103 154.5 t154.5 103.5t188 38.5zM506.5 1023q-89.5 0 -165.5 -44t-120 -120.5t-44 -166t44 -165.5t120 -120t165.5 -44t166 44t120.5 120t44 165.5t-44 166t-120.5 120.5t-166 44zM425 900h150q10 0 17.5 -7.5t7.5 -17.5v-75h75q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5 t-17.5 -7.5h-75v-75q0 -10 -7.5 -17.5t-17.5 -7.5h-150q-10 0 -17.5 7.5t-7.5 17.5v75h-75q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5h75v75q0 10 7.5 17.5t17.5 7.5z" /> +<glyph unicode="" d="M507 1177q98 0 187.5 -38.5t154.5 -103.5t103.5 -154.5t38.5 -187.5q0 -141 -78 -262l300 -299q8 -8 8 -18.5t-8 -18.5l-109 -108q-7 -8 -17.5 -8t-18.5 8l-300 299q-119 -77 -261 -77q-98 0 -188 38.5t-154.5 103t-103 154.5t-38.5 188t38.5 187.5t103 154.5 t154.5 103.5t188 38.5zM506.5 1023q-89.5 0 -165.5 -44t-120 -120.5t-44 -166t44 -165.5t120 -120t165.5 -44t166 44t120.5 120t44 165.5t-44 166t-120.5 120.5t-166 44zM325 800h350q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-350q-10 0 -17.5 7.5 t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5z" /> +<glyph unicode="" d="M550 1200h100q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5zM800 975v166q167 -62 272 -209.5t105 -331.5q0 -117 -45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5 t-184.5 123t-123 184.5t-45.5 224q0 184 105 331.5t272 209.5v-166q-103 -55 -165 -155t-62 -220q0 -116 57 -214.5t155.5 -155.5t214.5 -57t214.5 57t155.5 155.5t57 214.5q0 120 -62 220t-165 155z" /> +<glyph unicode="" d="M1025 1200h150q10 0 17.5 -7.5t7.5 -17.5v-1150q0 -10 -7.5 -17.5t-17.5 -7.5h-150q-10 0 -17.5 7.5t-7.5 17.5v1150q0 10 7.5 17.5t17.5 7.5zM725 800h150q10 0 17.5 -7.5t7.5 -17.5v-750q0 -10 -7.5 -17.5t-17.5 -7.5h-150q-10 0 -17.5 7.5t-7.5 17.5v750 q0 10 7.5 17.5t17.5 7.5zM425 500h150q10 0 17.5 -7.5t7.5 -17.5v-450q0 -10 -7.5 -17.5t-17.5 -7.5h-150q-10 0 -17.5 7.5t-7.5 17.5v450q0 10 7.5 17.5t17.5 7.5zM125 300h150q10 0 17.5 -7.5t7.5 -17.5v-250q0 -10 -7.5 -17.5t-17.5 -7.5h-150q-10 0 -17.5 7.5t-7.5 17.5 v250q0 10 7.5 17.5t17.5 7.5z" /> +<glyph unicode="" d="M600 1174q33 0 74 -5l38 -152l5 -1q49 -14 94 -39l5 -2l134 80q61 -48 104 -105l-80 -134l3 -5q25 -44 39 -93l1 -6l152 -38q5 -43 5 -73q0 -34 -5 -74l-152 -38l-1 -6q-15 -49 -39 -93l-3 -5l80 -134q-48 -61 -104 -105l-134 81l-5 -3q-44 -25 -94 -39l-5 -2l-38 -151 q-43 -5 -74 -5q-33 0 -74 5l-38 151l-5 2q-49 14 -94 39l-5 3l-134 -81q-60 48 -104 105l80 134l-3 5q-25 45 -38 93l-2 6l-151 38q-6 42 -6 74q0 33 6 73l151 38l2 6q13 48 38 93l3 5l-80 134q47 61 105 105l133 -80l5 2q45 25 94 39l5 1l38 152q43 5 74 5zM600 815 q-89 0 -152 -63t-63 -151.5t63 -151.5t152 -63t152 63t63 151.5t-63 151.5t-152 63z" /> +<glyph unicode="" d="M500 1300h300q41 0 70.5 -29.5t29.5 -70.5v-100h275q10 0 17.5 -7.5t7.5 -17.5v-75h-1100v75q0 10 7.5 17.5t17.5 7.5h275v100q0 41 29.5 70.5t70.5 29.5zM500 1200v-100h300v100h-300zM1100 900v-800q0 -41 -29.5 -70.5t-70.5 -29.5h-700q-41 0 -70.5 29.5t-29.5 70.5 v800h900zM300 800v-700h100v700h-100zM500 800v-700h100v700h-100zM700 800v-700h100v700h-100zM900 800v-700h100v700h-100z" /> +<glyph unicode="" d="M18 618l620 608q8 7 18.5 7t17.5 -7l608 -608q8 -8 5.5 -13t-12.5 -5h-175v-575q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5t-7.5 17.5v375h-300v-375q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5t-7.5 17.5v575h-175q-10 0 -12.5 5t5.5 13z" /> +<glyph unicode="" d="M600 1200v-400q0 -41 29.5 -70.5t70.5 -29.5h300v-650q0 -21 -14.5 -35.5t-35.5 -14.5h-800q-21 0 -35.5 14.5t-14.5 35.5v1100q0 21 14.5 35.5t35.5 14.5h450zM1000 800h-250q-21 0 -35.5 14.5t-14.5 35.5v250z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM600 1027q-116 0 -214.5 -57t-155.5 -155.5t-57 -214.5t57 -214.5 t155.5 -155.5t214.5 -57t214.5 57t155.5 155.5t57 214.5t-57 214.5t-155.5 155.5t-214.5 57zM525 900h50q10 0 17.5 -7.5t7.5 -17.5v-275h175q10 0 17.5 -7.5t7.5 -17.5v-50q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5t-7.5 17.5v350q0 10 7.5 17.5t17.5 7.5z" /> +<glyph unicode="" d="M1300 0h-538l-41 400h-242l-41 -400h-538l431 1200h209l-21 -300h162l-20 300h208zM515 800l-27 -300h224l-27 300h-170z" /> +<glyph unicode="" d="M550 1200h200q21 0 35.5 -14.5t14.5 -35.5v-450h191q20 0 25.5 -11.5t-7.5 -27.5l-327 -400q-13 -16 -32 -16t-32 16l-327 400q-13 16 -7.5 27.5t25.5 11.5h191v450q0 21 14.5 35.5t35.5 14.5zM1125 400h50q10 0 17.5 -7.5t7.5 -17.5v-350q0 -10 -7.5 -17.5t-17.5 -7.5 h-1050q-10 0 -17.5 7.5t-7.5 17.5v350q0 10 7.5 17.5t17.5 7.5h50q10 0 17.5 -7.5t7.5 -17.5v-175h900v175q0 10 7.5 17.5t17.5 7.5z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM600 1027q-116 0 -214.5 -57t-155.5 -155.5t-57 -214.5t57 -214.5 t155.5 -155.5t214.5 -57t214.5 57t155.5 155.5t57 214.5t-57 214.5t-155.5 155.5t-214.5 57zM525 900h150q10 0 17.5 -7.5t7.5 -17.5v-275h137q21 0 26 -11.5t-8 -27.5l-223 -275q-13 -16 -32 -16t-32 16l-223 275q-13 16 -8 27.5t26 11.5h137v275q0 10 7.5 17.5t17.5 7.5z " /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM600 1027q-116 0 -214.5 -57t-155.5 -155.5t-57 -214.5t57 -214.5 t155.5 -155.5t214.5 -57t214.5 57t155.5 155.5t57 214.5t-57 214.5t-155.5 155.5t-214.5 57zM632 914l223 -275q13 -16 8 -27.5t-26 -11.5h-137v-275q0 -10 -7.5 -17.5t-17.5 -7.5h-150q-10 0 -17.5 7.5t-7.5 17.5v275h-137q-21 0 -26 11.5t8 27.5l223 275q13 16 32 16 t32 -16z" /> +<glyph unicode="" d="M225 1200h750q10 0 19.5 -7t12.5 -17l186 -652q7 -24 7 -49v-425q0 -12 -4 -27t-9 -17q-12 -6 -37 -6h-1100q-12 0 -27 4t-17 8q-6 13 -6 38l1 425q0 25 7 49l185 652q3 10 12.5 17t19.5 7zM878 1000h-556q-10 0 -19 -7t-11 -18l-87 -450q-2 -11 4 -18t16 -7h150 q10 0 19.5 -7t11.5 -17l38 -152q2 -10 11.5 -17t19.5 -7h250q10 0 19.5 7t11.5 17l38 152q2 10 11.5 17t19.5 7h150q10 0 16 7t4 18l-87 450q-2 11 -11 18t-19 7z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM600 1027q-116 0 -214.5 -57t-155.5 -155.5t-57 -214.5t57 -214.5 t155.5 -155.5t214.5 -57t214.5 57t155.5 155.5t57 214.5t-57 214.5t-155.5 155.5t-214.5 57zM540 820l253 -190q17 -12 17 -30t-17 -30l-253 -190q-16 -12 -28 -6.5t-12 26.5v400q0 21 12 26.5t28 -6.5z" /> +<glyph unicode="" d="M947 1060l135 135q7 7 12.5 5t5.5 -13v-362q0 -10 -7.5 -17.5t-17.5 -7.5h-362q-11 0 -13 5.5t5 12.5l133 133q-109 76 -238 76q-116 0 -214.5 -57t-155.5 -155.5t-57 -214.5t57 -214.5t155.5 -155.5t214.5 -57t214.5 57t155.5 155.5t57 214.5h150q0 -117 -45.5 -224 t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5q192 0 347 -117z" /> +<glyph unicode="" d="M947 1060l135 135q7 7 12.5 5t5.5 -13v-361q0 -11 -7.5 -18.5t-18.5 -7.5h-361q-11 0 -13 5.5t5 12.5l134 134q-110 75 -239 75q-116 0 -214.5 -57t-155.5 -155.5t-57 -214.5h-150q0 117 45.5 224t123 184.5t184.5 123t224 45.5q192 0 347 -117zM1027 600h150 q0 -117 -45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5q-192 0 -348 118l-134 -134q-7 -8 -12.5 -5.5t-5.5 12.5v360q0 11 7.5 18.5t18.5 7.5h360q10 0 12.5 -5.5t-5.5 -12.5l-133 -133q110 -76 240 -76q116 0 214.5 57t155.5 155.5t57 214.5z" /> +<glyph unicode="" d="M125 1200h1050q10 0 17.5 -7.5t7.5 -17.5v-1150q0 -10 -7.5 -17.5t-17.5 -7.5h-1050q-10 0 -17.5 7.5t-7.5 17.5v1150q0 10 7.5 17.5t17.5 7.5zM1075 1000h-850q-10 0 -17.5 -7.5t-7.5 -17.5v-850q0 -10 7.5 -17.5t17.5 -7.5h850q10 0 17.5 7.5t7.5 17.5v850 q0 10 -7.5 17.5t-17.5 7.5zM325 900h50q10 0 17.5 -7.5t7.5 -17.5v-50q0 -10 -7.5 -17.5t-17.5 -7.5h-50q-10 0 -17.5 7.5t-7.5 17.5v50q0 10 7.5 17.5t17.5 7.5zM525 900h450q10 0 17.5 -7.5t7.5 -17.5v-50q0 -10 -7.5 -17.5t-17.5 -7.5h-450q-10 0 -17.5 7.5t-7.5 17.5v50 q0 10 7.5 17.5t17.5 7.5zM325 700h50q10 0 17.5 -7.5t7.5 -17.5v-50q0 -10 -7.5 -17.5t-17.5 -7.5h-50q-10 0 -17.5 7.5t-7.5 17.5v50q0 10 7.5 17.5t17.5 7.5zM525 700h450q10 0 17.5 -7.5t7.5 -17.5v-50q0 -10 -7.5 -17.5t-17.5 -7.5h-450q-10 0 -17.5 7.5t-7.5 17.5v50 q0 10 7.5 17.5t17.5 7.5zM325 500h50q10 0 17.5 -7.5t7.5 -17.5v-50q0 -10 -7.5 -17.5t-17.5 -7.5h-50q-10 0 -17.5 7.5t-7.5 17.5v50q0 10 7.5 17.5t17.5 7.5zM525 500h450q10 0 17.5 -7.5t7.5 -17.5v-50q0 -10 -7.5 -17.5t-17.5 -7.5h-450q-10 0 -17.5 7.5t-7.5 17.5v50 q0 10 7.5 17.5t17.5 7.5zM325 300h50q10 0 17.5 -7.5t7.5 -17.5v-50q0 -10 -7.5 -17.5t-17.5 -7.5h-50q-10 0 -17.5 7.5t-7.5 17.5v50q0 10 7.5 17.5t17.5 7.5zM525 300h450q10 0 17.5 -7.5t7.5 -17.5v-50q0 -10 -7.5 -17.5t-17.5 -7.5h-450q-10 0 -17.5 7.5t-7.5 17.5v50 q0 10 7.5 17.5t17.5 7.5z" /> +<glyph unicode="" d="M900 800v200q0 83 -58.5 141.5t-141.5 58.5h-300q-82 0 -141 -59t-59 -141v-200h-100q-41 0 -70.5 -29.5t-29.5 -70.5v-600q0 -41 29.5 -70.5t70.5 -29.5h900q41 0 70.5 29.5t29.5 70.5v600q0 41 -29.5 70.5t-70.5 29.5h-100zM400 800v150q0 21 15 35.5t35 14.5h200 q20 0 35 -14.5t15 -35.5v-150h-300z" /> +<glyph unicode="" d="M125 1100h50q10 0 17.5 -7.5t7.5 -17.5v-1075h-100v1075q0 10 7.5 17.5t17.5 7.5zM1075 1052q4 0 9 -2q16 -6 16 -23v-421q0 -6 -3 -12q-33 -59 -66.5 -99t-65.5 -58t-56.5 -24.5t-52.5 -6.5q-26 0 -57.5 6.5t-52.5 13.5t-60 21q-41 15 -63 22.5t-57.5 15t-65.5 7.5 q-85 0 -160 -57q-7 -5 -15 -5q-6 0 -11 3q-14 7 -14 22v438q22 55 82 98.5t119 46.5q23 2 43 0.5t43 -7t32.5 -8.5t38 -13t32.5 -11q41 -14 63.5 -21t57 -14t63.5 -7q103 0 183 87q7 8 18 8z" /> +<glyph unicode="" d="M600 1175q116 0 227 -49.5t192.5 -131t131 -192.5t49.5 -227v-300q0 -10 -7.5 -17.5t-17.5 -7.5h-50q-10 0 -17.5 7.5t-7.5 17.5v300q0 127 -70.5 231.5t-184.5 161.5t-245 57t-245 -57t-184.5 -161.5t-70.5 -231.5v-300q0 -10 -7.5 -17.5t-17.5 -7.5h-50 q-10 0 -17.5 7.5t-7.5 17.5v300q0 116 49.5 227t131 192.5t192.5 131t227 49.5zM220 500h160q8 0 14 -6t6 -14v-460q0 -8 -6 -14t-14 -6h-160q-8 0 -14 6t-6 14v460q0 8 6 14t14 6zM820 500h160q8 0 14 -6t6 -14v-460q0 -8 -6 -14t-14 -6h-160q-8 0 -14 6t-6 14v460 q0 8 6 14t14 6z" /> +<glyph unicode="" d="M321 814l258 172q9 6 15 2.5t6 -13.5v-750q0 -10 -6 -13.5t-15 2.5l-258 172q-21 14 -46 14h-250q-10 0 -17.5 7.5t-7.5 17.5v350q0 10 7.5 17.5t17.5 7.5h250q25 0 46 14zM900 668l120 120q7 7 17 7t17 -7l34 -34q7 -7 7 -17t-7 -17l-120 -120l120 -120q7 -7 7 -17 t-7 -17l-34 -34q-7 -7 -17 -7t-17 7l-120 119l-120 -119q-7 -7 -17 -7t-17 7l-34 34q-7 7 -7 17t7 17l119 120l-119 120q-7 7 -7 17t7 17l34 34q7 8 17 8t17 -8z" /> +<glyph unicode="" d="M321 814l258 172q9 6 15 2.5t6 -13.5v-750q0 -10 -6 -13.5t-15 2.5l-258 172q-21 14 -46 14h-250q-10 0 -17.5 7.5t-7.5 17.5v350q0 10 7.5 17.5t17.5 7.5h250q25 0 46 14zM766 900h4q10 -1 16 -10q96 -129 96 -290q0 -154 -90 -281q-6 -9 -17 -10l-3 -1q-9 0 -16 6 l-29 23q-7 7 -8.5 16.5t4.5 17.5q72 103 72 229q0 132 -78 238q-6 8 -4.5 18t9.5 17l29 22q7 5 15 5z" /> +<glyph unicode="" d="M967 1004h3q11 -1 17 -10q135 -179 135 -396q0 -105 -34 -206.5t-98 -185.5q-7 -9 -17 -10h-3q-9 0 -16 6l-42 34q-8 6 -9 16t5 18q111 150 111 328q0 90 -29.5 176t-84.5 157q-6 9 -5 19t10 16l42 33q7 5 15 5zM321 814l258 172q9 6 15 2.5t6 -13.5v-750q0 -10 -6 -13.5 t-15 2.5l-258 172q-21 14 -46 14h-250q-10 0 -17.5 7.5t-7.5 17.5v350q0 10 7.5 17.5t17.5 7.5h250q25 0 46 14zM766 900h4q10 -1 16 -10q96 -129 96 -290q0 -154 -90 -281q-6 -9 -17 -10l-3 -1q-9 0 -16 6l-29 23q-7 7 -8.5 16.5t4.5 17.5q72 103 72 229q0 132 -78 238 q-6 8 -4.5 18.5t9.5 16.5l29 22q7 5 15 5z" /> +<glyph unicode="" d="M500 900h100v-100h-100v-100h-400v-100h-100v600h500v-300zM1200 700h-200v-100h200v-200h-300v300h-200v300h-100v200h600v-500zM100 1100v-300h300v300h-300zM800 1100v-300h300v300h-300zM300 900h-100v100h100v-100zM1000 900h-100v100h100v-100zM300 500h200v-500 h-500v500h200v100h100v-100zM800 300h200v-100h-100v-100h-200v100h-100v100h100v200h-200v100h300v-300zM100 400v-300h300v300h-300zM300 200h-100v100h100v-100zM1200 200h-100v100h100v-100zM700 0h-100v100h100v-100zM1200 0h-300v100h300v-100z" /> +<glyph unicode="" d="M100 200h-100v1000h100v-1000zM300 200h-100v1000h100v-1000zM700 200h-200v1000h200v-1000zM900 200h-100v1000h100v-1000zM1200 200h-200v1000h200v-1000zM400 0h-300v100h300v-100zM600 0h-100v91h100v-91zM800 0h-100v91h100v-91zM1100 0h-200v91h200v-91z" /> +<glyph unicode="" d="M500 1200l682 -682q8 -8 8 -18t-8 -18l-464 -464q-8 -8 -18 -8t-18 8l-682 682l1 475q0 10 7.5 17.5t17.5 7.5h474zM319.5 1024.5q-29.5 29.5 -71 29.5t-71 -29.5t-29.5 -71.5t29.5 -71.5t71 -29.5t71 29.5t29.5 71.5t-29.5 71.5z" /> +<glyph unicode="" d="M500 1200l682 -682q8 -8 8 -18t-8 -18l-464 -464q-8 -8 -18 -8t-18 8l-682 682l1 475q0 10 7.5 17.5t17.5 7.5h474zM800 1200l682 -682q8 -8 8 -18t-8 -18l-464 -464q-8 -8 -18 -8t-18 8l-56 56l424 426l-700 700h150zM319.5 1024.5q-29.5 29.5 -71 29.5t-71 -29.5 t-29.5 -71.5t29.5 -71.5t71 -29.5t71 29.5t29.5 71.5t-29.5 71.5z" /> +<glyph unicode="" d="M300 1200h825q75 0 75 -75v-900q0 -25 -18 -43l-64 -64q-8 -8 -13 -5.5t-5 12.5v950q0 10 -7.5 17.5t-17.5 7.5h-700q-25 0 -43 -18l-64 -64q-8 -8 -5.5 -13t12.5 -5h700q10 0 17.5 -7.5t7.5 -17.5v-950q0 -10 -7.5 -17.5t-17.5 -7.5h-850q-10 0 -17.5 7.5t-7.5 17.5v975 q0 25 18 43l139 139q18 18 43 18z" /> +<glyph unicode="" d="M250 1200h800q21 0 35.5 -14.5t14.5 -35.5v-1150l-450 444l-450 -445v1151q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M822 1200h-444q-11 0 -19 -7.5t-9 -17.5l-78 -301q-7 -24 7 -45l57 -108q6 -9 17.5 -15t21.5 -6h450q10 0 21.5 6t17.5 15l62 108q14 21 7 45l-83 301q-1 10 -9 17.5t-19 7.5zM1175 800h-150q-10 0 -21 -6.5t-15 -15.5l-78 -156q-4 -9 -15 -15.5t-21 -6.5h-550 q-10 0 -21 6.5t-15 15.5l-78 156q-4 9 -15 15.5t-21 6.5h-150q-10 0 -17.5 -7.5t-7.5 -17.5v-650q0 -10 7.5 -17.5t17.5 -7.5h150q10 0 17.5 7.5t7.5 17.5v150q0 10 7.5 17.5t17.5 7.5h750q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 7.5 -17.5t17.5 -7.5h150q10 0 17.5 7.5 t7.5 17.5v650q0 10 -7.5 17.5t-17.5 7.5zM850 200h-500q-10 0 -19.5 -7t-11.5 -17l-38 -152q-2 -10 3.5 -17t15.5 -7h600q10 0 15.5 7t3.5 17l-38 152q-2 10 -11.5 17t-19.5 7z" /> +<glyph unicode="" d="M500 1100h200q56 0 102.5 -20.5t72.5 -50t44 -59t25 -50.5l6 -20h150q41 0 70.5 -29.5t29.5 -70.5v-600q0 -41 -29.5 -70.5t-70.5 -29.5h-1000q-41 0 -70.5 29.5t-29.5 70.5v600q0 41 29.5 70.5t70.5 29.5h150q2 8 6.5 21.5t24 48t45 61t72 48t102.5 21.5zM900 800v-100 h100v100h-100zM600 730q-95 0 -162.5 -67.5t-67.5 -162.5t67.5 -162.5t162.5 -67.5t162.5 67.5t67.5 162.5t-67.5 162.5t-162.5 67.5zM600 603q43 0 73 -30t30 -73t-30 -73t-73 -30t-73 30t-30 73t30 73t73 30z" /> +<glyph unicode="" d="M681 1199l385 -998q20 -50 60 -92q18 -19 36.5 -29.5t27.5 -11.5l10 -2v-66h-417v66q53 0 75 43.5t5 88.5l-82 222h-391q-58 -145 -92 -234q-11 -34 -6.5 -57t25.5 -37t46 -20t55 -6v-66h-365v66q56 24 84 52q12 12 25 30.5t20 31.5l7 13l399 1006h93zM416 521h340 l-162 457z" /> +<glyph unicode="" d="M753 641q5 -1 14.5 -4.5t36 -15.5t50.5 -26.5t53.5 -40t50.5 -54.5t35.5 -70t14.5 -87q0 -67 -27.5 -125.5t-71.5 -97.5t-98.5 -66.5t-108.5 -40.5t-102 -13h-500v89q41 7 70.5 32.5t29.5 65.5v827q0 24 -0.5 34t-3.5 24t-8.5 19.5t-17 13.5t-28 12.5t-42.5 11.5v71 l471 -1q57 0 115.5 -20.5t108 -57t80.5 -94t31 -124.5q0 -51 -15.5 -96.5t-38 -74.5t-45 -50.5t-38.5 -30.5zM400 700h139q78 0 130.5 48.5t52.5 122.5q0 41 -8.5 70.5t-29.5 55.5t-62.5 39.5t-103.5 13.5h-118v-350zM400 200h216q80 0 121 50.5t41 130.5q0 90 -62.5 154.5 t-156.5 64.5h-159v-400z" /> +<glyph unicode="" d="M877 1200l2 -57q-83 -19 -116 -45.5t-40 -66.5l-132 -839q-9 -49 13 -69t96 -26v-97h-500v97q186 16 200 98l173 832q3 17 3 30t-1.5 22.5t-9 17.5t-13.5 12.5t-21.5 10t-26 8.5t-33.5 10q-13 3 -19 5v57h425z" /> +<glyph unicode="" d="M1300 900h-50q0 21 -4 37t-9.5 26.5t-18 17.5t-22 11t-28.5 5.5t-31 2t-37 0.5h-200v-850q0 -22 25 -34.5t50 -13.5l25 -2v-100h-400v100q4 0 11 0.5t24 3t30 7t24 15t11 24.5v850h-200q-25 0 -37 -0.5t-31 -2t-28.5 -5.5t-22 -11t-18 -17.5t-9.5 -26.5t-4 -37h-50v300 h1000v-300zM175 1000h-75v-800h75l-125 -167l-125 167h75v800h-75l125 167z" /> +<glyph unicode="" d="M1100 900h-50q0 21 -4 37t-9.5 26.5t-18 17.5t-22 11t-28.5 5.5t-31 2t-37 0.5h-200v-650q0 -22 25 -34.5t50 -13.5l25 -2v-100h-400v100q4 0 11 0.5t24 3t30 7t24 15t11 24.5v650h-200q-25 0 -37 -0.5t-31 -2t-28.5 -5.5t-22 -11t-18 -17.5t-9.5 -26.5t-4 -37h-50v300 h1000v-300zM1167 50l-167 -125v75h-800v-75l-167 125l167 125v-75h800v75z" /> +<glyph unicode="" d="M50 1100h600q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-600q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM50 800h1000q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-1000q-21 0 -35.5 14.5t-14.5 35.5v100 q0 21 14.5 35.5t35.5 14.5zM50 500h800q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-800q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM50 200h1100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-1100 q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M250 1100h700q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-700q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM50 800h1100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-1100q-21 0 -35.5 14.5t-14.5 35.5v100 q0 21 14.5 35.5t35.5 14.5zM250 500h700q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-700q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM50 200h1100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-1100 q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M500 950v100q0 21 14.5 35.5t35.5 14.5h600q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-600q-21 0 -35.5 14.5t-14.5 35.5zM100 650v100q0 21 14.5 35.5t35.5 14.5h1000q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-1000 q-21 0 -35.5 14.5t-14.5 35.5zM300 350v100q0 21 14.5 35.5t35.5 14.5h800q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-800q-21 0 -35.5 14.5t-14.5 35.5zM0 50v100q0 21 14.5 35.5t35.5 14.5h1100q21 0 35.5 -14.5t14.5 -35.5v-100 q0 -21 -14.5 -35.5t-35.5 -14.5h-1100q-21 0 -35.5 14.5t-14.5 35.5z" /> +<glyph unicode="" d="M50 1100h1100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-1100q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM50 800h1100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-1100q-21 0 -35.5 14.5t-14.5 35.5v100 q0 21 14.5 35.5t35.5 14.5zM50 500h1100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-1100q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM50 200h1100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-1100 q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M50 1100h100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM350 1100h800q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-800q-21 0 -35.5 14.5t-14.5 35.5v100 q0 21 14.5 35.5t35.5 14.5zM50 800h100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM350 800h800q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-800 q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM50 500h100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM350 500h800q21 0 35.5 -14.5t14.5 -35.5v-100 q0 -21 -14.5 -35.5t-35.5 -14.5h-800q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM50 200h100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM350 200h800 q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-800q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M400 0h-100v1100h100v-1100zM550 1100h100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM550 800h500q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-500 q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM267 550l-167 -125v75h-200v100h200v75zM550 500h300q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-300q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM550 200h600 q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-600q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M50 1100h100q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM900 0h-100v1100h100v-1100zM50 800h500q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-500 q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM1100 600h200v-100h-200v-75l-167 125l167 125v-75zM50 500h300q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-300q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5zM50 200h600 q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-600q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M75 1000h750q31 0 53 -22t22 -53v-650q0 -31 -22 -53t-53 -22h-750q-31 0 -53 22t-22 53v650q0 31 22 53t53 22zM1200 300l-300 300l300 300v-600z" /> +<glyph unicode="" d="M44 1100h1112q18 0 31 -13t13 -31v-1012q0 -18 -13 -31t-31 -13h-1112q-18 0 -31 13t-13 31v1012q0 18 13 31t31 13zM100 1000v-737l247 182l298 -131l-74 156l293 318l236 -288v500h-1000zM342 884q56 0 95 -39t39 -94.5t-39 -95t-95 -39.5t-95 39.5t-39 95t39 94.5 t95 39z" /> +<glyph unicode="" d="M648 1169q117 0 216 -60t156.5 -161t57.5 -218q0 -115 -70 -258q-69 -109 -158 -225.5t-143 -179.5l-54 -62q-9 8 -25.5 24.5t-63.5 67.5t-91 103t-98.5 128t-95.5 148q-60 132 -60 249q0 88 34 169.5t91.5 142t137 96.5t166.5 36zM652.5 974q-91.5 0 -156.5 -65 t-65 -157t65 -156.5t156.5 -64.5t156.5 64.5t65 156.5t-65 157t-156.5 65z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM600 173v854q-116 0 -214.5 -57t-155.5 -155.5t-57 -214.5t57 -214.5 t155.5 -155.5t214.5 -57z" /> +<glyph unicode="" d="M554 1295q21 -72 57.5 -143.5t76 -130t83 -118t82.5 -117t70 -116t49.5 -126t18.5 -136.5q0 -71 -25.5 -135t-68.5 -111t-99 -82t-118.5 -54t-125.5 -23q-84 5 -161.5 34t-139.5 78.5t-99 125t-37 164.5q0 69 18 136.5t49.5 126.5t69.5 116.5t81.5 117.5t83.5 119 t76.5 131t58.5 143zM344 710q-23 -33 -43.5 -70.5t-40.5 -102.5t-17 -123q1 -37 14.5 -69.5t30 -52t41 -37t38.5 -24.5t33 -15q21 -7 32 -1t13 22l6 34q2 10 -2.5 22t-13.5 19q-5 4 -14 12t-29.5 40.5t-32.5 73.5q-26 89 6 271q2 11 -6 11q-8 1 -15 -10z" /> +<glyph unicode="" d="M1000 1013l108 115q2 1 5 2t13 2t20.5 -1t25 -9.5t28.5 -21.5q22 -22 27 -43t0 -32l-6 -10l-108 -115zM350 1100h400q50 0 105 -13l-187 -187h-368q-41 0 -70.5 -29.5t-29.5 -70.5v-500q0 -41 29.5 -70.5t70.5 -29.5h500q41 0 70.5 29.5t29.5 70.5v182l200 200v-332 q0 -165 -93.5 -257.5t-256.5 -92.5h-400q-165 0 -257.5 92.5t-92.5 257.5v400q0 165 92.5 257.5t257.5 92.5zM1009 803l-362 -362l-161 -50l55 170l355 355z" /> +<glyph unicode="" d="M350 1100h361q-164 -146 -216 -200h-195q-41 0 -70.5 -29.5t-29.5 -70.5v-500q0 -41 29.5 -70.5t70.5 -29.5h500q41 0 70.5 29.5t29.5 70.5l200 153v-103q0 -165 -92.5 -257.5t-257.5 -92.5h-400q-165 0 -257.5 92.5t-92.5 257.5v400q0 165 92.5 257.5t257.5 92.5z M824 1073l339 -301q8 -7 8 -17.5t-8 -17.5l-340 -306q-7 -6 -12.5 -4t-6.5 11v203q-26 1 -54.5 0t-78.5 -7.5t-92 -17.5t-86 -35t-70 -57q10 59 33 108t51.5 81.5t65 58.5t68.5 40.5t67 24.5t56 13.5t40 4.5v210q1 10 6.5 12.5t13.5 -4.5z" /> +<glyph unicode="" d="M350 1100h350q60 0 127 -23l-178 -177h-349q-41 0 -70.5 -29.5t-29.5 -70.5v-500q0 -41 29.5 -70.5t70.5 -29.5h500q41 0 70.5 29.5t29.5 70.5v69l200 200v-219q0 -165 -92.5 -257.5t-257.5 -92.5h-400q-165 0 -257.5 92.5t-92.5 257.5v400q0 165 92.5 257.5t257.5 92.5z M643 639l395 395q7 7 17.5 7t17.5 -7l101 -101q7 -7 7 -17.5t-7 -17.5l-531 -532q-7 -7 -17.5 -7t-17.5 7l-248 248q-7 7 -7 17.5t7 17.5l101 101q7 7 17.5 7t17.5 -7l111 -111q8 -7 18 -7t18 7z" /> +<glyph unicode="" d="M318 918l264 264q8 8 18 8t18 -8l260 -264q7 -8 4.5 -13t-12.5 -5h-170v-200h200v173q0 10 5 12t13 -5l264 -260q8 -7 8 -17.5t-8 -17.5l-264 -265q-8 -7 -13 -5t-5 12v173h-200v-200h170q10 0 12.5 -5t-4.5 -13l-260 -264q-8 -8 -18 -8t-18 8l-264 264q-8 8 -5.5 13 t12.5 5h175v200h-200v-173q0 -10 -5 -12t-13 5l-264 265q-8 7 -8 17.5t8 17.5l264 260q8 7 13 5t5 -12v-173h200v200h-175q-10 0 -12.5 5t5.5 13z" /> +<glyph unicode="" d="M250 1100h100q21 0 35.5 -14.5t14.5 -35.5v-438l464 453q15 14 25.5 10t10.5 -25v-1000q0 -21 -10.5 -25t-25.5 10l-464 453v-438q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v1000q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M50 1100h100q21 0 35.5 -14.5t14.5 -35.5v-438l464 453q15 14 25.5 10t10.5 -25v-438l464 453q15 14 25.5 10t10.5 -25v-1000q0 -21 -10.5 -25t-25.5 10l-464 453v-438q0 -21 -10.5 -25t-25.5 10l-464 453v-438q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5 t-14.5 35.5v1000q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M1200 1050v-1000q0 -21 -10.5 -25t-25.5 10l-464 453v-438q0 -21 -10.5 -25t-25.5 10l-492 480q-15 14 -15 35t15 35l492 480q15 14 25.5 10t10.5 -25v-438l464 453q15 14 25.5 10t10.5 -25z" /> +<glyph unicode="" d="M243 1074l814 -498q18 -11 18 -26t-18 -26l-814 -498q-18 -11 -30.5 -4t-12.5 28v1000q0 21 12.5 28t30.5 -4z" /> +<glyph unicode="" d="M250 1000h200q21 0 35.5 -14.5t14.5 -35.5v-800q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v800q0 21 14.5 35.5t35.5 14.5zM650 1000h200q21 0 35.5 -14.5t14.5 -35.5v-800q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v800 q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M1100 950v-800q0 -21 -14.5 -35.5t-35.5 -14.5h-800q-21 0 -35.5 14.5t-14.5 35.5v800q0 21 14.5 35.5t35.5 14.5h800q21 0 35.5 -14.5t14.5 -35.5z" /> +<glyph unicode="" d="M500 612v438q0 21 10.5 25t25.5 -10l492 -480q15 -14 15 -35t-15 -35l-492 -480q-15 -14 -25.5 -10t-10.5 25v438l-464 -453q-15 -14 -25.5 -10t-10.5 25v1000q0 21 10.5 25t25.5 -10z" /> +<glyph unicode="" d="M1048 1102l100 1q20 0 35 -14.5t15 -35.5l5 -1000q0 -21 -14.5 -35.5t-35.5 -14.5l-100 -1q-21 0 -35.5 14.5t-14.5 35.5l-2 437l-463 -454q-14 -15 -24.5 -10.5t-10.5 25.5l-2 437l-462 -455q-15 -14 -25.5 -9.5t-10.5 24.5l-5 1000q0 21 10.5 25.5t25.5 -10.5l466 -450 l-2 438q0 20 10.5 24.5t25.5 -9.5l466 -451l-2 438q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M850 1100h100q21 0 35.5 -14.5t14.5 -35.5v-1000q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v438l-464 -453q-15 -14 -25.5 -10t-10.5 25v1000q0 21 10.5 25t25.5 -10l464 -453v438q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M686 1081l501 -540q15 -15 10.5 -26t-26.5 -11h-1042q-22 0 -26.5 11t10.5 26l501 540q15 15 36 15t36 -15zM150 400h1000q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-1000q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M885 900l-352 -353l352 -353l-197 -198l-552 552l552 550z" /> +<glyph unicode="" d="M1064 547l-551 -551l-198 198l353 353l-353 353l198 198z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM650 900h-100q-21 0 -35.5 -14.5t-14.5 -35.5v-150h-150 q-21 0 -35.5 -14.5t-14.5 -35.5v-100q0 -21 14.5 -35.5t35.5 -14.5h150v-150q0 -21 14.5 -35.5t35.5 -14.5h100q21 0 35.5 14.5t14.5 35.5v150h150q21 0 35.5 14.5t14.5 35.5v100q0 21 -14.5 35.5t-35.5 14.5h-150v150q0 21 -14.5 35.5t-35.5 14.5z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM850 700h-500q-21 0 -35.5 -14.5t-14.5 -35.5v-100q0 -21 14.5 -35.5 t35.5 -14.5h500q21 0 35.5 14.5t14.5 35.5v100q0 21 -14.5 35.5t-35.5 14.5z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM741.5 913q-12.5 0 -21.5 -9l-120 -120l-120 120q-9 9 -21.5 9 t-21.5 -9l-141 -141q-9 -9 -9 -21.5t9 -21.5l120 -120l-120 -120q-9 -9 -9 -21.5t9 -21.5l141 -141q9 -9 21.5 -9t21.5 9l120 120l120 -120q9 -9 21.5 -9t21.5 9l141 141q9 9 9 21.5t-9 21.5l-120 120l120 120q9 9 9 21.5t-9 21.5l-141 141q-9 9 -21.5 9z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM546 623l-84 85q-7 7 -17.5 7t-18.5 -7l-139 -139q-7 -8 -7 -18t7 -18 l242 -241q7 -8 17.5 -8t17.5 8l375 375q7 7 7 17.5t-7 18.5l-139 139q-7 7 -17.5 7t-17.5 -7z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM588 941q-29 0 -59 -5.5t-63 -20.5t-58 -38.5t-41.5 -63t-16.5 -89.5 q0 -25 20 -25h131q30 -5 35 11q6 20 20.5 28t45.5 8q20 0 31.5 -10.5t11.5 -28.5q0 -23 -7 -34t-26 -18q-1 0 -13.5 -4t-19.5 -7.5t-20 -10.5t-22 -17t-18.5 -24t-15.5 -35t-8 -46q-1 -8 5.5 -16.5t20.5 -8.5h173q7 0 22 8t35 28t37.5 48t29.5 74t12 100q0 47 -17 83 t-42.5 57t-59.5 34.5t-64 18t-59 4.5zM675 400h-150q-10 0 -17.5 -7.5t-7.5 -17.5v-150q0 -10 7.5 -17.5t17.5 -7.5h150q10 0 17.5 7.5t7.5 17.5v150q0 10 -7.5 17.5t-17.5 7.5z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM675 1000h-150q-10 0 -17.5 -7.5t-7.5 -17.5v-150q0 -10 7.5 -17.5 t17.5 -7.5h150q10 0 17.5 7.5t7.5 17.5v150q0 10 -7.5 17.5t-17.5 7.5zM675 700h-250q-10 0 -17.5 -7.5t-7.5 -17.5v-50q0 -10 7.5 -17.5t17.5 -7.5h75v-200h-75q-10 0 -17.5 -7.5t-7.5 -17.5v-50q0 -10 7.5 -17.5t17.5 -7.5h350q10 0 17.5 7.5t7.5 17.5v50q0 10 -7.5 17.5 t-17.5 7.5h-75v275q0 10 -7.5 17.5t-17.5 7.5z" /> +<glyph unicode="" d="M525 1200h150q10 0 17.5 -7.5t7.5 -17.5v-194q103 -27 178.5 -102.5t102.5 -178.5h194q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-194q-27 -103 -102.5 -178.5t-178.5 -102.5v-194q0 -10 -7.5 -17.5t-17.5 -7.5h-150q-10 0 -17.5 7.5t-7.5 17.5v194 q-103 27 -178.5 102.5t-102.5 178.5h-194q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5h194q27 103 102.5 178.5t178.5 102.5v194q0 10 7.5 17.5t17.5 7.5zM700 893v-168q0 -10 -7.5 -17.5t-17.5 -7.5h-150q-10 0 -17.5 7.5t-7.5 17.5v168q-68 -23 -119 -74 t-74 -119h168q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-168q23 -68 74 -119t119 -74v168q0 10 7.5 17.5t17.5 7.5h150q10 0 17.5 -7.5t7.5 -17.5v-168q68 23 119 74t74 119h-168q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5h168 q-23 68 -74 119t-119 74z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM600 1027q-116 0 -214.5 -57t-155.5 -155.5t-57 -214.5t57 -214.5 t155.5 -155.5t214.5 -57t214.5 57t155.5 155.5t57 214.5t-57 214.5t-155.5 155.5t-214.5 57zM759 823l64 -64q7 -7 7 -17.5t-7 -17.5l-124 -124l124 -124q7 -7 7 -17.5t-7 -17.5l-64 -64q-7 -7 -17.5 -7t-17.5 7l-124 124l-124 -124q-7 -7 -17.5 -7t-17.5 7l-64 64 q-7 7 -7 17.5t7 17.5l124 124l-124 124q-7 7 -7 17.5t7 17.5l64 64q7 7 17.5 7t17.5 -7l124 -124l124 124q7 7 17.5 7t17.5 -7z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM600 1027q-116 0 -214.5 -57t-155.5 -155.5t-57 -214.5t57 -214.5 t155.5 -155.5t214.5 -57t214.5 57t155.5 155.5t57 214.5t-57 214.5t-155.5 155.5t-214.5 57zM782 788l106 -106q7 -7 7 -17.5t-7 -17.5l-320 -321q-8 -7 -18 -7t-18 7l-202 203q-8 7 -8 17.5t8 17.5l106 106q7 8 17.5 8t17.5 -8l79 -79l197 197q7 7 17.5 7t17.5 -7z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM600 1027q-116 0 -214.5 -57t-155.5 -155.5t-57 -214.5q0 -120 65 -225 l587 587q-105 65 -225 65zM965 819l-584 -584q104 -62 219 -62q116 0 214.5 57t155.5 155.5t57 214.5q0 115 -62 219z" /> +<glyph unicode="" d="M39 582l522 427q16 13 27.5 8t11.5 -26v-291h550q21 0 35.5 -14.5t14.5 -35.5v-200q0 -21 -14.5 -35.5t-35.5 -14.5h-550v-291q0 -21 -11.5 -26t-27.5 8l-522 427q-16 13 -16 32t16 32z" /> +<glyph unicode="" d="M639 1009l522 -427q16 -13 16 -32t-16 -32l-522 -427q-16 -13 -27.5 -8t-11.5 26v291h-550q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5h550v291q0 21 11.5 26t27.5 -8z" /> +<glyph unicode="" d="M682 1161l427 -522q13 -16 8 -27.5t-26 -11.5h-291v-550q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v550h-291q-21 0 -26 11.5t8 27.5l427 522q13 16 32 16t32 -16z" /> +<glyph unicode="" d="M550 1200h200q21 0 35.5 -14.5t14.5 -35.5v-550h291q21 0 26 -11.5t-8 -27.5l-427 -522q-13 -16 -32 -16t-32 16l-427 522q-13 16 -8 27.5t26 11.5h291v550q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M639 1109l522 -427q16 -13 16 -32t-16 -32l-522 -427q-16 -13 -27.5 -8t-11.5 26v291q-94 -2 -182 -20t-170.5 -52t-147 -92.5t-100.5 -135.5q5 105 27 193.5t67.5 167t113 135t167 91.5t225.5 42v262q0 21 11.5 26t27.5 -8z" /> +<glyph unicode="" d="M850 1200h300q21 0 35.5 -14.5t14.5 -35.5v-300q0 -21 -10.5 -25t-24.5 10l-94 94l-249 -249q-8 -7 -18 -7t-18 7l-106 106q-7 8 -7 18t7 18l249 249l-94 94q-14 14 -10 24.5t25 10.5zM350 0h-300q-21 0 -35.5 14.5t-14.5 35.5v300q0 21 10.5 25t24.5 -10l94 -94l249 249 q8 7 18 7t18 -7l106 -106q7 -8 7 -18t-7 -18l-249 -249l94 -94q14 -14 10 -24.5t-25 -10.5z" /> +<glyph unicode="" d="M1014 1120l106 -106q7 -8 7 -18t-7 -18l-249 -249l94 -94q14 -14 10 -24.5t-25 -10.5h-300q-21 0 -35.5 14.5t-14.5 35.5v300q0 21 10.5 25t24.5 -10l94 -94l249 249q8 7 18 7t18 -7zM250 600h300q21 0 35.5 -14.5t14.5 -35.5v-300q0 -21 -10.5 -25t-24.5 10l-94 94 l-249 -249q-8 -7 -18 -7t-18 7l-106 106q-7 8 -7 18t7 18l249 249l-94 94q-14 14 -10 24.5t25 10.5z" /> +<glyph unicode="" d="M600 1177q117 0 224 -45.5t184.5 -123t123 -184.5t45.5 -224t-45.5 -224t-123 -184.5t-184.5 -123t-224 -45.5t-224 45.5t-184.5 123t-123 184.5t-45.5 224t45.5 224t123 184.5t184.5 123t224 45.5zM704 900h-208q-20 0 -32 -14.5t-8 -34.5l58 -302q4 -20 21.5 -34.5 t37.5 -14.5h54q20 0 37.5 14.5t21.5 34.5l58 302q4 20 -8 34.5t-32 14.5zM675 400h-150q-10 0 -17.5 -7.5t-7.5 -17.5v-150q0 -10 7.5 -17.5t17.5 -7.5h150q10 0 17.5 7.5t7.5 17.5v150q0 10 -7.5 17.5t-17.5 7.5z" /> +<glyph unicode="" d="M260 1200q9 0 19 -2t15 -4l5 -2q22 -10 44 -23l196 -118q21 -13 36 -24q29 -21 37 -12q11 13 49 35l196 118q22 13 45 23q17 7 38 7q23 0 47 -16.5t37 -33.5l13 -16q14 -21 18 -45l25 -123l8 -44q1 -9 8.5 -14.5t17.5 -5.5h61q10 0 17.5 -7.5t7.5 -17.5v-50 q0 -10 -7.5 -17.5t-17.5 -7.5h-50q-10 0 -17.5 -7.5t-7.5 -17.5v-175h-400v300h-200v-300h-400v175q0 10 -7.5 17.5t-17.5 7.5h-50q-10 0 -17.5 7.5t-7.5 17.5v50q0 10 7.5 17.5t17.5 7.5h61q11 0 18 3t7 8q0 4 9 52l25 128q5 25 19 45q2 3 5 7t13.5 15t21.5 19.5t26.5 15.5 t29.5 7zM915 1079l-166 -162q-7 -7 -5 -12t12 -5h219q10 0 15 7t2 17l-51 149q-3 10 -11 12t-15 -6zM463 917l-177 157q-8 7 -16 5t-11 -12l-51 -143q-3 -10 2 -17t15 -7h231q11 0 12.5 5t-5.5 12zM500 0h-375q-10 0 -17.5 7.5t-7.5 17.5v375h400v-400zM1100 400v-375 q0 -10 -7.5 -17.5t-17.5 -7.5h-375v400h400z" /> +<glyph unicode="" d="M1165 1190q8 3 21 -6.5t13 -17.5q-2 -178 -24.5 -323.5t-55.5 -245.5t-87 -174.5t-102.5 -118.5t-118 -68.5t-118.5 -33t-120 -4.5t-105 9.5t-90 16.5q-61 12 -78 11q-4 1 -12.5 0t-34 -14.5t-52.5 -40.5l-153 -153q-26 -24 -37 -14.5t-11 43.5q0 64 42 102q8 8 50.5 45 t66.5 58q19 17 35 47t13 61q-9 55 -10 102.5t7 111t37 130t78 129.5q39 51 80 88t89.5 63.5t94.5 45t113.5 36t129 31t157.5 37t182 47.5zM1116 1098q-8 9 -22.5 -3t-45.5 -50q-38 -47 -119 -103.5t-142 -89.5l-62 -33q-56 -30 -102 -57t-104 -68t-102.5 -80.5t-85.5 -91 t-64 -104.5q-24 -56 -31 -86t2 -32t31.5 17.5t55.5 59.5q25 30 94 75.5t125.5 77.5t147.5 81q70 37 118.5 69t102 79.5t99 111t86.5 148.5q22 50 24 60t-6 19z" /> +<glyph unicode="" d="M653 1231q-39 -67 -54.5 -131t-10.5 -114.5t24.5 -96.5t47.5 -80t63.5 -62.5t68.5 -46.5t65 -30q-4 7 -17.5 35t-18.5 39.5t-17 39.5t-17 43t-13 42t-9.5 44.5t-2 42t4 43t13.5 39t23 38.5q96 -42 165 -107.5t105 -138t52 -156t13 -159t-19 -149.5q-13 -55 -44 -106.5 t-68 -87t-78.5 -64.5t-72.5 -45t-53 -22q-72 -22 -127 -11q-31 6 -13 19q6 3 17 7q13 5 32.5 21t41 44t38.5 63.5t21.5 81.5t-6.5 94.5t-50 107t-104 115.5q10 -104 -0.5 -189t-37 -140.5t-65 -93t-84 -52t-93.5 -11t-95 24.5q-80 36 -131.5 114t-53.5 171q-2 23 0 49.5 t4.5 52.5t13.5 56t27.5 60t46 64.5t69.5 68.5q-8 -53 -5 -102.5t17.5 -90t34 -68.5t44.5 -39t49 -2q31 13 38.5 36t-4.5 55t-29 64.5t-36 75t-26 75.5q-15 85 2 161.5t53.5 128.5t85.5 92.5t93.5 61t81.5 25.5z" /> +<glyph unicode="" d="M600 1094q82 0 160.5 -22.5t140 -59t116.5 -82.5t94.5 -95t68 -95t42.5 -82.5t14 -57.5t-14 -57.5t-43 -82.5t-68.5 -95t-94.5 -95t-116.5 -82.5t-140 -59t-159.5 -22.5t-159.5 22.5t-140 59t-116.5 82.5t-94.5 95t-68.5 95t-43 82.5t-14 57.5t14 57.5t42.5 82.5t68 95 t94.5 95t116.5 82.5t140 59t160.5 22.5zM888 829q-15 15 -18 12t5 -22q25 -57 25 -119q0 -124 -88 -212t-212 -88t-212 88t-88 212q0 59 23 114q8 19 4.5 22t-17.5 -12q-70 -69 -160 -184q-13 -16 -15 -40.5t9 -42.5q22 -36 47 -71t70 -82t92.5 -81t113 -58.5t133.5 -24.5 t133.5 24t113 58.5t92.5 81.5t70 81.5t47 70.5q11 18 9 42.5t-14 41.5q-90 117 -163 189zM448 727l-35 -36q-15 -15 -19.5 -38.5t4.5 -41.5q37 -68 93 -116q16 -13 38.5 -11t36.5 17l35 34q14 15 12.5 33.5t-16.5 33.5q-44 44 -89 117q-11 18 -28 20t-32 -12z" /> +<glyph unicode="" d="M592 0h-148l31 120q-91 20 -175.5 68.5t-143.5 106.5t-103.5 119t-66.5 110t-22 76q0 21 14 57.5t42.5 82.5t68 95t94.5 95t116.5 82.5t140 59t160.5 22.5q61 0 126 -15l32 121h148zM944 770l47 181q108 -85 176.5 -192t68.5 -159q0 -26 -19.5 -71t-59.5 -102t-93 -112 t-129 -104.5t-158 -75.5l46 173q77 49 136 117t97 131q11 18 9 42.5t-14 41.5q-54 70 -107 130zM310 824q-70 -69 -160 -184q-13 -16 -15 -40.5t9 -42.5q18 -30 39 -60t57 -70.5t74 -73t90 -61t105 -41.5l41 154q-107 18 -178.5 101.5t-71.5 193.5q0 59 23 114q8 19 4.5 22 t-17.5 -12zM448 727l-35 -36q-15 -15 -19.5 -38.5t4.5 -41.5q37 -68 93 -116q16 -13 38.5 -11t36.5 17l12 11l22 86l-3 4q-44 44 -89 117q-11 18 -28 20t-32 -12z" /> +<glyph unicode="" d="M-90 100l642 1066q20 31 48 28.5t48 -35.5l642 -1056q21 -32 7.5 -67.5t-50.5 -35.5h-1294q-37 0 -50.5 34t7.5 66zM155 200h345v75q0 10 7.5 17.5t17.5 7.5h150q10 0 17.5 -7.5t7.5 -17.5v-75h345l-445 723zM496 700h208q20 0 32 -14.5t8 -34.5l-58 -252 q-4 -20 -21.5 -34.5t-37.5 -14.5h-54q-20 0 -37.5 14.5t-21.5 34.5l-58 252q-4 20 8 34.5t32 14.5z" /> +<glyph unicode="" d="M650 1200q62 0 106 -44t44 -106v-339l363 -325q15 -14 26 -38.5t11 -44.5v-41q0 -20 -12 -26.5t-29 5.5l-359 249v-263q100 -93 100 -113v-64q0 -21 -13 -29t-32 1l-205 128l-205 -128q-19 -9 -32 -1t-13 29v64q0 20 100 113v263l-359 -249q-17 -12 -29 -5.5t-12 26.5v41 q0 20 11 44.5t26 38.5l363 325v339q0 62 44 106t106 44z" /> +<glyph unicode="" d="M850 1200h100q21 0 35.5 -14.5t14.5 -35.5v-50h50q21 0 35.5 -14.5t14.5 -35.5v-150h-1100v150q0 21 14.5 35.5t35.5 14.5h50v50q0 21 14.5 35.5t35.5 14.5h100q21 0 35.5 -14.5t14.5 -35.5v-50h500v50q0 21 14.5 35.5t35.5 14.5zM1100 800v-750q0 -21 -14.5 -35.5 t-35.5 -14.5h-1000q-21 0 -35.5 14.5t-14.5 35.5v750h1100zM100 600v-100h100v100h-100zM300 600v-100h100v100h-100zM500 600v-100h100v100h-100zM700 600v-100h100v100h-100zM900 600v-100h100v100h-100zM100 400v-100h100v100h-100zM300 400v-100h100v100h-100zM500 400 v-100h100v100h-100zM700 400v-100h100v100h-100zM900 400v-100h100v100h-100zM100 200v-100h100v100h-100zM300 200v-100h100v100h-100zM500 200v-100h100v100h-100zM700 200v-100h100v100h-100zM900 200v-100h100v100h-100z" /> +<glyph unicode="" d="M1135 1165l249 -230q15 -14 15 -35t-15 -35l-249 -230q-14 -14 -24.5 -10t-10.5 25v150h-159l-600 -600h-291q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5h209l600 600h241v150q0 21 10.5 25t24.5 -10zM522 819l-141 -141l-122 122h-209q-21 0 -35.5 14.5 t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5h291zM1135 565l249 -230q15 -14 15 -35t-15 -35l-249 -230q-14 -14 -24.5 -10t-10.5 25v150h-241l-181 181l141 141l122 -122h159v150q0 21 10.5 25t24.5 -10z" /> +<glyph unicode="" d="M100 1100h1000q41 0 70.5 -29.5t29.5 -70.5v-600q0 -41 -29.5 -70.5t-70.5 -29.5h-596l-304 -300v300h-100q-41 0 -70.5 29.5t-29.5 70.5v600q0 41 29.5 70.5t70.5 29.5z" /> +<glyph unicode="" d="M150 1200h200q21 0 35.5 -14.5t14.5 -35.5v-250h-300v250q0 21 14.5 35.5t35.5 14.5zM850 1200h200q21 0 35.5 -14.5t14.5 -35.5v-250h-300v250q0 21 14.5 35.5t35.5 14.5zM1100 800v-300q0 -41 -3 -77.5t-15 -89.5t-32 -96t-58 -89t-89 -77t-129 -51t-174 -20t-174 20 t-129 51t-89 77t-58 89t-32 96t-15 89.5t-3 77.5v300h300v-250v-27v-42.5t1.5 -41t5 -38t10 -35t16.5 -30t25.5 -24.5t35 -19t46.5 -12t60 -4t60 4.5t46.5 12.5t35 19.5t25 25.5t17 30.5t10 35t5 38t2 40.5t-0.5 42v25v250h300z" /> +<glyph unicode="" d="M1100 411l-198 -199l-353 353l-353 -353l-197 199l551 551z" /> +<glyph unicode="" d="M1101 789l-550 -551l-551 551l198 199l353 -353l353 353z" /> +<glyph unicode="" d="M404 1000h746q21 0 35.5 -14.5t14.5 -35.5v-551h150q21 0 25 -10.5t-10 -24.5l-230 -249q-14 -15 -35 -15t-35 15l-230 249q-14 14 -10 24.5t25 10.5h150v401h-381zM135 984l230 -249q14 -14 10 -24.5t-25 -10.5h-150v-400h385l215 -200h-750q-21 0 -35.5 14.5 t-14.5 35.5v550h-150q-21 0 -25 10.5t10 24.5l230 249q14 15 35 15t35 -15z" /> +<glyph unicode="" d="M56 1200h94q17 0 31 -11t18 -27l38 -162h896q24 0 39 -18.5t10 -42.5l-100 -475q-5 -21 -27 -42.5t-55 -21.5h-633l48 -200h535q21 0 35.5 -14.5t14.5 -35.5t-14.5 -35.5t-35.5 -14.5h-50v-50q0 -21 -14.5 -35.5t-35.5 -14.5t-35.5 14.5t-14.5 35.5v50h-300v-50 q0 -21 -14.5 -35.5t-35.5 -14.5t-35.5 14.5t-14.5 35.5v50h-31q-18 0 -32.5 10t-20.5 19l-5 10l-201 961h-54q-20 0 -35 14.5t-15 35.5t15 35.5t35 14.5z" /> +<glyph unicode="" d="M1200 1000v-100h-1200v100h200q0 41 29.5 70.5t70.5 29.5h300q41 0 70.5 -29.5t29.5 -70.5h500zM0 800h1200v-800h-1200v800z" /> +<glyph unicode="" d="M200 800l-200 -400v600h200q0 41 29.5 70.5t70.5 29.5h300q42 0 71 -29.5t29 -70.5h500v-200h-1000zM1500 700l-300 -700h-1200l300 700h1200z" /> +<glyph unicode="" d="M635 1184l230 -249q14 -14 10 -24.5t-25 -10.5h-150v-601h150q21 0 25 -10.5t-10 -24.5l-230 -249q-14 -15 -35 -15t-35 15l-230 249q-14 14 -10 24.5t25 10.5h150v601h-150q-21 0 -25 10.5t10 24.5l230 249q14 15 35 15t35 -15z" /> +<glyph unicode="" d="M936 864l249 -229q14 -15 14 -35.5t-14 -35.5l-249 -229q-15 -15 -25.5 -10.5t-10.5 24.5v151h-600v-151q0 -20 -10.5 -24.5t-25.5 10.5l-249 229q-14 15 -14 35.5t14 35.5l249 229q15 15 25.5 10.5t10.5 -25.5v-149h600v149q0 21 10.5 25.5t25.5 -10.5z" /> +<glyph unicode="" d="M1169 400l-172 732q-5 23 -23 45.5t-38 22.5h-672q-20 0 -38 -20t-23 -41l-172 -739h1138zM1100 300h-1000q-41 0 -70.5 -29.5t-29.5 -70.5v-100q0 -41 29.5 -70.5t70.5 -29.5h1000q41 0 70.5 29.5t29.5 70.5v100q0 41 -29.5 70.5t-70.5 29.5zM800 100v100h100v-100h-100 zM1000 100v100h100v-100h-100z" /> +<glyph unicode="" d="M1150 1100q21 0 35.5 -14.5t14.5 -35.5v-850q0 -21 -14.5 -35.5t-35.5 -14.5t-35.5 14.5t-14.5 35.5v850q0 21 14.5 35.5t35.5 14.5zM1000 200l-675 200h-38l47 -276q3 -16 -5.5 -20t-29.5 -4h-7h-84q-20 0 -34.5 14t-18.5 35q-55 337 -55 351v250v6q0 16 1 23.5t6.5 14 t17.5 6.5h200l675 250v-850zM0 750v-250q-4 0 -11 0.5t-24 6t-30 15t-24 30t-11 48.5v50q0 26 10.5 46t25 30t29 16t25.5 7z" /> +<glyph unicode="" d="M553 1200h94q20 0 29 -10.5t3 -29.5l-18 -37q83 -19 144 -82.5t76 -140.5l63 -327l118 -173h17q19 0 33 -14.5t14 -35t-13 -40.5t-31 -27q-8 -4 -23 -9.5t-65 -19.5t-103 -25t-132.5 -20t-158.5 -9q-57 0 -115 5t-104 12t-88.5 15.5t-73.5 17.5t-54.5 16t-35.5 12l-11 4 q-18 8 -31 28t-13 40.5t14 35t33 14.5h17l118 173l63 327q15 77 76 140t144 83l-18 32q-6 19 3.5 32t28.5 13zM498 110q50 -6 102 -6q53 0 102 6q-12 -49 -39.5 -79.5t-62.5 -30.5t-63 30.5t-39 79.5z" /> +<glyph unicode="" d="M800 946l224 78l-78 -224l234 -45l-180 -155l180 -155l-234 -45l78 -224l-224 78l-45 -234l-155 180l-155 -180l-45 234l-224 -78l78 224l-234 45l180 155l-180 155l234 45l-78 224l224 -78l45 234l155 -180l155 180z" /> +<glyph unicode="" d="M650 1200h50q40 0 70 -40.5t30 -84.5v-150l-28 -125h328q40 0 70 -40.5t30 -84.5v-100q0 -45 -29 -74l-238 -344q-16 -24 -38 -40.5t-45 -16.5h-250q-7 0 -42 25t-66 50l-31 25h-61q-45 0 -72.5 18t-27.5 57v400q0 36 20 63l145 196l96 198q13 28 37.5 48t51.5 20z M650 1100l-100 -212l-150 -213v-375h100l136 -100h214l250 375v125h-450l50 225v175h-50zM50 800h100q21 0 35.5 -14.5t14.5 -35.5v-500q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v500q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M600 1100h250q23 0 45 -16.5t38 -40.5l238 -344q29 -29 29 -74v-100q0 -44 -30 -84.5t-70 -40.5h-328q28 -118 28 -125v-150q0 -44 -30 -84.5t-70 -40.5h-50q-27 0 -51.5 20t-37.5 48l-96 198l-145 196q-20 27 -20 63v400q0 39 27.5 57t72.5 18h61q124 100 139 100z M50 1000h100q21 0 35.5 -14.5t14.5 -35.5v-500q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v500q0 21 14.5 35.5t35.5 14.5zM636 1000l-136 -100h-100v-375l150 -213l100 -212h50v175l-50 225h450v125l-250 375h-214z" /> +<glyph unicode="" d="M356 873l363 230q31 16 53 -6l110 -112q13 -13 13.5 -32t-11.5 -34l-84 -121h302q84 0 138 -38t54 -110t-55 -111t-139 -39h-106l-131 -339q-6 -21 -19.5 -41t-28.5 -20h-342q-7 0 -90 81t-83 94v525q0 17 14 35.5t28 28.5zM400 792v-503l100 -89h293l131 339 q6 21 19.5 41t28.5 20h203q21 0 30.5 25t0.5 50t-31 25h-456h-7h-6h-5.5t-6 0.5t-5 1.5t-5 2t-4 2.5t-4 4t-2.5 4.5q-12 25 5 47l146 183l-86 83zM50 800h100q21 0 35.5 -14.5t14.5 -35.5v-500q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v500 q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M475 1103l366 -230q2 -1 6 -3.5t14 -10.5t18 -16.5t14.5 -20t6.5 -22.5v-525q0 -13 -86 -94t-93 -81h-342q-15 0 -28.5 20t-19.5 41l-131 339h-106q-85 0 -139.5 39t-54.5 111t54 110t138 38h302l-85 121q-11 15 -10.5 34t13.5 32l110 112q22 22 53 6zM370 945l146 -183 q17 -22 5 -47q-2 -2 -3.5 -4.5t-4 -4t-4 -2.5t-5 -2t-5 -1.5t-6 -0.5h-6h-6.5h-6h-475v-100h221q15 0 29 -20t20 -41l130 -339h294l106 89v503l-342 236zM1050 800h100q21 0 35.5 -14.5t14.5 -35.5v-500q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5 v500q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M550 1294q72 0 111 -55t39 -139v-106l339 -131q21 -6 41 -19.5t20 -28.5v-342q0 -7 -81 -90t-94 -83h-525q-17 0 -35.5 14t-28.5 28l-9 14l-230 363q-16 31 6 53l112 110q13 13 32 13.5t34 -11.5l121 -84v302q0 84 38 138t110 54zM600 972v203q0 21 -25 30.5t-50 0.5 t-25 -31v-456v-7v-6v-5.5t-0.5 -6t-1.5 -5t-2 -5t-2.5 -4t-4 -4t-4.5 -2.5q-25 -12 -47 5l-183 146l-83 -86l236 -339h503l89 100v293l-339 131q-21 6 -41 19.5t-20 28.5zM450 200h500q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-500 q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M350 1100h500q21 0 35.5 14.5t14.5 35.5v100q0 21 -14.5 35.5t-35.5 14.5h-500q-21 0 -35.5 -14.5t-14.5 -35.5v-100q0 -21 14.5 -35.5t35.5 -14.5zM600 306v-106q0 -84 -39 -139t-111 -55t-110 54t-38 138v302l-121 -84q-15 -12 -34 -11.5t-32 13.5l-112 110 q-22 22 -6 53l230 363q1 2 3.5 6t10.5 13.5t16.5 17t20 13.5t22.5 6h525q13 0 94 -83t81 -90v-342q0 -15 -20 -28.5t-41 -19.5zM308 900l-236 -339l83 -86l183 146q22 17 47 5q2 -1 4.5 -2.5t4 -4t2.5 -4t2 -5t1.5 -5t0.5 -6v-5.5v-6v-7v-456q0 -22 25 -31t50 0.5t25 30.5 v203q0 15 20 28.5t41 19.5l339 131v293l-89 100h-503z" /> +<glyph unicode="" d="M600 1178q118 0 225 -45.5t184.5 -123t123 -184.5t45.5 -225t-45.5 -225t-123 -184.5t-184.5 -123t-225 -45.5t-225 45.5t-184.5 123t-123 184.5t-45.5 225t45.5 225t123 184.5t184.5 123t225 45.5zM914 632l-275 223q-16 13 -27.5 8t-11.5 -26v-137h-275 q-10 0 -17.5 -7.5t-7.5 -17.5v-150q0 -10 7.5 -17.5t17.5 -7.5h275v-137q0 -21 11.5 -26t27.5 8l275 223q16 13 16 32t-16 32z" /> +<glyph unicode="" d="M600 1178q118 0 225 -45.5t184.5 -123t123 -184.5t45.5 -225t-45.5 -225t-123 -184.5t-184.5 -123t-225 -45.5t-225 45.5t-184.5 123t-123 184.5t-45.5 225t45.5 225t123 184.5t184.5 123t225 45.5zM561 855l-275 -223q-16 -13 -16 -32t16 -32l275 -223q16 -13 27.5 -8 t11.5 26v137h275q10 0 17.5 7.5t7.5 17.5v150q0 10 -7.5 17.5t-17.5 7.5h-275v137q0 21 -11.5 26t-27.5 -8z" /> +<glyph unicode="" d="M600 1178q118 0 225 -45.5t184.5 -123t123 -184.5t45.5 -225t-45.5 -225t-123 -184.5t-184.5 -123t-225 -45.5t-225 45.5t-184.5 123t-123 184.5t-45.5 225t45.5 225t123 184.5t184.5 123t225 45.5zM855 639l-223 275q-13 16 -32 16t-32 -16l-223 -275q-13 -16 -8 -27.5 t26 -11.5h137v-275q0 -10 7.5 -17.5t17.5 -7.5h150q10 0 17.5 7.5t7.5 17.5v275h137q21 0 26 11.5t-8 27.5z" /> +<glyph unicode="" d="M600 1178q118 0 225 -45.5t184.5 -123t123 -184.5t45.5 -225t-45.5 -225t-123 -184.5t-184.5 -123t-225 -45.5t-225 45.5t-184.5 123t-123 184.5t-45.5 225t45.5 225t123 184.5t184.5 123t225 45.5zM675 900h-150q-10 0 -17.5 -7.5t-7.5 -17.5v-275h-137q-21 0 -26 -11.5 t8 -27.5l223 -275q13 -16 32 -16t32 16l223 275q13 16 8 27.5t-26 11.5h-137v275q0 10 -7.5 17.5t-17.5 7.5z" /> +<glyph unicode="" d="M600 1176q116 0 222.5 -46t184 -123.5t123.5 -184t46 -222.5t-46 -222.5t-123.5 -184t-184 -123.5t-222.5 -46t-222.5 46t-184 123.5t-123.5 184t-46 222.5t46 222.5t123.5 184t184 123.5t222.5 46zM627 1101q-15 -12 -36.5 -20.5t-35.5 -12t-43 -8t-39 -6.5 q-15 -3 -45.5 0t-45.5 -2q-20 -7 -51.5 -26.5t-34.5 -34.5q-3 -11 6.5 -22.5t8.5 -18.5q-3 -34 -27.5 -91t-29.5 -79q-9 -34 5 -93t8 -87q0 -9 17 -44.5t16 -59.5q12 0 23 -5t23.5 -15t19.5 -14q16 -8 33 -15t40.5 -15t34.5 -12q21 -9 52.5 -32t60 -38t57.5 -11 q7 -15 -3 -34t-22.5 -40t-9.5 -38q13 -21 23 -34.5t27.5 -27.5t36.5 -18q0 -7 -3.5 -16t-3.5 -14t5 -17q104 -2 221 112q30 29 46.5 47t34.5 49t21 63q-13 8 -37 8.5t-36 7.5q-15 7 -49.5 15t-51.5 19q-18 0 -41 -0.5t-43 -1.5t-42 -6.5t-38 -16.5q-51 -35 -66 -12 q-4 1 -3.5 25.5t0.5 25.5q-6 13 -26.5 17.5t-24.5 6.5q1 15 -0.5 30.5t-7 28t-18.5 11.5t-31 -21q-23 -25 -42 4q-19 28 -8 58q6 16 22 22q6 -1 26 -1.5t33.5 -4t19.5 -13.5q7 -12 18 -24t21.5 -20.5t20 -15t15.5 -10.5l5 -3q2 12 7.5 30.5t8 34.5t-0.5 32q-3 18 3.5 29 t18 22.5t15.5 24.5q6 14 10.5 35t8 31t15.5 22.5t34 22.5q-6 18 10 36q8 0 24 -1.5t24.5 -1.5t20 4.5t20.5 15.5q-10 23 -31 42.5t-37.5 29.5t-49 27t-43.5 23q0 1 2 8t3 11.5t1.5 10.5t-1 9.5t-4.5 4.5q31 -13 58.5 -14.5t38.5 2.5l12 5q5 28 -9.5 46t-36.5 24t-50 15 t-41 20q-18 -4 -37 0zM613 994q0 -17 8 -42t17 -45t9 -23q-8 1 -39.5 5.5t-52.5 10t-37 16.5q3 11 16 29.5t16 25.5q10 -10 19 -10t14 6t13.5 14.5t16.5 12.5z" /> +<glyph unicode="" d="M756 1157q164 92 306 -9l-259 -138l145 -232l251 126q6 -89 -34 -156.5t-117 -110.5q-60 -34 -127 -39.5t-126 16.5l-596 -596q-15 -16 -36.5 -16t-36.5 16l-111 110q-15 15 -15 36.5t15 37.5l600 599q-34 101 5.5 201.5t135.5 154.5z" /> +<glyph unicode="" horiz-adv-x="1220" d="M100 1196h1000q41 0 70.5 -29.5t29.5 -70.5v-100q0 -41 -29.5 -70.5t-70.5 -29.5h-1000q-41 0 -70.5 29.5t-29.5 70.5v100q0 41 29.5 70.5t70.5 29.5zM1100 1096h-200v-100h200v100zM100 796h1000q41 0 70.5 -29.5t29.5 -70.5v-100q0 -41 -29.5 -70.5t-70.5 -29.5h-1000 q-41 0 -70.5 29.5t-29.5 70.5v100q0 41 29.5 70.5t70.5 29.5zM1100 696h-500v-100h500v100zM100 396h1000q41 0 70.5 -29.5t29.5 -70.5v-100q0 -41 -29.5 -70.5t-70.5 -29.5h-1000q-41 0 -70.5 29.5t-29.5 70.5v100q0 41 29.5 70.5t70.5 29.5zM1100 296h-300v-100h300v100z " /> +<glyph unicode="" d="M150 1200h900q21 0 35.5 -14.5t14.5 -35.5t-14.5 -35.5t-35.5 -14.5h-900q-21 0 -35.5 14.5t-14.5 35.5t14.5 35.5t35.5 14.5zM700 500v-300l-200 -200v500l-350 500h900z" /> +<glyph unicode="" d="M500 1200h200q41 0 70.5 -29.5t29.5 -70.5v-100h300q41 0 70.5 -29.5t29.5 -70.5v-400h-500v100h-200v-100h-500v400q0 41 29.5 70.5t70.5 29.5h300v100q0 41 29.5 70.5t70.5 29.5zM500 1100v-100h200v100h-200zM1200 400v-200q0 -41 -29.5 -70.5t-70.5 -29.5h-1000 q-41 0 -70.5 29.5t-29.5 70.5v200h1200z" /> +<glyph unicode="" d="M50 1200h300q21 0 25 -10.5t-10 -24.5l-94 -94l199 -199q7 -8 7 -18t-7 -18l-106 -106q-8 -7 -18 -7t-18 7l-199 199l-94 -94q-14 -14 -24.5 -10t-10.5 25v300q0 21 14.5 35.5t35.5 14.5zM850 1200h300q21 0 35.5 -14.5t14.5 -35.5v-300q0 -21 -10.5 -25t-24.5 10l-94 94 l-199 -199q-8 -7 -18 -7t-18 7l-106 106q-7 8 -7 18t7 18l199 199l-94 94q-14 14 -10 24.5t25 10.5zM364 470l106 -106q7 -8 7 -18t-7 -18l-199 -199l94 -94q14 -14 10 -24.5t-25 -10.5h-300q-21 0 -35.5 14.5t-14.5 35.5v300q0 21 10.5 25t24.5 -10l94 -94l199 199 q8 7 18 7t18 -7zM1071 271l94 94q14 14 24.5 10t10.5 -25v-300q0 -21 -14.5 -35.5t-35.5 -14.5h-300q-21 0 -25 10.5t10 24.5l94 94l-199 199q-7 8 -7 18t7 18l106 106q8 7 18 7t18 -7z" /> +<glyph unicode="" d="M596 1192q121 0 231.5 -47.5t190 -127t127 -190t47.5 -231.5t-47.5 -231.5t-127 -190.5t-190 -127t-231.5 -47t-231.5 47t-190.5 127t-127 190.5t-47 231.5t47 231.5t127 190t190.5 127t231.5 47.5zM596 1010q-112 0 -207.5 -55.5t-151 -151t-55.5 -207.5t55.5 -207.5 t151 -151t207.5 -55.5t207.5 55.5t151 151t55.5 207.5t-55.5 207.5t-151 151t-207.5 55.5zM454.5 905q22.5 0 38.5 -16t16 -38.5t-16 -39t-38.5 -16.5t-38.5 16.5t-16 39t16 38.5t38.5 16zM754.5 905q22.5 0 38.5 -16t16 -38.5t-16 -39t-38 -16.5q-14 0 -29 10l-55 -145 q17 -23 17 -51q0 -36 -25.5 -61.5t-61.5 -25.5t-61.5 25.5t-25.5 61.5q0 32 20.5 56.5t51.5 29.5l122 126l1 1q-9 14 -9 28q0 23 16 39t38.5 16zM345.5 709q22.5 0 38.5 -16t16 -38.5t-16 -38.5t-38.5 -16t-38.5 16t-16 38.5t16 38.5t38.5 16zM854.5 709q22.5 0 38.5 -16 t16 -38.5t-16 -38.5t-38.5 -16t-38.5 16t-16 38.5t16 38.5t38.5 16z" /> +<glyph unicode="" d="M546 173l469 470q91 91 99 192q7 98 -52 175.5t-154 94.5q-22 4 -47 4q-34 0 -66.5 -10t-56.5 -23t-55.5 -38t-48 -41.5t-48.5 -47.5q-376 -375 -391 -390q-30 -27 -45 -41.5t-37.5 -41t-32 -46.5t-16 -47.5t-1.5 -56.5q9 -62 53.5 -95t99.5 -33q74 0 125 51l548 548 q36 36 20 75q-7 16 -21.5 26t-32.5 10q-26 0 -50 -23q-13 -12 -39 -38l-341 -338q-15 -15 -35.5 -15.5t-34.5 13.5t-14 34.5t14 34.5q327 333 361 367q35 35 67.5 51.5t78.5 16.5q14 0 29 -1q44 -8 74.5 -35.5t43.5 -68.5q14 -47 2 -96.5t-47 -84.5q-12 -11 -32 -32 t-79.5 -81t-114.5 -115t-124.5 -123.5t-123 -119.5t-96.5 -89t-57 -45q-56 -27 -120 -27q-70 0 -129 32t-93 89q-48 78 -35 173t81 163l511 511q71 72 111 96q91 55 198 55q80 0 152 -33q78 -36 129.5 -103t66.5 -154q17 -93 -11 -183.5t-94 -156.5l-482 -476 q-15 -15 -36 -16t-37 14t-17.5 34t14.5 35z" /> +<glyph unicode="" d="M649 949q48 68 109.5 104t121.5 38.5t118.5 -20t102.5 -64t71 -100.5t27 -123q0 -57 -33.5 -117.5t-94 -124.5t-126.5 -127.5t-150 -152.5t-146 -174q-62 85 -145.5 174t-150 152.5t-126.5 127.5t-93.5 124.5t-33.5 117.5q0 64 28 123t73 100.5t104 64t119 20 t120.5 -38.5t104.5 -104zM896 972q-33 0 -64.5 -19t-56.5 -46t-47.5 -53.5t-43.5 -45.5t-37.5 -19t-36 19t-40 45.5t-43 53.5t-54 46t-65.5 19q-67 0 -122.5 -55.5t-55.5 -132.5q0 -23 13.5 -51t46 -65t57.5 -63t76 -75l22 -22q15 -14 44 -44t50.5 -51t46 -44t41 -35t23 -12 t23.5 12t42.5 36t46 44t52.5 52t44 43q4 4 12 13q43 41 63.5 62t52 55t46 55t26 46t11.5 44q0 79 -53 133.5t-120 54.5z" /> +<glyph unicode="" d="M776.5 1214q93.5 0 159.5 -66l141 -141q66 -66 66 -160q0 -42 -28 -95.5t-62 -87.5l-29 -29q-31 53 -77 99l-18 18l95 95l-247 248l-389 -389l212 -212l-105 -106l-19 18l-141 141q-66 66 -66 159t66 159l283 283q65 66 158.5 66zM600 706l105 105q10 -8 19 -17l141 -141 q66 -66 66 -159t-66 -159l-283 -283q-66 -66 -159 -66t-159 66l-141 141q-66 66 -66 159.5t66 159.5l55 55q29 -55 75 -102l18 -17l-95 -95l247 -248l389 389z" /> +<glyph unicode="" d="M603 1200q85 0 162 -15t127 -38t79 -48t29 -46v-953q0 -41 -29.5 -70.5t-70.5 -29.5h-600q-41 0 -70.5 29.5t-29.5 70.5v953q0 21 30 46.5t81 48t129 37.5t163 15zM300 1000v-700h600v700h-600zM600 254q-43 0 -73.5 -30.5t-30.5 -73.5t30.5 -73.5t73.5 -30.5t73.5 30.5 t30.5 73.5t-30.5 73.5t-73.5 30.5z" /> +<glyph unicode="" d="M902 1185l283 -282q15 -15 15 -36t-14.5 -35.5t-35.5 -14.5t-35 15l-36 35l-279 -267v-300l-212 210l-308 -307l-280 -203l203 280l307 308l-210 212h300l267 279l-35 36q-15 14 -15 35t14.5 35.5t35.5 14.5t35 -15z" /> +<glyph unicode="" d="M700 1248v-78q38 -5 72.5 -14.5t75.5 -31.5t71 -53.5t52 -84t24 -118.5h-159q-4 36 -10.5 59t-21 45t-40 35.5t-64.5 20.5v-307l64 -13q34 -7 64 -16.5t70 -32t67.5 -52.5t47.5 -80t20 -112q0 -139 -89 -224t-244 -97v-77h-100v79q-150 16 -237 103q-40 40 -52.5 93.5 t-15.5 139.5h139q5 -77 48.5 -126t117.5 -65v335l-27 8q-46 14 -79 26.5t-72 36t-63 52t-40 72.5t-16 98q0 70 25 126t67.5 92t94.5 57t110 27v77h100zM600 754v274q-29 -4 -50 -11t-42 -21.5t-31.5 -41.5t-10.5 -65q0 -29 7 -50.5t16.5 -34t28.5 -22.5t31.5 -14t37.5 -10 q9 -3 13 -4zM700 547v-310q22 2 42.5 6.5t45 15.5t41.5 27t29 42t12 59.5t-12.5 59.5t-38 44.5t-53 31t-66.5 24.5z" /> +<glyph unicode="" d="M561 1197q84 0 160.5 -40t123.5 -109.5t47 -147.5h-153q0 40 -19.5 71.5t-49.5 48.5t-59.5 26t-55.5 9q-37 0 -79 -14.5t-62 -35.5q-41 -44 -41 -101q0 -26 13.5 -63t26.5 -61t37 -66q6 -9 9 -14h241v-100h-197q8 -50 -2.5 -115t-31.5 -95q-45 -62 -99 -112 q34 10 83 17.5t71 7.5q32 1 102 -16t104 -17q83 0 136 30l50 -147q-31 -19 -58 -30.5t-55 -15.5t-42 -4.5t-46 -0.5q-23 0 -76 17t-111 32.5t-96 11.5q-39 -3 -82 -16t-67 -25l-23 -11l-55 145q4 3 16 11t15.5 10.5t13 9t15.5 12t14.5 14t17.5 18.5q48 55 54 126.5 t-30 142.5h-221v100h166q-23 47 -44 104q-7 20 -12 41.5t-6 55.5t6 66.5t29.5 70.5t58.5 71q97 88 263 88z" /> +<glyph unicode="" d="M400 300h150q21 0 25 -11t-10 -25l-230 -250q-14 -15 -35 -15t-35 15l-230 250q-14 14 -10 25t25 11h150v900h200v-900zM935 1184l230 -249q14 -14 10 -24.5t-25 -10.5h-150v-900h-200v900h-150q-21 0 -25 10.5t10 24.5l230 249q14 15 35 15t35 -15z" /> +<glyph unicode="" d="M1000 700h-100v100h-100v-100h-100v500h300v-500zM400 300h150q21 0 25 -11t-10 -25l-230 -250q-14 -15 -35 -15t-35 15l-230 250q-14 14 -10 25t25 11h150v900h200v-900zM801 1100v-200h100v200h-100zM1000 350l-200 -250h200v-100h-300v150l200 250h-200v100h300v-150z " /> +<glyph unicode="" d="M400 300h150q21 0 25 -11t-10 -25l-230 -250q-14 -15 -35 -15t-35 15l-230 250q-14 14 -10 25t25 11h150v900h200v-900zM1000 1050l-200 -250h200v-100h-300v150l200 250h-200v100h300v-150zM1000 0h-100v100h-100v-100h-100v500h300v-500zM801 400v-200h100v200h-100z " /> +<glyph unicode="" d="M400 300h150q21 0 25 -11t-10 -25l-230 -250q-14 -15 -35 -15t-35 15l-230 250q-14 14 -10 25t25 11h150v900h200v-900zM1000 700h-100v400h-100v100h200v-500zM1100 0h-100v100h-200v400h300v-500zM901 400v-200h100v200h-100z" /> +<glyph unicode="" d="M400 300h150q21 0 25 -11t-10 -25l-230 -250q-14 -15 -35 -15t-35 15l-230 250q-14 14 -10 25t25 11h150v900h200v-900zM1100 700h-100v100h-200v400h300v-500zM901 1100v-200h100v200h-100zM1000 0h-100v400h-100v100h200v-500z" /> +<glyph unicode="" d="M400 300h150q21 0 25 -11t-10 -25l-230 -250q-14 -15 -35 -15t-35 15l-230 250q-14 14 -10 25t25 11h150v900h200v-900zM900 1000h-200v200h200v-200zM1000 700h-300v200h300v-200zM1100 400h-400v200h400v-200zM1200 100h-500v200h500v-200z" /> +<glyph unicode="" d="M400 300h150q21 0 25 -11t-10 -25l-230 -250q-14 -15 -35 -15t-35 15l-230 250q-14 14 -10 25t25 11h150v900h200v-900zM1200 1000h-500v200h500v-200zM1100 700h-400v200h400v-200zM1000 400h-300v200h300v-200zM900 100h-200v200h200v-200z" /> +<glyph unicode="" d="M350 1100h400q162 0 256 -93.5t94 -256.5v-400q0 -165 -93.5 -257.5t-256.5 -92.5h-400q-165 0 -257.5 92.5t-92.5 257.5v400q0 165 92.5 257.5t257.5 92.5zM800 900h-500q-41 0 -70.5 -29.5t-29.5 -70.5v-500q0 -41 29.5 -70.5t70.5 -29.5h500q41 0 70.5 29.5t29.5 70.5 v500q0 41 -29.5 70.5t-70.5 29.5z" /> +<glyph unicode="" d="M350 1100h400q165 0 257.5 -92.5t92.5 -257.5v-400q0 -165 -92.5 -257.5t-257.5 -92.5h-400q-163 0 -256.5 92.5t-93.5 257.5v400q0 163 94 256.5t256 93.5zM800 900h-500q-41 0 -70.5 -29.5t-29.5 -70.5v-500q0 -41 29.5 -70.5t70.5 -29.5h500q41 0 70.5 29.5t29.5 70.5 v500q0 41 -29.5 70.5t-70.5 29.5zM440 770l253 -190q17 -12 17 -30t-17 -30l-253 -190q-16 -12 -28 -6.5t-12 26.5v400q0 21 12 26.5t28 -6.5z" /> +<glyph unicode="" d="M350 1100h400q163 0 256.5 -94t93.5 -256v-400q0 -165 -92.5 -257.5t-257.5 -92.5h-400q-165 0 -257.5 92.5t-92.5 257.5v400q0 163 92.5 256.5t257.5 93.5zM800 900h-500q-41 0 -70.5 -29.5t-29.5 -70.5v-500q0 -41 29.5 -70.5t70.5 -29.5h500q41 0 70.5 29.5t29.5 70.5 v500q0 41 -29.5 70.5t-70.5 29.5zM350 700h400q21 0 26.5 -12t-6.5 -28l-190 -253q-12 -17 -30 -17t-30 17l-190 253q-12 16 -6.5 28t26.5 12z" /> +<glyph unicode="" d="M350 1100h400q165 0 257.5 -92.5t92.5 -257.5v-400q0 -163 -92.5 -256.5t-257.5 -93.5h-400q-163 0 -256.5 94t-93.5 256v400q0 165 92.5 257.5t257.5 92.5zM800 900h-500q-41 0 -70.5 -29.5t-29.5 -70.5v-500q0 -41 29.5 -70.5t70.5 -29.5h500q41 0 70.5 29.5t29.5 70.5 v500q0 41 -29.5 70.5t-70.5 29.5zM580 693l190 -253q12 -16 6.5 -28t-26.5 -12h-400q-21 0 -26.5 12t6.5 28l190 253q12 17 30 17t30 -17z" /> +<glyph unicode="" d="M550 1100h400q165 0 257.5 -92.5t92.5 -257.5v-400q0 -165 -92.5 -257.5t-257.5 -92.5h-400q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5h450q41 0 70.5 29.5t29.5 70.5v500q0 41 -29.5 70.5t-70.5 29.5h-450q-21 0 -35.5 14.5t-14.5 35.5v100 q0 21 14.5 35.5t35.5 14.5zM338 867l324 -284q16 -14 16 -33t-16 -33l-324 -284q-16 -14 -27 -9t-11 26v150h-250q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5h250v150q0 21 11 26t27 -9z" /> +<glyph unicode="" d="M793 1182l9 -9q8 -10 5 -27q-3 -11 -79 -225.5t-78 -221.5l300 1q24 0 32.5 -17.5t-5.5 -35.5q-1 0 -133.5 -155t-267 -312.5t-138.5 -162.5q-12 -15 -26 -15h-9l-9 8q-9 11 -4 32q2 9 42 123.5t79 224.5l39 110h-302q-23 0 -31 19q-10 21 6 41q75 86 209.5 237.5 t228 257t98.5 111.5q9 16 25 16h9z" /> +<glyph unicode="" d="M350 1100h400q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-450q-41 0 -70.5 -29.5t-29.5 -70.5v-500q0 -41 29.5 -70.5t70.5 -29.5h450q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-400q-165 0 -257.5 92.5t-92.5 257.5v400 q0 165 92.5 257.5t257.5 92.5zM938 867l324 -284q16 -14 16 -33t-16 -33l-324 -284q-16 -14 -27 -9t-11 26v150h-250q-21 0 -35.5 14.5t-14.5 35.5v200q0 21 14.5 35.5t35.5 14.5h250v150q0 21 11 26t27 -9z" /> +<glyph unicode="" d="M750 1200h400q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -10.5 -25t-24.5 10l-109 109l-312 -312q-15 -15 -35.5 -15t-35.5 15l-141 141q-15 15 -15 35.5t15 35.5l312 312l-109 109q-14 14 -10 24.5t25 10.5zM456 900h-156q-41 0 -70.5 -29.5t-29.5 -70.5v-500 q0 -41 29.5 -70.5t70.5 -29.5h500q41 0 70.5 29.5t29.5 70.5v148l200 200v-298q0 -165 -93.5 -257.5t-256.5 -92.5h-400q-165 0 -257.5 92.5t-92.5 257.5v400q0 165 92.5 257.5t257.5 92.5h300z" /> +<glyph unicode="" d="M600 1186q119 0 227.5 -46.5t187 -125t125 -187t46.5 -227.5t-46.5 -227.5t-125 -187t-187 -125t-227.5 -46.5t-227.5 46.5t-187 125t-125 187t-46.5 227.5t46.5 227.5t125 187t187 125t227.5 46.5zM600 1022q-115 0 -212 -56.5t-153.5 -153.5t-56.5 -212t56.5 -212 t153.5 -153.5t212 -56.5t212 56.5t153.5 153.5t56.5 212t-56.5 212t-153.5 153.5t-212 56.5zM600 794q80 0 137 -57t57 -137t-57 -137t-137 -57t-137 57t-57 137t57 137t137 57z" /> +<glyph unicode="" d="M450 1200h200q21 0 35.5 -14.5t14.5 -35.5v-350h245q20 0 25 -11t-9 -26l-383 -426q-14 -15 -33.5 -15t-32.5 15l-379 426q-13 15 -8.5 26t25.5 11h250v350q0 21 14.5 35.5t35.5 14.5zM50 300h1000q21 0 35.5 -14.5t14.5 -35.5v-250h-1100v250q0 21 14.5 35.5t35.5 14.5z M900 200v-50h100v50h-100z" /> +<glyph unicode="" d="M583 1182l378 -435q14 -15 9 -31t-26 -16h-244v-250q0 -20 -17 -35t-39 -15h-200q-20 0 -32 14.5t-12 35.5v250h-250q-20 0 -25.5 16.5t8.5 31.5l383 431q14 16 33.5 17t33.5 -14zM50 300h1000q21 0 35.5 -14.5t14.5 -35.5v-250h-1100v250q0 21 14.5 35.5t35.5 14.5z M900 200v-50h100v50h-100z" /> +<glyph unicode="" d="M396 723l369 369q7 7 17.5 7t17.5 -7l139 -139q7 -8 7 -18.5t-7 -17.5l-525 -525q-7 -8 -17.5 -8t-17.5 8l-292 291q-7 8 -7 18t7 18l139 139q8 7 18.5 7t17.5 -7zM50 300h1000q21 0 35.5 -14.5t14.5 -35.5v-250h-1100v250q0 21 14.5 35.5t35.5 14.5zM900 200v-50h100v50 h-100z" /> +<glyph unicode="" d="M135 1023l142 142q14 14 35 14t35 -14l77 -77l-212 -212l-77 76q-14 15 -14 36t14 35zM655 855l210 210q14 14 24.5 10t10.5 -25l-2 -599q-1 -20 -15.5 -35t-35.5 -15l-597 -1q-21 0 -25 10.5t10 24.5l208 208l-154 155l212 212zM50 300h1000q21 0 35.5 -14.5t14.5 -35.5 v-250h-1100v250q0 21 14.5 35.5t35.5 14.5zM900 200v-50h100v50h-100z" /> +<glyph unicode="" d="M350 1200l599 -2q20 -1 35 -15.5t15 -35.5l1 -597q0 -21 -10.5 -25t-24.5 10l-208 208l-155 -154l-212 212l155 154l-210 210q-14 14 -10 24.5t25 10.5zM524 512l-76 -77q-15 -14 -36 -14t-35 14l-142 142q-14 14 -14 35t14 35l77 77zM50 300h1000q21 0 35.5 -14.5 t14.5 -35.5v-250h-1100v250q0 21 14.5 35.5t35.5 14.5zM900 200v-50h100v50h-100z" /> +<glyph unicode="" d="M1200 103l-483 276l-314 -399v423h-399l1196 796v-1096zM483 424v-230l683 953z" /> +<glyph unicode="" d="M1100 1000v-850q0 -21 -14.5 -35.5t-35.5 -14.5h-150v400h-700v-400h-150q-21 0 -35.5 14.5t-14.5 35.5v1000q0 20 14.5 35t35.5 15h250v-300h500v300h100zM700 1000h-100v200h100v-200z" /> +<glyph unicode="" d="M1100 1000l-2 -149l-299 -299l-95 95q-9 9 -21.5 9t-21.5 -9l-149 -147h-312v-400h-150q-21 0 -35.5 14.5t-14.5 35.5v1000q0 20 14.5 35t35.5 15h250v-300h500v300h100zM700 1000h-100v200h100v-200zM1132 638l106 -106q7 -7 7 -17.5t-7 -17.5l-420 -421q-8 -7 -18 -7 t-18 7l-202 203q-8 7 -8 17.5t8 17.5l106 106q7 8 17.5 8t17.5 -8l79 -79l297 297q7 7 17.5 7t17.5 -7z" /> +<glyph unicode="" d="M1100 1000v-269l-103 -103l-134 134q-15 15 -33.5 16.5t-34.5 -12.5l-266 -266h-329v-400h-150q-21 0 -35.5 14.5t-14.5 35.5v1000q0 20 14.5 35t35.5 15h250v-300h500v300h100zM700 1000h-100v200h100v-200zM1202 572l70 -70q15 -15 15 -35.5t-15 -35.5l-131 -131 l131 -131q15 -15 15 -35.5t-15 -35.5l-70 -70q-15 -15 -35.5 -15t-35.5 15l-131 131l-131 -131q-15 -15 -35.5 -15t-35.5 15l-70 70q-15 15 -15 35.5t15 35.5l131 131l-131 131q-15 15 -15 35.5t15 35.5l70 70q15 15 35.5 15t35.5 -15l131 -131l131 131q15 15 35.5 15 t35.5 -15z" /> +<glyph unicode="" d="M1100 1000v-300h-350q-21 0 -35.5 -14.5t-14.5 -35.5v-150h-500v-400h-150q-21 0 -35.5 14.5t-14.5 35.5v1000q0 20 14.5 35t35.5 15h250v-300h500v300h100zM700 1000h-100v200h100v-200zM850 600h100q21 0 35.5 -14.5t14.5 -35.5v-250h150q21 0 25 -10.5t-10 -24.5 l-230 -230q-14 -14 -35 -14t-35 14l-230 230q-14 14 -10 24.5t25 10.5h150v250q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M1100 1000v-400l-165 165q-14 15 -35 15t-35 -15l-263 -265h-402v-400h-150q-21 0 -35.5 14.5t-14.5 35.5v1000q0 20 14.5 35t35.5 15h250v-300h500v300h100zM700 1000h-100v200h100v-200zM935 565l230 -229q14 -15 10 -25.5t-25 -10.5h-150v-250q0 -20 -14.5 -35 t-35.5 -15h-100q-21 0 -35.5 15t-14.5 35v250h-150q-21 0 -25 10.5t10 25.5l230 229q14 15 35 15t35 -15z" /> +<glyph unicode="" d="M50 1100h1100q21 0 35.5 -14.5t14.5 -35.5v-150h-1200v150q0 21 14.5 35.5t35.5 14.5zM1200 800v-550q0 -21 -14.5 -35.5t-35.5 -14.5h-1100q-21 0 -35.5 14.5t-14.5 35.5v550h1200zM100 500v-200h400v200h-400z" /> +<glyph unicode="" d="M935 1165l248 -230q14 -14 14 -35t-14 -35l-248 -230q-14 -14 -24.5 -10t-10.5 25v150h-400v200h400v150q0 21 10.5 25t24.5 -10zM200 800h-50q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5h50v-200zM400 800h-100v200h100v-200zM18 435l247 230 q14 14 24.5 10t10.5 -25v-150h400v-200h-400v-150q0 -21 -10.5 -25t-24.5 10l-247 230q-15 14 -15 35t15 35zM900 300h-100v200h100v-200zM1000 500h51q20 0 34.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-34.5 -14.5h-51v200z" /> +<glyph unicode="" d="M862 1073l276 116q25 18 43.5 8t18.5 -41v-1106q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v397q-4 1 -11 5t-24 17.5t-30 29t-24 42t-11 56.5v359q0 31 18.5 65t43.5 52zM550 1200q22 0 34.5 -12.5t14.5 -24.5l1 -13v-450q0 -28 -10.5 -59.5 t-25 -56t-29 -45t-25.5 -31.5l-10 -11v-447q0 -21 -14.5 -35.5t-35.5 -14.5h-200q-21 0 -35.5 14.5t-14.5 35.5v447q-4 4 -11 11.5t-24 30.5t-30 46t-24 55t-11 60v450q0 2 0.5 5.5t4 12t8.5 15t14.5 12t22.5 5.5q20 0 32.5 -12.5t14.5 -24.5l3 -13v-350h100v350v5.5t2.5 12 t7 15t15 12t25.5 5.5q23 0 35.5 -12.5t13.5 -24.5l1 -13v-350h100v350q0 2 0.5 5.5t3 12t7 15t15 12t24.5 5.5z" /> +<glyph unicode="" d="M1200 1100v-56q-4 0 -11 -0.5t-24 -3t-30 -7.5t-24 -15t-11 -24v-888q0 -22 25 -34.5t50 -13.5l25 -2v-56h-400v56q75 0 87.5 6.5t12.5 43.5v394h-500v-394q0 -37 12.5 -43.5t87.5 -6.5v-56h-400v56q4 0 11 0.5t24 3t30 7.5t24 15t11 24v888q0 22 -25 34.5t-50 13.5 l-25 2v56h400v-56q-75 0 -87.5 -6.5t-12.5 -43.5v-394h500v394q0 37 -12.5 43.5t-87.5 6.5v56h400z" /> +<glyph unicode="" d="M675 1000h375q21 0 35.5 -14.5t14.5 -35.5v-150h-105l-295 -98v98l-200 200h-400l100 100h375zM100 900h300q41 0 70.5 -29.5t29.5 -70.5v-500q0 -41 -29.5 -70.5t-70.5 -29.5h-300q-41 0 -70.5 29.5t-29.5 70.5v500q0 41 29.5 70.5t70.5 29.5zM100 800v-200h300v200 h-300zM1100 535l-400 -133v163l400 133v-163zM100 500v-200h300v200h-300zM1100 398v-248q0 -21 -14.5 -35.5t-35.5 -14.5h-375l-100 -100h-375l-100 100h400l200 200h105z" /> +<glyph unicode="" d="M17 1007l162 162q17 17 40 14t37 -22l139 -194q14 -20 11 -44.5t-20 -41.5l-119 -118q102 -142 228 -268t267 -227l119 118q17 17 42.5 19t44.5 -12l192 -136q19 -14 22.5 -37.5t-13.5 -40.5l-163 -162q-3 -1 -9.5 -1t-29.5 2t-47.5 6t-62.5 14.5t-77.5 26.5t-90 42.5 t-101.5 60t-111 83t-119 108.5q-74 74 -133.5 150.5t-94.5 138.5t-60 119.5t-34.5 100t-15 74.5t-4.5 48z" /> +<glyph unicode="" d="M600 1100q92 0 175 -10.5t141.5 -27t108.5 -36.5t81.5 -40t53.5 -37t31 -27l9 -10v-200q0 -21 -14.5 -33t-34.5 -9l-202 34q-20 3 -34.5 20t-14.5 38v146q-141 24 -300 24t-300 -24v-146q0 -21 -14.5 -38t-34.5 -20l-202 -34q-20 -3 -34.5 9t-14.5 33v200q3 4 9.5 10.5 t31 26t54 37.5t80.5 39.5t109 37.5t141 26.5t175 10.5zM600 795q56 0 97 -9.5t60 -23.5t30 -28t12 -24l1 -10v-50l365 -303q14 -15 24.5 -40t10.5 -45v-212q0 -21 -14.5 -35.5t-35.5 -14.5h-1100q-21 0 -35.5 14.5t-14.5 35.5v212q0 20 10.5 45t24.5 40l365 303v50 q0 4 1 10.5t12 23t30 29t60 22.5t97 10z" /> +<glyph unicode="" d="M1100 700l-200 -200h-600l-200 200v500h200v-200h200v200h200v-200h200v200h200v-500zM250 400h700q21 0 35.5 -14.5t14.5 -35.5t-14.5 -35.5t-35.5 -14.5h-12l137 -100h-950l137 100h-12q-21 0 -35.5 14.5t-14.5 35.5t14.5 35.5t35.5 14.5zM50 100h1100q21 0 35.5 -14.5 t14.5 -35.5v-50h-1200v50q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M700 1100h-100q-41 0 -70.5 -29.5t-29.5 -70.5v-1000h300v1000q0 41 -29.5 70.5t-70.5 29.5zM1100 800h-100q-41 0 -70.5 -29.5t-29.5 -70.5v-700h300v700q0 41 -29.5 70.5t-70.5 29.5zM400 0h-300v400q0 41 29.5 70.5t70.5 29.5h100q41 0 70.5 -29.5t29.5 -70.5v-400z " /> +<glyph unicode="" d="M200 1100h700q124 0 212 -88t88 -212v-500q0 -124 -88 -212t-212 -88h-700q-124 0 -212 88t-88 212v500q0 124 88 212t212 88zM100 900v-700h900v700h-900zM500 700h-200v-100h200v-300h-300v100h200v100h-200v300h300v-100zM900 700v-300l-100 -100h-200v500h200z M700 700v-300h100v300h-100z" /> +<glyph unicode="" d="M200 1100h700q124 0 212 -88t88 -212v-500q0 -124 -88 -212t-212 -88h-700q-124 0 -212 88t-88 212v500q0 124 88 212t212 88zM100 900v-700h900v700h-900zM500 300h-100v200h-100v-200h-100v500h100v-200h100v200h100v-500zM900 700v-300l-100 -100h-200v500h200z M700 700v-300h100v300h-100z" /> +<glyph unicode="" d="M200 1100h700q124 0 212 -88t88 -212v-500q0 -124 -88 -212t-212 -88h-700q-124 0 -212 88t-88 212v500q0 124 88 212t212 88zM100 900v-700h900v700h-900zM500 700h-200v-300h200v-100h-300v500h300v-100zM900 700h-200v-300h200v-100h-300v500h300v-100z" /> +<glyph unicode="" d="M200 1100h700q124 0 212 -88t88 -212v-500q0 -124 -88 -212t-212 -88h-700q-124 0 -212 88t-88 212v500q0 124 88 212t212 88zM100 900v-700h900v700h-900zM500 400l-300 150l300 150v-300zM900 550l-300 -150v300z" /> +<glyph unicode="" d="M200 1100h700q124 0 212 -88t88 -212v-500q0 -124 -88 -212t-212 -88h-700q-124 0 -212 88t-88 212v500q0 124 88 212t212 88zM100 900v-700h900v700h-900zM900 300h-700v500h700v-500zM800 700h-130q-38 0 -66.5 -43t-28.5 -108t27 -107t68 -42h130v300zM300 700v-300 h130q41 0 68 42t27 107t-28.5 108t-66.5 43h-130z" /> +<glyph unicode="" d="M200 1100h700q124 0 212 -88t88 -212v-500q0 -124 -88 -212t-212 -88h-700q-124 0 -212 88t-88 212v500q0 124 88 212t212 88zM100 900v-700h900v700h-900zM500 700h-200v-100h200v-300h-300v100h200v100h-200v300h300v-100zM900 300h-100v400h-100v100h200v-500z M700 300h-100v100h100v-100z" /> +<glyph unicode="" d="M200 1100h700q124 0 212 -88t88 -212v-500q0 -124 -88 -212t-212 -88h-700q-124 0 -212 88t-88 212v500q0 124 88 212t212 88zM100 900v-700h900v700h-900zM300 700h200v-400h-300v500h100v-100zM900 300h-100v400h-100v100h200v-500zM300 600v-200h100v200h-100z M700 300h-100v100h100v-100z" /> +<glyph unicode="" d="M200 1100h700q124 0 212 -88t88 -212v-500q0 -124 -88 -212t-212 -88h-700q-124 0 -212 88t-88 212v500q0 124 88 212t212 88zM100 900v-700h900v700h-900zM500 500l-199 -200h-100v50l199 200v150h-200v100h300v-300zM900 300h-100v400h-100v100h200v-500zM701 300h-100 v100h100v-100z" /> +<glyph unicode="" d="M600 1191q120 0 229.5 -47t188.5 -126t126 -188.5t47 -229.5t-47 -229.5t-126 -188.5t-188.5 -126t-229.5 -47t-229.5 47t-188.5 126t-126 188.5t-47 229.5t47 229.5t126 188.5t188.5 126t229.5 47zM600 1021q-114 0 -211 -56.5t-153.5 -153.5t-56.5 -211t56.5 -211 t153.5 -153.5t211 -56.5t211 56.5t153.5 153.5t56.5 211t-56.5 211t-153.5 153.5t-211 56.5zM800 700h-300v-200h300v-100h-300l-100 100v200l100 100h300v-100z" /> +<glyph unicode="" d="M600 1191q120 0 229.5 -47t188.5 -126t126 -188.5t47 -229.5t-47 -229.5t-126 -188.5t-188.5 -126t-229.5 -47t-229.5 47t-188.5 126t-126 188.5t-47 229.5t47 229.5t126 188.5t188.5 126t229.5 47zM600 1021q-114 0 -211 -56.5t-153.5 -153.5t-56.5 -211t56.5 -211 t153.5 -153.5t211 -56.5t211 56.5t153.5 153.5t56.5 211t-56.5 211t-153.5 153.5t-211 56.5zM800 700v-100l-50 -50l100 -100v-50h-100l-100 100h-150v-100h-100v400h300zM500 700v-100h200v100h-200z" /> +<glyph unicode="" d="M503 1089q110 0 200.5 -59.5t134.5 -156.5q44 14 90 14q120 0 205 -86.5t85 -207t-85 -207t-205 -86.5h-128v250q0 21 -14.5 35.5t-35.5 14.5h-300q-21 0 -35.5 -14.5t-14.5 -35.5v-250h-222q-80 0 -136 57.5t-56 136.5q0 69 43 122.5t108 67.5q-2 19 -2 37q0 100 49 185 t134 134t185 49zM525 500h150q10 0 17.5 -7.5t7.5 -17.5v-275h137q21 0 26 -11.5t-8 -27.5l-223 -244q-13 -16 -32 -16t-32 16l-223 244q-13 16 -8 27.5t26 11.5h137v275q0 10 7.5 17.5t17.5 7.5z" /> +<glyph unicode="" d="M502 1089q110 0 201 -59.5t135 -156.5q43 15 89 15q121 0 206 -86.5t86 -206.5q0 -99 -60 -181t-150 -110l-378 360q-13 16 -31.5 16t-31.5 -16l-381 -365h-9q-79 0 -135.5 57.5t-56.5 136.5q0 69 43 122.5t108 67.5q-2 19 -2 38q0 100 49 184.5t133.5 134t184.5 49.5z M632 467l223 -228q13 -16 8 -27.5t-26 -11.5h-137v-275q0 -10 -7.5 -17.5t-17.5 -7.5h-150q-10 0 -17.5 7.5t-7.5 17.5v275h-137q-21 0 -26 11.5t8 27.5q199 204 223 228q19 19 31.5 19t32.5 -19z" /> +<glyph unicode="" d="M700 100v100h400l-270 300h170l-270 300h170l-300 333l-300 -333h170l-270 -300h170l-270 -300h400v-100h-50q-21 0 -35.5 -14.5t-14.5 -35.5v-50h400v50q0 21 -14.5 35.5t-35.5 14.5h-50z" /> +<glyph unicode="" d="M600 1179q94 0 167.5 -56.5t99.5 -145.5q89 -6 150.5 -71.5t61.5 -155.5q0 -61 -29.5 -112.5t-79.5 -82.5q9 -29 9 -55q0 -74 -52.5 -126.5t-126.5 -52.5q-55 0 -100 30v-251q21 0 35.5 -14.5t14.5 -35.5v-50h-300v50q0 21 14.5 35.5t35.5 14.5v251q-45 -30 -100 -30 q-74 0 -126.5 52.5t-52.5 126.5q0 18 4 38q-47 21 -75.5 65t-28.5 97q0 74 52.5 126.5t126.5 52.5q5 0 23 -2q0 2 -1 10t-1 13q0 116 81.5 197.5t197.5 81.5z" /> +<glyph unicode="" d="M1010 1010q111 -111 150.5 -260.5t0 -299t-150.5 -260.5q-83 -83 -191.5 -126.5t-218.5 -43.5t-218.5 43.5t-191.5 126.5q-111 111 -150.5 260.5t0 299t150.5 260.5q83 83 191.5 126.5t218.5 43.5t218.5 -43.5t191.5 -126.5zM476 1065q-4 0 -8 -1q-121 -34 -209.5 -122.5 t-122.5 -209.5q-4 -12 2.5 -23t18.5 -14l36 -9q3 -1 7 -1q23 0 29 22q27 96 98 166q70 71 166 98q11 3 17.5 13.5t3.5 22.5l-9 35q-3 13 -14 19q-7 4 -15 4zM512 920q-4 0 -9 -2q-80 -24 -138.5 -82.5t-82.5 -138.5q-4 -13 2 -24t19 -14l34 -9q4 -1 8 -1q22 0 28 21 q18 58 58.5 98.5t97.5 58.5q12 3 18 13.5t3 21.5l-9 35q-3 12 -14 19q-7 4 -15 4zM719.5 719.5q-49.5 49.5 -119.5 49.5t-119.5 -49.5t-49.5 -119.5t49.5 -119.5t119.5 -49.5t119.5 49.5t49.5 119.5t-49.5 119.5zM855 551q-22 0 -28 -21q-18 -58 -58.5 -98.5t-98.5 -57.5 q-11 -4 -17 -14.5t-3 -21.5l9 -35q3 -12 14 -19q7 -4 15 -4q4 0 9 2q80 24 138.5 82.5t82.5 138.5q4 13 -2.5 24t-18.5 14l-34 9q-4 1 -8 1zM1000 515q-23 0 -29 -22q-27 -96 -98 -166q-70 -71 -166 -98q-11 -3 -17.5 -13.5t-3.5 -22.5l9 -35q3 -13 14 -19q7 -4 15 -4 q4 0 8 1q121 34 209.5 122.5t122.5 209.5q4 12 -2.5 23t-18.5 14l-36 9q-3 1 -7 1z" /> +<glyph unicode="" d="M700 800h300v-380h-180v200h-340v-200h-380v755q0 10 7.5 17.5t17.5 7.5h575v-400zM1000 900h-200v200zM700 300h162l-212 -212l-212 212h162v200h100v-200zM520 0h-395q-10 0 -17.5 7.5t-7.5 17.5v395zM1000 220v-195q0 -10 -7.5 -17.5t-17.5 -7.5h-195z" /> +<glyph unicode="" d="M700 800h300v-520l-350 350l-550 -550v1095q0 10 7.5 17.5t17.5 7.5h575v-400zM1000 900h-200v200zM862 200h-162v-200h-100v200h-162l212 212zM480 0h-355q-10 0 -17.5 7.5t-7.5 17.5v55h380v-80zM1000 80v-55q0 -10 -7.5 -17.5t-17.5 -7.5h-155v80h180z" /> +<glyph unicode="" d="M1162 800h-162v-200h100l100 -100h-300v300h-162l212 212zM200 800h200q27 0 40 -2t29.5 -10.5t23.5 -30t7 -57.5h300v-100h-600l-200 -350v450h100q0 36 7 57.5t23.5 30t29.5 10.5t40 2zM800 400h240l-240 -400h-800l300 500h500v-100z" /> +<glyph unicode="" d="M650 1100h100q21 0 35.5 -14.5t14.5 -35.5v-50h50q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-300q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5h50v50q0 21 14.5 35.5t35.5 14.5zM1000 850v150q41 0 70.5 -29.5t29.5 -70.5v-800 q0 -41 -29.5 -70.5t-70.5 -29.5h-600q-1 0 -20 4l246 246l-326 326v324q0 41 29.5 70.5t70.5 29.5v-150q0 -62 44 -106t106 -44h300q62 0 106 44t44 106zM412 250l-212 -212v162h-200v100h200v162z" /> +<glyph unicode="" d="M450 1100h100q21 0 35.5 -14.5t14.5 -35.5v-50h50q21 0 35.5 -14.5t14.5 -35.5v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-300q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5h50v50q0 21 14.5 35.5t35.5 14.5zM800 850v150q41 0 70.5 -29.5t29.5 -70.5v-500 h-200v-300h200q0 -36 -7 -57.5t-23.5 -30t-29.5 -10.5t-40 -2h-600q-41 0 -70.5 29.5t-29.5 70.5v800q0 41 29.5 70.5t70.5 29.5v-150q0 -62 44 -106t106 -44h300q62 0 106 44t44 106zM1212 250l-212 -212v162h-200v100h200v162z" /> +<glyph unicode="" d="M658 1197l637 -1104q23 -38 7 -65.5t-60 -27.5h-1276q-44 0 -60 27.5t7 65.5l637 1104q22 39 54 39t54 -39zM704 800h-208q-20 0 -32 -14.5t-8 -34.5l58 -302q4 -20 21.5 -34.5t37.5 -14.5h54q20 0 37.5 14.5t21.5 34.5l58 302q4 20 -8 34.5t-32 14.5zM500 300v-100h200 v100h-200z" /> +<glyph unicode="" d="M425 1100h250q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5zM425 800h250q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5 t17.5 7.5zM825 800h250q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5zM25 500h250q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5t-7.5 17.5v150 q0 10 7.5 17.5t17.5 7.5zM425 500h250q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5zM825 500h250q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5t-7.5 17.5 v150q0 10 7.5 17.5t17.5 7.5zM25 200h250q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5zM425 200h250q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5 t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5zM825 200h250q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-250q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5z" /> +<glyph unicode="" d="M700 1200h100v-200h-100v-100h350q62 0 86.5 -39.5t-3.5 -94.5l-66 -132q-41 -83 -81 -134h-772q-40 51 -81 134l-66 132q-28 55 -3.5 94.5t86.5 39.5h350v100h-100v200h100v100h200v-100zM250 400h700q21 0 35.5 -14.5t14.5 -35.5t-14.5 -35.5t-35.5 -14.5h-12l137 -100 h-950l138 100h-13q-21 0 -35.5 14.5t-14.5 35.5t14.5 35.5t35.5 14.5zM50 100h1100q21 0 35.5 -14.5t14.5 -35.5v-50h-1200v50q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M600 1300q40 0 68.5 -29.5t28.5 -70.5h-194q0 41 28.5 70.5t68.5 29.5zM443 1100h314q18 -37 18 -75q0 -8 -3 -25h328q41 0 44.5 -16.5t-30.5 -38.5l-175 -145h-678l-178 145q-34 22 -29 38.5t46 16.5h328q-3 17 -3 25q0 38 18 75zM250 700h700q21 0 35.5 -14.5 t14.5 -35.5t-14.5 -35.5t-35.5 -14.5h-150v-200l275 -200h-950l275 200v200h-150q-21 0 -35.5 14.5t-14.5 35.5t14.5 35.5t35.5 14.5zM50 100h1100q21 0 35.5 -14.5t14.5 -35.5v-50h-1200v50q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M600 1181q75 0 128 -53t53 -128t-53 -128t-128 -53t-128 53t-53 128t53 128t128 53zM602 798h46q34 0 55.5 -28.5t21.5 -86.5q0 -76 39 -183h-324q39 107 39 183q0 58 21.5 86.5t56.5 28.5h45zM250 400h700q21 0 35.5 -14.5t14.5 -35.5t-14.5 -35.5t-35.5 -14.5h-13 l138 -100h-950l137 100h-12q-21 0 -35.5 14.5t-14.5 35.5t14.5 35.5t35.5 14.5zM50 100h1100q21 0 35.5 -14.5t14.5 -35.5v-50h-1200v50q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M600 1300q47 0 92.5 -53.5t71 -123t25.5 -123.5q0 -78 -55.5 -133.5t-133.5 -55.5t-133.5 55.5t-55.5 133.5q0 62 34 143l144 -143l111 111l-163 163q34 26 63 26zM602 798h46q34 0 55.5 -28.5t21.5 -86.5q0 -76 39 -183h-324q39 107 39 183q0 58 21.5 86.5t56.5 28.5h45 zM250 400h700q21 0 35.5 -14.5t14.5 -35.5t-14.5 -35.5t-35.5 -14.5h-13l138 -100h-950l137 100h-12q-21 0 -35.5 14.5t-14.5 35.5t14.5 35.5t35.5 14.5zM50 100h1100q21 0 35.5 -14.5t14.5 -35.5v-50h-1200v50q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M600 1200l300 -161v-139h-300q0 -57 18.5 -108t50 -91.5t63 -72t70 -67.5t57.5 -61h-530q-60 83 -90.5 177.5t-30.5 178.5t33 164.5t87.5 139.5t126 96.5t145.5 41.5v-98zM250 400h700q21 0 35.5 -14.5t14.5 -35.5t-14.5 -35.5t-35.5 -14.5h-13l138 -100h-950l137 100 h-12q-21 0 -35.5 14.5t-14.5 35.5t14.5 35.5t35.5 14.5zM50 100h1100q21 0 35.5 -14.5t14.5 -35.5v-50h-1200v50q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M600 1300q41 0 70.5 -29.5t29.5 -70.5v-78q46 -26 73 -72t27 -100v-50h-400v50q0 54 27 100t73 72v78q0 41 29.5 70.5t70.5 29.5zM400 800h400q54 0 100 -27t72 -73h-172v-100h200v-100h-200v-100h200v-100h-200v-100h200q0 -83 -58.5 -141.5t-141.5 -58.5h-400 q-83 0 -141.5 58.5t-58.5 141.5v400q0 83 58.5 141.5t141.5 58.5z" /> +<glyph unicode="" d="M150 1100h900q21 0 35.5 -14.5t14.5 -35.5v-500q0 -21 -14.5 -35.5t-35.5 -14.5h-900q-21 0 -35.5 14.5t-14.5 35.5v500q0 21 14.5 35.5t35.5 14.5zM125 400h950q10 0 17.5 -7.5t7.5 -17.5v-50q0 -10 -7.5 -17.5t-17.5 -7.5h-283l224 -224q13 -13 13 -31.5t-13 -32 t-31.5 -13.5t-31.5 13l-88 88h-524l-87 -88q-13 -13 -32 -13t-32 13.5t-13 32t13 31.5l224 224h-289q-10 0 -17.5 7.5t-7.5 17.5v50q0 10 7.5 17.5t17.5 7.5zM541 300l-100 -100h324l-100 100h-124z" /> +<glyph unicode="" d="M200 1100h800q83 0 141.5 -58.5t58.5 -141.5v-200h-100q0 41 -29.5 70.5t-70.5 29.5h-250q-41 0 -70.5 -29.5t-29.5 -70.5h-100q0 41 -29.5 70.5t-70.5 29.5h-250q-41 0 -70.5 -29.5t-29.5 -70.5h-100v200q0 83 58.5 141.5t141.5 58.5zM100 600h1000q41 0 70.5 -29.5 t29.5 -70.5v-300h-1200v300q0 41 29.5 70.5t70.5 29.5zM300 100v-50q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v50h200zM1100 100v-50q0 -21 -14.5 -35.5t-35.5 -14.5h-100q-21 0 -35.5 14.5t-14.5 35.5v50h200z" /> +<glyph unicode="" d="M480 1165l682 -683q31 -31 31 -75.5t-31 -75.5l-131 -131h-481l-517 518q-32 31 -32 75.5t32 75.5l295 296q31 31 75.5 31t76.5 -31zM108 794l342 -342l303 304l-341 341zM250 100h800q21 0 35.5 -14.5t14.5 -35.5v-50h-900v50q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M1057 647l-189 506q-8 19 -27.5 33t-40.5 14h-400q-21 0 -40.5 -14t-27.5 -33l-189 -506q-8 -19 1.5 -33t30.5 -14h625v-150q0 -21 14.5 -35.5t35.5 -14.5t35.5 14.5t14.5 35.5v150h125q21 0 30.5 14t1.5 33zM897 0h-595v50q0 21 14.5 35.5t35.5 14.5h50v50 q0 21 14.5 35.5t35.5 14.5h48v300h200v-300h47q21 0 35.5 -14.5t14.5 -35.5v-50h50q21 0 35.5 -14.5t14.5 -35.5v-50z" /> +<glyph unicode="" d="M900 800h300v-575q0 -10 -7.5 -17.5t-17.5 -7.5h-375v591l-300 300v84q0 10 7.5 17.5t17.5 7.5h375v-400zM1200 900h-200v200zM400 600h300v-575q0 -10 -7.5 -17.5t-17.5 -7.5h-650q-10 0 -17.5 7.5t-7.5 17.5v950q0 10 7.5 17.5t17.5 7.5h375v-400zM700 700h-200v200z " /> +<glyph unicode="" d="M484 1095h195q75 0 146 -32.5t124 -86t89.5 -122.5t48.5 -142q18 -14 35 -20q31 -10 64.5 6.5t43.5 48.5q10 34 -15 71q-19 27 -9 43q5 8 12.5 11t19 -1t23.5 -16q41 -44 39 -105q-3 -63 -46 -106.5t-104 -43.5h-62q-7 -55 -35 -117t-56 -100l-39 -234q-3 -20 -20 -34.5 t-38 -14.5h-100q-21 0 -33 14.5t-9 34.5l12 70q-49 -14 -91 -14h-195q-24 0 -65 8l-11 -64q-3 -20 -20 -34.5t-38 -14.5h-100q-21 0 -33 14.5t-9 34.5l26 157q-84 74 -128 175l-159 53q-19 7 -33 26t-14 40v50q0 21 14.5 35.5t35.5 14.5h124q11 87 56 166l-111 95 q-16 14 -12.5 23.5t24.5 9.5h203q116 101 250 101zM675 1000h-250q-10 0 -17.5 -7.5t-7.5 -17.5v-50q0 -10 7.5 -17.5t17.5 -7.5h250q10 0 17.5 7.5t7.5 17.5v50q0 10 -7.5 17.5t-17.5 7.5z" /> +<glyph unicode="" d="M641 900l423 247q19 8 42 2.5t37 -21.5l32 -38q14 -15 12.5 -36t-17.5 -34l-139 -120h-390zM50 1100h106q67 0 103 -17t66 -71l102 -212h823q21 0 35.5 -14.5t14.5 -35.5v-50q0 -21 -14 -40t-33 -26l-737 -132q-23 -4 -40 6t-26 25q-42 67 -100 67h-300q-62 0 -106 44 t-44 106v200q0 62 44 106t106 44zM173 928h-80q-19 0 -28 -14t-9 -35v-56q0 -51 42 -51h134q16 0 21.5 8t5.5 24q0 11 -16 45t-27 51q-18 28 -43 28zM550 727q-32 0 -54.5 -22.5t-22.5 -54.5t22.5 -54.5t54.5 -22.5t54.5 22.5t22.5 54.5t-22.5 54.5t-54.5 22.5zM130 389 l152 130q18 19 34 24t31 -3.5t24.5 -17.5t25.5 -28q28 -35 50.5 -51t48.5 -13l63 5l48 -179q13 -61 -3.5 -97.5t-67.5 -79.5l-80 -69q-47 -40 -109 -35.5t-103 51.5l-130 151q-40 47 -35.5 109.5t51.5 102.5zM380 377l-102 -88q-31 -27 2 -65l37 -43q13 -15 27.5 -19.5 t31.5 6.5l61 53q19 16 14 49q-2 20 -12 56t-17 45q-11 12 -19 14t-23 -8z" /> +<glyph unicode="" d="M625 1200h150q10 0 17.5 -7.5t7.5 -17.5v-109q79 -33 131 -87.5t53 -128.5q1 -46 -15 -84.5t-39 -61t-46 -38t-39 -21.5l-17 -6q6 0 15 -1.5t35 -9t50 -17.5t53 -30t50 -45t35.5 -64t14.5 -84q0 -59 -11.5 -105.5t-28.5 -76.5t-44 -51t-49.5 -31.5t-54.5 -16t-49.5 -6.5 t-43.5 -1v-75q0 -10 -7.5 -17.5t-17.5 -7.5h-150q-10 0 -17.5 7.5t-7.5 17.5v75h-100v-75q0 -10 -7.5 -17.5t-17.5 -7.5h-150q-10 0 -17.5 7.5t-7.5 17.5v75h-175q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5h75v600h-75q-10 0 -17.5 7.5t-7.5 17.5v150 q0 10 7.5 17.5t17.5 7.5h175v75q0 10 7.5 17.5t17.5 7.5h150q10 0 17.5 -7.5t7.5 -17.5v-75h100v75q0 10 7.5 17.5t17.5 7.5zM400 900v-200h263q28 0 48.5 10.5t30 25t15 29t5.5 25.5l1 10q0 4 -0.5 11t-6 24t-15 30t-30 24t-48.5 11h-263zM400 500v-200h363q28 0 48.5 10.5 t30 25t15 29t5.5 25.5l1 10q0 4 -0.5 11t-6 24t-15 30t-30 24t-48.5 11h-363z" /> +<glyph unicode="" d="M212 1198h780q86 0 147 -61t61 -147v-416q0 -51 -18 -142.5t-36 -157.5l-18 -66q-29 -87 -93.5 -146.5t-146.5 -59.5h-572q-82 0 -147 59t-93 147q-8 28 -20 73t-32 143.5t-20 149.5v416q0 86 61 147t147 61zM600 1045q-70 0 -132.5 -11.5t-105.5 -30.5t-78.5 -41.5 t-57 -45t-36 -41t-20.5 -30.5l-6 -12l156 -243h560l156 243q-2 5 -6 12.5t-20 29.5t-36.5 42t-57 44.5t-79 42t-105 29.5t-132.5 12zM762 703h-157l195 261z" /> +<glyph unicode="" d="M475 1300h150q103 0 189 -86t86 -189v-500q0 -41 -42 -83t-83 -42h-450q-41 0 -83 42t-42 83v500q0 103 86 189t189 86zM700 300v-225q0 -21 -27 -48t-48 -27h-150q-21 0 -48 27t-27 48v225h300z" /> +<glyph unicode="" d="M475 1300h96q0 -150 89.5 -239.5t239.5 -89.5v-446q0 -41 -42 -83t-83 -42h-450q-41 0 -83 42t-42 83v500q0 103 86 189t189 86zM700 300v-225q0 -21 -27 -48t-48 -27h-150q-21 0 -48 27t-27 48v225h300z" /> +<glyph unicode="" d="M1294 767l-638 -283l-378 170l-78 -60v-224l100 -150v-199l-150 148l-150 -149v200l100 150v250q0 4 -0.5 10.5t0 9.5t1 8t3 8t6.5 6l47 40l-147 65l642 283zM1000 380l-350 -166l-350 166v147l350 -165l350 165v-147z" /> +<glyph unicode="" d="M250 800q62 0 106 -44t44 -106t-44 -106t-106 -44t-106 44t-44 106t44 106t106 44zM650 800q62 0 106 -44t44 -106t-44 -106t-106 -44t-106 44t-44 106t44 106t106 44zM1050 800q62 0 106 -44t44 -106t-44 -106t-106 -44t-106 44t-44 106t44 106t106 44z" /> +<glyph unicode="" d="M550 1100q62 0 106 -44t44 -106t-44 -106t-106 -44t-106 44t-44 106t44 106t106 44zM550 700q62 0 106 -44t44 -106t-44 -106t-106 -44t-106 44t-44 106t44 106t106 44zM550 300q62 0 106 -44t44 -106t-44 -106t-106 -44t-106 44t-44 106t44 106t106 44z" /> +<glyph unicode="" d="M125 1100h950q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-950q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5zM125 700h950q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-950q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5 t17.5 7.5zM125 300h950q10 0 17.5 -7.5t7.5 -17.5v-150q0 -10 -7.5 -17.5t-17.5 -7.5h-950q-10 0 -17.5 7.5t-7.5 17.5v150q0 10 7.5 17.5t17.5 7.5z" /> +<glyph unicode="" d="M350 1200h500q162 0 256 -93.5t94 -256.5v-500q0 -165 -93.5 -257.5t-256.5 -92.5h-500q-165 0 -257.5 92.5t-92.5 257.5v500q0 165 92.5 257.5t257.5 92.5zM900 1000h-600q-41 0 -70.5 -29.5t-29.5 -70.5v-600q0 -41 29.5 -70.5t70.5 -29.5h600q41 0 70.5 29.5 t29.5 70.5v600q0 41 -29.5 70.5t-70.5 29.5zM350 900h500q21 0 35.5 -14.5t14.5 -35.5v-300q0 -21 -14.5 -35.5t-35.5 -14.5h-500q-21 0 -35.5 14.5t-14.5 35.5v300q0 21 14.5 35.5t35.5 14.5zM400 800v-200h400v200h-400z" /> +<glyph unicode="" d="M150 1100h1000q21 0 35.5 -14.5t14.5 -35.5t-14.5 -35.5t-35.5 -14.5h-50v-200h50q21 0 35.5 -14.5t14.5 -35.5t-14.5 -35.5t-35.5 -14.5h-50v-200h50q21 0 35.5 -14.5t14.5 -35.5t-14.5 -35.5t-35.5 -14.5h-50v-200h50q21 0 35.5 -14.5t14.5 -35.5t-14.5 -35.5 t-35.5 -14.5h-1000q-21 0 -35.5 14.5t-14.5 35.5t14.5 35.5t35.5 14.5h50v200h-50q-21 0 -35.5 14.5t-14.5 35.5t14.5 35.5t35.5 14.5h50v200h-50q-21 0 -35.5 14.5t-14.5 35.5t14.5 35.5t35.5 14.5h50v200h-50q-21 0 -35.5 14.5t-14.5 35.5t14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M650 1187q87 -67 118.5 -156t0 -178t-118.5 -155q-87 66 -118.5 155t0 178t118.5 156zM300 800q124 0 212 -88t88 -212q-124 0 -212 88t-88 212zM1000 800q0 -124 -88 -212t-212 -88q0 124 88 212t212 88zM300 500q124 0 212 -88t88 -212q-124 0 -212 88t-88 212z M1000 500q0 -124 -88 -212t-212 -88q0 124 88 212t212 88zM700 199v-144q0 -21 -14.5 -35.5t-35.5 -14.5t-35.5 14.5t-14.5 35.5v142q40 -4 43 -4q17 0 57 6z" /> +<glyph unicode="" d="M745 878l69 19q25 6 45 -12l298 -295q11 -11 15 -26.5t-2 -30.5q-5 -14 -18 -23.5t-28 -9.5h-8q1 0 1 -13q0 -29 -2 -56t-8.5 -62t-20 -63t-33 -53t-51 -39t-72.5 -14h-146q-184 0 -184 288q0 24 10 47q-20 4 -62 4t-63 -4q11 -24 11 -47q0 -288 -184 -288h-142 q-48 0 -84.5 21t-56 51t-32 71.5t-16 75t-3.5 68.5q0 13 2 13h-7q-15 0 -27.5 9.5t-18.5 23.5q-6 15 -2 30.5t15 25.5l298 296q20 18 46 11l76 -19q20 -5 30.5 -22.5t5.5 -37.5t-22.5 -31t-37.5 -5l-51 12l-182 -193h891l-182 193l-44 -12q-20 -5 -37.5 6t-22.5 31t6 37.5 t31 22.5z" /> +<glyph unicode="" d="M1200 900h-50q0 21 -4 37t-9.5 26.5t-18 17.5t-22 11t-28.5 5.5t-31 2t-37 0.5h-200v-850q0 -22 25 -34.5t50 -13.5l25 -2v-100h-400v100q4 0 11 0.5t24 3t30 7t24 15t11 24.5v850h-200q-25 0 -37 -0.5t-31 -2t-28.5 -5.5t-22 -11t-18 -17.5t-9.5 -26.5t-4 -37h-50v300 h1000v-300zM500 450h-25q0 15 -4 24.5t-9 14.5t-17 7.5t-20 3t-25 0.5h-100v-425q0 -11 12.5 -17.5t25.5 -7.5h12v-50h-200v50q50 0 50 25v425h-100q-17 0 -25 -0.5t-20 -3t-17 -7.5t-9 -14.5t-4 -24.5h-25v150h500v-150z" /> +<glyph unicode="" d="M1000 300v50q-25 0 -55 32q-14 14 -25 31t-16 27l-4 11l-289 747h-69l-300 -754q-18 -35 -39 -56q-9 -9 -24.5 -18.5t-26.5 -14.5l-11 -5v-50h273v50q-49 0 -78.5 21.5t-11.5 67.5l69 176h293l61 -166q13 -34 -3.5 -66.5t-55.5 -32.5v-50h312zM412 691l134 342l121 -342 h-255zM1100 150v-100q0 -21 -14.5 -35.5t-35.5 -14.5h-1000q-21 0 -35.5 14.5t-14.5 35.5v100q0 21 14.5 35.5t35.5 14.5h1000q21 0 35.5 -14.5t14.5 -35.5z" /> +<glyph unicode="" d="M50 1200h1100q21 0 35.5 -14.5t14.5 -35.5v-1100q0 -21 -14.5 -35.5t-35.5 -14.5h-1100q-21 0 -35.5 14.5t-14.5 35.5v1100q0 21 14.5 35.5t35.5 14.5zM611 1118h-70q-13 0 -18 -12l-299 -753q-17 -32 -35 -51q-18 -18 -56 -34q-12 -5 -12 -18v-50q0 -8 5.5 -14t14.5 -6 h273q8 0 14 6t6 14v50q0 8 -6 14t-14 6q-55 0 -71 23q-10 14 0 39l63 163h266l57 -153q11 -31 -6 -55q-12 -17 -36 -17q-8 0 -14 -6t-6 -14v-50q0 -8 6 -14t14 -6h313q8 0 14 6t6 14v50q0 7 -5.5 13t-13.5 7q-17 0 -42 25q-25 27 -40 63h-1l-288 748q-5 12 -19 12zM639 611 h-197l103 264z" /> +<glyph unicode="" d="M1200 1100h-1200v100h1200v-100zM50 1000h400q21 0 35.5 -14.5t14.5 -35.5v-900q0 -21 -14.5 -35.5t-35.5 -14.5h-400q-21 0 -35.5 14.5t-14.5 35.5v900q0 21 14.5 35.5t35.5 14.5zM650 1000h400q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-400 q-21 0 -35.5 14.5t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5zM700 900v-300h300v300h-300z" /> +<glyph unicode="" d="M50 1200h400q21 0 35.5 -14.5t14.5 -35.5v-900q0 -21 -14.5 -35.5t-35.5 -14.5h-400q-21 0 -35.5 14.5t-14.5 35.5v900q0 21 14.5 35.5t35.5 14.5zM650 700h400q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-400q-21 0 -35.5 14.5t-14.5 35.5v400 q0 21 14.5 35.5t35.5 14.5zM700 600v-300h300v300h-300zM1200 0h-1200v100h1200v-100z" /> +<glyph unicode="" d="M50 1000h400q21 0 35.5 -14.5t14.5 -35.5v-350h100v150q0 21 14.5 35.5t35.5 14.5h400q21 0 35.5 -14.5t14.5 -35.5v-150h100v-100h-100v-150q0 -21 -14.5 -35.5t-35.5 -14.5h-400q-21 0 -35.5 14.5t-14.5 35.5v150h-100v-350q0 -21 -14.5 -35.5t-35.5 -14.5h-400 q-21 0 -35.5 14.5t-14.5 35.5v800q0 21 14.5 35.5t35.5 14.5zM700 700v-300h300v300h-300z" /> +<glyph unicode="" d="M100 0h-100v1200h100v-1200zM250 1100h400q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-400q-21 0 -35.5 14.5t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5zM300 1000v-300h300v300h-300zM250 500h900q21 0 35.5 -14.5t14.5 -35.5v-400 q0 -21 -14.5 -35.5t-35.5 -14.5h-900q-21 0 -35.5 14.5t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M600 1100h150q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-150v-100h450q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-900q-21 0 -35.5 14.5t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5h350v100h-150q-21 0 -35.5 14.5 t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5h150v100h100v-100zM400 1000v-300h300v300h-300z" /> +<glyph unicode="" d="M1200 0h-100v1200h100v-1200zM550 1100h400q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-400q-21 0 -35.5 14.5t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5zM600 1000v-300h300v300h-300zM50 500h900q21 0 35.5 -14.5t14.5 -35.5v-400 q0 -21 -14.5 -35.5t-35.5 -14.5h-900q-21 0 -35.5 14.5t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5z" /> +<glyph unicode="" d="M865 565l-494 -494q-23 -23 -41 -23q-14 0 -22 13.5t-8 38.5v1000q0 25 8 38.5t22 13.5q18 0 41 -23l494 -494q14 -14 14 -35t-14 -35z" /> +<glyph unicode="" d="M335 635l494 494q29 29 50 20.5t21 -49.5v-1000q0 -41 -21 -49.5t-50 20.5l-494 494q-14 14 -14 35t14 35z" /> +<glyph unicode="" d="M100 900h1000q41 0 49.5 -21t-20.5 -50l-494 -494q-14 -14 -35 -14t-35 14l-494 494q-29 29 -20.5 50t49.5 21z" /> +<glyph unicode="" d="M635 865l494 -494q29 -29 20.5 -50t-49.5 -21h-1000q-41 0 -49.5 21t20.5 50l494 494q14 14 35 14t35 -14z" /> +<glyph unicode="" d="M700 741v-182l-692 -323v221l413 193l-413 193v221zM1200 0h-800v200h800v-200z" /> +<glyph unicode="" d="M1200 900h-200v-100h200v-100h-300v300h200v100h-200v100h300v-300zM0 700h50q0 21 4 37t9.5 26.5t18 17.5t22 11t28.5 5.5t31 2t37 0.5h100v-550q0 -22 -25 -34.5t-50 -13.5l-25 -2v-100h400v100q-4 0 -11 0.5t-24 3t-30 7t-24 15t-11 24.5v550h100q25 0 37 -0.5t31 -2 t28.5 -5.5t22 -11t18 -17.5t9.5 -26.5t4 -37h50v300h-800v-300z" /> +<glyph unicode="" d="M800 700h-50q0 21 -4 37t-9.5 26.5t-18 17.5t-22 11t-28.5 5.5t-31 2t-37 0.5h-100v-550q0 -22 25 -34.5t50 -14.5l25 -1v-100h-400v100q4 0 11 0.5t24 3t30 7t24 15t11 24.5v550h-100q-25 0 -37 -0.5t-31 -2t-28.5 -5.5t-22 -11t-18 -17.5t-9.5 -26.5t-4 -37h-50v300 h800v-300zM1100 200h-200v-100h200v-100h-300v300h200v100h-200v100h300v-300z" /> +<glyph unicode="" d="M701 1098h160q16 0 21 -11t-7 -23l-464 -464l464 -464q12 -12 7 -23t-21 -11h-160q-13 0 -23 9l-471 471q-7 8 -7 18t7 18l471 471q10 9 23 9z" /> +<glyph unicode="" d="M339 1098h160q13 0 23 -9l471 -471q7 -8 7 -18t-7 -18l-471 -471q-10 -9 -23 -9h-160q-16 0 -21 11t7 23l464 464l-464 464q-12 12 -7 23t21 11z" /> +<glyph unicode="" d="M1087 882q11 -5 11 -21v-160q0 -13 -9 -23l-471 -471q-8 -7 -18 -7t-18 7l-471 471q-9 10 -9 23v160q0 16 11 21t23 -7l464 -464l464 464q12 12 23 7z" /> +<glyph unicode="" d="M618 993l471 -471q9 -10 9 -23v-160q0 -16 -11 -21t-23 7l-464 464l-464 -464q-12 -12 -23 -7t-11 21v160q0 13 9 23l471 471q8 7 18 7t18 -7z" /> +<glyph unicode="" d="M1000 1200q0 -124 -88 -212t-212 -88q0 124 88 212t212 88zM450 1000h100q21 0 40 -14t26 -33l79 -194q5 1 16 3q34 6 54 9.5t60 7t65.5 1t61 -10t56.5 -23t42.5 -42t29 -64t5 -92t-19.5 -121.5q-1 -7 -3 -19.5t-11 -50t-20.5 -73t-32.5 -81.5t-46.5 -83t-64 -70 t-82.5 -50q-13 -5 -42 -5t-65.5 2.5t-47.5 2.5q-14 0 -49.5 -3.5t-63 -3.5t-43.5 7q-57 25 -104.5 78.5t-75 111.5t-46.5 112t-26 90l-7 35q-15 63 -18 115t4.5 88.5t26 64t39.5 43.5t52 25.5t58.5 13t62.5 2t59.5 -4.5t55.5 -8l-147 192q-12 18 -5.5 30t27.5 12z" /> +<glyph unicode="🔑" d="M250 1200h600q21 0 35.5 -14.5t14.5 -35.5v-400q0 -21 -14.5 -35.5t-35.5 -14.5h-150v-500l-255 -178q-19 -9 -32 -1t-13 29v650h-150q-21 0 -35.5 14.5t-14.5 35.5v400q0 21 14.5 35.5t35.5 14.5zM400 1100v-100h300v100h-300z" /> +<glyph unicode="🚪" d="M250 1200h750q39 0 69.5 -40.5t30.5 -84.5v-933l-700 -117v950l600 125h-700v-1000h-100v1025q0 23 15.5 49t34.5 26zM500 525v-100l100 20v100z" /> +</font> +</defs></svg>
\ No newline at end of file diff --git a/python/vespa/docs/fonts/glyphicons-halflings-regular.ttf b/python/vespa/docs/fonts/glyphicons-halflings-regular.ttf Binary files differnew file mode 100644 index 00000000000..1413fc609ab --- /dev/null +++ b/python/vespa/docs/fonts/glyphicons-halflings-regular.ttf diff --git a/python/vespa/docs/fonts/glyphicons-halflings-regular.woff b/python/vespa/docs/fonts/glyphicons-halflings-regular.woff Binary files differnew file mode 100644 index 00000000000..9e612858f80 --- /dev/null +++ b/python/vespa/docs/fonts/glyphicons-halflings-regular.woff diff --git a/python/vespa/docs/fonts/glyphicons-halflings-regular.woff2 b/python/vespa/docs/fonts/glyphicons-halflings-regular.woff2 Binary files differnew file mode 100644 index 00000000000..64539b54c37 --- /dev/null +++ b/python/vespa/docs/fonts/glyphicons-halflings-regular.woff2 diff --git a/python/vespa/docs/images/company_logo.png b/python/vespa/docs/images/company_logo.png Binary files differnew file mode 100644 index 00000000000..4cc1ab88d7d --- /dev/null +++ b/python/vespa/docs/images/company_logo.png diff --git a/python/vespa/docs/images/company_logo_big.png b/python/vespa/docs/images/company_logo_big.png Binary files differnew file mode 100644 index 00000000000..4cc1ab88d7d --- /dev/null +++ b/python/vespa/docs/images/company_logo_big.png diff --git a/python/vespa/docs/images/doc_example.png b/python/vespa/docs/images/doc_example.png Binary files differnew file mode 100644 index 00000000000..d4aa7321569 --- /dev/null +++ b/python/vespa/docs/images/doc_example.png diff --git a/python/vespa/docs/images/export_example.png b/python/vespa/docs/images/export_example.png Binary files differnew file mode 100644 index 00000000000..1c2829c4c1f --- /dev/null +++ b/python/vespa/docs/images/export_example.png diff --git a/python/vespa/docs/images/favicon.ico b/python/vespa/docs/images/favicon.ico Binary files differnew file mode 100644 index 00000000000..c35e9559b4e --- /dev/null +++ b/python/vespa/docs/images/favicon.ico diff --git a/python/vespa/docs/images/workflowarrow.png b/python/vespa/docs/images/workflowarrow.png Binary files differnew file mode 100644 index 00000000000..91a3e816183 --- /dev/null +++ b/python/vespa/docs/images/workflowarrow.png diff --git a/python/vespa/docs/index.html b/python/vespa/docs/index.html new file mode 100644 index 00000000000..7c55143b923 --- /dev/null +++ b/python/vespa/docs/index.html @@ -0,0 +1,301 @@ +--- + +title: Vespa library for data analysis + +keywords: fastai +sidebar: home_sidebar + +summary: "Provide data analysis support for Vespa applications" +description: "Provide data analysis support for Vespa applications" +--- +<!-- + +################################################# +### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ### +################################################# +# file to edit: notebooks/index.ipynb +# command to build the docs after a change: nbdev_build_docs + +--> + +<div class="container" id="notebook-container"> + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Install">Install<a class="anchor-link" href="#Install"> </a></h2> +</div> +</div> +</div> +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p><code>pip install pyvespa</code></p> + +</div> +</div> +</div> +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Connect-to-a-Vespa-app">Connect to a Vespa app<a class="anchor-link" href="#Connect-to-a-Vespa-app"> </a></h2><blockquote><p>Connect to a running Vespa application</p> +</blockquote> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.application</span> <span class="kn">import</span> <span class="n">Vespa</span> + +<span class="n">app</span> <span class="o">=</span> <span class="n">Vespa</span><span class="p">(</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"https://api.cord19.vespa.ai"</span><span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Define-a-Query-model">Define a Query model<a class="anchor-link" href="#Define-a-Query-model"> </a></h2><blockquote><p>Easily define matching and ranking criteria</p> +</blockquote> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.query</span> <span class="kn">import</span> <span class="n">Query</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">WeakAnd</span><span class="p">,</span> <span class="n">ANN</span><span class="p">,</span> <span class="n">RankProfile</span> +<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span> + +<span class="n">match_phase</span> <span class="o">=</span> <span class="n">Union</span><span class="p">(</span> + <span class="n">WeakAnd</span><span class="p">(</span><span class="n">hits</span> <span class="o">=</span> <span class="mi">10</span><span class="p">),</span> + <span class="n">ANN</span><span class="p">(</span> + <span class="n">doc_vector</span><span class="o">=</span><span class="s2">"title_embedding"</span><span class="p">,</span> + <span class="n">query_vector</span><span class="o">=</span><span class="s2">"title_vector"</span><span class="p">,</span> + <span class="n">embedding_model</span><span class="o">=</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="p">[</span><span class="n">random</span><span class="p">()</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">768</span><span class="p">)],</span> + <span class="n">hits</span> <span class="o">=</span> <span class="mi">10</span><span class="p">,</span> + <span class="n">label</span><span class="o">=</span><span class="s2">"title"</span> + <span class="p">)</span> +<span class="p">)</span> + +<span class="n">rank_profile</span> <span class="o">=</span> <span class="n">RankProfile</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"bm25"</span><span class="p">,</span> <span class="n">list_features</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> + +<span class="n">query_model</span> <span class="o">=</span> <span class="n">Query</span><span class="p">(</span><span class="n">match_phase</span><span class="o">=</span><span class="n">match_phase</span><span class="p">,</span> <span class="n">rank_profile</span><span class="o">=</span><span class="n">rank_profile</span><span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Query-the-vespa-app">Query the vespa app<a class="anchor-link" href="#Query-the-vespa-app"> </a></h2><blockquote><p>Send queries via the query API. See the <a href="/vespa/query">query page</a> for more examples.</p> +</blockquote> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">query_result</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">query</span><span class="p">(</span> + <span class="n">query</span><span class="o">=</span><span class="s2">"Is remdesivir an effective treatment for COVID-19?"</span><span class="p">,</span> + <span class="n">query_model</span><span class="o">=</span><span class="n">query_model</span> +<span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">query_result</span><span class="o">.</span><span class="n">number_documents_retrieved</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Labelled-data">Labelled data<a class="anchor-link" href="#Labelled-data"> </a></h2><blockquote><p>How to structure labelled data</p> +</blockquote> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">labelled_data</span> <span class="o">=</span> <span class="p">[</span> + <span class="p">{</span> + <span class="s2">"query_id"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> + <span class="s2">"query"</span><span class="p">:</span> <span class="s2">"Intrauterine virus infections and congenital heart disease"</span><span class="p">,</span> + <span class="s2">"relevant_docs"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span> <span class="p">{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">}]</span> + <span class="p">},</span> + <span class="p">{</span> + <span class="s2">"query_id"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> + <span class="s2">"query"</span><span class="p">:</span> <span class="s2">"Clinical and immunologic studies in identical twins discordant for systemic lupus erythematosus"</span><span class="p">,</span> + <span class="s2">"relevant_docs"</span><span class="p">:</span> <span class="p">[{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span> <span class="p">{</span><span class="s2">"id"</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> <span class="s2">"score"</span><span class="p">:</span> <span class="mi">1</span><span class="p">}]</span> + <span class="p">}</span> +<span class="p">]</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>Non-relevant documents are assigned <code>"score": 0</code> by default. Relevant documents will be assigned <code>"score": 1</code> by default if the field is missing from the labelled data. The defaults for both relevant and non-relevant documents can be modified on the appropriate methods.</p> + +</div> +</div> +</div> +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Collect-training-data">Collect training data<a class="anchor-link" href="#Collect-training-data"> </a></h2><blockquote><p>Collect training data to analyse and/or improve ranking functions. See the <a href="/vespa/collect_training_data">collect training data page</a> for more examples.</p> +</blockquote> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">training_data_batch</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">collect_training_data</span><span class="p">(</span> + <span class="n">labelled_data</span> <span class="o">=</span> <span class="n">labelled_data</span><span class="p">,</span> + <span class="n">id_field</span> <span class="o">=</span> <span class="s2">"id"</span><span class="p">,</span> + <span class="n">query_model</span> <span class="o">=</span> <span class="n">query_model</span><span class="p">,</span> + <span class="n">number_additional_docs</span> <span class="o">=</span> <span class="mi">2</span> +<span class="p">)</span> +<span class="n">training_data_batch</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Evaluating-a-query-model">Evaluating a query model<a class="anchor-link" href="#Evaluating-a-query-model"> </a></h2><blockquote><p>Define metrics and evaluate query models. See the <a href="/vespa/evaluation">evaluation page</a> for more examples.</p> +</blockquote> + +</div> +</div> +</div> +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>We will define the following evaluation metrics:</p> +<ul> +<li>% of documents retrieved per query</li> +<li>recall @ 10 per query</li> +<li>MRR @ 10 per query</li> +</ul> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.evaluation</span> <span class="kn">import</span> <span class="n">MatchRatio</span><span class="p">,</span> <span class="n">Recall</span><span class="p">,</span> <span class="n">ReciprocalRank</span> + +<span class="n">eval_metrics</span> <span class="o">=</span> <span class="p">[</span><span class="n">MatchRatio</span><span class="p">(),</span> <span class="n">Recall</span><span class="p">(</span><span class="n">at</span><span class="o">=</span><span class="mi">10</span><span class="p">),</span> <span class="n">ReciprocalRank</span><span class="p">(</span><span class="n">at</span><span class="o">=</span><span class="mi">10</span><span class="p">)]</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>Evaluate:</p> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">evaluation</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">evaluate</span><span class="p">(</span> + <span class="n">labelled_data</span> <span class="o">=</span> <span class="n">labelled_data</span><span class="p">,</span> + <span class="n">eval_metrics</span> <span class="o">=</span> <span class="n">eval_metrics</span><span class="p">,</span> + <span class="n">query_model</span> <span class="o">=</span> <span class="n">query_model</span><span class="p">,</span> + <span class="n">id_field</span> <span class="o">=</span> <span class="s2">"id"</span><span class="p">,</span> +<span class="p">)</span> +<span class="n">evaluation</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +</div> + + diff --git a/python/vespa/docs/js/customscripts.js b/python/vespa/docs/js/customscripts.js new file mode 100644 index 00000000000..27701a35d6b --- /dev/null +++ b/python/vespa/docs/js/customscripts.js @@ -0,0 +1,54 @@ +$('#mysidebar').height($(".nav").height()); + + +$( document ).ready(function() { + + //this script says, if the height of the viewport is greater than 800px, then insert affix class, which makes the nav bar float in a fixed + // position as your scroll. if you have a lot of nav items, this height may not work for you. + var h = $(window).height(); + //console.log (h); + if (h > 800) { + $( "#mysidebar" ).attr("class", "nav affix"); + } + // activate tooltips. although this is a bootstrap js function, it must be activated this way in your theme. + $('[data-toggle="tooltip"]').tooltip({ + placement : 'top' + }); + + /** + * AnchorJS + */ + anchors.add('h2,h3,h4,h5'); + +}); + +// needed for nav tabs on pages. See Formatting > Nav tabs for more details. +// script from http://stackoverflow.com/questions/10523433/how-do-i-keep-the-current-tab-active-with-twitter-bootstrap-after-a-page-reload +$(function() { + var json, tabsState; + $('a[data-toggle="pill"], a[data-toggle="tab"]').on('shown.bs.tab', function(e) { + var href, json, parentId, tabsState; + + tabsState = localStorage.getItem("tabs-state"); + json = JSON.parse(tabsState || "{}"); + parentId = $(e.target).parents("ul.nav.nav-pills, ul.nav.nav-tabs").attr("id"); + href = $(e.target).attr('href'); + json[parentId] = href; + + return localStorage.setItem("tabs-state", JSON.stringify(json)); + }); + + tabsState = localStorage.getItem("tabs-state"); + json = JSON.parse(tabsState || "{}"); + + $.each(json, function(containerId, href) { + return $("#" + containerId + " a[href=" + href + "]").tab('show'); + }); + + $("ul.nav.nav-pills, ul.nav.nav-tabs").each(function() { + var $this = $(this); + if (!json[$this.attr("id")]) { + return $this.find("a[data-toggle=tab]:first, a[data-toggle=pill]:first").tab("show"); + } + }); +}); diff --git a/python/vespa/docs/js/jekyll-search.js b/python/vespa/docs/js/jekyll-search.js new file mode 100644 index 00000000000..d884a244e4a --- /dev/null +++ b/python/vespa/docs/js/jekyll-search.js @@ -0,0 +1 @@ +!function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a="function"==typeof require&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}for(var i="function"==typeof require&&require,o=0;o<r.length;o++)s(r[o]);return s}({1:[function(require,module){module.exports=function(){function receivedResponse(xhr){return 200==xhr.status&&4==xhr.readyState}function handleResponse(xhr,callback){xhr.onreadystatechange=function(){if(receivedResponse(xhr))try{callback(null,JSON.parse(xhr.responseText))}catch(err){callback(err,null)}}}var self=this;self.load=function(location,callback){var xhr=window.XMLHttpRequest?new XMLHttpRequest:new ActiveXObject("Microsoft.XMLHTTP");xhr.open("GET",location,!0),handleResponse(xhr,callback),xhr.send()}}},{}],2:[function(require,module){function FuzzySearchStrategy(){function createFuzzyRegExpFromString(string){return new RegExp(string.split("").join(".*?"),"gi")}var self=this;self.matches=function(string,crit){return"string"!=typeof string?!1:(string=string.trim(),!!string.match(createFuzzyRegExpFromString(crit)))}}module.exports=new FuzzySearchStrategy},{}],3:[function(require,module){function LiteralSearchStrategy(){function doMatch(string,crit){return string.toLowerCase().indexOf(crit.toLowerCase())>=0}var self=this;self.matches=function(string,crit){return"string"!=typeof string?!1:(string=string.trim(),doMatch(string,crit))}}module.exports=new LiteralSearchStrategy},{}],4:[function(require,module){module.exports=function(){function findMatches(store,crit,strategy){for(var data=store.get(),i=0;i<data.length&&matches.length<limit;i++)findMatchesInObject(data[i],crit,strategy);return matches}function findMatchesInObject(obj,crit,strategy){for(var key in obj)if(strategy.matches(obj[key],crit)){matches.push(obj);break}}function getSearchStrategy(){return fuzzy?fuzzySearchStrategy:literalSearchStrategy}var self=this,matches=[],fuzzy=!1,limit=10,fuzzySearchStrategy=require("./SearchStrategies/fuzzy"),literalSearchStrategy=require("./SearchStrategies/literal");self.setFuzzy=function(_fuzzy){fuzzy=!!_fuzzy},self.setLimit=function(_limit){limit=parseInt(_limit,10)||limit},self.search=function(data,crit){return crit?(matches.length=0,findMatches(data,crit,getSearchStrategy())):[]}}},{"./SearchStrategies/fuzzy":2,"./SearchStrategies/literal":3}],5:[function(require,module){module.exports=function(_store){function isObject(obj){return!!obj&&"[object Object]"==Object.prototype.toString.call(obj)}function isArray(obj){return!!obj&&"[object Array]"==Object.prototype.toString.call(obj)}function addObject(data){return store.push(data),data}function addArray(data){for(var added=[],i=0;i<data.length;i++)isObject(data[i])&&added.push(addObject(data[i]));return added}var self=this,store=[];isArray(_store)&&addArray(_store),self.clear=function(){return store.length=0,store},self.get=function(){return store},self.put=function(data){return isObject(data)?addObject(data):isArray(data)?addArray(data):void 0}}},{}],6:[function(require,module){module.exports=function(){var self=this,templatePattern=/\{(.*?)\}/g;self.setTemplatePattern=function(newTemplatePattern){templatePattern=newTemplatePattern},self.render=function(t,data){return t.replace(templatePattern,function(match,prop){return data[prop]||match})}}},{}],7:[function(require){!function(window){"use strict";function SimpleJekyllSearch(){function initWithJSON(){store.put(opt.dataSource),registerInput()}function initWithURL(url){jsonLoader.load(url,function(err,json){err?throwError("failed to get JSON ("+url+")"):(store.put(json),registerInput())})}function throwError(message){throw new Error("SimpleJekyllSearch --- "+message)}function validateOptions(_opt){for(var i=0;i<requiredOptions.length;i++){var req=requiredOptions[i];_opt[req]||throwError("You must specify a "+req)}}function assignOptions(_opt){for(var option in opt)opt[option]=_opt[option]||opt[option]}function isJSON(json){try{return json instanceof Object&&JSON.parse(JSON.stringify(json))}catch(e){return!1}}function emptyResultsContainer(){opt.resultsContainer.innerHTML=""}function appendToResultsContainer(text){opt.resultsContainer.innerHTML+=text}function registerInput(){opt.searchInput.addEventListener("keyup",function(e){return 0==e.target.value.length?void emptyResultsContainer():void render(searcher.search(store,e.target.value))})}function render(results){if(emptyResultsContainer(),0==results.length)return appendToResultsContainer(opt.noResultsText);for(var i=0;i<results.length;i++)appendToResultsContainer(templater.render(opt.searchResultTemplate,results[i]))}var self=this,requiredOptions=["searchInput","resultsContainer","dataSource"],opt={searchInput:null,resultsContainer:null,dataSource:[],searchResultTemplate:'<li><a href="{url}" title="{desc}">{title}</a></li>',noResultsText:"No results found",limit:10,fuzzy:!1};self.init=function(_opt){validateOptions(_opt),assignOptions(_opt),isJSON(opt.dataSource)?initWithJSON(opt.dataSource):initWithURL(opt.dataSource)}}var Searcher=require("./Searcher"),Templater=require("./Templater"),Store=require("./Store"),JSONLoader=require("./JSONLoader"),searcher=new Searcher,templater=new Templater,store=new Store,jsonLoader=new JSONLoader;window.SimpleJekyllSearch=new SimpleJekyllSearch}(window,document)},{"./JSONLoader":1,"./Searcher":4,"./Store":5,"./Templater":6}]},{},[7]); diff --git a/python/vespa/docs/js/jquery.ba-throttle-debounce.min.js b/python/vespa/docs/js/jquery.ba-throttle-debounce.min.js new file mode 100644 index 00000000000..07205508eb6 --- /dev/null +++ b/python/vespa/docs/js/jquery.ba-throttle-debounce.min.js @@ -0,0 +1,9 @@ +/* + * jQuery throttle / debounce - v1.1 - 3/7/2010 + * http://benalman.com/projects/jquery-throttle-debounce-plugin/ + * + * Copyright (c) 2010 "Cowboy" Ben Alman + * Dual licensed under the MIT and GPL licenses. + * http://benalman.com/about/license/ + */ +(function(b,c){var $=b.jQuery||b.Cowboy||(b.Cowboy={}),a;$.throttle=a=function(e,f,j,i){var h,d=0;if(typeof f!=="boolean"){i=j;j=f;f=c}function g(){var o=this,m=+new Date()-d,n=arguments;function l(){d=+new Date();j.apply(o,n)}function k(){h=c}if(i&&!h){l()}h&&clearTimeout(h);if(i===c&&m>e){l()}else{if(f!==true){h=setTimeout(i?k:l,i===c?e-m:e)}}}if($.guid){g.guid=j.guid=j.guid||$.guid++}return g};$.debounce=function(d,e,f){return f===c?a(d,e,false):a(d,f,e!==false)}})(this);
\ No newline at end of file diff --git a/python/vespa/docs/js/jquery.navgoco.min.js b/python/vespa/docs/js/jquery.navgoco.min.js new file mode 100755 index 00000000000..4ba44753323 --- /dev/null +++ b/python/vespa/docs/js/jquery.navgoco.min.js @@ -0,0 +1,8 @@ +/* + * jQuery Navgoco Menus Plugin v0.2.1 (2014-04-11) + * https://github.com/tefra/navgoco + * + * Copyright (c) 2014 Chris T (@tefra) + * BSD - https://github.com/tefra/navgoco/blob/master/LICENSE-BSD + */ +!function(a){"use strict";var b=function(b,c,d){return this.el=b,this.$el=a(b),this.options=c,this.uuid=this.$el.attr("id")?this.$el.attr("id"):d,this.state={},this.init(),this};b.prototype={init:function(){var b=this;b._load(),b.$el.find("ul").each(function(c){var d=a(this);d.attr("data-index",c),b.options.save&&b.state.hasOwnProperty(c)?(d.parent().addClass(b.options.openClass),d.show()):d.parent().hasClass(b.options.openClass)?(d.show(),b.state[c]=1):d.hide()});var c=a("<span></span>").prepend(b.options.caretHtml),d=b.$el.find("li > a");b._trigger(c,!1),b._trigger(d,!0),b.$el.find("li:has(ul) > a").prepend(c)},_trigger:function(b,c){var d=this;b.on("click",function(b){b.stopPropagation();var e=c?a(this).next():a(this).parent().next(),f=!1;if(c){var g=a(this).attr("href");f=void 0===g||""===g||"#"===g}if(e=e.length>0?e:!1,d.options.onClickBefore.call(this,b,e),!c||e&&f)b.preventDefault(),d._toggle(e,e.is(":hidden")),d._save();else if(d.options.accordion){var h=d.state=d._parents(a(this));d.$el.find("ul").filter(":visible").each(function(){var b=a(this),c=b.attr("data-index");h.hasOwnProperty(c)||d._toggle(b,!1)}),d._save()}d.options.onClickAfter.call(this,b,e)})},_toggle:function(b,c){var d=this,e=b.attr("data-index"),f=b.parent();if(d.options.onToggleBefore.call(this,b,c),c){if(f.addClass(d.options.openClass),b.slideDown(d.options.slide),d.state[e]=1,d.options.accordion){var g=d.state=d._parents(b);g[e]=d.state[e]=1,d.$el.find("ul").filter(":visible").each(function(){var b=a(this),c=b.attr("data-index");g.hasOwnProperty(c)||d._toggle(b,!1)})}}else f.removeClass(d.options.openClass),b.slideUp(d.options.slide),d.state[e]=0;d.options.onToggleAfter.call(this,b,c)},_parents:function(b,c){var d={},e=b.parent(),f=e.parents("ul");return f.each(function(){var b=a(this),e=b.attr("data-index");return e?void(d[e]=c?b:1):!1}),d},_save:function(){if(this.options.save){var b={};for(var d in this.state)1===this.state[d]&&(b[d]=1);c[this.uuid]=this.state=b,a.cookie(this.options.cookie.name,JSON.stringify(c),this.options.cookie)}},_load:function(){if(this.options.save){if(null===c){var b=a.cookie(this.options.cookie.name);c=b?JSON.parse(b):{}}this.state=c.hasOwnProperty(this.uuid)?c[this.uuid]:{}}},toggle:function(b){var c=this,d=arguments.length;if(1>=d)c.$el.find("ul").each(function(){var d=a(this);c._toggle(d,b)});else{var e,f={},g=Array.prototype.slice.call(arguments,1);d--;for(var h=0;d>h;h++){e=g[h];var i=c.$el.find('ul[data-index="'+e+'"]').first();if(i&&(f[e]=i,b)){var j=c._parents(i,!0);for(var k in j)f.hasOwnProperty(k)||(f[k]=j[k])}}for(e in f)c._toggle(f[e],b)}c._save()},destroy:function(){a.removeData(this.$el),this.$el.find("li:has(ul) > a").unbind("click"),this.$el.find("li:has(ul) > a > span").unbind("click")}},a.fn.navgoco=function(c){if("string"==typeof c&&"_"!==c.charAt(0)&&"init"!==c)var d=!0,e=Array.prototype.slice.call(arguments,1);else c=a.extend({},a.fn.navgoco.defaults,c||{}),a.cookie||(c.save=!1);return this.each(function(f){var g=a(this),h=g.data("navgoco");h||(h=new b(this,d?a.fn.navgoco.defaults:c,f),g.data("navgoco",h)),d&&h[c].apply(h,e)})};var c=null;a.fn.navgoco.defaults={caretHtml:"",accordion:!1,openClass:"open",save:!0,cookie:{name:"navgoco",expires:!1,path:"/"},slide:{duration:400,easing:"swing"},onClickBefore:a.noop,onClickAfter:a.noop,onToggleBefore:a.noop,onToggleAfter:a.noop}}(jQuery);
\ No newline at end of file diff --git a/python/vespa/docs/js/jquery.shuffle.min.js b/python/vespa/docs/js/jquery.shuffle.min.js new file mode 100644 index 00000000000..d1031271996 --- /dev/null +++ b/python/vespa/docs/js/jquery.shuffle.min.js @@ -0,0 +1,1588 @@ +/*! + * Shuffle.js by @Vestride + * Categorize, sort, and filter a responsive grid of items. + * Dependencies: jQuery 1.9+, Modernizr 2.6.2+ + * @license MIT license + * @version 3.0.0 + */ + +/* Modernizr 2.6.2 (Custom Build) | MIT & BSD + * Build: http://modernizr.com/download/#-csstransforms-csstransforms3d-csstransitions-cssclasses-prefixed-teststyles-testprop-testallprops-prefixes-domprefixes + */ +window.Modernizr=function(a,b,c){function z(a){j.cssText=a}function A(a,b){return z(m.join(a+";")+(b||""))}function B(a,b){return typeof a===b}function C(a,b){return!!~(""+a).indexOf(b)}function D(a,b){for(var d in a){var e=a[d];if(!C(e,"-")&&j[e]!==c)return b=="pfx"?e:!0}return!1}function E(a,b,d){for(var e in a){var f=b[a[e]];if(f!==c)return d===!1?a[e]:B(f,"function")?f.bind(d||b):f}return!1}function F(a,b,c){var d=a.charAt(0).toUpperCase()+a.slice(1),e=(a+" "+o.join(d+" ")+d).split(" ");return B(b,"string")||B(b,"undefined")?D(e,b):(e=(a+" "+p.join(d+" ")+d).split(" "),E(e,b,c))}var d="2.6.2",e={},f=!0,g=b.documentElement,h="modernizr",i=b.createElement(h),j=i.style,k,l={}.toString,m=" -webkit- -moz- -o- -ms- ".split(" "),n="Webkit Moz O ms",o=n.split(" "),p=n.toLowerCase().split(" "),q={},r={},s={},t=[],u=t.slice,v,w=function(a,c,d,e){var f,i,j,k,l=b.createElement("div"),m=b.body,n=m||b.createElement("body");if(parseInt(d,10))while(d--)j=b.createElement("div"),j.id=e?e[d]:h+(d+1),l.appendChild(j);return f=["­",'<style id="s',h,'">',a,"</style>"].join(""),l.id=h,(m?l:n).innerHTML+=f,n.appendChild(l),m||(n.style.background="",n.style.overflow="hidden",k=g.style.overflow,g.style.overflow="hidden",g.appendChild(n)),i=c(l,a),m?l.parentNode.removeChild(l):(n.parentNode.removeChild(n),g.style.overflow=k),!!i},x={}.hasOwnProperty,y;!B(x,"undefined")&&!B(x.call,"undefined")?y=function(a,b){return x.call(a,b)}:y=function(a,b){return b in a&&B(a.constructor.prototype[b],"undefined")},Function.prototype.bind||(Function.prototype.bind=function(b){var c=this;if(typeof c!="function")throw new TypeError;var d=u.call(arguments,1),e=function(){if(this instanceof e){var a=function(){};a.prototype=c.prototype;var f=new a,g=c.apply(f,d.concat(u.call(arguments)));return Object(g)===g?g:f}return c.apply(b,d.concat(u.call(arguments)))};return e}),q.csstransforms=function(){return!!F("transform")},q.csstransforms3d=function(){var a=!!F("perspective");return a&&"webkitPerspective"in g.style&&w("@media (transform-3d),(-webkit-transform-3d){#modernizr{left:9px;position:absolute;height:3px;}}",function(b,c){a=b.offsetLeft===9&&b.offsetHeight===3}),a},q.csstransitions=function(){return F("transition")};for(var G in q)y(q,G)&&(v=G.toLowerCase(),e[v]=q[G](),t.push((e[v]?"":"no-")+v));return e.addTest=function(a,b){if(typeof a=="object")for(var d in a)y(a,d)&&e.addTest(d,a[d]);else{a=a.toLowerCase();if(e[a]!==c)return e;b=typeof b=="function"?b():b,typeof f!="undefined"&&f&&(g.className+=" "+(b?"":"no-")+a),e[a]=b}return e},z(""),i=k=null,e._version=d,e._prefixes=m,e._domPrefixes=p,e._cssomPrefixes=o,e.testProp=function(a){return D([a])},e.testAllProps=F,e.testStyles=w,e.prefixed=function(a,b,c){return b?F(a,b,c):F(a,"pfx")},g.className=g.className.replace(/(^|\s)no-js(\s|$)/,"$1$2")+(f?" js "+t.join(" "):""),e}(this,this.document); + +(function (factory) { + if (typeof define === 'function' && define.amd) { + define(['jquery', 'modernizr'], factory); + } else { + window.Shuffle = factory(window.jQuery, window.Modernizr); + } +})(function($, Modernizr, undefined) { + +'use strict'; + + +// Validate Modernizr exists. +// Shuffle requires `csstransitions`, `csstransforms`, `csstransforms3d`, +// and `prefixed` to exist on the Modernizr object. +if (typeof Modernizr !== 'object') { + throw new Error('Shuffle.js requires Modernizr.\n' + + 'http://vestride.github.io/Shuffle/#dependencies'); +} + + +/** + * Returns css prefixed properties like `-webkit-transition` or `box-sizing` + * from `transition` or `boxSizing`, respectively. + * @param {(string|boolean)} prop Property to be prefixed. + * @return {string} The prefixed css property. + */ +function dashify( prop ) { + if (!prop) { + return ''; + } + + // Replace upper case with dash-lowercase, + // then fix ms- prefixes because they're not capitalized. + return prop.replace(/([A-Z])/g, function( str, m1 ) { + return '-' + m1.toLowerCase(); + }).replace(/^ms-/,'-ms-'); +} + +// Constant, prefixed variables. +var TRANSITION = Modernizr.prefixed('transition'); +var TRANSITION_DELAY = Modernizr.prefixed('transitionDelay'); +var TRANSITION_DURATION = Modernizr.prefixed('transitionDuration'); + +// Note(glen): Stock Android 4.1.x browser will fail here because it wrongly +// says it supports non-prefixed transitions. +// https://github.com/Modernizr/Modernizr/issues/897 +var TRANSITIONEND = { + 'WebkitTransition' : 'webkitTransitionEnd', + 'transition' : 'transitionend' +}[ TRANSITION ]; + +var TRANSFORM = Modernizr.prefixed('transform'); +var CSS_TRANSFORM = dashify(TRANSFORM); + +// Constants +var CAN_TRANSITION_TRANSFORMS = Modernizr.csstransforms && Modernizr.csstransitions; +var HAS_TRANSFORMS_3D = Modernizr.csstransforms3d; +var SHUFFLE = 'shuffle'; +var COLUMN_THRESHOLD = 0.3; + +// Configurable. You can change these constants to fit your application. +// The default scale and concealed scale, however, have to be different values. +var ALL_ITEMS = 'all'; +var FILTER_ATTRIBUTE_KEY = 'groups'; +var DEFAULT_SCALE = 1; +var CONCEALED_SCALE = 0.001; + + +// Underscore's throttle function. +function throttle(func, wait, options) { + var context, args, result; + var timeout = null; + var previous = 0; + options = options || {}; + var later = function() { + previous = options.leading === false ? 0 : $.now(); + timeout = null; + result = func.apply(context, args); + context = args = null; + }; + return function() { + var now = $.now(); + if (!previous && options.leading === false) { + previous = now; + } + var remaining = wait - (now - previous); + context = this; + args = arguments; + if (remaining <= 0 || remaining > wait) { + clearTimeout(timeout); + timeout = null; + previous = now; + result = func.apply(context, args); + context = args = null; + } else if (!timeout && options.trailing !== false) { + timeout = setTimeout(later, remaining); + } + return result; + }; +} + +function each(obj, iterator, context) { + for (var i = 0, length = obj.length; i < length; i++) { + if (iterator.call(context, obj[i], i, obj) === {}) { + return; + } + } +} + +function defer(fn, context, wait) { + return setTimeout( $.proxy( fn, context ), wait ); +} + +function arrayMax( array ) { + return Math.max.apply( Math, array ); +} + +function arrayMin( array ) { + return Math.min.apply( Math, array ); +} + + +/** + * Always returns a numeric value, given a value. + * @param {*} value Possibly numeric value. + * @return {number} `value` or zero if `value` isn't numeric. + * @private + */ +function getNumber(value) { + return $.isNumeric(value) ? value : 0; +} + + +/** + * Represents a coordinate pair. + * @param {number} [x=0] X. + * @param {number} [y=0] Y. + */ +var Point = function(x, y) { + this.x = getNumber( x ); + this.y = getNumber( y ); +}; + + +/** + * Whether two points are equal. + * @param {Point} a Point A. + * @param {Point} b Point B. + * @return {boolean} + */ +Point.equals = function(a, b) { + return a.x === b.x && a.y === b.y; +}; + + +// Used for unique instance variables +var id = 0; +var $window = $( window ); + + +/** + * Categorize, sort, and filter a responsive grid of items. + * + * @param {Element} element An element which is the parent container for the grid items. + * @param {Object} [options=Shuffle.options] Options object. + * @constructor + */ +var Shuffle = function( element, options ) { + options = options || {}; + $.extend( this, Shuffle.options, options, Shuffle.settings ); + + this.$el = $(element); + this.element = element; + this.unique = 'shuffle_' + id++; + + this._fire( Shuffle.EventType.LOADING ); + this._init(); + + // Dispatch the done event asynchronously so that people can bind to it after + // Shuffle has been initialized. + defer(function() { + this.initialized = true; + this._fire( Shuffle.EventType.DONE ); + }, this, 16); +}; + + +/** + * Events the container element emits with the .shuffle namespace. + * For example, "done.shuffle". + * @enum {string} + */ +Shuffle.EventType = { + LOADING: 'loading', + DONE: 'done', + LAYOUT: 'layout', + REMOVED: 'removed' +}; + + +/** @enum {string} */ +Shuffle.ClassName = { + BASE: SHUFFLE, + SHUFFLE_ITEM: 'shuffle-item', + FILTERED: 'filtered', + CONCEALED: 'concealed' +}; + + +// Overrideable options +Shuffle.options = { + group: ALL_ITEMS, // Initial filter group. + speed: 250, // Transition/animation speed (milliseconds). + easing: 'ease-out', // CSS easing function to use. + itemSelector: '', // e.g. '.picture-item'. + sizer: null, // Sizer element. Use an element to determine the size of columns and gutters. + gutterWidth: 0, // A static number or function that tells the plugin how wide the gutters between columns are (in pixels). + columnWidth: 0, // A static number or function that returns a number which tells the plugin how wide the columns are (in pixels). + delimeter: null, // If your group is not json, and is comma delimeted, you could set delimeter to ','. + buffer: 0, // Useful for percentage based heights when they might not always be exactly the same (in pixels). + initialSort: null, // Shuffle can be initialized with a sort object. It is the same object given to the sort method. + throttle: throttle, // By default, shuffle will throttle resize events. This can be changed or removed. + throttleTime: 300, // How often shuffle can be called on resize (in milliseconds). + sequentialFadeDelay: 150, // Delay between each item that fades in when adding items. + supported: CAN_TRANSITION_TRANSFORMS // Whether to use transforms or absolute positioning. +}; + + +// Not overrideable +Shuffle.settings = { + useSizer: false, + itemCss : { // default CSS for each item + position: 'absolute', + top: 0, + left: 0, + visibility: 'visible' + }, + revealAppendedDelay: 300, + lastSort: {}, + lastFilter: ALL_ITEMS, + enabled: true, + destroyed: false, + initialized: false, + _animations: [], + styleQueue: [] +}; + + +// Expose for testing. +Shuffle.Point = Point; + + +/** + * Static methods. + */ + +/** + * If the browser has 3d transforms available, build a string with those, + * otherwise use 2d transforms. + * @param {Point} point X and Y positions. + * @param {number} scale Scale amount. + * @return {string} A normalized string which can be used with the transform style. + * @private + */ +Shuffle._getItemTransformString = function(point, scale) { + if ( HAS_TRANSFORMS_3D ) { + return 'translate3d(' + point.x + 'px, ' + point.y + 'px, 0) scale3d(' + scale + ', ' + scale + ', 1)'; + } else { + return 'translate(' + point.x + 'px, ' + point.y + 'px) scale(' + scale + ')'; + } +}; + + +/** + * Retrieve the computed style for an element, parsed as a float. This should + * not be used for width or height values because jQuery mangles them and they + * are not precise enough. + * @param {Element} element Element to get style for. + * @param {string} style Style property. + * @return {number} The parsed computed value or zero if that fails because IE + * will return 'auto' when the element doesn't have margins instead of + * the computed style. + * @private + */ +Shuffle._getNumberStyle = function( element, style ) { + return Shuffle._getFloat( $( element ).css( style ) ); +}; + + +/** + * Parse a string as an integer. + * @param {string} value String integer. + * @return {number} The string as an integer or zero. + * @private + */ +Shuffle._getInt = function(value) { + return getNumber( parseInt( value, 10 ) ); +}; + +/** + * Parse a string as an float. + * @param {string} value String float. + * @return {number} The string as an float or zero. + * @private + */ +Shuffle._getFloat = function(value) { + return getNumber( parseFloat( value ) ); +}; + + +/** + * Returns the outer width of an element, optionally including its margins. + * The `offsetWidth` property must be used because having a scale transform + * on the element affects the bounding box. Sadly, Firefox doesn't return an + * integer value for offsetWidth (yet). + * @param {Element} element The element. + * @param {boolean} [includeMargins] Whether to include margins. Default is false. + * @return {number} The width. + */ +Shuffle._getOuterWidth = function( element, includeMargins ) { + var width = element.offsetWidth; + + // Use jQuery here because it uses getComputedStyle internally and is + // cross-browser. Using the style property of the element will only work + // if there are inline styles. + if ( includeMargins ) { + var marginLeft = Shuffle._getNumberStyle( element, 'marginLeft'); + var marginRight = Shuffle._getNumberStyle( element, 'marginRight'); + width += marginLeft + marginRight; + } + + return width; +}; + + +/** + * Returns the outer height of an element, optionally including its margins. + * @param {Element} element The element. + * @param {boolean} [includeMargins] Whether to include margins. Default is false. + * @return {number} The height. + */ +Shuffle._getOuterHeight = function( element, includeMargins ) { + var height = element.offsetHeight; + + if ( includeMargins ) { + var marginTop = Shuffle._getNumberStyle( element, 'marginTop'); + var marginBottom = Shuffle._getNumberStyle( element, 'marginBottom'); + height += marginTop + marginBottom; + } + + return height; +}; + + +/** + * Change a property or execute a function which will not have a transition + * @param {Element} element DOM element that won't be transitioned + * @param {Function} callback A function which will be called while transition + * is set to 0ms. + * @param {Object} [context] Optional context for the callback function. + * @private + */ +Shuffle._skipTransition = function( element, callback, context ) { + var duration = element.style[ TRANSITION_DURATION ]; + + // Set the duration to zero so it happens immediately + element.style[ TRANSITION_DURATION ] = '0ms'; // ms needed for firefox! + + callback.call( context ); + + // Force reflow + var reflow = element.offsetWidth; + // Avoid jshint warnings: unused variables and expressions. + reflow = null; + + // Put the duration back + element.style[ TRANSITION_DURATION ] = duration; +}; + + +/** + * Instance methods. + */ + +Shuffle.prototype._init = function() { + this.$items = this._getItems(); + + this.sizer = this._getElementOption( this.sizer ); + + if ( this.sizer ) { + this.useSizer = true; + } + + // Add class and invalidate styles + this.$el.addClass( Shuffle.ClassName.BASE ); + + // Set initial css for each item + this._initItems(); + + // Bind resize events + // http://stackoverflow.com/questions/1852751/window-resize-event-firing-in-internet-explorer + $window.on('resize.' + SHUFFLE + '.' + this.unique, this._getResizeFunction()); + + // Get container css all in one request. Causes reflow + var containerCSS = this.$el.css(['position', 'overflow']); + var containerWidth = Shuffle._getOuterWidth( this.element ); + + // Add styles to the container if it doesn't have them. + this._validateStyles( containerCSS ); + + // We already got the container's width above, no need to cause another reflow getting it again... + // Calculate the number of columns there will be + this._setColumns( containerWidth ); + + // Kick off! + this.shuffle( this.group, this.initialSort ); + + // The shuffle items haven't had transitions set on them yet + // so the user doesn't see the first layout. Set them now that the first layout is done. + if ( this.supported ) { + defer(function() { + this._setTransitions(); + this.element.style[ TRANSITION ] = 'height ' + this.speed + 'ms ' + this.easing; + }, this); + } +}; + + +/** + * Returns a throttled and proxied function for the resize handler. + * @return {Function} + * @private + */ +Shuffle.prototype._getResizeFunction = function() { + var resizeFunction = $.proxy( this._onResize, this ); + return this.throttle ? + this.throttle( resizeFunction, this.throttleTime ) : + resizeFunction; +}; + + +/** + * Retrieve an element from an option. + * @param {string|jQuery|Element} option The option to check. + * @return {?Element} The plain element or null. + * @private + */ +Shuffle.prototype._getElementOption = function( option ) { + // If column width is a string, treat is as a selector and search for the + // sizer element within the outermost container + if ( typeof option === 'string' ) { + return this.$el.find( option )[0] || null; + + // Check for an element + } else if ( option && option.nodeType && option.nodeType === 1 ) { + return option; + + // Check for jQuery object + } else if ( option && option.jquery ) { + return option[0]; + } + + return null; +}; + + +/** + * Ensures the shuffle container has the css styles it needs applied to it. + * @param {Object} styles Key value pairs for position and overflow. + * @private + */ +Shuffle.prototype._validateStyles = function(styles) { + // Position cannot be static. + if ( styles.position === 'static' ) { + this.element.style.position = 'relative'; + } + + // Overflow has to be hidden + if ( styles.overflow !== 'hidden' ) { + this.element.style.overflow = 'hidden'; + } +}; + + +/** + * Filter the elements by a category. + * @param {string} [category] Category to filter by. If it's given, the last + * category will be used to filter the items. + * @param {ArrayLike} [$collection] Optionally filter a collection. Defaults to + * all the items. + * @return {jQuery} Filtered items. + * @private + */ +Shuffle.prototype._filter = function( category, $collection ) { + category = category || this.lastFilter; + $collection = $collection || this.$items; + + var set = this._getFilteredSets( category, $collection ); + + // Individually add/remove concealed/filtered classes + this._toggleFilterClasses( set.filtered, set.concealed ); + + // Save the last filter in case elements are appended. + this.lastFilter = category; + + // This is saved mainly because providing a filter function (like searching) + // will overwrite the `lastFilter` property every time its called. + if ( typeof category === 'string' ) { + this.group = category; + } + + return set.filtered; +}; + + +/** + * Returns an object containing the filtered and concealed elements. + * @param {string|Function} category Category or function to filter by. + * @param {ArrayLike.<Element>} $items A collection of items to filter. + * @return {!{filtered: jQuery, concealed: jQuery}} + * @private + */ +Shuffle.prototype._getFilteredSets = function( category, $items ) { + var $filtered = $(); + var $concealed = $(); + + // category === 'all', add filtered class to everything + if ( category === ALL_ITEMS ) { + $filtered = $items; + + // Loop through each item and use provided function to determine + // whether to hide it or not. + } else { + each($items, function( el ) { + var $item = $(el); + if ( this._doesPassFilter( category, $item ) ) { + $filtered = $filtered.add( $item ); + } else { + $concealed = $concealed.add( $item ); + } + }, this); + } + + return { + filtered: $filtered, + concealed: $concealed + }; +}; + + +/** + * Test an item to see if it passes a category. + * @param {string|Function} category Category or function to filter by. + * @param {jQuery} $item A single item, wrapped with jQuery. + * @return {boolean} Whether it passes the category/filter. + * @private + */ +Shuffle.prototype._doesPassFilter = function( category, $item ) { + if ( $.isFunction( category ) ) { + return category.call( $item[0], $item, this ); + + // Check each element's data-groups attribute against the given category. + } else { + var groups = $item.data( FILTER_ATTRIBUTE_KEY ); + var keys = this.delimeter && !$.isArray( groups ) ? + groups.split( this.delimeter ) : + groups; + return $.inArray(category, keys) > -1; + } +}; + + +/** + * Toggles the filtered and concealed class names. + * @param {jQuery} $filtered Filtered set. + * @param {jQuery} $concealed Concealed set. + * @private + */ +Shuffle.prototype._toggleFilterClasses = function( $filtered, $concealed ) { + $filtered + .removeClass( Shuffle.ClassName.CONCEALED ) + .addClass( Shuffle.ClassName.FILTERED ); + $concealed + .removeClass( Shuffle.ClassName.FILTERED ) + .addClass( Shuffle.ClassName.CONCEALED ); +}; + + +/** + * Set the initial css for each item + * @param {jQuery} [$items] Optionally specifiy at set to initialize + */ +Shuffle.prototype._initItems = function( $items ) { + $items = $items || this.$items; + $items.addClass([ + Shuffle.ClassName.SHUFFLE_ITEM, + Shuffle.ClassName.FILTERED + ].join(' ')); + $items.css( this.itemCss ).data('point', new Point()).data('scale', DEFAULT_SCALE); +}; + + +/** + * Updates the filtered item count. + * @private + */ +Shuffle.prototype._updateItemCount = function() { + this.visibleItems = this._getFilteredItems().length; +}; + + +/** + * Sets css transform transition on a an element. + * @param {Element} element Element to set transition on. + * @private + */ +Shuffle.prototype._setTransition = function( element ) { + element.style[ TRANSITION ] = CSS_TRANSFORM + ' ' + this.speed + 'ms ' + + this.easing + ', opacity ' + this.speed + 'ms ' + this.easing; +}; + + +/** + * Sets css transform transition on a group of elements. + * @param {ArrayLike.<Element>} $items Elements to set transitions on. + * @private + */ +Shuffle.prototype._setTransitions = function( $items ) { + $items = $items || this.$items; + each($items, function( el ) { + this._setTransition( el ); + }, this); +}; + + +/** + * Sets a transition delay on a collection of elements, making each delay + * greater than the last. + * @param {ArrayLike.<Element>} $collection Array to iterate over. + */ +Shuffle.prototype._setSequentialDelay = function( $collection ) { + if ( !this.supported ) { + return; + } + + // $collection can be an array of dom elements or jquery object + each($collection, function( el, i ) { + // This works because the transition-property: transform, opacity; + el.style[ TRANSITION_DELAY ] = '0ms,' + ((i + 1) * this.sequentialFadeDelay) + 'ms'; + }, this); +}; + + +Shuffle.prototype._getItems = function() { + return this.$el.children( this.itemSelector ); +}; + + +Shuffle.prototype._getFilteredItems = function() { + return this.$items.filter('.' + Shuffle.ClassName.FILTERED); +}; + + +Shuffle.prototype._getConcealedItems = function() { + return this.$items.filter('.' + Shuffle.ClassName.CONCEALED); +}; + + +/** + * Returns the column size, based on column width and sizer options. + * @param {number} containerWidth Size of the parent container. + * @param {number} gutterSize Size of the gutters. + * @return {number} + * @private + */ +Shuffle.prototype._getColumnSize = function( containerWidth, gutterSize ) { + var size; + + // If the columnWidth property is a function, then the grid is fluid + if ( $.isFunction( this.columnWidth ) ) { + size = this.columnWidth(containerWidth); + + // columnWidth option isn't a function, are they using a sizing element? + } else if ( this.useSizer ) { + size = Shuffle._getOuterWidth(this.sizer); + + // if not, how about the explicitly set option? + } else if ( this.columnWidth ) { + size = this.columnWidth; + + // or use the size of the first item + } else if ( this.$items.length > 0 ) { + size = Shuffle._getOuterWidth(this.$items[0], true); + + // if there's no items, use size of container + } else { + size = containerWidth; + } + + // Don't let them set a column width of zero. + if ( size === 0 ) { + size = containerWidth; + } + + return size + gutterSize; +}; + + +/** + * Returns the gutter size, based on gutter width and sizer options. + * @param {number} containerWidth Size of the parent container. + * @return {number} + * @private + */ +Shuffle.prototype._getGutterSize = function( containerWidth ) { + var size; + if ( $.isFunction( this.gutterWidth ) ) { + size = this.gutterWidth(containerWidth); + } else if ( this.useSizer ) { + size = Shuffle._getNumberStyle(this.sizer, 'marginLeft'); + } else { + size = this.gutterWidth; + } + + return size; +}; + + +/** + * Calculate the number of columns to be used. Gets css if using sizer element. + * @param {number} [theContainerWidth] Optionally specify a container width if it's already available. + */ +Shuffle.prototype._setColumns = function( theContainerWidth ) { + var containerWidth = theContainerWidth || Shuffle._getOuterWidth( this.element ); + var gutter = this._getGutterSize( containerWidth ); + var columnWidth = this._getColumnSize( containerWidth, gutter ); + var calculatedColumns = (containerWidth + gutter) / columnWidth; + + // Widths given from getComputedStyle are not precise enough... + if ( Math.abs(Math.round(calculatedColumns) - calculatedColumns) < COLUMN_THRESHOLD ) { + // e.g. calculatedColumns = 11.998876 + calculatedColumns = Math.round( calculatedColumns ); + } + + this.cols = Math.max( Math.floor(calculatedColumns), 1 ); + this.containerWidth = containerWidth; + this.colWidth = columnWidth; +}; + +/** + * Adjust the height of the grid + */ +Shuffle.prototype._setContainerSize = function() { + this.$el.css( 'height', this._getContainerSize() ); +}; + + +/** + * Based on the column heights, it returns the biggest one. + * @return {number} + * @private + */ +Shuffle.prototype._getContainerSize = function() { + return arrayMax( this.positions ); +}; + + +/** + * Fire events with .shuffle namespace + */ +Shuffle.prototype._fire = function( name, args ) { + this.$el.trigger( name + '.' + SHUFFLE, args && args.length ? args : [ this ] ); +}; + + +/** + * Zeros out the y columns array, which is used to determine item placement. + * @private + */ +Shuffle.prototype._resetCols = function() { + var i = this.cols; + this.positions = []; + while (i--) { + this.positions.push( 0 ); + } +}; + + +/** + * Loops through each item that should be shown and calculates the x, y position. + * @param {Array.<Element>} items Array of items that will be shown/layed out in order in their array. + * Because jQuery collection are always ordered in DOM order, we can't pass a jq collection. + * @param {boolean} [isOnlyPosition=false] If true this will position the items with zero opacity. + */ +Shuffle.prototype._layout = function( items, isOnlyPosition ) { + each(items, function( item ) { + this._layoutItem( item, !!isOnlyPosition ); + }, this); + + // `_layout` always happens after `_shrink`, so it's safe to process the style + // queue here with styles from the shrink method. + this._processStyleQueue(); + + // Adjust the height of the container. + this._setContainerSize(); +}; + + +/** + * Calculates the position of the item and pushes it onto the style queue. + * @param {Element} item Element which is being positioned. + * @param {boolean} isOnlyPosition Whether to position the item, but with zero + * opacity so that it can fade in later. + * @private + */ +Shuffle.prototype._layoutItem = function( item, isOnlyPosition ) { + var $item = $(item); + var itemData = $item.data(); + var currPos = itemData.point; + var currScale = itemData.scale; + var itemSize = { + width: Shuffle._getOuterWidth( item, true ), + height: Shuffle._getOuterHeight( item, true ) + }; + var pos = this._getItemPosition( itemSize ); + + // If the item will not change its position, do not add it to the render + // queue. Transitions don't fire when setting a property to the same value. + if ( Point.equals(currPos, pos) && currScale === DEFAULT_SCALE ) { + return; + } + + // Save data for shrink + itemData.point = pos; + itemData.scale = DEFAULT_SCALE; + + this.styleQueue.push({ + $item: $item, + point: pos, + scale: DEFAULT_SCALE, + opacity: isOnlyPosition ? 0 : 1, + skipTransition: isOnlyPosition, + callfront: function() { + if ( !isOnlyPosition ) { + $item.css( 'visibility', 'visible' ); + } + }, + callback: function() { + if ( isOnlyPosition ) { + $item.css( 'visibility', 'hidden' ); + } + } + }); +}; + + +/** + * Determine the location of the next item, based on its size. + * @param {{width: number, height: number}} itemSize Object with width and height. + * @return {Point} + * @private + */ +Shuffle.prototype._getItemPosition = function( itemSize ) { + var columnSpan = this._getColumnSpan( itemSize.width, this.colWidth, this.cols ); + + var setY = this._getColumnSet( columnSpan, this.cols ); + + // Finds the index of the smallest number in the set. + var shortColumnIndex = this._getShortColumn( setY, this.buffer ); + + // Position the item + var point = new Point( + Math.round( this.colWidth * shortColumnIndex ), + Math.round( setY[shortColumnIndex] )); + + // Update the columns array with the new values for each column. + // e.g. before the update the columns could be [250, 0, 0, 0] for an item + // which spans 2 columns. After it would be [250, itemHeight, itemHeight, 0]. + var setHeight = setY[shortColumnIndex] + itemSize.height; + var setSpan = this.cols + 1 - setY.length; + for ( var i = 0; i < setSpan; i++ ) { + this.positions[ shortColumnIndex + i ] = setHeight; + } + + return point; +}; + + +/** + * Determine the number of columns an items spans. + * @param {number} itemWidth Width of the item. + * @param {number} columnWidth Width of the column (includes gutter). + * @param {number} columns Total number of columns + * @return {number} + * @private + */ +Shuffle.prototype._getColumnSpan = function( itemWidth, columnWidth, columns ) { + var columnSpan = itemWidth / columnWidth; + + // If the difference between the rounded column span number and the + // calculated column span number is really small, round the number to + // make it fit. + if ( Math.abs(Math.round( columnSpan ) - columnSpan ) < COLUMN_THRESHOLD ) { + // e.g. columnSpan = 4.0089945390298745 + columnSpan = Math.round( columnSpan ); + } + + // Ensure the column span is not more than the amount of columns in the whole layout. + return Math.min( Math.ceil( columnSpan ), columns ); +}; + + +/** + * Retrieves the column set to use for placement. + * @param {number} columnSpan The number of columns this current item spans. + * @param {number} columns The total columns in the grid. + * @return {Array.<number>} An array of numbers represeting the column set. + * @private + */ +Shuffle.prototype._getColumnSet = function( columnSpan, columns ) { + // The item spans only one column. + if ( columnSpan === 1 ) { + return this.positions; + + // The item spans more than one column, figure out how many different + // places it could fit horizontally. + // The group count is the number of places within the positions this block + // could fit, ignoring the current positions of items. + // Imagine a 2 column brick as the second item in a 4 column grid with + // 10px height each. Find the places it would fit: + // [10, 0, 0, 0] + // | | | + // * * * + // + // Then take the places which fit and get the bigger of the two: + // max([10, 0]), max([0, 0]), max([0, 0]) = [10, 0, 0] + // + // Next, find the first smallest number (the short column). + // [10, 0, 0] + // | + // * + // + // And that's where it should be placed! + } else { + var groupCount = columns + 1 - columnSpan; + var groupY = []; + + // For how many possible positions for this item there are. + for ( var i = 0; i < groupCount; i++ ) { + // Find the bigger value for each place it could fit. + groupY[i] = arrayMax( this.positions.slice( i, i + columnSpan ) ); + } + + return groupY; + } +}; + + +/** + * Find index of short column, the first from the left where this item will go. + * + * @param {Array.<number>} positions The array to search for the smallest number. + * @param {number} buffer Optional buffer which is very useful when the height + * is a percentage of the width. + * @return {number} Index of the short column. + * @private + */ +Shuffle.prototype._getShortColumn = function( positions, buffer ) { + var minPosition = arrayMin( positions ); + for (var i = 0, len = positions.length; i < len; i++) { + if ( positions[i] >= minPosition - buffer && positions[i] <= minPosition + buffer ) { + return i; + } + } + return 0; +}; + + +/** + * Hides the elements that don't match our filter. + * @param {jQuery} $collection jQuery collection to shrink. + * @private + */ +Shuffle.prototype._shrink = function( $collection ) { + var $concealed = $collection || this._getConcealedItems(); + + each($concealed, function( item ) { + var $item = $(item); + var itemData = $item.data(); + + // Continuing would add a transitionend event listener to the element, but + // that listener would not execute because the transform and opacity would + // stay the same. + if ( itemData.scale === CONCEALED_SCALE ) { + return; + } + + itemData.scale = CONCEALED_SCALE; + + this.styleQueue.push({ + $item: $item, + point: itemData.point, + scale : CONCEALED_SCALE, + opacity: 0, + callback: function() { + $item.css( 'visibility', 'hidden' ); + } + }); + }, this); +}; + + +/** + * Resize handler. + * @private + */ +Shuffle.prototype._onResize = function() { + // If shuffle is disabled, destroyed, don't do anything + if ( !this.enabled || this.destroyed || this.isTransitioning ) { + return; + } + + // Will need to check height in the future if it's layed out horizontaly + var containerWidth = Shuffle._getOuterWidth( this.element ); + + // containerWidth hasn't changed, don't do anything + if ( containerWidth === this.containerWidth ) { + return; + } + + this.update(); +}; + + +/** + * Returns styles for either jQuery animate or transition. + * @param {Object} opts Transition options. + * @return {!Object} Transforms for transitions, left/top for animate. + * @private + */ +Shuffle.prototype._getStylesForTransition = function( opts ) { + var styles = { + opacity: opts.opacity + }; + + if ( this.supported ) { + styles[ TRANSFORM ] = Shuffle._getItemTransformString( opts.point, opts.scale ); + } else { + styles.left = opts.point.x; + styles.top = opts.point.y; + } + + return styles; +}; + + +/** + * Transitions an item in the grid + * + * @param {Object} opts options. + * @param {jQuery} opts.$item jQuery object representing the current item. + * @param {Point} opts.point A point object with the x and y coordinates. + * @param {number} opts.scale Amount to scale the item. + * @param {number} opts.opacity Opacity of the item. + * @param {Function} opts.callback Complete function for the animation. + * @param {Function} opts.callfront Function to call before transitioning. + * @private + */ +Shuffle.prototype._transition = function( opts ) { + var styles = this._getStylesForTransition( opts ); + this._startItemAnimation( opts.$item, styles, opts.callfront || $.noop, opts.callback || $.noop ); +}; + + +Shuffle.prototype._startItemAnimation = function( $item, styles, callfront, callback ) { + // Transition end handler removes its listener. + function handleTransitionEnd( evt ) { + // Make sure this event handler has not bubbled up from a child. + if ( evt.target === evt.currentTarget ) { + $( evt.target ).off( TRANSITIONEND, handleTransitionEnd ); + callback(); + } + } + + callfront(); + + // Transitions are not set until shuffle has loaded to avoid the initial transition. + if ( !this.initialized ) { + $item.css( styles ); + callback(); + return; + } + + // Use CSS Transforms if we have them + if ( this.supported ) { + $item.css( styles ); + $item.on( TRANSITIONEND, handleTransitionEnd ); + + // Use jQuery to animate left/top + } else { + // Save the deferred object which jQuery returns. + var anim = $item.stop( true ).animate( styles, this.speed, 'swing', callback ); + // Push the animation to the list of pending animations. + this._animations.push( anim.promise() ); + } +}; + + +/** + * Execute the styles gathered in the style queue. This applies styles to elements, + * triggering transitions. + * @param {boolean} noLayout Whether to trigger a layout event. + * @private + */ +Shuffle.prototype._processStyleQueue = function( noLayout ) { + var $transitions = $(); + + // Iterate over the queue and keep track of ones that use transitions. + each(this.styleQueue, function( transitionObj ) { + if ( transitionObj.skipTransition ) { + this._styleImmediately( transitionObj ); + } else { + $transitions = $transitions.add( transitionObj.$item ); + this._transition( transitionObj ); + } + }, this); + + + if ( $transitions.length > 0 && this.initialized ) { + // Set flag that shuffle is currently in motion. + this.isTransitioning = true; + + if ( this.supported ) { + this._whenCollectionDone( $transitions, TRANSITIONEND, this._movementFinished ); + + // The _transition function appends a promise to the animations array. + // When they're all complete, do things. + } else { + this._whenAnimationsDone( this._movementFinished ); + } + + // A call to layout happened, but none of the newly filtered items will + // change position. Asynchronously fire the callback here. + } else if ( !noLayout ) { + defer( this._layoutEnd, this ); + } + + // Remove everything in the style queue + this.styleQueue.length = 0; +}; + + +/** + * Apply styles without a transition. + * @param {Object} opts Transitions options object. + * @private + */ +Shuffle.prototype._styleImmediately = function( opts ) { + Shuffle._skipTransition(opts.$item[0], function() { + opts.$item.css( this._getStylesForTransition( opts ) ); + }, this); +}; + +Shuffle.prototype._movementFinished = function() { + this.isTransitioning = false; + this._layoutEnd(); +}; + +Shuffle.prototype._layoutEnd = function() { + this._fire( Shuffle.EventType.LAYOUT ); +}; + +Shuffle.prototype._addItems = function( $newItems, addToEnd, isSequential ) { + // Add classes and set initial positions. + this._initItems( $newItems ); + + // Add transition to each item. + this._setTransitions( $newItems ); + + // Update the list of + this.$items = this._getItems(); + + // Shrink all items (without transitions). + this._shrink( $newItems ); + each(this.styleQueue, function( transitionObj ) { + transitionObj.skipTransition = true; + }); + + // Apply shrink positions, but do not cause a layout event. + this._processStyleQueue( true ); + + if ( addToEnd ) { + this._addItemsToEnd( $newItems, isSequential ); + } else { + this.shuffle( this.lastFilter ); + } +}; + + +Shuffle.prototype._addItemsToEnd = function( $newItems, isSequential ) { + // Get ones that passed the current filter + var $passed = this._filter( null, $newItems ); + var passed = $passed.get(); + + // How many filtered elements? + this._updateItemCount(); + + this._layout( passed, true ); + + if ( isSequential && this.supported ) { + this._setSequentialDelay( passed ); + } + + this._revealAppended( passed ); +}; + + +/** + * Triggers appended elements to fade in. + * @param {ArrayLike.<Element>} $newFilteredItems Collection of elements. + * @private + */ +Shuffle.prototype._revealAppended = function( newFilteredItems ) { + defer(function() { + each(newFilteredItems, function( el ) { + var $item = $( el ); + this._transition({ + $item: $item, + opacity: 1, + point: $item.data('point'), + scale: DEFAULT_SCALE + }); + }, this); + + this._whenCollectionDone($(newFilteredItems), TRANSITIONEND, function() { + $(newFilteredItems).css( TRANSITION_DELAY, '0ms' ); + this._movementFinished(); + }); + }, this, this.revealAppendedDelay); +}; + + +/** + * Execute a function when an event has been triggered for every item in a collection. + * @param {jQuery} $collection Collection of elements. + * @param {string} eventName Event to listen for. + * @param {Function} callback Callback to execute when they're done. + * @private + */ +Shuffle.prototype._whenCollectionDone = function( $collection, eventName, callback ) { + var done = 0; + var items = $collection.length; + var self = this; + + function handleEventName( evt ) { + if ( evt.target === evt.currentTarget ) { + $( evt.target ).off( eventName, handleEventName ); + done++; + + // Execute callback if all items have emitted the correct event. + if ( done === items ) { + callback.call( self ); + } + } + } + + // Bind the event to all items. + $collection.on( eventName, handleEventName ); +}; + + +/** + * Execute a callback after jQuery `animate` for a collection has finished. + * @param {Function} callback Callback to execute when they're done. + * @private + */ +Shuffle.prototype._whenAnimationsDone = function( callback ) { + $.when.apply( null, this._animations ).always( $.proxy( function() { + this._animations.length = 0; + callback.call( this ); + }, this )); +}; + + +/** + * Public Methods + */ + +/** + * The magic. This is what makes the plugin 'shuffle' + * @param {string|Function} [category] Category to filter by. Can be a function + * @param {Object} [sortObj] A sort object which can sort the filtered set + */ +Shuffle.prototype.shuffle = function( category, sortObj ) { + if ( !this.enabled || this.isTransitioning ) { + return; + } + + if ( !category ) { + category = ALL_ITEMS; + } + + this._filter( category ); + + // How many filtered elements? + this._updateItemCount(); + + // Shrink each concealed item + this._shrink(); + + // Update transforms on .filtered elements so they will animate to their new positions + this.sort( sortObj ); +}; + + +/** + * Gets the .filtered elements, sorts them, and passes them to layout. + * @param {Object} opts the options object for the sorted plugin + */ +Shuffle.prototype.sort = function( opts ) { + if ( this.enabled && !this.isTransitioning ) { + this._resetCols(); + + var sortOptions = opts || this.lastSort; + var items = this._getFilteredItems().sorted( sortOptions ); + + this._layout( items ); + + this.lastSort = sortOptions; + } +}; + + +/** + * Reposition everything. + * @param {boolean} isOnlyLayout If true, column and gutter widths won't be + * recalculated. + */ +Shuffle.prototype.update = function( isOnlyLayout ) { + if ( this.enabled && !this.isTransitioning ) { + + if ( !isOnlyLayout ) { + // Get updated colCount + this._setColumns(); + } + + // Layout items + this.sort(); + } +}; + + +/** + * Use this instead of `update()` if you don't need the columns and gutters updated + * Maybe an image inside `shuffle` loaded (and now has a height), which means calculations + * could be off. + */ +Shuffle.prototype.layout = function() { + this.update( true ); +}; + + +/** + * New items have been appended to shuffle. Fade them in sequentially + * @param {jQuery} $newItems jQuery collection of new items + * @param {boolean} [addToEnd=false] If true, new items will be added to the end / bottom + * of the items. If not true, items will be mixed in with the current sort order. + * @param {boolean} [isSequential=true] If false, new items won't sequentially fade in + */ +Shuffle.prototype.appended = function( $newItems, addToEnd, isSequential ) { + this._addItems( $newItems, addToEnd === true, isSequential !== false ); +}; + + +/** + * Disables shuffle from updating dimensions and layout on resize + */ +Shuffle.prototype.disable = function() { + this.enabled = false; +}; + + +/** + * Enables shuffle again + * @param {boolean} [isUpdateLayout=true] if undefined, shuffle will update columns and gutters + */ +Shuffle.prototype.enable = function( isUpdateLayout ) { + this.enabled = true; + if ( isUpdateLayout !== false ) { + this.update(); + } +}; + + +/** + * Remove 1 or more shuffle items + * @param {jQuery} $collection A jQuery object containing one or more element in shuffle + * @return {Shuffle} The shuffle object + */ +Shuffle.prototype.remove = function( $collection ) { + + // If this isn't a jquery object, exit + if ( !$collection.length || !$collection.jquery ) { + return; + } + + function handleRemoved() { + // Remove the collection in the callback + $collection.remove(); + + // Update things now that elements have been removed. + this.$items = this._getItems(); + this._updateItemCount(); + + this._fire( Shuffle.EventType.REMOVED, [ $collection, this ] ); + + // Let it get garbage collected + $collection = null; + } + + // Hide collection first. + this._toggleFilterClasses( $(), $collection ); + this._shrink( $collection ); + + this.sort(); + + this.$el.one( Shuffle.EventType.LAYOUT + '.' + SHUFFLE, $.proxy( handleRemoved, this ) ); +}; + + +/** + * Destroys shuffle, removes events, styles, and classes + */ +Shuffle.prototype.destroy = function() { + // If there is more than one shuffle instance on the page, + // removing the resize handler from the window would remove them + // all. This is why a unique value is needed. + $window.off('.' + this.unique); + + // Reset container styles + this.$el + .removeClass( SHUFFLE ) + .removeAttr('style') + .removeData( SHUFFLE ); + + // Reset individual item styles + this.$items + .removeAttr('style') + .removeData('point') + .removeData('scale') + .removeClass([ + Shuffle.ClassName.CONCEALED, + Shuffle.ClassName.FILTERED, + Shuffle.ClassName.SHUFFLE_ITEM + ].join(' ')); + + // Null DOM references + this.$items = null; + this.$el = null; + this.sizer = null; + this.element = null; + + // Set a flag so if a debounced resize has been triggered, + // it can first check if it is actually destroyed and not doing anything + this.destroyed = true; +}; + + +// Plugin definition +$.fn.shuffle = function( opts ) { + var args = Array.prototype.slice.call( arguments, 1 ); + return this.each(function() { + var $this = $( this ); + var shuffle = $this.data( SHUFFLE ); + + // If we don't have a stored shuffle, make a new one and save it + if ( !shuffle ) { + shuffle = new Shuffle( this, opts ); + $this.data( SHUFFLE, shuffle ); + } else if ( typeof opts === 'string' && shuffle[ opts ] ) { + shuffle[ opts ].apply( shuffle, args ); + } + }); +}; + + +// http://stackoverflow.com/a/962890/373422 +function randomize( array ) { + var tmp, current; + var top = array.length; + + if ( !top ) { + return array; + } + + while ( --top ) { + current = Math.floor( Math.random() * (top + 1) ); + tmp = array[ current ]; + array[ current ] = array[ top ]; + array[ top ] = tmp; + } + + return array; +} + + +// You can return `undefined` from the `by` function to revert to DOM order +// This plugin does NOT return a jQuery object. It returns a plain array because +// jQuery sorts everything in DOM order. +$.fn.sorted = function(options) { + var opts = $.extend({}, $.fn.sorted.defaults, options); + var arr = this.get(); + var revert = false; + + if ( !arr.length ) { + return []; + } + + if ( opts.randomize ) { + return randomize( arr ); + } + + // Sort the elements by the opts.by function. + // If we don't have opts.by, default to DOM order + if ( $.isFunction( opts.by ) ) { + arr.sort(function(a, b) { + + // Exit early if we already know we want to revert + if ( revert ) { + return 0; + } + + var valA = opts.by($(a)); + var valB = opts.by($(b)); + + // If both values are undefined, use the DOM order + if ( valA === undefined && valB === undefined ) { + revert = true; + return 0; + } + + if ( valA < valB || valA === 'sortFirst' || valB === 'sortLast' ) { + return -1; + } + + if ( valA > valB || valA === 'sortLast' || valB === 'sortFirst' ) { + return 1; + } + + return 0; + }); + } + + // Revert to the original array if necessary + if ( revert ) { + return this.get(); + } + + if ( opts.reverse ) { + arr.reverse(); + } + + return arr; +}; + + +$.fn.sorted.defaults = { + reverse: false, // Use array.reverse() to reverse the results + by: null, // Sorting function + randomize: false // If true, this will skip the sorting and return a randomized order in the array +}; + +return Shuffle; + +});
\ No newline at end of file diff --git a/python/vespa/docs/js/toc.js b/python/vespa/docs/js/toc.js new file mode 100644 index 00000000000..b5244bb0b03 --- /dev/null +++ b/python/vespa/docs/js/toc.js @@ -0,0 +1,90 @@ +// https://github.com/ghiculescu/jekyll-table-of-contents +// this library modified by fastai to: +// - update the location.href with the correct anchor when a toc item is clicked on +(function($){ + $.fn.toc = function(options) { + var defaults = { + noBackToTopLinks: false, + title: '', + minimumHeaders: 3, + headers: 'h1, h2, h3, h4', + listType: 'ol', // values: [ol|ul] + showEffect: 'show', // values: [show|slideDown|fadeIn|none] + showSpeed: 'slow' // set to 0 to deactivate effect + }, + settings = $.extend(defaults, options); + + var headers = $(settings.headers).filter(function() { + // get all headers with an ID + var previousSiblingName = $(this).prev().attr( "name" ); + if (!this.id && previousSiblingName) { + this.id = $(this).attr( "id", previousSiblingName.replace(/\./g, "-") ); + } + return this.id; + }), output = $(this); + if (!headers.length || headers.length < settings.minimumHeaders || !output.length) { + return; + } + + if (0 === settings.showSpeed) { + settings.showEffect = 'none'; + } + + var render = { + show: function() { output.hide().html(html).show(settings.showSpeed); }, + slideDown: function() { output.hide().html(html).slideDown(settings.showSpeed); }, + fadeIn: function() { output.hide().html(html).fadeIn(settings.showSpeed); }, + none: function() { output.html(html); } + }; + + var get_level = function(ele) { return parseInt(ele.nodeName.replace("H", ""), 10); } + var highest_level = headers.map(function(_, ele) { return get_level(ele); }).get().sort()[0]; + //var return_to_top = '<i class="glyphicon glyphicon-upload back-to-top"></i>'; + // other nice icons that can be used instead: glyphicon-upload glyphicon-hand-up glyphicon-chevron-up glyphicon-menu-up glyphicon-triangle-top + var level = get_level(headers[0]), + this_level, + html = settings.title + " <"+settings.listType+">"; + headers.on('click', function() { + if (!settings.noBackToTopLinks) { + var pos = $(window).scrollTop(); + window.location.hash = this.id; + $(window).scrollTop(pos); + } + }) + .addClass('clickable-header') + .each(function(_, header) { + base_url = window.location.href; + base_url = base_url.replace(/#.*$/, ""); + this_level = get_level(header); + //if (!settings.noBackToTopLinks && this_level > 1) { + // $(header).addClass('top-level-header').before(return_to_top); + //} + txt = header.textContent.split('¶')[0].split(/\[(test|source)\]/)[0]; + if (!txt) {return;} + if (this_level === level) // same level as before; same indenting + html += "<li><a href='" + base_url + "#" + header.id + "'>" + txt + "</a>"; + else if (this_level <= level){ // higher level than before; end parent ol + for(i = this_level; i < level; i++) { + html += "</li></"+settings.listType+">" + } + html += "<li><a href='" + base_url + "#" + header.id + "'>" + txt + "</a>"; + } + else if (this_level > level) { // lower level than before; expand the previous to contain a ol + for(i = this_level; i > level; i--) { + html += "<"+settings.listType+">"+((i-level == 2) ? "<li class=\"hide_content\">" : "<li>") + } + html += "<a href='" + base_url + "#" + header.id + "'>" + txt + "</a>"; + } + level = this_level; // update for the next one + }); + html += "</"+settings.listType+">"; + if (!settings.noBackToTopLinks) { + $(document).on('click', '.back-to-top', function() { + $(window).scrollTop(0); + window.location.hash = ''; + }); + } + + render[settings.showEffect](); + }; +})(jQuery); diff --git a/python/vespa/docs/licenses/LICENSE b/python/vespa/docs/licenses/LICENSE new file mode 100644 index 00000000000..21e88dc044b --- /dev/null +++ b/python/vespa/docs/licenses/LICENSE @@ -0,0 +1,24 @@ +/* This license pertains to the docs template, except for the Navgoco jQuery component. */ + +The MIT License (MIT) + +Original theme: Copyright (c) 2016 Tom Johnson +Modifications: Copyright (c) 2017 onwards fast.ai, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/python/vespa/docs/licenses/LICENSE-BSD-NAVGOCO.txt b/python/vespa/docs/licenses/LICENSE-BSD-NAVGOCO.txt new file mode 100644 index 00000000000..7fdefc39032 --- /dev/null +++ b/python/vespa/docs/licenses/LICENSE-BSD-NAVGOCO.txt @@ -0,0 +1,27 @@ +/* This license pertains to the Navgoco jQuery component used for the sidebar. */ + +Copyright (c) 2013, Christodoulos Tsoulloftas, http://www.komposta.net +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the <Christodoulos Tsoulloftas> nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file diff --git a/python/vespa/docs/query.html b/python/vespa/docs/query.html new file mode 100644 index 00000000000..a92f85ad04b --- /dev/null +++ b/python/vespa/docs/query.html @@ -0,0 +1,411 @@ +--- + +title: Query API + +keywords: fastai +sidebar: home_sidebar + +summary: "Python query API" +description: "Python query API" +--- +<!-- + +################################################# +### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ### +################################################# +# file to edit: notebooks/query.ipynb +# command to build the docs after a change: nbdev_build_docs + +--> + +<div class="container" id="notebook-container"> + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>We can connect to the CORD-19 Search app and use it to exemplify the query API</p> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.application</span> <span class="k">import</span> <span class="n">Vespa</span> + +<span class="n">app</span> <span class="o">=</span> <span class="n">Vespa</span><span class="p">(</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"https://api.cord19.vespa.ai"</span><span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Specify-the-request-body">Specify the request body<a class="anchor-link" href="#Specify-the-request-body"> </a></h2><blockquote><p>Full flexibility by specifying the entire request body</p> +</blockquote> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">body</span> <span class="o">=</span> <span class="p">{</span> + <span class="s1">'yql'</span><span class="p">:</span> <span class="s1">'select title, abstract from sources * where userQuery();'</span><span class="p">,</span> + <span class="s1">'hits'</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> + <span class="s1">'query'</span><span class="p">:</span> <span class="s1">'Is remdesivir an effective treatment for COVID-19?'</span><span class="p">,</span> + <span class="s1">'type'</span><span class="p">:</span> <span class="s1">'any'</span><span class="p">,</span> + <span class="s1">'ranking'</span><span class="p">:</span> <span class="s1">'bm25'</span> +<span class="p">}</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">results</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">results</span><span class="p">[</span><span class="s2">"root"</span><span class="p">][</span><span class="s2">"fields"</span><span class="p">]</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + + +<div class="output_text output_subarea output_execute_result"> +<pre>{'totalCount': 52387}</pre> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Specify-a-query-model">Specify a query model<a class="anchor-link" href="#Specify-a-query-model"> </a></h2> +</div> +</div> +</div> +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h3 id="Query-+-term-matching-+-rank-profile">Query + term-matching + rank profile<a class="anchor-link" href="#Query-+-term-matching-+-rank-profile"> </a></h3> +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.query</span> <span class="k">import</span> <span class="n">Query</span><span class="p">,</span> <span class="n">OR</span><span class="p">,</span> <span class="n">RankProfile</span> + +<span class="n">results</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">query</span><span class="p">(</span> + <span class="n">query</span><span class="o">=</span><span class="s2">"Is remdesivir an effective treatment for COVID-19?"</span><span class="p">,</span> + <span class="n">query_model</span> <span class="o">=</span> <span class="n">Query</span><span class="p">(</span> + <span class="n">match_phase</span><span class="o">=</span><span class="n">OR</span><span class="p">(),</span> + <span class="n">rank_profile</span><span class="o">=</span><span class="n">RankProfile</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"bm25"</span><span class="p">)</span> + <span class="p">)</span> +<span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">results</span><span class="p">[</span><span class="s2">"root"</span><span class="p">][</span><span class="s2">"fields"</span><span class="p">]</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + + +<div class="output_text output_subarea output_execute_result"> +<pre>{'totalCount': 52387}</pre> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h3 id="Query-+-term-matching-+-ann-operator-+-rank_profile">Query + term-matching + ann operator + rank_profile<a class="anchor-link" href="#Query-+-term-matching-+-ann-operator-+-rank_profile"> </a></h3> +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.query</span> <span class="k">import</span> <span class="n">Query</span><span class="p">,</span> <span class="n">ANN</span><span class="p">,</span> <span class="n">WeakAnd</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">RankProfile</span> +<span class="kn">from</span> <span class="nn">random</span> <span class="k">import</span> <span class="n">random</span> + +<span class="n">match_phase</span> <span class="o">=</span> <span class="n">Union</span><span class="p">(</span> + <span class="n">WeakAnd</span><span class="p">(</span><span class="n">hits</span> <span class="o">=</span> <span class="mi">10</span><span class="p">),</span> + <span class="n">ANN</span><span class="p">(</span> + <span class="n">doc_vector</span><span class="o">=</span><span class="s2">"title_embedding"</span><span class="p">,</span> + <span class="n">query_vector</span><span class="o">=</span><span class="s2">"title_vector"</span><span class="p">,</span> + <span class="n">embedding_model</span><span class="o">=</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="p">[</span><span class="n">random</span><span class="p">()</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">768</span><span class="p">)],</span> + <span class="n">hits</span> <span class="o">=</span> <span class="mi">10</span><span class="p">,</span> + <span class="n">label</span><span class="o">=</span><span class="s2">"title"</span> + <span class="p">)</span> +<span class="p">)</span> +<span class="n">rank_profile</span> <span class="o">=</span> <span class="n">RankProfile</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"bm25"</span><span class="p">,</span> <span class="n">list_features</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> +<span class="n">query_model</span> <span class="o">=</span> <span class="n">Query</span><span class="p">(</span><span class="n">match_phase</span><span class="o">=</span><span class="n">match_phase</span><span class="p">,</span> <span class="n">rank_profile</span><span class="o">=</span><span class="n">rank_profile</span><span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">results</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="s2">"Is remdesivir an effective treatment for COVID-19?"</span><span class="p">,</span> + <span class="n">query_model</span><span class="o">=</span><span class="n">query_model</span><span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">results</span><span class="p">[</span><span class="s2">"root"</span><span class="p">][</span><span class="s2">"fields"</span><span class="p">]</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + + +<div class="output_text output_subarea output_execute_result"> +<pre>{'totalCount': 1084}</pre> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Recall-specific-documents">Recall specific documents<a class="anchor-link" href="#Recall-specific-documents"> </a></h2> +</div> +</div> +</div> +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>Let's take a look at the top 3 ids from the last query.</p> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">top_ids</span> <span class="o">=</span> <span class="p">[</span><span class="n">hit</span><span class="p">[</span><span class="s2">"fields"</span><span class="p">][</span><span class="s2">"id"</span><span class="p">]</span> <span class="k">for</span> <span class="n">hit</span> <span class="ow">in</span> <span class="n">results</span><span class="p">[</span><span class="s2">"root"</span><span class="p">][</span><span class="s2">"children"</span><span class="p">][</span><span class="mi">0</span><span class="p">:</span><span class="mi">3</span><span class="p">]]</span> +<span class="n">top_ids</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + + +<div class="output_text output_subarea output_execute_result"> +<pre>[40215, 18456, 33692]</pre> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>Assume that we now want to retrieve the second and third ids above. We can do so with the <code>recall</code> argument.</p> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">results_with_recall</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="s2">"Is remdesivir an effective treatment for COVID-19?"</span><span class="p">,</span> + <span class="n">query_model</span><span class="o">=</span><span class="n">query_model</span><span class="p">,</span> + <span class="n">recall</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"id"</span><span class="p">,</span> <span class="n">top_ids</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="mi">3</span><span class="p">]))</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>It will only retrieve the documents with Vespa field <code>id</code> that is defined on the list that is inside the tuple.</p> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">id_recalled</span> <span class="o">=</span> <span class="p">[</span><span class="n">hit</span><span class="p">[</span><span class="s2">"fields"</span><span class="p">][</span><span class="s2">"id"</span><span class="p">]</span> <span class="k">for</span> <span class="n">hit</span> <span class="ow">in</span> <span class="n">results_with_recall</span><span class="p">[</span><span class="s2">"root"</span><span class="p">][</span><span class="s2">"children"</span><span class="p">]]</span> +<span class="n">id_recalled</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + + +<div class="output_text output_subarea output_execute_result"> +<pre>[18456, 33692]</pre> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +</div> + + diff --git a/python/vespa/docs/search.html b/python/vespa/docs/search.html new file mode 100644 index 00000000000..e3200f5ed61 --- /dev/null +++ b/python/vespa/docs/search.html @@ -0,0 +1,291 @@ +--- + +title: Search API + +keywords: fastai +sidebar: home_sidebar + +summary: "Python search API" +description: "Python search API" +--- +<!-- + +################################################# +### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ### +################################################# +# file to edit: notebooks/search.ipynb +# command to build the docs after a change: nbdev_build_docs + +--> + +<div class="container" id="notebook-container"> + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<p>We can connect to the CORD-19 Search app and use it to exemplify the search API</p> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.application</span> <span class="kn">import</span> <span class="n">Vespa</span> + +<span class="n">app</span> <span class="o">=</span> <span class="n">Vespa</span><span class="p">(</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"https://api.cord19.vespa.ai"</span><span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h2 id="Specify-the-request-body">Specify the request body<a class="anchor-link" href="#Specify-the-request-body"> </a></h2><blockquote><p>Full flexibility by specifying the entire request body</p> +</blockquote> + +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">body</span> <span class="o">=</span> <span class="p">{</span> + <span class="s1">'yql'</span><span class="p">:</span> <span class="s1">'select title, abstract from sources * where userQuery();'</span><span class="p">,</span> + <span class="s1">'hits'</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> + <span class="s1">'query'</span><span class="p">:</span> <span class="s1">'coronavirus temperature sensitivity'</span><span class="p">,</span> + <span class="s1">'type'</span><span class="p">:</span> <span class="s1">'any'</span><span class="p">,</span> + <span class="s1">'ranking'</span><span class="p">:</span> <span class="s1">'bm25'</span> +<span class="p">}</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">results</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">results</span><span class="p">[</span><span class="s2">"root"</span><span class="p">][</span><span class="s2">"fields"</span><span class="p">]</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + + +<div class="output_text output_subarea output_execute_result"> +<pre>{'totalCount': 46917}</pre> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h3 id="Query-+-term-matching-+-rank-profile">Query + term-matching + rank profile<a class="anchor-link" href="#Query-+-term-matching-+-rank-profile"> </a></h3> +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.search</span> <span class="kn">import</span> <span class="n">Search</span><span class="p">,</span> <span class="n">OR</span><span class="p">,</span> <span class="n">RankProfile</span> + +<span class="n">results</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">search</span><span class="p">(</span> + <span class="n">query</span><span class="o">=</span><span class="s2">"Is remdesivir an effective treatment for COVID-19?"</span><span class="p">,</span> + <span class="n">search_model</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span> + <span class="n">match_phase</span><span class="o">=</span><span class="n">OR</span><span class="p">(),</span> + <span class="n">rank_profile</span><span class="o">=</span><span class="n">RankProfile</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"bm25"</span><span class="p">)</span> + <span class="p">)</span> +<span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">results</span><span class="p">[</span><span class="s2">"root"</span><span class="p">][</span><span class="s2">"fields"</span><span class="p">]</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + + +<div class="output_text output_subarea output_execute_result"> +<pre>{'totalCount': 46917}</pre> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell"> +<div class="text_cell_render border-box-sizing rendered_html"> +<h3 id="Query-+-term-matching-+-ann-operator-+-rank_profile">Query + term-matching + ann operator + rank_profile<a class="anchor-link" href="#Query-+-term-matching-+-ann-operator-+-rank_profile"> </a></h3> +</div> +</div> +</div> + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">vespa.search</span> <span class="kn">import</span> <span class="n">Search</span><span class="p">,</span> <span class="n">ANN</span><span class="p">,</span> <span class="n">WeakAnd</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">RankProfile</span> +<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span> + +<span class="n">match_phase</span> <span class="o">=</span> <span class="n">Union</span><span class="p">(</span> + <span class="n">WeakAnd</span><span class="p">(</span><span class="n">hits</span> <span class="o">=</span> <span class="mi">10</span><span class="p">),</span> + <span class="n">ANN</span><span class="p">(</span> + <span class="n">doc_vector</span><span class="o">=</span><span class="s2">"title_embedding"</span><span class="p">,</span> + <span class="n">query_vector</span><span class="o">=</span><span class="s2">"title_vector"</span><span class="p">,</span> + <span class="n">embedding_model</span><span class="o">=</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="p">[</span><span class="n">random</span><span class="p">()</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">768</span><span class="p">)],</span> + <span class="n">hits</span> <span class="o">=</span> <span class="mi">10</span><span class="p">,</span> + <span class="n">label</span><span class="o">=</span><span class="s2">"title"</span> + <span class="p">)</span> +<span class="p">)</span> +<span class="n">rank_profile</span> <span class="o">=</span> <span class="n">RankProfile</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">"bm25"</span><span class="p">,</span> <span class="n">list_features</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> +<span class="n">search_model</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">match_phase</span><span class="o">=</span><span class="n">match_phase</span><span class="p">,</span> <span class="n">rank_profile</span><span class="o">=</span><span class="n">rank_profile</span><span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">results</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="s2">"Is remdesivir an effective treatment for COVID-19?"</span><span class="p">,</span> + <span class="n">search_model</span><span class="o">=</span><span class="n">search_model</span><span class="p">)</span> +</pre></div> + + </div> +</div> +</div> + +</div> + {% endraw %} + + {% raw %} + +<div class="cell border-box-sizing code_cell rendered"> +<div class="input"> + +<div class="inner_cell"> + <div class="input_area"> +<div class=" highlight hl-ipython3"><pre><span></span><span class="n">results</span><span class="p">[</span><span class="s2">"root"</span><span class="p">][</span><span class="s2">"fields"</span><span class="p">]</span> +</pre></div> + + </div> +</div> +</div> + +<div class="output_wrapper"> +<div class="output"> + +<div class="output_area"> + + + +<div class="output_text output_subarea output_execute_result"> +<pre>{'totalCount': 1049}</pre> +</div> + +</div> + +</div> +</div> + +</div> + {% endraw %} + +</div> + + diff --git a/python/vespa/docs/sidebar.json b/python/vespa/docs/sidebar.json new file mode 100644 index 00000000000..2acf7014eff --- /dev/null +++ b/python/vespa/docs/sidebar.json @@ -0,0 +1,8 @@ +{ + "pyvespa": { + "Overview": "/", + "Vespa - collect training data": "/collect_training_data", + "Vespa - Evaluate query models": "/evaluation", + "Query API": "/query" + } +}
\ No newline at end of file diff --git a/python/vespa/docs/sitemap.xml b/python/vespa/docs/sitemap.xml new file mode 100644 index 00000000000..38a04d6c4e8 --- /dev/null +++ b/python/vespa/docs/sitemap.xml @@ -0,0 +1,24 @@ +--- +layout: none +search: exclude +--- + +<?xml version="1.0" encoding="UTF-8"?> +<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> + {% for post in site.posts %} + {% unless post.search == "exclude" %} + <url> + <loc>{{site.url}}{{post.url}}</loc> + </url> + {% endunless %} + {% endfor %} + + + {% for page in site.pages %} + {% unless page.search == "exclude" %} + <url> + <loc>{{site.url}}{{ page.url}}</loc> + </url> + {% endunless %} + {% endfor %} +</urlset>
\ No newline at end of file diff --git a/python/vespa/docs/tooltips.json b/python/vespa/docs/tooltips.json new file mode 100644 index 00000000000..cee21d3fcc8 --- /dev/null +++ b/python/vespa/docs/tooltips.json @@ -0,0 +1,19 @@ +--- +layout: null +search: exclude +--- + +{ +"entries": +[ +{% for page in site.tooltips %} +{ +"doc_id": "{{ page.doc_id }}", +"body": "{{ page.content | strip_newlines | replace: '\', '\\\\' | replace: '"', '\\"' }}" +} {% unless forloop.last %},{% endunless %} +{% endfor %} +] +} + + + diff --git a/python/vespa/notebooks/collect_training_data.ipynb b/python/vespa/notebooks/collect_training_data.ipynb new file mode 100644 index 00000000000..ab0952bb11c --- /dev/null +++ b/python/vespa/notebooks/collect_training_data.ipynb @@ -0,0 +1,1231 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# hide\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Vespa - collect training data\n", + "\n", + "> Collect training data to analyse and/or improve ranking functions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Connect to the application and define a query model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from vespa.application import Vespa\n", + "from vespa.query import Query, RankProfile, OR\n", + "\n", + "app = Vespa(url = \"https://api.cord19.vespa.ai\")\n", + "query_model = Query(\n", + " match_phase = OR(),\n", + " rank_profile = RankProfile(name=\"bm25\", list_features=True))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define some labelled data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "labelled_data = [\n", + " {\n", + " \"query_id\": 0, \n", + " \"query\": \"Intrauterine virus infections and congenital heart disease\",\n", + " \"relevant_docs\": [{\"id\": 0, \"score\": 1}, {\"id\": 3, \"score\": 1}]\n", + " },\n", + " {\n", + " \"query_id\": 1, \n", + " \"query\": \"Clinical and immunologic studies in identical twins discordant for systemic lupus erythematosus\",\n", + " \"relevant_docs\": [{\"id\": 1, \"score\": 1}, {\"id\": 5, \"score\": 1}]\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Collect training data in batch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>attributeMatch(authors.first)</th>\n", + " <th>attributeMatch(authors.first).averageWeight</th>\n", + " <th>attributeMatch(authors.first).completeness</th>\n", + " <th>attributeMatch(authors.first).fieldCompleteness</th>\n", + " <th>attributeMatch(authors.first).importance</th>\n", + " <th>attributeMatch(authors.first).matches</th>\n", + " <th>attributeMatch(authors.first).maxWeight</th>\n", + " <th>attributeMatch(authors.first).normalizedWeight</th>\n", + " <th>attributeMatch(authors.first).normalizedWeightedWeight</th>\n", + " <th>attributeMatch(authors.first).queryCompleteness</th>\n", + " <th>...</th>\n", + " <th>textSimilarity(results).queryCoverage</th>\n", + " <th>textSimilarity(results).score</th>\n", + " <th>textSimilarity(title).fieldCoverage</th>\n", + " <th>textSimilarity(title).order</th>\n", + " <th>textSimilarity(title).proximity</th>\n", + " <th>textSimilarity(title).queryCoverage</th>\n", + " <th>textSimilarity(title).score</th>\n", + " <th>document_id</th>\n", + " <th>query_id</th>\n", + " <th>relevant</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.000000</td>\n", + " <td>0.000000</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>56212</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.187500</td>\n", + " <td>0.5</td>\n", + " <td>0.617188</td>\n", + " <td>0.428571</td>\n", + " <td>0.457087</td>\n", + " <td>34026</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.000000</td>\n", + " <td>0.000000</td>\n", + " <td>3</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>56212</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>5</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.187500</td>\n", + " <td>0.5</td>\n", + " <td>0.617188</td>\n", + " <td>0.428571</td>\n", + " <td>0.457087</td>\n", + " <td>34026</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>6</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.071429</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.083333</td>\n", + " <td>0.039286</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " </tr>\n", + " <tr>\n", + " <th>7</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>29774</td>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>8</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.500000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>0.333333</td>\n", + " <td>0.700000</td>\n", + " <td>22787</td>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>9</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.058824</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.083333</td>\n", + " <td>0.036765</td>\n", + " <td>5</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " </tr>\n", + " <tr>\n", + " <th>10</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>29774</td>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>11</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.500000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>0.333333</td>\n", + " <td>0.700000</td>\n", + " <td>22787</td>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "<p>12 rows × 984 columns</p>\n", + "</div>" + ], + "text/plain": [ + " attributeMatch(authors.first) \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).averageWeight \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).completeness \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).fieldCompleteness \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).importance \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).matches \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).maxWeight \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).normalizedWeight \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).normalizedWeightedWeight \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).queryCompleteness ... \\\n", + "0 0.0 ... \n", + "1 0.0 ... \n", + "2 0.0 ... \n", + "3 0.0 ... \n", + "4 0.0 ... \n", + "5 0.0 ... \n", + "6 0.0 ... \n", + "7 0.0 ... \n", + "8 0.0 ... \n", + "9 0.0 ... \n", + "10 0.0 ... \n", + "11 0.0 ... \n", + "\n", + " textSimilarity(results).queryCoverage textSimilarity(results).score \\\n", + "0 0.0 0.0 \n", + "1 0.0 0.0 \n", + "2 0.0 0.0 \n", + "3 0.0 0.0 \n", + "4 0.0 0.0 \n", + "5 0.0 0.0 \n", + "6 0.0 0.0 \n", + "7 0.0 0.0 \n", + "8 0.0 0.0 \n", + "9 0.0 0.0 \n", + "10 0.0 0.0 \n", + "11 0.0 0.0 \n", + "\n", + " textSimilarity(title).fieldCoverage textSimilarity(title).order \\\n", + "0 0.000000 0.0 \n", + "1 1.000000 1.0 \n", + "2 0.187500 0.5 \n", + "3 0.000000 0.0 \n", + "4 1.000000 1.0 \n", + "5 0.187500 0.5 \n", + "6 0.071429 0.0 \n", + "7 1.000000 1.0 \n", + "8 0.500000 1.0 \n", + "9 0.058824 0.0 \n", + "10 1.000000 1.0 \n", + "11 0.500000 1.0 \n", + "\n", + " textSimilarity(title).proximity textSimilarity(title).queryCoverage \\\n", + "0 0.000000 0.000000 \n", + "1 1.000000 1.000000 \n", + "2 0.617188 0.428571 \n", + "3 0.000000 0.000000 \n", + "4 1.000000 1.000000 \n", + "5 0.617188 0.428571 \n", + "6 0.000000 0.083333 \n", + "7 1.000000 1.000000 \n", + "8 1.000000 0.333333 \n", + "9 0.000000 0.083333 \n", + "10 1.000000 1.000000 \n", + "11 1.000000 0.333333 \n", + "\n", + " textSimilarity(title).score document_id query_id relevant \n", + "0 0.000000 0 0 1 \n", + "1 1.000000 56212 0 0 \n", + "2 0.457087 34026 0 0 \n", + "3 0.000000 3 0 1 \n", + "4 1.000000 56212 0 0 \n", + "5 0.457087 34026 0 0 \n", + "6 0.039286 1 1 1 \n", + "7 1.000000 29774 1 0 \n", + "8 0.700000 22787 1 0 \n", + "9 0.036765 5 1 1 \n", + "10 1.000000 29774 1 0 \n", + "11 0.700000 22787 1 0 \n", + "\n", + "[12 rows x 984 columns]" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "training_data_batch = app.collect_training_data(\n", + " labelled_data = labelled_data,\n", + " id_field = \"id\",\n", + " query_model = query_model,\n", + " number_additional_docs = 2\n", + ")\n", + "training_data_batch" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Collect training data point\n", + "\n", + "> You can have finer control with the `collect_training_data_point` method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>attributeMatch(authors.first)</th>\n", + " <th>attributeMatch(authors.first).averageWeight</th>\n", + " <th>attributeMatch(authors.first).completeness</th>\n", + " <th>attributeMatch(authors.first).fieldCompleteness</th>\n", + " <th>attributeMatch(authors.first).importance</th>\n", + " <th>attributeMatch(authors.first).matches</th>\n", + " <th>attributeMatch(authors.first).maxWeight</th>\n", + " <th>attributeMatch(authors.first).normalizedWeight</th>\n", + " <th>attributeMatch(authors.first).normalizedWeightedWeight</th>\n", + " <th>attributeMatch(authors.first).queryCompleteness</th>\n", + " <th>...</th>\n", + " <th>textSimilarity(results).queryCoverage</th>\n", + " <th>textSimilarity(results).score</th>\n", + " <th>textSimilarity(title).fieldCoverage</th>\n", + " <th>textSimilarity(title).order</th>\n", + " <th>textSimilarity(title).proximity</th>\n", + " <th>textSimilarity(title).queryCoverage</th>\n", + " <th>textSimilarity(title).score</th>\n", + " <th>document_id</th>\n", + " <th>query_id</th>\n", + " <th>relevant</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.000000</td>\n", + " <td>0.000000</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>56212</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.187500</td>\n", + " <td>0.5</td>\n", + " <td>0.617188</td>\n", + " <td>0.428571</td>\n", + " <td>0.457087</td>\n", + " <td>34026</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.000000</td>\n", + " <td>0.000000</td>\n", + " <td>3</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>56212</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>5</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.187500</td>\n", + " <td>0.5</td>\n", + " <td>0.617188</td>\n", + " <td>0.428571</td>\n", + " <td>0.457087</td>\n", + " <td>34026</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>6</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.071429</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.083333</td>\n", + " <td>0.039286</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " </tr>\n", + " <tr>\n", + " <th>7</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>29774</td>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>8</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.500000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>0.333333</td>\n", + " <td>0.700000</td>\n", + " <td>22787</td>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>9</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.058824</td>\n", + " <td>0.0</td>\n", + " <td>0.000000</td>\n", + " <td>0.083333</td>\n", + " <td>0.036765</td>\n", + " <td>5</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " </tr>\n", + " <tr>\n", + " <th>10</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>1.000000</td>\n", + " <td>29774</td>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>11</th>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>...</td>\n", + " <td>0.0</td>\n", + " <td>0.0</td>\n", + " <td>0.500000</td>\n", + " <td>1.0</td>\n", + " <td>1.000000</td>\n", + " <td>0.333333</td>\n", + " <td>0.700000</td>\n", + " <td>22787</td>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "<p>12 rows × 984 columns</p>\n", + "</div>" + ], + "text/plain": [ + " attributeMatch(authors.first) \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).averageWeight \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).completeness \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).fieldCompleteness \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).importance \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).matches \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).maxWeight \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).normalizedWeight \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).normalizedWeightedWeight \\\n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "6 0.0 \n", + "7 0.0 \n", + "8 0.0 \n", + "9 0.0 \n", + "10 0.0 \n", + "11 0.0 \n", + "\n", + " attributeMatch(authors.first).queryCompleteness ... \\\n", + "0 0.0 ... \n", + "1 0.0 ... \n", + "2 0.0 ... \n", + "3 0.0 ... \n", + "4 0.0 ... \n", + "5 0.0 ... \n", + "6 0.0 ... \n", + "7 0.0 ... \n", + "8 0.0 ... \n", + "9 0.0 ... \n", + "10 0.0 ... \n", + "11 0.0 ... \n", + "\n", + " textSimilarity(results).queryCoverage textSimilarity(results).score \\\n", + "0 0.0 0.0 \n", + "1 0.0 0.0 \n", + "2 0.0 0.0 \n", + "3 0.0 0.0 \n", + "4 0.0 0.0 \n", + "5 0.0 0.0 \n", + "6 0.0 0.0 \n", + "7 0.0 0.0 \n", + "8 0.0 0.0 \n", + "9 0.0 0.0 \n", + "10 0.0 0.0 \n", + "11 0.0 0.0 \n", + "\n", + " textSimilarity(title).fieldCoverage textSimilarity(title).order \\\n", + "0 0.000000 0.0 \n", + "1 1.000000 1.0 \n", + "2 0.187500 0.5 \n", + "3 0.000000 0.0 \n", + "4 1.000000 1.0 \n", + "5 0.187500 0.5 \n", + "6 0.071429 0.0 \n", + "7 1.000000 1.0 \n", + "8 0.500000 1.0 \n", + "9 0.058824 0.0 \n", + "10 1.000000 1.0 \n", + "11 0.500000 1.0 \n", + "\n", + " textSimilarity(title).proximity textSimilarity(title).queryCoverage \\\n", + "0 0.000000 0.000000 \n", + "1 1.000000 1.000000 \n", + "2 0.617188 0.428571 \n", + "3 0.000000 0.000000 \n", + "4 1.000000 1.000000 \n", + "5 0.617188 0.428571 \n", + "6 0.000000 0.083333 \n", + "7 1.000000 1.000000 \n", + "8 1.000000 0.333333 \n", + "9 0.000000 0.083333 \n", + "10 1.000000 1.000000 \n", + "11 1.000000 0.333333 \n", + "\n", + " textSimilarity(title).score document_id query_id relevant \n", + "0 0.000000 0 0 1 \n", + "1 1.000000 56212 0 0 \n", + "2 0.457087 34026 0 0 \n", + "3 0.000000 3 0 1 \n", + "4 1.000000 56212 0 0 \n", + "5 0.457087 34026 0 0 \n", + "6 0.039286 1 1 1 \n", + "7 1.000000 29774 1 0 \n", + "8 0.700000 22787 1 0 \n", + "9 0.036765 5 1 1 \n", + "10 1.000000 29774 1 0 \n", + "11 0.700000 22787 1 0 \n", + "\n", + "[12 rows x 984 columns]" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pandas import concat, DataFrame\n", + "\n", + "\n", + "training_data = []\n", + "for query_data in labelled_data:\n", + " for doc_data in query_data[\"relevant_docs\"]:\n", + " training_data_point = app.collect_training_data_point(\n", + " query = query_data[\"query\"],\n", + " query_id = query_data[\"query_id\"],\n", + " relevant_id = doc_data[\"id\"],\n", + " id_field = \"id\",\n", + " query_model = query_model,\n", + " number_additional_docs = 2\n", + " )\n", + " training_data.extend(training_data_point)\n", + "training_data = DataFrame.from_records(training_data)\n", + "training_data" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "vespa", + "language": "python", + "name": "vespa" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/vespa/notebooks/evaluation.ipynb b/python/vespa/notebooks/evaluation.ipynb new file mode 100644 index 00000000000..9a37effc691 --- /dev/null +++ b/python/vespa/notebooks/evaluation.ipynb @@ -0,0 +1,296 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# hide\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Vespa - Evaluate query models\n", + "\n", + "> Define metrics and evaluate query models" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Connect to the application and define a query model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from vespa.application import Vespa\n", + "from vespa.query import Query, RankProfile, OR\n", + "\n", + "app = Vespa(url = \"https://api.cord19.vespa.ai\")\n", + "query_model = Query(\n", + " match_phase = OR(),\n", + " rank_profile = RankProfile(name=\"bm25\", list_features=True))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define some labelled data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "labelled_data = [\n", + " {\n", + " \"query_id\": 0, \n", + " \"query\": \"Intrauterine virus infections and congenital heart disease\",\n", + " \"relevant_docs\": [{\"id\": 0, \"score\": 1}, {\"id\": 3, \"score\": 1}]\n", + " },\n", + " {\n", + " \"query_id\": 1, \n", + " \"query\": \"Clinical and immunologic studies in identical twins discordant for systemic lupus erythematosus\",\n", + " \"relevant_docs\": [{\"id\": 1, \"score\": 1}, {\"id\": 5, \"score\": 1}]\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from vespa.evaluation import MatchRatio, Recall, ReciprocalRank\n", + "\n", + "eval_metrics = [MatchRatio(), Recall(at=10), ReciprocalRank(at=10)]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluate in batch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>query_id</th>\n", + " <th>match_ratio_retrieved_docs</th>\n", + " <th>match_ratio_docs_available</th>\n", + " <th>match_ratio_value</th>\n", + " <th>recall_10_value</th>\n", + " <th>reciprocal_rank_10_value</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>0</td>\n", + " <td>52526</td>\n", + " <td>58692</td>\n", + " <td>0.894943</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>1</td>\n", + " <td>54048</td>\n", + " <td>58692</td>\n", + " <td>0.920875</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " query_id match_ratio_retrieved_docs match_ratio_docs_available \\\n", + "0 0 52526 58692 \n", + "1 1 54048 58692 \n", + "\n", + " match_ratio_value recall_10_value reciprocal_rank_10_value \n", + "0 0.894943 0 0 \n", + "1 0.920875 0 0 " + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "evaluation = app.evaluate(\n", + " labelled_data = labelled_data,\n", + " eval_metrics = eval_metrics, \n", + " query_model = query_model, \n", + " id_field = \"id\",\n", + ")\n", + "evaluation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluate specific query\n", + "\n", + "> You can have finer control with the `evaluate_query` method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>query_id</th>\n", + " <th>match_ratio_retrieved_docs</th>\n", + " <th>match_ratio_docs_available</th>\n", + " <th>match_ratio_value</th>\n", + " <th>recall_10_value</th>\n", + " <th>reciprocal_rank_10_value</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>0</td>\n", + " <td>52526</td>\n", + " <td>58692</td>\n", + " <td>0.894943</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>1</td>\n", + " <td>54048</td>\n", + " <td>58692</td>\n", + " <td>0.920875</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " query_id match_ratio_retrieved_docs match_ratio_docs_available \\\n", + "0 0 52526 58692 \n", + "1 1 54048 58692 \n", + "\n", + " match_ratio_value recall_10_value reciprocal_rank_10_value \n", + "0 0.894943 0 0 \n", + "1 0.920875 0 0 " + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pandas import concat, DataFrame\n", + "\n", + "evaluation = []\n", + "for query_data in labelled_data:\n", + " query_evaluation = app.evaluate_query(\n", + " eval_metrics = eval_metrics, \n", + " query_model = query_model, \n", + " query_id = query_data[\"query_id\"], \n", + " query = query_data[\"query\"], \n", + " id_field = \"id\",\n", + " relevant_docs = query_data[\"relevant_docs\"],\n", + " default_score = 0\n", + " )\n", + " evaluation.append(query_evaluation)\n", + "evaluation = DataFrame.from_records(evaluation)\n", + "evaluation" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "vespa", + "language": "python", + "name": "vespa" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/vespa/notebooks/index.ipynb b/python/vespa/notebooks/index.ipynb new file mode 100644 index 00000000000..e6ffdc538e2 --- /dev/null +++ b/python/vespa/notebooks/index.ipynb @@ -0,0 +1,243 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# hide\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Vespa library for data analysis\n", + "\n", + "> Provide data analysis support for Vespa applications" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`pip install pyvespa`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Connect to a Vespa app\n", + "\n", + "> Connect to a running Vespa application" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from vespa.application import Vespa\n", + "\n", + "app = Vespa(url = \"https://api.cord19.vespa.ai\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define a Query model\n", + "\n", + "> Easily define matching and ranking criteria" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from vespa.query import Query, Union, WeakAnd, ANN, RankProfile\n", + "from random import random\n", + "\n", + "match_phase = Union(\n", + " WeakAnd(hits = 10), \n", + " ANN(\n", + " doc_vector=\"title_embedding\", \n", + " query_vector=\"title_vector\", \n", + " embedding_model=lambda x: [random() for x in range(768)],\n", + " hits = 10,\n", + " label=\"title\"\n", + " )\n", + ")\n", + "\n", + "rank_profile = RankProfile(name=\"bm25\", list_features=True)\n", + "\n", + "query_model = Query(match_phase=match_phase, rank_profile=rank_profile)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Query the vespa app\n", + "\n", + "> Send queries via the query API. See the [query page](/vespa/query) for more examples." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query_result = app.query(\n", + " query=\"Is remdesivir an effective treatment for COVID-19?\", \n", + " query_model=query_model\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query_result.number_documents_retrieved" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Labelled data\n", + "\n", + "> How to structure labelled data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "labelled_data = [\n", + " {\n", + " \"query_id\": 0, \n", + " \"query\": \"Intrauterine virus infections and congenital heart disease\",\n", + " \"relevant_docs\": [{\"id\": 0, \"score\": 1}, {\"id\": 3, \"score\": 1}]\n", + " },\n", + " {\n", + " \"query_id\": 1, \n", + " \"query\": \"Clinical and immunologic studies in identical twins discordant for systemic lupus erythematosus\",\n", + " \"relevant_docs\": [{\"id\": 1, \"score\": 1}, {\"id\": 5, \"score\": 1}]\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Non-relevant documents are assigned `\"score\": 0` by default. Relevant documents will be assigned `\"score\": 1` by default if the field is missing from the labelled data. The defaults for both relevant and non-relevant documents can be modified on the appropriate methods." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Collect training data\n", + "\n", + "> Collect training data to analyse and/or improve ranking functions. See the [collect training data page](/vespa/collect_training_data) for more examples." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "training_data_batch = app.collect_training_data(\n", + " labelled_data = labelled_data,\n", + " id_field = \"id\",\n", + " query_model = query_model,\n", + " number_additional_docs = 2\n", + ")\n", + "training_data_batch" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluating a query model\n", + "\n", + "> Define metrics and evaluate query models. See the [evaluation page](/vespa/evaluation) for more examples." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will define the following evaluation metrics:\n", + "* % of documents retrieved per query\n", + "* recall @ 10 per query\n", + "* MRR @ 10 per query" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from vespa.evaluation import MatchRatio, Recall, ReciprocalRank\n", + "\n", + "eval_metrics = [MatchRatio(), Recall(at=10), ReciprocalRank(at=10)]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Evaluate:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "evaluation = app.evaluate(\n", + " labelled_data = labelled_data,\n", + " eval_metrics = eval_metrics, \n", + " query_model = query_model, \n", + " id_field = \"id\",\n", + ")\n", + "evaluation" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "vespa", + "language": "python", + "name": "vespa" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/vespa/notebooks/query.ipynb b/python/vespa/notebooks/query.ipynb new file mode 100644 index 00000000000..27053527b52 --- /dev/null +++ b/python/vespa/notebooks/query.ipynb @@ -0,0 +1,308 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# hide\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Query API\n", + "\n", + "> Python query API" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can connect to the CORD-19 Search app and use it to exemplify the query API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from vespa.application import Vespa\n", + "\n", + "app = Vespa(url = \"https://api.cord19.vespa.ai\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Specify the request body\n", + "\n", + "> Full flexibility by specifying the entire request body" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "body = {\n", + " 'yql': 'select title, abstract from sources * where userQuery();',\n", + " 'hits': 5,\n", + " 'query': 'Is remdesivir an effective treatment for COVID-19?',\n", + " 'type': 'any',\n", + " 'ranking': 'bm25'\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results = app.query(body=body)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "108882" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results.number_documents_retrieved" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Specify a query model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Query + term-matching + rank profile" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from vespa.query import Query, OR, RankProfile\n", + "\n", + "results = app.query(\n", + " query=\"Is remdesivir an effective treatment for COVID-19?\", \n", + " query_model = Query(\n", + " match_phase=OR(), \n", + " rank_profile=RankProfile(name=\"bm25\")\n", + " )\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "108882" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results.number_documents_retrieved" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Query + term-matching + ann operator + rank_profile" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from vespa.query import Query, ANN, WeakAnd, Union, RankProfile\n", + "from random import random\n", + "\n", + "match_phase = Union(\n", + " WeakAnd(hits = 10), \n", + " ANN(\n", + " doc_vector=\"title_embedding\", \n", + " query_vector=\"title_vector\", \n", + " embedding_model=lambda x: [random() for x in range(768)],\n", + " hits = 10,\n", + " label=\"title\"\n", + " )\n", + ")\n", + "rank_profile = RankProfile(name=\"bm25\", list_features=True)\n", + "query_model = Query(match_phase=match_phase, rank_profile=rank_profile)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results = app.query(query=\"Is remdesivir an effective treatment for COVID-19?\", \n", + " query_model=query_model)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "947" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results.number_documents_retrieved" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Recall specific documents" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's take a look at the top 3 ids from the last query." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[117166, 60125, 28903]" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "top_ids = [hit[\"fields\"][\"id\"] for hit in results.hits[0:3]]\n", + "top_ids" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Assume that we now want to retrieve the second and third ids above. We can do so with the `recall` argument." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results_with_recall = app.query(query=\"Is remdesivir an effective treatment for COVID-19?\", \n", + " query_model=query_model,\n", + " recall = (\"id\", top_ids[1:3]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It will only retrieve the documents with Vespa field `id` that is defined on the list that is inside the tuple." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[60125, 28903]" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "id_recalled = [hit[\"fields\"][\"id\"] for hit in results_with_recall.hits]\n", + "id_recalled" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#hide\n", + "from fastcore.test import all_equal, test\n", + "\n", + "test(id_recalled, top_ids[1:3], all_equal)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "vespa", + "language": "python", + "name": "vespa" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/vespa/settings.ini b/python/vespa/settings.ini new file mode 100644 index 00000000000..60394dcb425 --- /dev/null +++ b/python/vespa/settings.ini @@ -0,0 +1,56 @@ +[DEFAULT] +# All sections below are required unless otherwise specified +host = github +lib_name = pyvespa +user = vespa-engine +description = Vespa python API +keywords = vespa, search engine, data science +author = Thiago G. Martins +author_email = tmartins@verizonmedia.com +copyright = Verizon Media +branch = master +version = 0.0.1 +min_python = 3.6 +audience = Developers +language = English +# Set to True if you want to create a more fancy sidebar.json than the default +custom_sidebar = False +# Add licenses and see current list in `setup.py` +license = apache2 +# From 1-7: Planning Pre-Alpha Alpha Beta Production Mature Inactive +status = 2 + +# Optional. Same format as setuptools requirements +requirements = requests pandas +# Optional. Same format as setuptools console_scripts +# console_scripts = +# Optional. Same format as setuptools dependency-links +# dep_links = + +### +# You probably won't need to change anything under here, +# unless you have some special requirements +### + +# Change to, e.g. "nbs", to put your notebooks in nbs dir instead of repo root +nbs_path = notebooks +doc_path = docs + +# Anything shown as '%(...)s' is substituted with that setting automatically +doc_host = https://%(user)s.github.io +doc_baseurl = /%(lib_name)s/ +git_url = https://github.com/%(user)s/%(lib_name)s/tree/%(branch)s/ +lib_path = %(lib_name)s +title = %(lib_name)s + +#Optional advanced parameters +#Monospace docstings: adds <pre> tags around the doc strings, preserving newlines/indentation. +#monospace_docstrings = False +#Test flags: introduce here the test flags you want to use separated by | +#tst_flags = +#Custom sidebar: customize sidebar.json yourself for advanced sidebars (False/True) +#custom_sidebar = +#Cell spacing: if you want cell blocks in code separated by more than one new line +#cell_spacing = +#Custom jekyll styles: if you want more jekyll styles than tip/important/warning, set them here +#jekyll_styles = note,warning,tip,important
\ No newline at end of file diff --git a/python/vespa/setup.py b/python/vespa/setup.py new file mode 100644 index 00000000000..2dea754602e --- /dev/null +++ b/python/vespa/setup.py @@ -0,0 +1,77 @@ +import os +from pkg_resources import parse_version +from configparser import ConfigParser +import setuptools + +assert parse_version(setuptools.__version__) >= parse_version("36.2") + +# note: all settings are in settings.ini; edit there, not here +config = ConfigParser(delimiters=["="]) +config.read("settings.ini") +cfg = config["DEFAULT"] + +cfg_keys = "description keywords author author_email".split() +expected = ( + cfg_keys + + "lib_name user branch license status min_python audience language".split() +) +for o in expected: + assert o in cfg, "missing expected setting: {}".format(o) +setup_cfg = {o: cfg[o] for o in cfg_keys} + +licenses = { + "apache2": ( + "Apache Software License 2.0", + "OSI Approved :: Apache Software License", + ), +} +statuses = [ + "1 - Planning", + "2 - Pre-Alpha", + "3 - Alpha", + "4 - Beta", + "5 - Production/Stable", + "6 - Mature", + "7 - Inactive", +] +py_versions = ( + "2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8".split() +) + +requirements = cfg.get("requirements", "").split() +lic = licenses[cfg["license"]] +min_python = cfg["min_python"] + + +def get_target_version(): + build_nr = os.environ.get("GITHUB_RUN_NUMBER", "0+dev") + version = "0.1" + return "{}.{}".format(version, build_nr) + + +setuptools.setup( + name=cfg["lib_name"], + version=get_target_version(), + license=lic[0], + classifiers=[ + "Development Status :: " + statuses[int(cfg["status"])], + "Intended Audience :: " + cfg["audience"].title(), + "License :: " + lic[1], + "Natural Language :: " + cfg["language"].title(), + ] + + [ + "Programming Language :: Python :: " + o + for o in py_versions[py_versions.index(min_python) :] + ], + url=cfg["git_url"], + packages=setuptools.find_packages(), + include_package_data=True, + install_requires=requirements, + dependency_links=cfg.get("dep_links", "").split(), + python_requires=">=" + cfg["min_python"], + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + zip_safe=False, + entry_points={"console_scripts": cfg.get("console_scripts", "").split()}, + **setup_cfg +) diff --git a/python/vespa/vespa/__init__.py b/python/vespa/vespa/__init__.py new file mode 100644 index 00000000000..b506a040722 --- /dev/null +++ b/python/vespa/vespa/__init__.py @@ -0,0 +1,3 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +__version__ = "0.0.1" diff --git a/python/vespa/vespa/_nbdev.py b/python/vespa/vespa/_nbdev.py new file mode 100644 index 00000000000..b68d7b2f4bc --- /dev/null +++ b/python/vespa/vespa/_nbdev.py @@ -0,0 +1,13 @@ +# AUTOGENERATED BY NBDEV! DO NOT EDIT! + +__all__ = ["index", "modules", "custom_doc_links", "git_url"] + +index = {} + +modules = [] + +doc_url = "https://vespa-engine.github.io/vespa/" + +git_url = "https://github.com/vespa-engine/vespa/tree/master/" + +def custom_doc_links(name): return None diff --git a/python/vespa/vespa/application.py b/python/vespa/vespa/application.py new file mode 100644 index 00000000000..3646cf87bf2 --- /dev/null +++ b/python/vespa/vespa/application.py @@ -0,0 +1,267 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +from typing import Optional, Dict, Tuple, List +from requests import post +from pandas import DataFrame + +from vespa.query import Query, VespaResult +from vespa.evaluation import EvalMetric + + +class Vespa(object): + def __init__(self, url: str, port: Optional[int] = None) -> None: + """ + Establish a connection with a Vespa application. + + :param url: URL + :param port: Port + + >>> Vespa(url = "https://cord19.vespa.ai") + >>> Vespa(url = "http://localhost", port = 8080) + + """ + self.url = url + self.port = port + + if port is None: + self.end_point = self.url + else: + self.end_point = str(url).rstrip("/") + ":" + str(port) + self.search_end_point = self.end_point + "/search/" + + def query( + self, + body: Optional[Dict] = None, + query: Optional[str] = None, + query_model: Optional[Query] = None, + debug_request: bool = False, + recall: Optional[Tuple] = None, + **kwargs + ) -> VespaResult: + """ + Send a query request to the Vespa application. + + Either send 'body' containing all the request parameters or specify 'query' and 'query_model'. + + :param body: Dict containing all the request parameters. + :param query: Query string + :param query_model: Query model + :param debug_request: return request body for debugging instead of sending the request. + :param recall: Tuple of size 2 where the first element is the name of the field to use to recall and the + second element is a list of the values to be recalled. + :param kwargs: Additional parameters to be sent along the request. + :return: Either the request body if debug_request is True or the result from the Vespa application + """ + + if body is None: + assert query is not None, "No 'query' specified." + assert query_model is not None, "No 'query_model' specified." + body = query_model.create_body(query=query) + if recall is not None: + body.update( + { + "recall": "+(" + + " ".join( + ["{}:{}".format(recall[0], str(doc)) for doc in recall[1]] + ) + + ")" + } + ) + + body.update(kwargs) + + if debug_request: + return VespaResult(vespa_result={}, request_body=body) + else: + r = post(self.search_end_point, json=body) + return VespaResult(vespa_result=r.json()) + + def collect_training_data_point( + self, + query: str, + query_id: str, + relevant_id: str, + id_field: str, + query_model: Query, + number_additional_docs: int, + relevant_score: int = 1, + default_score: int = 0, + **kwargs + ) -> List[Dict]: + """ + Collect training data based on a single query + + :param query: Query string. + :param query_id: Query id represented as str. + :param relevant_id: Relevant id represented as a str. + :param id_field: The Vespa field representing the document id. + :param query_model: Query model. + :param number_additional_docs: Number of additional documents to retrieve for each relevant document. + :param relevant_score: Score to assign to relevant documents. Default to 1. + :param default_score: Score to assign to the additional documents that are not relevant. Default to 0. + :param kwargs: Extra keyword arguments to be included in the Vespa Query. + :return: List of dicts containing the document id (document_id), query id (query_id), scores (relevant) + and vespa rank features returned by the Query model RankProfile used. + """ + + assert ( + query_model.rank_profile.list_features == "true" + ), "Enable rank features via RankProfile is necessary." + + relevant_id_result = self.query( + query=query, + query_model=query_model, + recall=(id_field, [relevant_id]), + **kwargs + ) + hits = relevant_id_result.hits + features = [] + if len(hits) == 1 and hits[0]["fields"][id_field] == relevant_id: + random_hits_result = self.query( + query=query, + query_model=query_model, + hits=number_additional_docs, + **kwargs + ) + hits.extend(random_hits_result.hits) + + features = annotate_data( + hits=hits, + query_id=query_id, + id_field=id_field, + relevant_id=relevant_id, + relevant_score=relevant_score, + default_score=default_score, + ) + return features + + def collect_training_data( + self, + labelled_data: List[Dict], + id_field: str, + query_model: Query, + number_additional_docs: int, + relevant_score: int = 1, + default_score: int = 0, + **kwargs + ) -> DataFrame: + """ + Collect training data based on a set of labelled data. + + :param labelled_data: Labelled data containing query, query_id and relevant ids. + :param id_field: The Vespa field representing the document id. + :param query_model: Query model. + :param number_additional_docs: Number of additional documents to retrieve for each relevant document. + :param relevant_score: Score to assign to relevant documents. Default to 1. + :param default_score: Score to assign to the additional documents that are not relevant. Default to 0. + :param kwargs: Extra keyword arguments to be included in the Vespa Query. + :return: DataFrame containing document id (document_id), query id (query_id), scores (relevant) + and vespa rank features returned by the Query model RankProfile used. + """ + + training_data = [] + for query_data in labelled_data: + for doc_data in query_data["relevant_docs"]: + training_data_point = self.collect_training_data_point( + query=query_data["query"], + query_id=query_data["query_id"], + relevant_id=doc_data["id"], + id_field=id_field, + query_model=query_model, + number_additional_docs=number_additional_docs, + relevant_score=doc_data.get("score", relevant_score), + default_score=default_score, + **kwargs + ) + training_data.extend(training_data_point) + training_data = DataFrame.from_records(training_data) + return training_data + + def evaluate_query( + self, + eval_metrics: List[EvalMetric], + query_model: Query, + query_id: str, + query: str, + id_field: str, + relevant_docs: List[Dict], + default_score: int = 0, + **kwargs + ) -> Dict: + """ + Evaluate a query according to evaluation metrics + + :param eval_metrics: A list of evaluation metrics. + :param query_model: Query model. + :param query_id: Query id represented as str. + :param query: Query string. + :param id_field: The Vespa field representing the document id. + :param relevant_docs: A list with dicts where each dict contains a doc id a optionally a doc score. + :param default_score: Score to assign to the additional documents that are not relevant. Default to 0. + :param kwargs: Extra keyword arguments to be included in the Vespa Query. + :return: Dict containing query_id and metrics according to the selected evaluation metrics. + """ + + query_results = self.query(query=query, query_model=query_model, **kwargs) + evaluation = {"query_id": query_id} + for evaluator in eval_metrics: + evaluation.update( + evaluator.evaluate_query( + query_results, relevant_docs, id_field, default_score + ) + ) + return evaluation + + def evaluate( + self, + labelled_data: List[Dict], + eval_metrics: List[EvalMetric], + query_model: Query, + id_field: str, + default_score: int = 0, + **kwargs + ) -> DataFrame: + """ + + :param labelled_data: Labelled data containing query, query_id and relevant ids. + :param eval_metrics: A list of evaluation metrics. + :param query_model: Query model. + :param id_field: The Vespa field representing the document id. + :param default_score: Score to assign to the additional documents that are not relevant. Default to 0. + :param kwargs: Extra keyword arguments to be included in the Vespa Query. + :return: DataFrame containing query_id and metrics according to the selected evaluation metrics. + """ + evaluation = [] + for query_data in labelled_data: + evaluation_query = self.evaluate_query( + eval_metrics=eval_metrics, + query_model=query_model, + query_id=query_data["query_id"], + query=query_data["query"], + id_field=id_field, + relevant_docs=query_data["relevant_docs"], + default_score=default_score, + **kwargs + ) + evaluation.append(evaluation_query) + evaluation = DataFrame.from_records(evaluation) + return evaluation + + +# todo: a better pattern for labelled data would be (query_id, query, doc_id, score) with the possibility od +# assigning a specific default value for those docs not mentioned +def annotate_data(hits, query_id, id_field, relevant_id, relevant_score, default_score): + data = [] + for h in hits: + rank_features = h["fields"]["rankfeatures"] + rank_features.update({"document_id": h["fields"][id_field]}) + rank_features.update({"query_id": query_id}) + rank_features.update( + { + "relevant": relevant_score + if h["fields"][id_field] == relevant_id + else default_score + } + ) + data.append(rank_features) + return data diff --git a/python/vespa/vespa/evaluation.py b/python/vespa/vespa/evaluation.py new file mode 100644 index 00000000000..4ca7a1d136b --- /dev/null +++ b/python/vespa/vespa/evaluation.py @@ -0,0 +1,132 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +from typing import Dict, List +from vespa.query import VespaResult + + +class EvalMetric(object): + def __init__(self) -> None: + pass + + def evaluate_query( + self, query_results, relevant_docs, id_field, default_score + ) -> Dict: + raise NotImplementedError + + +class MatchRatio(EvalMetric): + def __init__(self) -> None: + """ + Computes the ratio of documents retrieved by the match phase. + """ + super().__init__() + self.name = "match_ratio" + + def evaluate_query( + self, + query_results: VespaResult, + relevant_docs: List[Dict], + id_field: str, + default_score: int, + ) -> Dict: + """ + Evaluate query results. + + :param query_results: Raw query results returned by Vespa. + :param relevant_docs: A list with dicts where each dict contains a doc id a optionally a doc score. + :param id_field: The Vespa field representing the document id. + :param default_score: Score to assign to the additional documents that are not relevant. Default to 0. + :return: Dict containing the number of retrieved docs (_retrieved_docs), the number of docs available in + the corpus (_docs_available) and the match ratio (_value). + """ + retrieved_docs = query_results.number_documents_retrieved + docs_available = query_results.number_documents_indexed + value = 0 + if docs_available > 0: + value = retrieved_docs / docs_available + return { + str(self.name) + "_retrieved_docs": retrieved_docs, + str(self.name) + "_docs_available": docs_available, + str(self.name) + "_value": value, + } + + +class Recall(EvalMetric): + def __init__(self, at: int) -> None: + """ + Compute the recall at position `at` + + :param at: Maximum position on the resulting list to look for relevant docs. + """ + super().__init__() + self.name = "recall_" + str(at) + self.at = at + + def evaluate_query( + self, + query_results: VespaResult, + relevant_docs: List[Dict], + id_field: str, + default_score: int, + ) -> Dict: + """ + Evaluate query results. + + :param query_results: Raw query results returned by Vespa. + :param relevant_docs: A list with dicts where each dict contains a doc id a optionally a doc score. + :param id_field: The Vespa field representing the document id. + :param default_score: Score to assign to the additional documents that are not relevant. Default to 0. + :return: Dict containing the recall value (_value). + """ + + relevant_ids = {str(doc["id"]) for doc in relevant_docs} + try: + retrieved_ids = { + str(hit["fields"][id_field]) for hit in query_results.hits[: self.at] + } + except KeyError: + retrieved_ids = set() + + return { + str(self.name) + + "_value": len(relevant_ids & retrieved_ids) / len(relevant_ids) + } + + +class ReciprocalRank(EvalMetric): + def __init__(self, at: int): + """ + Compute the reciprocal rank at position `at` + + :param at: Maximum position on the resulting list to look for relevant docs. + """ + super().__init__() + self.name = "reciprocal_rank_" + str(at) + self.at = at + + def evaluate_query( + self, + query_results: VespaResult, + relevant_docs: List[Dict], + id_field: str, + default_score: int, + ) -> Dict: + """ + Evaluate query results. + + :param query_results: Raw query results returned by Vespa. + :param relevant_docs: A list with dicts where each dict contains a doc id a optionally a doc score. + :param id_field: The Vespa field representing the document id. + :param default_score: Score to assign to the additional documents that are not relevant. Default to 0. + :return: Dict containing the reciprocal rank value (_value). + """ + + relevant_ids = {str(doc["id"]) for doc in relevant_docs} + rr = 0 + hits = query_results.hits[: self.at] + for index, hit in enumerate(hits): + if hit["fields"][id_field] in relevant_ids: + rr = 1 / (index + 1) + break + + return {str(self.name) + "_value": rr} diff --git a/python/vespa/vespa/query.py b/python/vespa/vespa/query.py new file mode 100644 index 00000000000..ed67819b821 --- /dev/null +++ b/python/vespa/vespa/query.py @@ -0,0 +1,229 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +from typing import Callable, List, Optional, Dict + + +# +# Match phase +# +class MatchFilter(object): + """ + Abstract class for match filters. + """ + + def create_match_filter(self, query: str) -> str: + """ + Create part of the YQL expression related to the filter. + + :param query: Query input. + :return: Part of the YQL expression related to the filter. + """ + raise NotImplementedError + + def get_query_properties(self, query: Optional[str] = None) -> Dict: + """ + Relevant request properties associated with the filter. + + :param query: Query input. + :return: dict containing the relevant request properties associated with the filter. + """ + raise NotImplementedError + + +class AND(MatchFilter): + def __init__(self) -> None: + """ + Filter that match document containing all the query terms. + """ + super().__init__() + + def create_match_filter(self, query: str) -> str: + return '(userInput("{}"))'.format(query) + + def get_query_properties(self, query: Optional[str] = None) -> Dict: + return {} + + +class OR(MatchFilter): + def __init__(self) -> None: + """ + Filter that match any document containing at least one query term. + """ + super().__init__() + + def create_match_filter(self, query: str) -> str: + return '([{{"grammar": "any"}}]userInput("{}"))'.format(query) + + def get_query_properties(self, query: Optional[str] = None) -> Dict: + return {} + + +class WeakAnd(MatchFilter): + def __init__(self, hits: int, field: str = "default") -> None: + """ + Match documents according to the weakAND algorithm. + + Reference: https://docs.vespa.ai/documentation/using-wand-with-vespa.html + + :param hits: Lower bound on the number of hits to be retrieved. + :param field: Which Vespa field to search. + """ + super().__init__() + self.hits = hits + self.field = field + + def create_match_filter(self, query: str) -> str: + query_tokens = query.split(" ") + terms = ", ".join( + ['{} contains "{}"'.format(self.field, token) for token in query_tokens] + ) + return '([{{"targetNumHits": {}}}]weakAnd({}))'.format(self.hits, terms) + + def get_query_properties(self, query: Optional[str] = None) -> Dict: + return {} + + +class ANN(MatchFilter): + def __init__( + self, + doc_vector: str, + query_vector: str, + embedding_model: Callable[[str], List[float]], + hits: int, + label: str, + ) -> None: + """ + Match documents according to the nearest neighbor operator. + + Reference: https://docs.vespa.ai/documentation/reference/query-language-reference.html#nearestneighbor + + :param doc_vector: Name of the document field to be used in the distance calculation. + :param query_vector: Name of the query field to be used in the distance calculation. + :param embedding_model: Model that takes query str as input and return list of floats as output. + :param hits: Lower bound on the number of hits to return. + :param label: A label to identify this specific operator instance. + """ + super().__init__() + self.doc_vector = doc_vector + self.query_vector = query_vector + self.embedding_model = embedding_model + self.hits = hits + self.label = label + + def create_match_filter(self, query: str) -> str: + return '([{{"targetNumHits": {}, "label": "{}"}}]nearestNeighbor({}, {}))'.format( + self.hits, self.label, self.doc_vector, self.query_vector + ) + + def get_query_properties(self, query: Optional[str] = None) -> Dict[str, str]: + embedding_vector = self.embedding_model(query) + return { + "ranking.features.query({})".format(self.query_vector): str( + embedding_vector + ) + } + + +class Union(MatchFilter): + def __init__(self, *args: MatchFilter) -> None: + """ + Match documents that belongs to the union of many match filters. + + :param args: Match filters to be taken the union of. + """ + super().__init__() + self.operators = args + + def create_match_filter(self, query: str) -> str: + match_filters = [] + for operator in self.operators: + match_filter = operator.create_match_filter(query=query) + if match_filter is not None: + match_filters.append(match_filter) + return " or ".join(match_filters) + + def get_query_properties(self, query: Optional[str] = None) -> Dict[str, str]: + query_properties = {} + for operator in self.operators: + query_properties.update(operator.get_query_properties(query=query)) + return query_properties + + +# +# Ranking phase +# +class RankProfile(object): + def __init__(self, name: str = "default", list_features: bool = False) -> None: + """ + Define a rank profile. + + :param name: Name of the rank profile as defined in a Vespa search definition. + :param list_features: Should the ranking features be returned. Either 'true' or 'false'. + """ + self.name = name + self.list_features = "false" + if list_features: + self.list_features = "true" + + +class Query(object): + def __init__( + self, + match_phase: MatchFilter = AND(), + rank_profile: RankProfile = RankProfile(), + ) -> None: + """ + Define a query model. + + :param match_phase: Define the match criteria. One of the MatchFilter options available. + :param rank_profile: Define the rank criteria. + """ + self.match_phase = match_phase + self.rank_profile = rank_profile + + def create_body(self, query: str) -> Dict[str, str]: + """ + Create the appropriate request body to be sent to Vespa. + + :param query: Query input. + :return: dict representing the request body. + """ + + match_filter = self.match_phase.create_match_filter(query=query) + query_properties = self.match_phase.get_query_properties(query=query) + + body = { + "yql": "select * from sources * where {};".format(match_filter), + "ranking": { + "profile": self.rank_profile.name, + "listFeatures": self.rank_profile.list_features, + }, + } + body.update(query_properties) + return body + + +class VespaResult(object): + def __init__(self, vespa_result, request_body=None): + self._vespa_result = vespa_result + self._request_body = request_body + + @property + def request_body(self) -> Optional[Dict]: + return self._request_body + + @property + def json(self) -> Dict: + return self._vespa_result + + @property + def hits(self) -> List: + return self._vespa_result.get("root", {}).get("children", []) + + @property + def number_documents_retrieved(self) -> int: + return self._vespa_result.get("root", {}).get("fields", {}).get("totalCount", 0) + + @property + def number_documents_indexed(self) -> int: + return self._vespa_result.get("root", {}).get("coverage", {}).get("documents", 0) diff --git a/python/vespa/vespa/test_application.py b/python/vespa/vespa/test_application.py new file mode 100644 index 00000000000..84bd1c0a6ad --- /dev/null +++ b/python/vespa/vespa/test_application.py @@ -0,0 +1,375 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import unittest +from unittest.mock import Mock, call +from pandas import DataFrame +from pandas.testing import assert_frame_equal + +from vespa.application import Vespa +from vespa.query import Query, OR, RankProfile, VespaResult + + +class TestVespa(unittest.TestCase): + def test_end_point(self): + self.assertEqual( + Vespa(url="https://cord19.vespa.ai").end_point, "https://cord19.vespa.ai" + ) + self.assertEqual( + Vespa(url="http://localhost", port=8080).end_point, "http://localhost:8080" + ) + self.assertEqual( + Vespa(url="http://localhost/", port=8080).end_point, "http://localhost:8080" + ) + + +class TestVespaQuery(unittest.TestCase): + def test_query(self): + app = Vespa(url="http://localhost", port=8080) + + body = {"yql": "select * from sources * where test"} + self.assertDictEqual( + app.query(body=body, debug_request=True).request_body, body + ) + + self.assertDictEqual( + app.query( + query="this is a test", + query_model=Query(match_phase=OR(), rank_profile=RankProfile()), + debug_request=True, + hits=10, + ).request_body, + { + "yql": 'select * from sources * where ([{"grammar": "any"}]userInput("this is a test"));', + "ranking": {"profile": "default", "listFeatures": "false"}, + "hits": 10, + }, + ) + + self.assertDictEqual( + app.query( + query="this is a test", + query_model=Query(match_phase=OR(), rank_profile=RankProfile()), + debug_request=True, + hits=10, + recall=("id", [1, 5]), + ).request_body, + { + "yql": 'select * from sources * where ([{"grammar": "any"}]userInput("this is a test"));', + "ranking": {"profile": "default", "listFeatures": "false"}, + "hits": 10, + "recall": "+(id:1 id:5)", + }, + ) + + +class TestVespaCollectData(unittest.TestCase): + def setUp(self) -> None: + self.app = Vespa(url="http://localhost", port=8080) + self.raw_vespa_result_recall = { + "root": { + "id": "toplevel", + "relevance": 1.0, + "fields": {"totalCount": 1083}, + "coverage": { + "coverage": 100, + "documents": 62529, + "full": True, + "nodes": 2, + "results": 1, + "resultsFull": 1, + }, + "children": [ + { + "id": "id:covid-19:doc::40215", + "relevance": 30.368213170494712, + "source": "content", + "fields": { + "vespa_id_field": "abc", + "sddocname": "doc", + "body_text": "this is a body", + "title": "this is a title", + "rankfeatures": {"a": 1, "b": 2}, + }, + } + ], + } + } + + self.raw_vespa_result_additional = { + "root": { + "id": "toplevel", + "relevance": 1.0, + "fields": {"totalCount": 1083}, + "coverage": { + "coverage": 100, + "documents": 62529, + "full": True, + "nodes": 2, + "results": 1, + "resultsFull": 1, + }, + "children": [ + { + "id": "id:covid-19:doc::40216", + "relevance": 10, + "source": "content", + "fields": { + "vespa_id_field": "def", + "sddocname": "doc", + "body_text": "this is a body 2", + "title": "this is a title 2", + "rankfeatures": {"a": 3, "b": 4}, + }, + }, + { + "id": "id:covid-19:doc::40217", + "relevance": 8, + "source": "content", + "fields": { + "vespa_id_field": "ghi", + "sddocname": "doc", + "body_text": "this is a body 3", + "title": "this is a title 3", + "rankfeatures": {"a": 5, "b": 6}, + }, + }, + ], + } + } + + def test_disable_rank_features(self): + with self.assertRaises(AssertionError): + self.app.collect_training_data_point( + query="this is a query", + query_id="123", + relevant_id="abc", + id_field="vespa_id_field", + query_model=Query(), + number_additional_docs=2, + ) + + def test_collect_training_data_point(self): + + self.app.query = Mock( + side_effect=[ + VespaResult(self.raw_vespa_result_recall), + VespaResult(self.raw_vespa_result_additional), + ] + ) + query_model = Query(rank_profile=RankProfile(list_features=True)) + data = self.app.collect_training_data_point( + query="this is a query", + query_id="123", + relevant_id="abc", + id_field="vespa_id_field", + query_model=query_model, + number_additional_docs=2, + timeout="15s", + ) + + self.assertEqual(self.app.query.call_count, 2) + self.app.query.assert_has_calls( + [ + call( + query="this is a query", + query_model=query_model, + recall=("vespa_id_field", ["abc"]), + timeout="15s", + ), + call( + query="this is a query", + query_model=query_model, + hits=2, + timeout="15s", + ), + ] + ) + expected_data = [ + {"document_id": "abc", "query_id": "123", "relevant": 1, "a": 1, "b": 2}, + {"document_id": "def", "query_id": "123", "relevant": 0, "a": 3, "b": 4}, + {"document_id": "ghi", "query_id": "123", "relevant": 0, "a": 5, "b": 6}, + ] + self.assertEqual(data, expected_data) + + def test_collect_training_data_point_0_recall_hits(self): + + self.raw_vespa_result_recall = { + "root": { + "id": "toplevel", + "relevance": 1.0, + "fields": {"totalCount": 0}, + "coverage": { + "coverage": 100, + "documents": 62529, + "full": True, + "nodes": 2, + "results": 1, + "resultsFull": 1, + }, + } + } + self.app.query = Mock( + side_effect=[ + VespaResult(self.raw_vespa_result_recall), + VespaResult(self.raw_vespa_result_additional), + ] + ) + query_model = Query(rank_profile=RankProfile(list_features=True)) + data = self.app.collect_training_data_point( + query="this is a query", + query_id="123", + relevant_id="abc", + id_field="vespa_id_field", + query_model=query_model, + number_additional_docs=2, + timeout="15s", + ) + + self.assertEqual(self.app.query.call_count, 1) + self.app.query.assert_has_calls( + [ + call( + query="this is a query", + query_model=query_model, + recall=("vespa_id_field", ["abc"]), + timeout="15s", + ), + ] + ) + expected_data = [] + self.assertEqual(data, expected_data) + + def test_collect_training_data(self): + + mock_return_value = [ + {"document_id": "abc", "query_id": "123", "relevant": 1, "a": 1, "b": 2,}, + {"document_id": "def", "query_id": "123", "relevant": 0, "a": 3, "b": 4,}, + {"document_id": "ghi", "query_id": "123", "relevant": 0, "a": 5, "b": 6,}, + ] + self.app.collect_training_data_point = Mock(return_value=mock_return_value) + labelled_data = [ + { + "query_id": 123, + "query": "this is a query", + "relevant_docs": [{"id": "abc", "score": 1}], + } + ] + query_model = Query(rank_profile=RankProfile(list_features=True)) + data = self.app.collect_training_data( + labelled_data=labelled_data, + id_field="vespa_id_field", + query_model=query_model, + number_additional_docs=2, + timeout="15s", + ) + self.app.collect_training_data_point.assert_has_calls( + [ + call( + query="this is a query", + query_id=123, + relevant_id="abc", + id_field="vespa_id_field", + query_model=query_model, + number_additional_docs=2, + relevant_score=1, + default_score=0, + timeout="15s", + ) + ] + ) + assert_frame_equal(data, DataFrame.from_records(mock_return_value)) + + +class TestVespaEvaluate(unittest.TestCase): + def setUp(self) -> None: + self.app = Vespa(url="http://localhost", port=8080) + + self.labelled_data = [ + { + "query_id": 0, + "query": "Intrauterine virus infections and congenital heart disease", + "relevant_docs": [{"id": "def", "score": 1}, {"id": "abc", "score": 1}], + }, + ] + + self.query_results = { + "root": { + "id": "toplevel", + "relevance": 1.0, + "fields": {"totalCount": 1083}, + "coverage": { + "coverage": 100, + "documents": 62529, + "full": True, + "nodes": 2, + "results": 1, + "resultsFull": 1, + }, + "children": [ + { + "id": "id:covid-19:doc::40216", + "relevance": 10, + "source": "content", + "fields": { + "vespa_id_field": "ghi", + "sddocname": "doc", + "body_text": "this is a body 2", + "title": "this is a title 2", + "rankfeatures": {"a": 3, "b": 4}, + }, + }, + { + "id": "id:covid-19:doc::40217", + "relevance": 8, + "source": "content", + "fields": { + "vespa_id_field": "def", + "sddocname": "doc", + "body_text": "this is a body 3", + "title": "this is a title 3", + "rankfeatures": {"a": 5, "b": 6}, + }, + }, + ], + } + } + + def test_evaluate_query(self): + self.app.query = Mock(return_value={}) + eval_metric = Mock() + eval_metric.evaluate_query = Mock(return_value={"metric": 1}) + eval_metric2 = Mock() + eval_metric2.evaluate_query = Mock(return_value={"metric_2": 2}) + query_model = Query() + evaluation = self.app.evaluate_query( + eval_metrics=[eval_metric, eval_metric2], + query_model=query_model, + query_id="0", + query="this is a test", + id_field="vespa_id_field", + relevant_docs=self.labelled_data[0]["relevant_docs"], + default_score=0, + hits=10, + ) + self.assertEqual(self.app.query.call_count, 1) + self.app.query.assert_has_calls( + [call(query="this is a test", query_model=query_model, hits=10),] + ) + self.assertEqual(eval_metric.evaluate_query.call_count, 1) + eval_metric.evaluate_query.assert_has_calls( + [call({}, self.labelled_data[0]["relevant_docs"], "vespa_id_field", 0),] + ) + self.assertDictEqual(evaluation, {"query_id": "0", "metric": 1, "metric_2": 2}) + + def test_evaluate(self): + self.app.evaluate_query = Mock(side_effect=[{"query_id": "0", "metric": 1},]) + evaluation = self.app.evaluate( + labelled_data=self.labelled_data, + eval_metrics=[Mock()], + query_model=Mock(), + id_field="mock", + default_score=0, + ) + assert_frame_equal( + evaluation, DataFrame.from_records([{"query_id": "0", "metric": 1}]) + ) diff --git a/python/vespa/vespa/test_evaluation.py b/python/vespa/vespa/test_evaluation.py new file mode 100644 index 00000000000..b6941985d94 --- /dev/null +++ b/python/vespa/vespa/test_evaluation.py @@ -0,0 +1,186 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import unittest + +from vespa.query import VespaResult +from vespa.evaluation import MatchRatio, Recall, ReciprocalRank + + +class TestEvalMetric(unittest.TestCase): + def setUp(self) -> None: + self.labelled_data = [ + { + "query_id": 0, + "query": "Intrauterine virus infections and congenital heart disease", + "relevant_docs": [{"id": "def", "score": 1}, {"id": "abc", "score": 1}], + }, + ] + + self.query_results = { + "root": { + "id": "toplevel", + "relevance": 1.0, + "fields": {"totalCount": 1083}, + "coverage": { + "coverage": 100, + "documents": 62529, + "full": True, + "nodes": 2, + "results": 1, + "resultsFull": 1, + }, + "children": [ + { + "id": "id:covid-19:doc::40216", + "relevance": 10, + "source": "content", + "fields": { + "vespa_id_field": "ghi", + "sddocname": "doc", + "body_text": "this is a body 2", + "title": "this is a title 2", + "rankfeatures": {"a": 3, "b": 4}, + }, + }, + { + "id": "id:covid-19:doc::40217", + "relevance": 8, + "source": "content", + "fields": { + "vespa_id_field": "def", + "sddocname": "doc", + "body_text": "this is a body 3", + "title": "this is a title 3", + "rankfeatures": {"a": 5, "b": 6}, + }, + }, + ], + } + } + + def test_match_ratio(self): + metric = MatchRatio() + + evaluation = metric.evaluate_query( + query_results=VespaResult(self.query_results), + relevant_docs=self.labelled_data[0]["relevant_docs"], + id_field="vespa_id_field", + default_score=0, + ) + + self.assertDictEqual( + evaluation, + { + "match_ratio_retrieved_docs": 1083, + "match_ratio_docs_available": 62529, + "match_ratio_value": 1083 / 62529, + }, + ) + + evaluation = metric.evaluate_query( + query_results=VespaResult( + { + "root": { + "id": "toplevel", + "relevance": 1.0, + "coverage": { + "coverage": 100, + "documents": 62529, + "full": True, + "nodes": 2, + "results": 1, + "resultsFull": 1, + }, + } + } + ), + relevant_docs=self.labelled_data[0]["relevant_docs"], + id_field="vespa_id_field", + default_score=0, + ) + + self.assertDictEqual( + evaluation, + { + "match_ratio_retrieved_docs": 0, + "match_ratio_docs_available": 62529, + "match_ratio_value": 0 / 62529, + }, + ) + + evaluation = metric.evaluate_query( + query_results=VespaResult( + { + "root": { + "id": "toplevel", + "relevance": 1.0, + "fields": {"totalCount": 1083}, + "coverage": { + "coverage": 100, + "full": True, + "nodes": 2, + "results": 1, + "resultsFull": 1, + }, + } + } + ), + relevant_docs=self.labelled_data[0]["relevant_docs"], + id_field="vespa_id_field", + default_score=0, + ) + + self.assertDictEqual( + evaluation, + { + "match_ratio_retrieved_docs": 1083, + "match_ratio_docs_available": 0, + "match_ratio_value": 0, + }, + ) + + def test_recall(self): + metric = Recall(at=2) + evaluation = metric.evaluate_query( + query_results=VespaResult(self.query_results), + relevant_docs=self.labelled_data[0]["relevant_docs"], + id_field="vespa_id_field", + default_score=0, + ) + self.assertDictEqual( + evaluation, {"recall_2_value": 0.5,}, + ) + + metric = Recall(at=1) + evaluation = metric.evaluate_query( + query_results=VespaResult(self.query_results), + relevant_docs=self.labelled_data[0]["relevant_docs"], + id_field="vespa_id_field", + default_score=0, + ) + self.assertDictEqual( + evaluation, {"recall_1_value": 0.0,}, + ) + + def test_reciprocal_rank(self): + metric = ReciprocalRank(at=2) + evaluation = metric.evaluate_query( + query_results=VespaResult(self.query_results), + relevant_docs=self.labelled_data[0]["relevant_docs"], + id_field="vespa_id_field", + default_score=0, + ) + self.assertDictEqual( + evaluation, {"reciprocal_rank_2_value": 0.5,}, + ) + + metric = ReciprocalRank(at=1) + evaluation = metric.evaluate_query( + query_results=VespaResult(self.query_results), + relevant_docs=self.labelled_data[0]["relevant_docs"], + id_field="vespa_id_field", + default_score=0, + ) + self.assertDictEqual( + evaluation, {"reciprocal_rank_1_value": 0.0,}, + ) diff --git a/python/vespa/vespa/test_query.py b/python/vespa/vespa/test_query.py new file mode 100644 index 00000000000..1e933f25c7d --- /dev/null +++ b/python/vespa/vespa/test_query.py @@ -0,0 +1,190 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import unittest + +from vespa.query import Query, OR, AND, WeakAnd, ANN, Union, RankProfile, VespaResult + + +class TestMatchFilter(unittest.TestCase): + def setUp(self) -> None: + self.query = "this is a test" + + def test_and(self): + match_filter = AND() + self.assertEqual( + match_filter.create_match_filter(query=self.query), + '(userInput("this is a test"))', + ) + self.assertDictEqual(match_filter.get_query_properties(query=self.query), {}) + + def test_or(self): + match_filter = OR() + self.assertEqual( + match_filter.create_match_filter(query=self.query), + '([{"grammar": "any"}]userInput("this is a test"))', + ) + self.assertDictEqual(match_filter.get_query_properties(query=self.query), {}) + + def test_weak_and(self): + match_filter = WeakAnd(hits=10, field="field_name") + self.assertEqual( + match_filter.create_match_filter(query=self.query), + '([{"targetNumHits": 10}]weakAnd(field_name contains "this", field_name contains "is", field_name contains "", ' + 'field_name contains "a", field_name contains "test"))', + ) + self.assertDictEqual(match_filter.get_query_properties(query=self.query), {}) + + def test_ann(self): + match_filter = ANN( + doc_vector="doc_vector", + query_vector="query_vector", + embedding_model=lambda x: [1, 2, 3], + hits=10, + label="label", + ) + self.assertEqual( + match_filter.create_match_filter(query=self.query), + '([{"targetNumHits": 10, "label": "label"}]nearestNeighbor(doc_vector, query_vector))', + ) + self.assertDictEqual( + match_filter.get_query_properties(query=self.query), + {"ranking.features.query(query_vector)": "[1, 2, 3]"}, + ) + + def test_union(self): + match_filter = Union( + WeakAnd(hits=10, field="field_name"), + ANN( + doc_vector="doc_vector", + query_vector="query_vector", + embedding_model=lambda x: [1, 2, 3], + hits=10, + label="label", + ), + ) + self.assertEqual( + match_filter.create_match_filter(query=self.query), + '([{"targetNumHits": 10}]weakAnd(field_name contains "this", field_name contains "is", ' + 'field_name contains "", ' + 'field_name contains "a", field_name contains "test")) or ' + '([{"targetNumHits": 10, "label": "label"}]nearestNeighbor(doc_vector, query_vector))', + ) + self.assertDictEqual( + match_filter.get_query_properties(query=self.query), + {"ranking.features.query(query_vector)": "[1, 2, 3]"}, + ) + + +class TestRankProfile(unittest.TestCase): + def test_rank_profile(self): + rank_profile = RankProfile(name="rank_profile", list_features=True) + self.assertEqual(rank_profile.name, "rank_profile") + self.assertEqual(rank_profile.list_features, "true") + + +class TestQuery(unittest.TestCase): + def setUp(self) -> None: + self.query = "this is a test" + + def test_default(self): + query = Query() + self.assertDictEqual( + query.create_body(query=self.query), + { + "yql": 'select * from sources * where (userInput("this is a test"));', + "ranking": {"profile": "default", "listFeatures": "false"}, + }, + ) + + def test_match_and_rank(self): + query = Query( + match_phase=ANN( + doc_vector="doc_vector", + query_vector="query_vector", + embedding_model=lambda x: [1, 2, 3], + hits=10, + label="label", + ), + rank_profile=RankProfile(name="bm25", list_features=True), + ) + self.assertDictEqual( + query.create_body(query=self.query), + { + "yql": 'select * from sources * where ([{"targetNumHits": 10, "label": "label"}]nearestNeighbor(doc_vector, query_vector));', + "ranking": {"profile": "bm25", "listFeatures": "true"}, + "ranking.features.query(query_vector)": "[1, 2, 3]", + }, + ) + + +class TestVespaResult(unittest.TestCase): + def setUp(self) -> None: + self.raw_vespa_result_empty_hits = { + "root": { + "id": "toplevel", + "relevance": 1.0, + "fields": {"totalCount": 0}, + "coverage": { + "coverage": 100, + "documents": 62529, + "full": True, + "nodes": 2, + "results": 1, + "resultsFull": 1, + }, + } + } + + self.raw_vespa_result = { + "root": { + "id": "toplevel", + "relevance": 1.0, + "fields": {"totalCount": 1083}, + "coverage": { + "coverage": 100, + "documents": 62529, + "full": True, + "nodes": 2, + "results": 1, + "resultsFull": 1, + }, + "children": [ + { + "id": "id:covid-19:doc::40215", + "relevance": 30.368213170494712, + "source": "content", + "fields": { + "sddocname": "doc", + "body_text": "this is a body", + "title": "this is a title", + }, + } + ], + } + } + + def test_json(self): + vespa_result = VespaResult(vespa_result=self.raw_vespa_result) + self.assertDictEqual(vespa_result.json, self.raw_vespa_result) + + def test_hits(self): + empty_hits_vespa_result = VespaResult( + vespa_result=self.raw_vespa_result_empty_hits + ) + self.assertEqual(empty_hits_vespa_result.hits, []) + vespa_result = VespaResult(vespa_result=self.raw_vespa_result) + self.assertEqual( + vespa_result.hits, + [ + { + "id": "id:covid-19:doc::40215", + "relevance": 30.368213170494712, + "source": "content", + "fields": { + "sddocname": "doc", + "body_text": "this is a body", + "title": "this is a title", + }, + } + ], + ) diff --git a/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h b/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h index 72ce1754d71..67b269139d3 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h +++ b/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h @@ -33,7 +33,7 @@ public: * Creates a new object with an initial capacity of 16 without dynamic allocation. **/ AttributeContent() : - _dynamicBuf(NULL), + _dynamicBuf(nullptr), _size(0), _capacity(16) { @@ -42,7 +42,7 @@ public: * Destructs the object. **/ ~AttributeContent() { - if (_dynamicBuf != NULL) { + if (_dynamicBuf != nullptr) { delete [] _dynamicBuf; } } @@ -53,7 +53,7 @@ public: * @return iterator **/ const T * begin() const { - if (_dynamicBuf != NULL) { + if (_dynamicBuf != nullptr) { return _dynamicBuf; } return _staticBuf; @@ -102,7 +102,7 @@ public: * @return read/write pointer. **/ T * data() { - if (_dynamicBuf != NULL) { + if (_dynamicBuf != nullptr) { return _dynamicBuf; } return _staticBuf; @@ -126,7 +126,7 @@ public: **/ void allocate(uint32_t n) { if (n > _capacity) { - if (_dynamicBuf != NULL) { + if (_dynamicBuf != nullptr) { delete [] _dynamicBuf; } _dynamicBuf = new T[n]; @@ -141,8 +141,7 @@ public: * @param attribute the attribute vector * @param docId the docId **/ - void fill(const search::attribute::IAttributeVector & attribute, - search::attribute::IAttributeVector::DocId docId) + void fill(const IAttributeVector & attribute, IAttributeVector::DocId docId) { uint32_t count = attribute.get(docId, data(), capacity()); while (count > capacity()) { diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp index b4e05875820..7752baa603a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp @@ -18,6 +18,7 @@ Config::Config() : _compactionStrategy(), _predicateParams(), _tensorType(vespalib::eval::ValueType::error_type()), + _distance_metric(DistanceMetric::Euclidean), _hnsw_index_params() { } @@ -36,6 +37,7 @@ Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_) _compactionStrategy(), _predicateParams(), _tensorType(vespalib::eval::ValueType::error_type()), + _distance_metric(DistanceMetric::Euclidean), _hnsw_index_params() { } @@ -63,6 +65,7 @@ Config::operator==(const Config &b) const _predicateParams == b._predicateParams && (_basicType.type() != BasicType::Type::TENSOR || _tensorType == b._tensorType) && + _distance_metric == b._distance_metric && _hnsw_index_params == b._hnsw_index_params; } diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h index df5aa9e217a..822a4e4e028 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchcommon/src/vespa/searchcommon/attribute/config.h @@ -9,6 +9,7 @@ #include <vespa/searchcommon/common/compaction_strategy.h> #include <vespa/searchcommon/common/growstrategy.h> #include <vespa/eval/eval/value_type.h> +#include <cassert> #include <optional> namespace search::attribute { @@ -35,6 +36,7 @@ public: bool huge() const { return _huge; } const PredicateParams &predicateParams() const { return _predicateParams; } vespalib::eval::ValueType tensorType() const { return _tensorType; } + DistanceMetric distance_metric() const { return _distance_metric; } const std::optional<HnswIndexParams>& hnsw_index_params() const { return _hnsw_index_params; } /** @@ -67,7 +69,12 @@ public: _tensorType = tensorType_in; return *this; } + Config& set_distance_metric(DistanceMetric value) { + _distance_metric = value; + return *this; + } Config& set_hnsw_index_params(const HnswIndexParams& params) { + assert(_distance_metric == params.distance_metric()); _hnsw_index_params = params; return *this; } @@ -122,6 +129,7 @@ private: CompactionStrategy _compactionStrategy; PredicateParams _predicateParams; vespalib::eval::ValueType _tensorType; + DistanceMetric _distance_metric; std::optional<HnswIndexParams> _hnsw_index_params; }; diff --git a/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h index 94d3fda49f3..c8b196023d6 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h +++ b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h @@ -14,25 +14,31 @@ class HnswIndexParams { private: uint32_t _max_links_per_node; uint32_t _neighbors_to_explore_at_insert; + // This is always the same as in the attribute config, and is duplicated here to simplify usage. DistanceMetric _distance_metric; + bool _allow_multi_threaded_indexing; public: HnswIndexParams(uint32_t max_links_per_node_in, uint32_t neighbors_to_explore_at_insert_in, - DistanceMetric distance_metric_in) + DistanceMetric distance_metric_in, + bool allow_multi_threaded_indexing_in = false) : _max_links_per_node(max_links_per_node_in), _neighbors_to_explore_at_insert(neighbors_to_explore_at_insert_in), - _distance_metric(distance_metric_in) + _distance_metric(distance_metric_in), + _allow_multi_threaded_indexing(allow_multi_threaded_indexing_in) {} uint32_t max_links_per_node() const { return _max_links_per_node; } uint32_t neighbors_to_explore_at_insert() const { return _neighbors_to_explore_at_insert; } DistanceMetric distance_metric() const { return _distance_metric; } + bool allow_multi_threaded_indexing() const { return _allow_multi_threaded_indexing; } bool operator==(const HnswIndexParams& rhs) const { return (_max_links_per_node == rhs._max_links_per_node && _neighbors_to_explore_at_insert == rhs._neighbors_to_explore_at_insert && - _distance_metric == rhs._distance_metric); + _distance_metric == rhs._distance_metric && + _allow_multi_threaded_indexing == rhs._allow_multi_threaded_indexing); } }; diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.cpp b/searchcommon/src/vespa/searchcommon/attribute/status.cpp index da13548ec2e..f2bb49c348a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/status.cpp @@ -20,6 +20,42 @@ Status::Status() { } +Status::Status(const Status& rhs) + : _numDocs(rhs._numDocs), + _numValues(rhs._numValues), + _numUniqueValues(rhs._numUniqueValues), + _allocated(rhs._allocated), + _used(rhs._used), + _dead(rhs._dead), + _unused(rhs._unused), + _onHold(rhs._onHold), + _onHoldMax(rhs._onHoldMax), + _lastSyncToken(rhs.getLastSyncToken()), + _updates(rhs._updates), + _nonIdempotentUpdates(rhs._nonIdempotentUpdates), + _bitVectors(rhs._bitVectors) +{ +} + +Status& +Status::operator=(const Status& rhs) +{ + _numDocs = rhs._numDocs; + _numValues = rhs._numValues; + _numUniqueValues = rhs._numUniqueValues; + _allocated = rhs._allocated; + _used = rhs._used; + _dead = rhs._dead; + _unused = rhs._unused; + _onHold = rhs._onHold; + _onHoldMax = rhs._onHoldMax; + setLastSyncToken(rhs.getLastSyncToken()); + _updates = rhs._updates; + _nonIdempotentUpdates = rhs._nonIdempotentUpdates; + _bitVectors = rhs._bitVectors; + return *this; +} + vespalib::string Status::createName(vespalib::stringref index, vespalib::stringref attr) { diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.h b/searchcommon/src/vespa/searchcommon/attribute/status.h index 888355b3f58..a624309da65 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.h +++ b/searchcommon/src/vespa/searchcommon/attribute/status.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/vespalib/stllike/string.h> +#include <atomic> namespace search::attribute { @@ -10,6 +11,8 @@ class Status { public: Status(); + Status(const Status& rhs); + Status& operator=(const Status& rhs); void updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, uint64_t used, uint64_t dead, uint64_t onHold); @@ -22,14 +25,15 @@ public: uint64_t getDead() const { return _dead; } uint64_t getOnHold() const { return _onHold; } uint64_t getOnHoldMax() const { return _onHoldMax; } - uint64_t getLastSyncToken() const { return _lastSyncToken; } + // This might be accessed from other threads than the writer thread. + uint64_t getLastSyncToken() const { return _lastSyncToken.load(std::memory_order_relaxed); } uint64_t getUpdateCount() const { return _updates; } uint64_t getNonIdempotentUpdateCount() const { return _nonIdempotentUpdates; } uint32_t getBitVectors() const { return _bitVectors; } void setNumDocs(uint64_t v) { _numDocs = v; } void incNumDocs() { ++_numDocs; } - void setLastSyncToken(uint64_t v) { _lastSyncToken = v; } + void setLastSyncToken(uint64_t v) { _lastSyncToken.store(v, std::memory_order_relaxed); } void incUpdates(uint64_t v=1) { _updates += v; } void incNonIdempotentUpdates(uint64_t v = 1) { _nonIdempotentUpdates += v; } void incBitVectors() { ++_bitVectors; } @@ -47,7 +51,7 @@ private: uint64_t _unused; uint64_t _onHold; uint64_t _onHoldMax; - uint64_t _lastSyncToken; + std::atomic<uint64_t> _lastSyncToken; uint64_t _updates; uint64_t _nonIdempotentUpdates; uint32_t _bitVectors; diff --git a/searchcommon/src/vespa/searchcommon/common/schema.cpp b/searchcommon/src/vespa/searchcommon/common/schema.cpp index a21cc43572e..c59edbef22f 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.cpp +++ b/searchcommon/src/vespa/searchcommon/common/schema.cpp @@ -70,16 +70,20 @@ namespace index { const uint32_t Schema::UNKNOWN_FIELD_ID(std::numeric_limits<uint32_t>::max()); Schema::Field::Field(vespalib::stringref n, DataType dt) - : _name(n), - _dataType(dt), - _collectionType(schema::CollectionType::SINGLE) + : Field(n, dt, schema::CollectionType::SINGLE, "") { } Schema::Field::Field(vespalib::stringref n, DataType dt, CollectionType ct) + : Field(n, dt, ct, "") +{ +} + +Schema::Field::Field(vespalib::stringref n, DataType dt, CollectionType ct, vespalib::stringref tensor_spec) : _name(n), _dataType(dt), - _collectionType(ct) + _collectionType(ct), + _tensor_spec(tensor_spec) { } @@ -111,15 +115,14 @@ Schema::Field::operator==(const Field &rhs) const { return _name == rhs._name && _dataType == rhs._dataType && - _collectionType == rhs._collectionType; + _collectionType == rhs._collectionType && + _tensor_spec == rhs._tensor_spec; } bool Schema::Field::operator!=(const Field &rhs) const { - return _name != rhs._name || - _dataType != rhs._dataType || - _collectionType != rhs._collectionType; + return !((*this) == rhs); } Schema::IndexField::IndexField(vespalib::stringref name, DataType dt) diff --git a/searchcommon/src/vespa/searchcommon/common/schema.h b/searchcommon/src/vespa/searchcommon/common/schema.h index e17d219d7e8..9003578adaf 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.h +++ b/searchcommon/src/vespa/searchcommon/common/schema.h @@ -35,10 +35,12 @@ public: vespalib::string _name; DataType _dataType; CollectionType _collectionType; + vespalib::string _tensor_spec; public: Field(vespalib::stringref n, DataType dt); Field(vespalib::stringref n, DataType dt, CollectionType ct); + Field(vespalib::stringref n, DataType dt, CollectionType ct, vespalib::stringref tensor_spec); /** * Create this field based on the given config lines. @@ -58,6 +60,7 @@ public: const vespalib::string &getName() const { return _name; } DataType getDataType() const { return _dataType; } CollectionType getCollectionType() const { return _collectionType; } + const vespalib::string& get_tensor_spec() const { return _tensor_spec; } bool matchingTypes(const Field &rhs) const { return getDataType() == rhs.getDataType() && diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 412652821d1..fe0504b25af 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -59,6 +59,7 @@ vespa_define_module( src/tests/proton/attribute/attribute_manager src/tests/proton/attribute/attribute_populator src/tests/proton/attribute/attribute_usage_filter + src/tests/proton/attribute/attribute_usage_sampler_functor src/tests/proton/attribute/attributes_state_explorer src/tests/proton/attribute/document_field_extractor src/tests/proton/attribute/document_field_populator diff --git a/searchcore/src/tests/proton/attribute/CMakeLists.txt b/searchcore/src/tests/proton/attribute/CMakeLists.txt index 79f81f3daa1..c23d97c6e88 100644 --- a/searchcore/src/tests/proton/attribute/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/CMakeLists.txt @@ -7,9 +7,11 @@ vespa_add_executable(searchcore_attribute_test_app TEST searchcore_attribute searchcore_flushengine searchcore_pcommon + searchlib_test + gtest ) -vespa_add_test(NAME searchcore_attribute_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/attribute_test.sh - DEPENDS searchcore_attribute_test_app) +vespa_add_test(NAME searchcore_attribute_test_app COMMAND searchcore_attribute_test_app) + vespa_add_executable(searchcore_attributeflush_test_app TEST SOURCES attributeflush_test.cpp diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 839ef14fcb0..91f580cd221 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -7,41 +7,43 @@ #include <vespa/document/update/arithmeticvalueupdate.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/tensor.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> -#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/test/attribute_utils.h> +#include <vespa/searchcore/proton/test/mock_attribute_manager.h> #include <vespa/searchcorespi/flush/iflushtarget.h> -#include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> +#include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/bitvector_search_cache.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/attribute/imported_attribute_vector_factory.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/predicate_attribute.h> -#include <vespa/vespalib/util/foregroundtaskexecutor.h> +#include <vespa/searchlib/attribute/singlenumericattribute.hpp> #include <vespa/searchlib/common/idestructorcallback.h> -#include <vespa/vespalib/util/sequencedtaskexecutorobserver.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/predicate/predicate_hash.h> #include <vespa/searchlib/predicate/predicate_index.h> +#include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/vespalib/btree/btreeroot.hpp> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchcommon/attribute/iattributevector.h> -#include <vespa/vespalib/btree/btreeroot.hpp> -#include <vespa/searchlib/attribute/singlenumericattribute.hpp> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/foregroundtaskexecutor.h> +#include <vespa/vespalib/util/sequencedtaskexecutorobserver.h> #include <vespa/log/log.h> LOG_SETUP("attribute_test"); @@ -57,8 +59,11 @@ using namespace vespa::config::search; using proton::ImportedAttributesRepo; using proton::test::AttributeUtils; +using proton::test::MockAttributeManager; using search::TuneFileAttributes; using search::attribute::BitVectorSearchCache; +using search::attribute::DistanceMetric; +using search::attribute::HnswIndexParams; using search::attribute::IAttributeVector; using search::attribute::ImportedAttributeVector; using search::attribute::ImportedAttributeVectorFactory; @@ -67,24 +72,25 @@ using search::index::DummyFileHeaderContext; using search::index::schema::CollectionType; using search::predicate::PredicateHash; using search::predicate::PredicateIndex; +using search::tensor::DenseTensorAttribute; +using search::tensor::PrepareResult; using search::tensor::TensorAttribute; using search::test::DirectoryHandler; using std::string; -using vespalib::eval::ValueType; -using vespalib::eval::TensorSpec; -using vespalib::tensor::Tensor; -using vespalib::tensor::DefaultTensorEngine; using vespalib::ForegroundTaskExecutor; using vespalib::SequencedTaskExecutorObserver; +using vespalib::eval::TensorSpec; +using vespalib::eval::ValueType; +using vespalib::tensor::DefaultTensorEngine; +using vespalib::tensor::Tensor; -using AVConfig = search::attribute::Config; using AVBasicType = search::attribute::BasicType; using AVCollectionType = search::attribute::CollectionType; +using AVConfig = search::attribute::Config; using Int32AttributeVector = SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> >; using LidVector = LidVectorContext::LidVector; -namespace -{ +namespace { const uint64_t createSerialNum = 42u; @@ -116,45 +122,48 @@ fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64 const std::shared_ptr<IDestructorCallback> emptyCallback; -struct Fixture -{ +class AttributeWriterTest : public ::testing::Test { +public: DirectoryHandler _dirHandler; - DummyFileHeaderContext _fileHeaderContext; - ForegroundTaskExecutor _attributeFieldWriterReal; - SequencedTaskExecutorObserver _attributeFieldWriter; - HwInfo _hwInfo; - proton::AttributeManager::SP _m; + std::unique_ptr<ForegroundTaskExecutor> _attributeFieldWriterReal; + std::unique_ptr<SequencedTaskExecutorObserver> _attributeFieldWriter; + std::shared_ptr<MockAttributeManager> _mgr; std::unique_ptr<AttributeWriter> _aw; - Fixture(uint32_t threads) + AttributeWriterTest() : _dirHandler(test_dir), - _fileHeaderContext(), - _attributeFieldWriterReal(threads), - _attributeFieldWriter(_attributeFieldWriterReal), - _hwInfo(), - _m(std::make_shared<proton::AttributeManager> - (test_dir, "test.subdb", TuneFileAttributes(), - _fileHeaderContext, _attributeFieldWriter, _hwInfo)), + _attributeFieldWriterReal(), + _attributeFieldWriter(), + _mgr(), _aw() { - allocAttributeWriter(); + setup(1); } - Fixture() - : Fixture(1) - { + ~AttributeWriterTest(); + void setup(uint32_t threads) { + _aw.reset(); + _attributeFieldWriterReal = std::make_unique<ForegroundTaskExecutor>(threads); + _attributeFieldWriter = std::make_unique<SequencedTaskExecutorObserver>(*_attributeFieldWriterReal); + _mgr = std::make_shared<MockAttributeManager>(); + _mgr->set_writer(*_attributeFieldWriter); + allocAttributeWriter(); } - ~Fixture(); void allocAttributeWriter() { - _aw = std::make_unique<AttributeWriter>(_m); + _aw = std::make_unique<AttributeWriter>(_mgr); } AttributeVector::SP addAttribute(const vespalib::string &name) { - return addAttribute({name, AVConfig(AVBasicType::INT32)}, createSerialNum); + return addAttribute({name, AVConfig(AVBasicType::INT32)}); } - AttributeVector::SP addAttribute(const AttributeSpec &spec, SerialNum serialNum) { - auto ret = _m->addAttribute(spec, serialNum); + AttributeVector::SP addAttribute(const AttributeSpec &spec) { + auto ret = _mgr->addAttribute(spec.getName(), + AttributeFactory::createAttribute(spec.getName(), spec.getConfig())); allocAttributeWriter(); return ret; } + void add_attribute(AttributeVector::SP attr) { + _mgr->addAttribute(attr->getName(), std::move(attr)); + allocAttributeWriter(); + } void put(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit = true) { _aw->put(serialNum, doc, lid, immediateCommit, emptyCallback); @@ -177,13 +186,13 @@ struct Fixture _aw->forceCommit(serialNum, emptyCallback); } void assertExecuteHistory(std::vector<uint32_t> expExecuteHistory) { - EXPECT_EQUAL(expExecuteHistory, _attributeFieldWriter.getExecuteHistory()); + EXPECT_EQ(expExecuteHistory, _attributeFieldWriter->getExecuteHistory()); } }; -Fixture::~Fixture() = default; +AttributeWriterTest::~AttributeWriterTest() = default; -TEST_F("require that attribute writer handles put", Fixture) +TEST_F(AttributeWriterTest, handles_put) { Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::INT32, CollectionType::SINGLE)); @@ -193,108 +202,108 @@ TEST_F("require that attribute writer handles put", Fixture) DocBuilder idb(s); - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute({"a2", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); - AttributeVector::SP a3 = f.addAttribute({"a3", AVConfig(AVBasicType::FLOAT)}, createSerialNum); - AttributeVector::SP a4 = f.addAttribute({"a4", AVConfig(AVBasicType::STRING)}, createSerialNum); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute({"a2", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); + auto a3 = addAttribute({"a3", AVConfig(AVBasicType::FLOAT)}); + auto a4 = addAttribute({"a4", AVConfig(AVBasicType::STRING)}); attribute::IntegerContent ibuf; attribute::FloatContent fbuf; attribute::ConstCharContent sbuf; { // empty document should give default values - EXPECT_EQUAL(1u, a1->getNumDocs()); - f.put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - EXPECT_EQUAL(2u, a2->getNumDocs()); - EXPECT_EQUAL(2u, a3->getNumDocs()); - EXPECT_EQUAL(2u, a4->getNumDocs()); - EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, a2->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, a3->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, a4->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a1->getNumDocs()); + put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); + EXPECT_EQ(2u, a1->getNumDocs()); + EXPECT_EQ(2u, a2->getNumDocs()); + EXPECT_EQ(2u, a3->getNumDocs()); + EXPECT_EQ(2u, a4->getNumDocs()); + EXPECT_EQ(1u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a2->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a3->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a4->getStatus().getLastSyncToken()); ibuf.fill(*a1, 1); - EXPECT_EQUAL(1u, ibuf.size()); + EXPECT_EQ(1u, ibuf.size()); EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ibuf[0])); ibuf.fill(*a2, 1); - EXPECT_EQUAL(0u, ibuf.size()); + EXPECT_EQ(0u, ibuf.size()); fbuf.fill(*a3, 1); - EXPECT_EQUAL(1u, fbuf.size()); + EXPECT_EQ(1u, fbuf.size()); EXPECT_TRUE(search::attribute::isUndefined<float>(fbuf[0])); sbuf.fill(*a4, 1); - EXPECT_EQUAL(1u, sbuf.size()); - EXPECT_EQUAL(strcmp("", sbuf[0]), 0); + EXPECT_EQ(1u, sbuf.size()); + EXPECT_EQ(strcmp("", sbuf[0]), 0); } { // document with single value & multi value attribute - Document::UP doc = idb.startDocument("id:ns:searchdocument::2"). + auto doc = idb.startDocument("id:ns:searchdocument::2"). startAttributeField("a1").addInt(10).endField(). startAttributeField("a2").startElement().addInt(20).endElement(). startElement().addInt(30).endElement().endField().endDocument(); - f.put(2, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a2->getNumDocs()); - EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(2u, a2->getStatus().getLastSyncToken()); - EXPECT_EQUAL(2u, a3->getStatus().getLastSyncToken()); - EXPECT_EQUAL(2u, a4->getStatus().getLastSyncToken()); + put(2, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a2->getNumDocs()); + EXPECT_EQ(2u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(2u, a2->getStatus().getLastSyncToken()); + EXPECT_EQ(2u, a3->getStatus().getLastSyncToken()); + EXPECT_EQ(2u, a4->getStatus().getLastSyncToken()); ibuf.fill(*a1, 2); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(10u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(10u, ibuf[0]); ibuf.fill(*a2, 2); - EXPECT_EQUAL(2u, ibuf.size()); - EXPECT_EQUAL(20u, ibuf[0]); - EXPECT_EQUAL(30u, ibuf[1]); + EXPECT_EQ(2u, ibuf.size()); + EXPECT_EQ(20u, ibuf[0]); + EXPECT_EQ(30u, ibuf[1]); } { // replace existing document - Document::UP doc = idb.startDocument("id:ns:searchdocument::2"). + auto doc = idb.startDocument("id:ns:searchdocument::2"). startAttributeField("a1").addInt(100).endField(). startAttributeField("a2").startElement().addInt(200).endElement(). startElement().addInt(300).endElement(). startElement().addInt(400).endElement().endField().endDocument(); - f.put(3, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a2->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(3u, a2->getStatus().getLastSyncToken()); - EXPECT_EQUAL(3u, a3->getStatus().getLastSyncToken()); - EXPECT_EQUAL(3u, a4->getStatus().getLastSyncToken()); + put(3, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a2->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a2->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a3->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a4->getStatus().getLastSyncToken()); ibuf.fill(*a1, 2); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(100u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(100u, ibuf[0]); ibuf.fill(*a2, 2); - EXPECT_EQUAL(3u, ibuf.size()); - EXPECT_EQUAL(200u, ibuf[0]); - EXPECT_EQUAL(300u, ibuf[1]); - EXPECT_EQUAL(400u, ibuf[2]); + EXPECT_EQ(3u, ibuf.size()); + EXPECT_EQ(200u, ibuf[0]); + EXPECT_EQ(300u, ibuf[1]); + EXPECT_EQ(400u, ibuf[2]); } } -TEST_F("require that attribute writer handles predicate put", Fixture) +TEST_F(AttributeWriterTest, handles_predicate_put) { Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE)); DocBuilder idb(s); - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); // empty document should give default values - EXPECT_EQUAL(1u, a1->getNumDocs()); - f.put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); + EXPECT_EQ(1u, a1->getNumDocs()); + put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); + EXPECT_EQ(2u, a1->getNumDocs()); + EXPECT_EQ(1u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(0u, index.getZeroConstraintDocs().size()); // document with single value attribute PredicateSlimeBuilder builder; - Document::UP doc = + auto doc = idb.startDocument("id:ns:searchdocument::2").startAttributeField("a1") .addPredicate(builder.true_predicate().build()) .endField().endDocument(); - f.put(2, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); + put(2, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(2u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, index.getZeroConstraintDocs().size()); auto it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")); EXPECT_FALSE(it.valid()); @@ -303,9 +312,9 @@ TEST_F("require that attribute writer handles predicate put", Fixture) doc = idb.startDocument("id:ns:searchdocument::2").startAttributeField("a1") .addPredicate(builder.feature("foo").value("bar").build()) .endField().endDocument(); - f.put(3, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + put(3, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")); EXPECT_TRUE(it.valid()); @@ -317,21 +326,21 @@ assertUndefined(const IAttributeVector &attr, uint32_t docId) EXPECT_TRUE(search::attribute::isUndefined<int32_t>(attr.getInt(docId))); } -TEST_F("require that attribute writer handles remove", Fixture) +TEST_F(AttributeWriterTest, handles_remove) { - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute("a2"); fillAttribute(a1, 1, 10, 1); fillAttribute(a2, 1, 20, 1); - f.remove(2, 0); + remove(2, 0); - TEST_DO(assertUndefined(*a1, 0)); - TEST_DO(assertUndefined(*a2, 0)); + assertUndefined(*a1, 0); + assertUndefined(*a2, 0); - f.remove(2, 0); // same sync token as previous + remove(2, 0); // same sync token as previous try { - f.remove(1, 0); // lower sync token than previous + remove(1, 0); // lower sync token than previous EXPECT_TRUE(true); // update is ignored } catch (vespalib::IllegalStateException & e) { LOG(info, "Got expected exception: '%s'", e.getMessage().c_str()); @@ -339,62 +348,63 @@ TEST_F("require that attribute writer handles remove", Fixture) } } -TEST_F("require that attribute writer handles batch remove", Fixture) +TEST_F(AttributeWriterTest, handles_batch_remove) { - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute("a2"); fillAttribute(a1, 4, 22, 1); fillAttribute(a2, 4, 33, 1); LidVector lidsToRemove = {1,3}; - f.remove(lidsToRemove, 2); - - TEST_DO(assertUndefined(*a1, 1)); - EXPECT_EQUAL(22, a1->getInt(2)); - TEST_DO(assertUndefined(*a1, 3)); - TEST_DO(assertUndefined(*a2, 1)); - EXPECT_EQUAL(33, a2->getInt(2)); - TEST_DO(assertUndefined(*a2, 3)); + remove(lidsToRemove, 2); + + assertUndefined(*a1, 1); + EXPECT_EQ(22, a1->getInt(2)); + assertUndefined(*a1, 3); + assertUndefined(*a2, 1); + EXPECT_EQ(33, a2->getInt(2)); + assertUndefined(*a2, 3); } -void verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::stringref expected) +void +verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::stringref expected) { attribute::ConstCharContent sbuf; sbuf.fill(v, lid); - EXPECT_EQUAL(1u, sbuf.size()); - EXPECT_EQUAL(expected, sbuf[0]); + EXPECT_EQ(1u, sbuf.size()); + EXPECT_EQ(expected, sbuf[0]); } -TEST_F("require that visibilitydelay is honoured", Fixture) +TEST_F(AttributeWriterTest, visibility_delay_is_honoured) { - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::STRING)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::STRING)}); Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::STRING, CollectionType::SINGLE)); DocBuilder idb(s); - EXPECT_EQUAL(1u, a1->getNumDocs()); - EXPECT_EQUAL(0u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a1->getNumDocs()); + EXPECT_EQ(0u, a1->getStatus().getLastSyncToken()); Document::UP doc = idb.startDocument("id:ns:searchdocument::1") .startAttributeField("a1").addStr("10").endField() .endDocument(); - f.put(3, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); - AttributeWriter awDelayed(f._m); + put(3, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); + AttributeWriter awDelayed(_mgr); awDelayed.put(4, *doc, 2, false, emptyCallback); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); awDelayed.put(5, *doc, 4, false, emptyCallback); - EXPECT_EQUAL(5u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(5u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); awDelayed.forceCommit(6, emptyCallback); - EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(6u, a1->getStatus().getLastSyncToken()); - AttributeWriter awDelayedShort(f._m); + AttributeWriter awDelayedShort(_mgr); awDelayedShort.put(7, *doc, 2, false, emptyCallback); - EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(6u, a1->getStatus().getLastSyncToken()); awDelayedShort.put(8, *doc, 2, false, emptyCallback); awDelayedShort.forceCommit(8, emptyCallback); - EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(8u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "10"); awDelayed.put(9, *idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1").addStr("11").endField().endDocument(), @@ -403,40 +413,39 @@ TEST_F("require that visibilitydelay is honoured", Fixture) 2, false, emptyCallback); awDelayed.put(11, *idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1").addStr("30").endField().endDocument(), 2, false, emptyCallback); - EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(8u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "10"); awDelayed.forceCommit(12, emptyCallback); - EXPECT_EQUAL(12u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(12u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "30"); - } -TEST_F("require that attribute writer handles predicate remove", Fixture) +TEST_F(AttributeWriterTest, handles_predicate_remove) { - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); Schema s; s.addAttributeField( Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE)); DocBuilder idb(s); PredicateSlimeBuilder builder; - Document::UP doc = + auto doc = idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1") .addPredicate(builder.true_predicate().build()) .endField().endDocument(); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); - EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); - f.remove(2, 1); - EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); + EXPECT_EQ(1u, index.getZeroConstraintDocs().size()); + remove(2, 1); + EXPECT_EQ(0u, index.getZeroConstraintDocs().size()); } -TEST_F("require that attribute writer handles update", Fixture) +TEST_F(AttributeWriterTest, handles_update) { - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute("a2"); fillAttribute(a1, 1, 10, 1); fillAttribute(a2, 1, 20, 1); @@ -454,19 +463,19 @@ TEST_F("require that attribute writer handles update", Fixture) DummyFieldUpdateCallback onUpdate; bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit, onUpdate); + update(2, upd, 1, immediateCommit, onUpdate); attribute::IntegerContent ibuf; ibuf.fill(*a1, 1); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(15u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(15u, ibuf[0]); ibuf.fill(*a2, 1); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(30u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(30u, ibuf[0]); - f.update(2, upd, 1, immediateCommit, onUpdate); // same sync token as previous + update(2, upd, 1, immediateCommit, onUpdate); // same sync token as previous try { - f.update(1, upd, 1, immediateCommit, onUpdate); // lower sync token than previous + update(1, upd, 1, immediateCommit, onUpdate); // lower sync token than previous EXPECT_TRUE(true); // update is ignored } catch (vespalib::IllegalStateException & e) { LOG(info, "Got expected exception: '%s'", e.getMessage().c_str()); @@ -474,20 +483,20 @@ TEST_F("require that attribute writer handles update", Fixture) } } -TEST_F("require that attribute writer handles predicate update", Fixture) +TEST_F(AttributeWriterTest, handles_predicate_update) { - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); Schema schema; schema.addAttributeField(Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE)); DocBuilder idb(schema); PredicateSlimeBuilder builder; - Document::UP doc = + auto doc = idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1") .addPredicate(builder.true_predicate().build()) .endField().endDocument(); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); const document::DocumentType &dt(idb.getDocumentType()); DocumentUpdate upd(*idb.getDocumentTypeRepo(), dt, DocumentId("id:ns:searchdocument::1")); @@ -496,20 +505,20 @@ TEST_F("require that attribute writer handles predicate update", Fixture) .addUpdate(AssignValueUpdate(new_value))); PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); - EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); + EXPECT_EQ(1u, index.getZeroConstraintDocs().size()); EXPECT_FALSE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); bool immediateCommit = true; DummyFieldUpdateCallback onUpdate; - f.update(2, upd, 1, immediateCommit, onUpdate); - EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); + update(2, upd, 1, immediateCommit, onUpdate); + EXPECT_EQ(0u, index.getZeroConstraintDocs().size()); EXPECT_TRUE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); } -struct AttributeCollectionSpecFixture -{ +class AttributeCollectionSpecTest : public ::testing::Test { +public: AttributesConfigBuilder _builder; AttributeCollectionSpecFactory _factory; - AttributeCollectionSpecFixture(bool fastAccessOnly) + AttributeCollectionSpecTest(bool fastAccessOnly) : _builder(), _factory(search::GrowStrategy(), 100, fastAccessOnly) { @@ -528,49 +537,47 @@ struct AttributeCollectionSpecFixture } }; -struct NormalAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture -{ - NormalAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(false) {} +class NormalAttributeCollectionSpecTest : public AttributeCollectionSpecTest { +public: + NormalAttributeCollectionSpecTest() : AttributeCollectionSpecTest(false) {} }; -struct FastAccessAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture +struct FastAccessAttributeCollectionSpecTest : public AttributeCollectionSpecTest { - FastAccessAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(true) {} + FastAccessAttributeCollectionSpecTest() : AttributeCollectionSpecTest(true) {} }; -TEST_F("require that normal attribute collection spec can be created", - NormalAttributeCollectionSpecFixture) +TEST_F(NormalAttributeCollectionSpecTest, spec_can_be_created) { - AttributeCollectionSpec::UP spec = f.create(10, 20); - EXPECT_EQUAL(2u, spec->getAttributes().size()); - EXPECT_EQUAL("a1", spec->getAttributes()[0].getName()); - EXPECT_EQUAL("a2", spec->getAttributes()[1].getName()); - EXPECT_EQUAL(10u, spec->getDocIdLimit()); - EXPECT_EQUAL(20u, spec->getCurrentSerialNum()); + AttributeCollectionSpec::UP spec = create(10, 20); + EXPECT_EQ(2u, spec->getAttributes().size()); + EXPECT_EQ("a1", spec->getAttributes()[0].getName()); + EXPECT_EQ("a2", spec->getAttributes()[1].getName()); + EXPECT_EQ(10u, spec->getDocIdLimit()); + EXPECT_EQ(20u, spec->getCurrentSerialNum()); } -TEST_F("require that fast access attribute collection spec can be created", - FastAccessAttributeCollectionSpecFixture) +TEST_F(FastAccessAttributeCollectionSpecTest, spec_can_be_created) { - AttributeCollectionSpec::UP spec = f.create(10, 20); - EXPECT_EQUAL(1u, spec->getAttributes().size()); - EXPECT_EQUAL("a2", spec->getAttributes()[0].getName()); - EXPECT_EQUAL(10u, spec->getDocIdLimit()); - EXPECT_EQUAL(20u, spec->getCurrentSerialNum()); + AttributeCollectionSpec::UP spec = create(10, 20); + EXPECT_EQ(1u, spec->getAttributes().size()); + EXPECT_EQ("a2", spec->getAttributes()[0].getName()); + EXPECT_EQ(10u, spec->getDocIdLimit()); + EXPECT_EQ(20u, spec->getCurrentSerialNum()); } const FilterAttributeManager::AttributeSet ACCEPTED_ATTRIBUTES = {"a2"}; -struct FilterFixture -{ +class FilterAttributeManagerTest : public ::testing::Test { +public: DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; HwInfo _hwInfo; - proton::AttributeManager::SP _baseMgr; FilterAttributeManager _filterMgr; - FilterFixture() + + FilterAttributeManagerTest() : _dirHandler(test_dir), _fileHeaderContext(), _attributeFieldWriter(), @@ -587,34 +594,34 @@ struct FilterFixture } }; -TEST_F("require that filter attribute manager can filter attributes", FilterFixture) +TEST_F(FilterAttributeManagerTest, filter_attributes) { - EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == nullptr); - EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != nullptr); + EXPECT_TRUE(_filterMgr.getAttribute("a1").get() == nullptr); + EXPECT_TRUE(_filterMgr.getAttribute("a2").get() != nullptr); std::vector<AttributeGuard> attrs; - f._filterMgr.getAttributeList(attrs); - EXPECT_EQUAL(1u, attrs.size()); - EXPECT_EQUAL("a2", attrs[0]->getName()); - searchcorespi::IFlushTarget::List targets = f._filterMgr.getFlushTargets(); - EXPECT_EQUAL(2u, targets.size()); - EXPECT_EQUAL("attribute.flush.a2", targets[0]->getName()); - EXPECT_EQUAL("attribute.shrink.a2", targets[1]->getName()); + _filterMgr.getAttributeList(attrs); + EXPECT_EQ(1u, attrs.size()); + EXPECT_EQ("a2", attrs[0]->getName()); + searchcorespi::IFlushTarget::List targets = _filterMgr.getFlushTargets(); + EXPECT_EQ(2u, targets.size()); + EXPECT_EQ("attribute.flush.a2", targets[0]->getName()); + EXPECT_EQ("attribute.shrink.a2", targets[1]->getName()); } -TEST_F("require that filter attribute manager can return flushed serial number", FilterFixture) +TEST_F(FilterAttributeManagerTest, returns_flushed_serial_number) { - f._baseMgr->flushAll(100); - EXPECT_EQUAL(0u, f._filterMgr.getFlushedSerialNum("a1")); - EXPECT_EQUAL(100u, f._filterMgr.getFlushedSerialNum("a2")); + _baseMgr->flushAll(100); + EXPECT_EQ(0u, _filterMgr.getFlushedSerialNum("a1")); + EXPECT_EQ(100u, _filterMgr.getFlushedSerialNum("a2")); } -TEST_F("readable_attribute_vector filters attributes", FilterFixture) +TEST_F(FilterAttributeManagerTest, readable_attribute_vector_filters_attributes) { - auto av = f._filterMgr.readable_attribute_vector("a2"); + auto av = _filterMgr.readable_attribute_vector("a2"); ASSERT_TRUE(av); - EXPECT_EQUAL("a2", av->makeReadGuard(false)->attribute()->getName()); + EXPECT_EQ("a2", av->makeReadGuard(false)->attribute()->getName()); - av = f._filterMgr.readable_attribute_vector("a1"); + av = _filterMgr.readable_attribute_vector("a1"); EXPECT_FALSE(av); } @@ -625,18 +632,20 @@ Tensor::UP make_tensor(const TensorSpec &spec) { return Tensor::UP(dynamic_cast<Tensor*>(tensor.release())); } +const vespalib::string sparse_tensor = "tensor(x{},y{})"; + AttributeVector::SP -createTensorAttribute(Fixture &f) { +createTensorAttribute(AttributeWriterTest &t) { AVConfig cfg(AVBasicType::TENSOR); - cfg.setTensorType(ValueType::from_spec("tensor(x{},y{})")); - auto ret = f.addAttribute({"a1", cfg}, createSerialNum); + cfg.setTensorType(ValueType::from_spec(sparse_tensor)); + auto ret = t.addAttribute({"a1", cfg}); return ret; } Schema -createTensorSchema() { +createTensorSchema(const vespalib::string& tensor_spec = sparse_tensor) { Schema schema; - schema.addAttributeField(Schema::AttributeField("a1", schema::DataType::TENSOR, CollectionType::SINGLE)); + schema.addAttributeField(Schema::AttributeField("a1", schema::DataType::TENSOR, CollectionType::SINGLE, tensor_spec)); return schema; } @@ -649,38 +658,34 @@ createTensorPutDoc(DocBuilder &builder, const Tensor &tensor) { } - -TEST_F("Test that we can use attribute writer to write to tensor attribute", - Fixture) +TEST_F(AttributeWriterTest, can_write_to_tensor_attribute) { - AttributeVector::SP a1 = createTensorAttribute(f); + auto a1 = createTensorAttribute(*this); Schema s = createTensorSchema(); DocBuilder builder(s); - auto tensor = make_tensor(TensorSpec("tensor(x{},y{})") + auto tensor = make_tensor(TensorSpec(sparse_tensor) .add({{"x", "4"}, {"y", "5"}}, 7)); Document::UP doc = createTensorPutDoc(builder, *tensor); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - TensorAttribute *tensorAttribute = - dynamic_cast<TensorAttribute *>(a1.get()); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); + auto *tensorAttribute = dynamic_cast<TensorAttribute *>(a1.get()); EXPECT_TRUE(tensorAttribute != nullptr); auto tensor2 = tensorAttribute->getTensor(1); EXPECT_TRUE(static_cast<bool>(tensor2)); EXPECT_TRUE(tensor->equals(*tensor2)); } -TEST_F("require that attribute writer handles tensor assign update", Fixture) +TEST_F(AttributeWriterTest, handles_tensor_assign_update) { - AttributeVector::SP a1 = createTensorAttribute(f); + auto a1 = createTensorAttribute(*this); Schema s = createTensorSchema(); DocBuilder builder(s); - auto tensor = make_tensor(TensorSpec("tensor(x{},y{})") + auto tensor = make_tensor(TensorSpec(sparse_tensor) .add({{"x", "6"}, {"y", "7"}}, 9)); - Document::UP doc = createTensorPutDoc(builder, *tensor); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - TensorAttribute *tensorAttribute = - dynamic_cast<TensorAttribute *>(a1.get()); + auto doc = createTensorPutDoc(builder, *tensor); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); + auto *tensorAttribute = dynamic_cast<TensorAttribute *>(a1.get()); EXPECT_TRUE(tensorAttribute != nullptr); auto tensor2 = tensorAttribute->getTensor(1); EXPECT_TRUE(static_cast<bool>(tensor2)); @@ -688,23 +693,22 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture) const document::DocumentType &dt(builder.getDocumentType()); DocumentUpdate upd(*builder.getDocumentTypeRepo(), dt, DocumentId("id:ns:searchdocument::1")); - auto new_tensor = make_tensor(TensorSpec("tensor(x{},y{})") + auto new_tensor = make_tensor(TensorSpec(sparse_tensor) .add({{"x", "8"}, {"y", "9"}}, 11)); - TensorDataType xySparseTensorDataType(vespalib::eval::ValueType::from_spec("tensor(x{},y{})")); + TensorDataType xySparseTensorDataType(vespalib::eval::ValueType::from_spec(sparse_tensor)); TensorFieldValue new_value(xySparseTensorDataType); new_value = new_tensor->clone(); upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(AssignValueUpdate(new_value))); bool immediateCommit = true; DummyFieldUpdateCallback onUpdate; - f.update(2, upd, 1, immediateCommit, onUpdate); - EXPECT_EQUAL(2u, a1->getNumDocs()); + update(2, upd, 1, immediateCommit, onUpdate); + EXPECT_EQ(2u, a1->getNumDocs()); EXPECT_TRUE(tensorAttribute != nullptr); tensor2 = tensorAttribute->getTensor(1); EXPECT_TRUE(static_cast<bool>(tensor2)); EXPECT_TRUE(!tensor->equals(*tensor2)); EXPECT_TRUE(new_tensor->equals(*tensor2)); - } namespace { @@ -712,16 +716,16 @@ namespace { void assertPutDone(AttributeVector &attr, int32_t expVal) { - EXPECT_EQUAL(2u, attr.getNumDocs()); - EXPECT_EQUAL(1u, attr.getStatus().getLastSyncToken()); + EXPECT_EQ(2u, attr.getNumDocs()); + EXPECT_EQ(1u, attr.getStatus().getLastSyncToken()); attribute::IntegerContent ibuf; ibuf.fill(attr, 1); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(expVal, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(expVal, ibuf[0]); } void -putAttributes(Fixture &f, std::vector<uint32_t> expExecuteHistory) +putAttributes(AttributeWriterTest &t, std::vector<uint32_t> expExecuteHistory) { Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::INT32, CollectionType::SINGLE)); @@ -730,41 +734,205 @@ putAttributes(Fixture &f, std::vector<uint32_t> expExecuteHistory) DocBuilder idb(s); - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); - AttributeVector::SP a3 = f.addAttribute("a3"); + auto a1 = t.addAttribute("a1"); + auto a2 = t.addAttribute("a2"); + auto a3 = t.addAttribute("a3"); - EXPECT_EQUAL(1u, a1->getNumDocs()); - EXPECT_EQUAL(1u, a2->getNumDocs()); - EXPECT_EQUAL(1u, a3->getNumDocs()); - f.put(1, *idb.startDocument("id:ns:searchdocument::1"). + EXPECT_EQ(1u, a1->getNumDocs()); + EXPECT_EQ(1u, a2->getNumDocs()); + EXPECT_EQ(1u, a3->getNumDocs()); + t.put(1, *idb.startDocument("id:ns:searchdocument::1"). startAttributeField("a1").addInt(10).endField(). startAttributeField("a2").addInt(15).endField(). startAttributeField("a3").addInt(20).endField(). endDocument(), 1); - TEST_DO(assertPutDone(*a1, 10)); - TEST_DO(assertPutDone(*a2, 15)); - TEST_DO(assertPutDone(*a3, 20)); - TEST_DO(f.assertExecuteHistory(expExecuteHistory)); + assertPutDone(*a1, 10); + assertPutDone(*a2, 15); + assertPutDone(*a3, 20); + t.assertExecuteHistory(expExecuteHistory); +} + +} + +TEST_F(AttributeWriterTest, spreads_write_over_1_write_context) +{ + putAttributes(*this, {0}); } +TEST_F(AttributeWriterTest, spreads_write_over_2_write_contexts) +{ + setup(2); + putAttributes(*this, {0, 1}); } -TEST_F("require that attribute writer spreads write over 1 write context", Fixture(1)) +TEST_F(AttributeWriterTest, spreads_write_over_3_write_contexts) { - TEST_DO(putAttributes(f, {0})); + setup(8); + putAttributes(*this, {0, 1, 2}); } -TEST_F("require that attribute writer spreads write over 2 write contexts", Fixture(2)) +struct MockPrepareResult : public PrepareResult { + uint32_t docid; + const Tensor& tensor; + MockPrepareResult(uint32_t docid_in, const Tensor& tensor_in) : docid(docid_in), tensor(tensor_in) {} +}; + +class MockDenseTensorAttribute : public DenseTensorAttribute { +public: + mutable size_t prepare_set_tensor_cnt; + mutable size_t complete_set_tensor_cnt; + size_t clear_doc_cnt; + + MockDenseTensorAttribute(vespalib::stringref name, const AVConfig& cfg) + : DenseTensorAttribute(name, cfg), + prepare_set_tensor_cnt(0), + complete_set_tensor_cnt(0), + clear_doc_cnt(0) + {} + uint32_t clearDoc(DocId docid) override { + ++clear_doc_cnt; + return DenseTensorAttribute::clearDoc(docid); + } + std::unique_ptr<PrepareResult> prepare_set_tensor(uint32_t docid, const Tensor& tensor) const override { + ++prepare_set_tensor_cnt; + return std::make_unique<MockPrepareResult>(docid, tensor); + } + + virtual void complete_set_tensor(DocId docid, const Tensor& tensor, std::unique_ptr<PrepareResult> prepare_result) override { + ++complete_set_tensor_cnt; + assert(prepare_result); + auto* mock_result = dynamic_cast<MockPrepareResult*>(prepare_result.get()); + assert(mock_result); + EXPECT_EQ(docid, mock_result->docid); + EXPECT_EQ(tensor, mock_result->tensor); + } +}; + +const vespalib::string dense_tensor = "tensor(x[2])"; + +AVConfig +get_tensor_config(bool allow_multi_threaded_indexing) { - TEST_DO(putAttributes(f, {0, 1})); + AVConfig cfg(AVBasicType::TENSOR); + cfg.setTensorType(ValueType::from_spec(dense_tensor)); + cfg.set_hnsw_index_params(HnswIndexParams(4, 4, DistanceMetric::Euclidean, allow_multi_threaded_indexing)); + return cfg; } -TEST_F("require that attribute writer spreads write over 3 write contexts", Fixture(8)) +std::shared_ptr<MockDenseTensorAttribute> +make_mock_tensor_attribute(const vespalib::string& name, bool allow_multi_threaded_indexing) { - TEST_DO(putAttributes(f, {0, 1, 2})); + auto cfg = get_tensor_config(allow_multi_threaded_indexing); + return std::make_shared<MockDenseTensorAttribute>(name, cfg); } +TEST_F(AttributeWriterTest, tensor_attributes_using_two_phase_put_are_in_separate_write_contexts) +{ + addAttribute("a1"); + addAttribute({"t1", get_tensor_config(true)}); + addAttribute({"t2", get_tensor_config(true)}); + addAttribute({"t3", get_tensor_config(false)}); + allocAttributeWriter(); + + const auto& ctx = _aw->get_write_contexts(); + EXPECT_EQ(3, ctx.size()); + EXPECT_FALSE(ctx[0].use_two_phase_put()); + EXPECT_EQ(2, ctx[0].getFields().size()); + + EXPECT_TRUE(ctx[1].use_two_phase_put()); + EXPECT_EQ(1, ctx[1].getFields().size()); + EXPECT_EQ("t1", ctx[1].getFields()[0].getAttribute().getName()); + + EXPECT_TRUE(ctx[2].use_two_phase_put()); + EXPECT_EQ(1, ctx[2].getFields().size()); + EXPECT_EQ("t2", ctx[2].getFields()[0].getAttribute().getName()); +} + +class TwoPhasePutTest : public AttributeWriterTest { +public: + Schema schema; + DocBuilder builder; + std::shared_ptr<MockDenseTensorAttribute> attr; + std::unique_ptr<Tensor> tensor; + + TwoPhasePutTest() + : AttributeWriterTest(), + schema(createTensorSchema(dense_tensor)), + builder(schema), + attr() + { + setup(2); + attr = make_mock_tensor_attribute("a1", true); + add_attribute(attr); + AttributeManager::padAttribute(*attr, 4); + attr->clear_doc_cnt = 0; + tensor = make_tensor(TensorSpec(dense_tensor) + .add({{"x", 0}}, 3).add({{"x", 1}}, 5)); + } + void expect_tensor_attr_calls(size_t exp_prepare_cnt, + size_t exp_complete_cnt, + size_t exp_clear_doc_cnt = 0) { + EXPECT_EQ(exp_prepare_cnt, attr->prepare_set_tensor_cnt); + EXPECT_EQ(exp_complete_cnt, attr->complete_set_tensor_cnt); + EXPECT_EQ(exp_clear_doc_cnt, attr->clear_doc_cnt); + } + Document::UP make_doc() { + return createTensorPutDoc(builder, *tensor); + } + Document::UP make_no_field_doc() { + return builder.startDocument("id:ns:searchdocument::1").endDocument(); + } + Document::UP make_no_tensor_doc() { + return builder.startDocument("id:ns:searchdocument::1"). + startAttributeField("a1"). + addTensor(std::unique_ptr<vespalib::tensor::Tensor>()).endField().endDocument(); + } +}; + +TEST_F(TwoPhasePutTest, handles_put_in_two_phases_when_specified_for_tensor_attribute) +{ + auto doc = make_doc(); + + put(1, *doc, 1); + expect_tensor_attr_calls(1, 1); + assertExecuteHistory({1, 0}); + + put(2, *doc, 2); + expect_tensor_attr_calls(2, 2); + assertExecuteHistory({1, 0, 0, 0}); + + put(3, *doc, 3); + expect_tensor_attr_calls(3, 3); + // Note that the prepare step is executed round-robin between the 2 threads. + assertExecuteHistory({1, 0, 0, 0, 1, 0}); +} + +TEST_F(TwoPhasePutTest, put_is_ignored_when_serial_number_is_older_or_equal_to_attribute) +{ + auto doc = make_doc(); + attr->commit(7, 7); + put(7, *doc, 1); + expect_tensor_attr_calls(0, 0); + assertExecuteHistory({1, 0}); +} + +TEST_F(TwoPhasePutTest, document_is_cleared_if_field_is_not_set) +{ + auto doc = make_no_field_doc(); + put(1, *doc, 1); + expect_tensor_attr_calls(0, 0, 1); + assertExecuteHistory({1, 0}); +} + +TEST_F(TwoPhasePutTest, document_is_cleared_if_tensor_in_field_is_not_set) +{ + auto doc = make_no_tensor_doc(); + put(1, *doc, 1); + expect_tensor_attr_calls(0, 0, 1); + assertExecuteHistory({1, 0}); +} + + ImportedAttributeVector::SP createImportedAttribute(const vespalib::string &name) { @@ -787,74 +955,66 @@ createImportedAttributesRepo() return result; } -TEST_F("require that AttributeWriter::forceCommit() clears search cache in imported attribute vectors", Fixture) +TEST_F(AttributeWriterTest, forceCommit_clears_search_cache_in_imported_attribute_vectors) { - f._m->setImportedAttributes(createImportedAttributesRepo()); - f.commit(10); - EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_a")->getSearchCache()->size()); - EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); + _mgr->setImportedAttributes(createImportedAttributesRepo()); + commit(10); + EXPECT_EQ(0u, _mgr->getImportedAttributes()->get("imported_a")->getSearchCache()->size()); + EXPECT_EQ(0u, _mgr->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); } -struct StructFixtureBase : public Fixture -{ +class StructWriterTestBase : public AttributeWriterTest { +public: DocumentType _type; const Field _valueField; StructDataType _structFieldType; - StructFixtureBase() - : Fixture(), + StructWriterTestBase() + : AttributeWriterTest(), _type("test"), _valueField("value", 2, *DataType::INT, true), _structFieldType("struct") { - addAttribute({"value", AVConfig(AVBasicType::INT32, AVCollectionType::SINGLE)}, createSerialNum); + addAttribute({"value", AVConfig(AVBasicType::INT32, AVCollectionType::SINGLE)}); _type.addField(_valueField); _structFieldType.addField(_valueField); } - ~StructFixtureBase(); + ~StructWriterTestBase(); - std::unique_ptr<StructFieldValue> - makeStruct() - { + std::unique_ptr<StructFieldValue> makeStruct() { return std::make_unique<StructFieldValue>(_structFieldType); } - std::unique_ptr<StructFieldValue> - makeStruct(const int32_t value) - { + std::unique_ptr<StructFieldValue> makeStruct(const int32_t value) { auto ret = makeStruct(); ret->setValue(_valueField, IntFieldValue(value)); return ret; } - std::unique_ptr<Document> - makeDoc() - { + std::unique_ptr<Document> makeDoc() { return std::make_unique<Document>(_type, DocumentId("id::test::1")); } }; -StructFixtureBase::~StructFixtureBase() = default; +StructWriterTestBase::~StructWriterTestBase() = default; -struct StructArrayFixture : public StructFixtureBase -{ - using StructFixtureBase::makeDoc; +class StructArrayWriterTest : public StructWriterTestBase { +public: + using StructWriterTestBase::makeDoc; const ArrayDataType _structArrayFieldType; const Field _structArrayField; - StructArrayFixture() - : StructFixtureBase(), + StructArrayWriterTest() + : StructWriterTestBase(), _structArrayFieldType(_structFieldType), _structArrayField("array", _structArrayFieldType, true) { - addAttribute({"array.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + addAttribute({"array.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); _type.addField(_structArrayField); } - ~StructArrayFixture(); + ~StructArrayWriterTest(); - std::unique_ptr<Document> - makeDoc(int32_t value, const std::vector<int32_t> &arrayValues) - { + std::unique_ptr<Document> makeDoc(int32_t value, const std::vector<int32_t> &arrayValues) { auto doc = makeDoc(); doc->setValue(_valueField, IntFieldValue(value)); ArrayFieldValue s(_structArrayFieldType); @@ -865,49 +1025,47 @@ struct StructArrayFixture : public StructFixtureBase return doc; } void checkAttrs(uint32_t lid, int32_t value, const std::vector<int32_t> &arrayValues) { - auto valueAttr = _m->getAttribute("value")->getSP(); - auto arrayValueAttr = _m->getAttribute("array.value")->getSP(); - EXPECT_EQUAL(value, valueAttr->getInt(lid)); + auto valueAttr = _mgr->getAttribute("value")->getSP(); + auto arrayValueAttr = _mgr->getAttribute("array.value")->getSP(); + EXPECT_EQ(value, valueAttr->getInt(lid)); attribute::IntegerContent ibuf; ibuf.fill(*arrayValueAttr, lid); - EXPECT_EQUAL(arrayValues.size(), ibuf.size()); + EXPECT_EQ(arrayValues.size(), ibuf.size()); for (size_t i = 0; i < arrayValues.size(); ++i) { - EXPECT_EQUAL(arrayValues[i], ibuf[i]); + EXPECT_EQ(arrayValues[i], ibuf[i]); } } }; -StructArrayFixture::~StructArrayFixture() = default; +StructArrayWriterTest::~StructArrayWriterTest() = default; -TEST_F("require that update with doc argument updates struct field attributes (array)", StructArrayFixture) +TEST_F(StructArrayWriterTest, update_with_doc_argument_updates_struct_field_attributes) { - auto doc = f.makeDoc(10, {11, 12}); - f.put(10, *doc, 1); - TEST_DO(f.checkAttrs(1, 10, {11, 12})); - doc = f.makeDoc(20, {21}); - f.update(11, *doc, 1, true); - TEST_DO(f.checkAttrs(1, 10, {21})); + auto doc = makeDoc(10, {11, 12}); + put(10, *doc, 1); + checkAttrs(1, 10, {11, 12}); + doc = makeDoc(20, {21}); + update(11, *doc, 1, true); + checkAttrs(1, 10, {21}); } -struct StructMapFixture : public StructFixtureBase -{ - using StructFixtureBase::makeDoc; +class StructMapWriterTest : public StructWriterTestBase { +public: + using StructWriterTestBase::makeDoc; const MapDataType _structMapFieldType; const Field _structMapField; - StructMapFixture() - : StructFixtureBase(), + StructMapWriterTest() + : StructWriterTestBase(), _structMapFieldType(*DataType::INT, _structFieldType), _structMapField("map", _structMapFieldType, true) { - addAttribute({"map.value.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); - addAttribute({"map.key", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + addAttribute({"map.value.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); + addAttribute({"map.key", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); _type.addField(_structMapField); } - std::unique_ptr<Document> - makeDoc(int32_t value, const std::map<int32_t, int32_t> &mapValues) - { + std::unique_ptr<Document> makeDoc(int32_t value, const std::map<int32_t, int32_t> &mapValues) { auto doc = makeDoc(); doc->setValue(_valueField, IntFieldValue(value)); MapFieldValue s(_structMapFieldType); @@ -917,38 +1075,35 @@ struct StructMapFixture : public StructFixtureBase doc->setValue(_structMapField, s); return doc; } + void checkAttrs(uint32_t lid, int32_t expValue, const std::map<int32_t, int32_t> &expMap) { - auto valueAttr = _m->getAttribute("value")->getSP(); - auto mapKeyAttr = _m->getAttribute("map.key")->getSP(); - auto mapValueAttr = _m->getAttribute("map.value.value")->getSP(); - EXPECT_EQUAL(expValue, valueAttr->getInt(lid)); + auto valueAttr = _mgr->getAttribute("value")->getSP(); + auto mapKeyAttr = _mgr->getAttribute("map.key")->getSP(); + auto mapValueAttr = _mgr->getAttribute("map.value.value")->getSP(); + EXPECT_EQ(expValue, valueAttr->getInt(lid)); attribute::IntegerContent mapKeys; mapKeys.fill(*mapKeyAttr, lid); attribute::IntegerContent mapValues; mapValues.fill(*mapValueAttr, lid); - EXPECT_EQUAL(expMap.size(), mapValues.size()); - EXPECT_EQUAL(expMap.size(), mapKeys.size()); + EXPECT_EQ(expMap.size(), mapValues.size()); + EXPECT_EQ(expMap.size(), mapKeys.size()); size_t i = 0; for (const auto &expMapElem : expMap) { - EXPECT_EQUAL(expMapElem.first, mapKeys[i]); - EXPECT_EQUAL(expMapElem.second, mapValues[i]); + EXPECT_EQ(expMapElem.first, mapKeys[i]); + EXPECT_EQ(expMapElem.second, mapValues[i]); ++i; } } }; -TEST_F("require that update with doc argument updates struct field attributes (map)", StructMapFixture) +TEST_F(StructMapWriterTest, update_with_doc_argument_updates_struct_field_attributes) { - auto doc = f.makeDoc(10, {{1, 11}, {2, 12}}); - f.put(10, *doc, 1); - TEST_DO(f.checkAttrs(1, 10, {{1, 11}, {2, 12}})); - doc = f.makeDoc(20, {{42, 21}}); - f.update(11, *doc, 1, true); - TEST_DO(f.checkAttrs(1, 10, {{42, 21}})); + auto doc = makeDoc(10, {{1, 11}, {2, 12}}); + put(10, *doc, 1); + checkAttrs(1, 10, {{1, 11}, {2, 12}}); + doc = makeDoc(20, {{42, 21}}); + update(11, *doc, 1, true); + checkAttrs(1, 10, {{42, 21}}); } -TEST_MAIN() -{ - vespalib::rmdir(test_dir, true); - TEST_RUN_ALL(); -} +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/attribute/attribute_test.sh b/searchcore/src/tests/proton/attribute/attribute_test.sh deleted file mode 100755 index 26aa6d5f57a..00000000000 --- a/searchcore/src/tests/proton/attribute/attribute_test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -set -e -rm -rf test_output -$VALGRIND ./searchcore_attribute_test_app diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt new file mode 100644 index 00000000000..2cb6bc65df3 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_attribute_usage_sampler_functor_test_app TEST + SOURCES + attribute_usage_sampler_functor_test.cpp + DEPENDS + searchcore_attribute + searchcore_pcommon + gtest +) +vespa_add_test(NAME searchcore_attribute_usage_sampler_functor_test_app COMMAND searchcore_attribute_usage_sampler_functor_test_app) diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp new file mode 100644 index 00000000000..cb092aaa910 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp @@ -0,0 +1,132 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/config-attributes.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> +#include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> +#include <vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h> +#include <vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h> +#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/stllike/string.h> + +using vespa::config::search::AttributesConfig; +using vespa::config::search::AttributesConfigBuilder; +using search::AttributeVector; +using search::attribute::Config; + +namespace proton { + +namespace { + +AttributesConfig::Attribute build_single_config(const vespalib::string& name, bool fast_search) +{ + AttributesConfigBuilder::Attribute builder; + builder.name = name; + builder.datatype = AttributesConfig::Attribute::Datatype::INT32; + builder.collectiontype = AttributesConfig::Attribute::Collectiontype::WEIGHTEDSET; + builder.fastsearch = fast_search; + return builder; +} + +AttributesConfig build_config(bool fast_search) +{ + AttributesConfigBuilder builder; + builder.attribute.emplace_back(build_single_config("a1", fast_search)); + builder.attribute.emplace_back(build_single_config("a2", fast_search)); + return builder; +} + +std::shared_ptr<AttributeVector> build_attribute_vector(const vespalib::string& name, const AttributeConfigInspector& attribute_config_inspector, uint32_t docs) +{ + auto attribute_vector = search::AttributeFactory::createAttribute(name, *attribute_config_inspector.get_config(name)); + attribute_vector->addReservedDoc(); + for (uint32_t wanted_doc_id = 1; wanted_doc_id <= docs; ++wanted_doc_id) { + uint32_t doc_id = 0; + attribute_vector->addDoc(doc_id); + assert(doc_id == wanted_doc_id); + attribute_vector->clearDoc(doc_id); + auto &integer_attribute_vector = dynamic_cast<search::IntegerAttribute &>(*attribute_vector); + integer_attribute_vector.append(doc_id, 10, 1); + integer_attribute_vector.append(doc_id, 11, 1); + } + attribute_vector->commit(true); + return attribute_vector; +} + +} + +class AttributeUsageSamplerFunctorTest : public ::testing::Test { +protected: + AttributeUsageFilter _filter; + std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; +public: + AttributeUsageSamplerFunctorTest(); + ~AttributeUsageSamplerFunctorTest(); +protected: + void sample_usage(bool sample_a1, bool sample_a2, bool old_fast_search, bool new_fast_search = true); + size_t get_transient_memory_usage() const { return _transient_memory_usage_provider->get_transient_memory_usage(); } +}; + +AttributeUsageSamplerFunctorTest::AttributeUsageSamplerFunctorTest() + : _filter(), + _transient_memory_usage_provider(std::make_shared<TransientMemoryUsageProvider>()) +{ +} + +AttributeUsageSamplerFunctorTest::~AttributeUsageSamplerFunctorTest() = default; + +void +AttributeUsageSamplerFunctorTest::sample_usage(bool sample_a1, bool sample_a2, bool old_fast_search, bool new_fast_search) +{ + auto old_config = build_config(old_fast_search); + auto old_inspector = std::make_shared<AttributeConfigInspector>(old_config); + auto av1 = build_attribute_vector("a1", *old_inspector, 1); + auto av2 = build_attribute_vector("a2", *old_inspector, 3); + EXPECT_EQ(av1->getEnumeratedSave(), old_fast_search); + auto new_config = build_config(new_fast_search); + auto new_inspector = std::make_shared<AttributeConfigInspector>(new_config); + auto context = std::make_shared<AttributeUsageSamplerContext>(_filter, new_inspector, _transient_memory_usage_provider); + if (sample_a1) { + AttributeUsageSamplerFunctor functor1(context, "ready"); + functor1(*av1); + } + if (sample_a2) { + AttributeUsageSamplerFunctor functor2(context, "ready"); + functor2(*av2); + } +} + +TEST_F(AttributeUsageSamplerFunctorTest, plain_attribute_vector_requires_no_transient_memory_for_load) +{ + sample_usage(true, true, false, false); + EXPECT_EQ(0u, get_transient_memory_usage()); +} + +TEST_F(AttributeUsageSamplerFunctorTest, fast_search_attribute_vector_requires_transient_memory_for_load) +{ + sample_usage(true, false, true, true); + EXPECT_EQ(24u, get_transient_memory_usage()); +} + +TEST_F(AttributeUsageSamplerFunctorTest, fast_search_attribute_vector_requires_more_transient_memory_for_load_from_unenumerated) +{ + sample_usage(true, false, false, true); + EXPECT_EQ(40u, get_transient_memory_usage()); +} + +TEST_F(AttributeUsageSamplerFunctorTest, transient_memory_aggregation_function_for_attribute_usage_sampler_context_is_max) +{ + sample_usage(true, false, true, true); + EXPECT_EQ(24u, get_transient_memory_usage()); + sample_usage(false, true, true, true); + EXPECT_EQ(72u, get_transient_memory_usage()); + sample_usage(true, true, true, true); + EXPECT_EQ(72u, get_transient_memory_usage()); +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index e1785e1e48d..b6d6d2437d8 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -662,10 +662,11 @@ void addField(Schema & s, const std::string &name, Schema::DataType dtype, - Schema::CollectionType ctype) + Schema::CollectionType ctype, + const std::string& tensor_spec = "") { - s.addSummaryField(Schema::SummaryField(name, dtype, ctype)); - s.addAttributeField(Schema::AttributeField(name, dtype, ctype)); + s.addSummaryField(Schema::SummaryField(name, dtype, ctype, tensor_spec)); + s.addAttributeField(Schema::AttributeField(name, dtype, ctype, tensor_spec)); } @@ -682,7 +683,7 @@ Test::requireThatAttributesAreUsed() addField(s, "bg", schema::DataType::INT32, CollectionType::WEIGHTEDSET); addField(s, "bh", schema::DataType::FLOAT, CollectionType::WEIGHTEDSET); addField(s, "bi", schema::DataType::STRING, CollectionType::WEIGHTEDSET); - addField(s, "bj", schema::DataType::TENSOR, CollectionType::SINGLE); + addField(s, "bj", schema::DataType::TENSOR, CollectionType::SINGLE, "tensor(x{},y{})"); BuildContext bc(s); DBContext dc(bc._repo, getDocTypeName()); diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 18235116d27..d1d08a332e3 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -285,8 +285,8 @@ SchemaContext::SchemaContext() : schema(new Schema()), builder() { - schema->addAttributeField(Schema::AttributeField("tensor", DataType::TENSOR, CollectionType::SINGLE)); - schema->addAttributeField(Schema::AttributeField("tensor2", DataType::TENSOR, CollectionType::SINGLE)); + schema->addAttributeField(Schema::AttributeField("tensor", DataType::TENSOR, CollectionType::SINGLE, "tensor(x{},y{})")); + schema->addAttributeField(Schema::AttributeField("tensor2", DataType::TENSOR, CollectionType::SINGLE, "tensor(x{},y{})")); addField("i1"); } diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 1f0f3566b1d..6eaa1bd373a 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -3,11 +3,14 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/fastos/thread.h> +#include <vespa/config-attributes.h> +#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> #include <vespa/searchcore/proton/bucketdb/bucket_create_notifier.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/searchcore/proton/common/feedtoken.h> +#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h> #include <vespa/searchcore/proton/documentmetastore/operation_listener.h> #include <vespa/searchcore/proton/feedoperation/moveoperation.h> #include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h> @@ -60,6 +63,7 @@ using storage::spi::Timestamp; using vespalib::Slime; using vespalib::makeClosure; using vespalib::makeTask; +using vespa::config::search::AttributesConfigBuilder; using BlockedReason = IBlockableMaintenanceJob::BlockedReason; @@ -885,6 +889,8 @@ MaintenanceControllerFixture::injectMaintenanceJobs() _jobTrackers, *this, _readyAttributeManager, _notReadyAttributeManager, + std::make_unique<const AttributeConfigInspector>(AttributesConfigBuilder()), + std::make_shared<TransientMemoryUsageProvider>(), _attributeUsageFilter); } } diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp index 0f55a4c30de..5f93f97f165 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/prepare_restart_flush_strategy_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.h> #include <vespa/searchcore/proton/flushengine/flush_target_candidates.h> +#include <vespa/searchcore/proton/flushengine/flush_target_candidate.h> #include <vespa/searchcore/proton/flushengine/tls_stats_map.h> #include <vespa/searchcore/proton/test/dummy_flush_handler.h> #include <vespa/searchcore/proton/test/dummy_flush_target.h> @@ -21,20 +22,16 @@ struct SimpleFlushTarget : public test::DummyFlushTarget { SerialNum flushedSerial; uint64_t approxDiskBytes; - SimpleFlushTarget(const vespalib::string &name, - SerialNum flushedSerial_, - uint64_t approxDiskBytes_) - : test::DummyFlushTarget(name), - flushedSerial(flushedSerial_), - approxDiskBytes(approxDiskBytes_) - {} + double replay_operation_cost; SimpleFlushTarget(const vespalib::string &name, const Type &type, SerialNum flushedSerial_, - uint64_t approxDiskBytes_) + uint64_t approxDiskBytes_, + double replay_operation_cost_) : test::DummyFlushTarget(name, type, Component::OTHER), flushedSerial(flushedSerial_), - approxDiskBytes(approxDiskBytes_) + approxDiskBytes(approxDiskBytes_), + replay_operation_cost(replay_operation_cost_) {} virtual SerialNum getFlushedSerialNum() const override { return flushedSerial; @@ -42,6 +39,9 @@ struct SimpleFlushTarget : public test::DummyFlushTarget virtual uint64_t getApproxBytesToWriteToDisk() const override { return approxDiskBytes; } + double get_replay_operation_cost() const override { + return replay_operation_cost; + } }; class ContextsBuilder @@ -66,30 +66,35 @@ public: const vespalib::string &targetName, IFlushTarget::Type targetType, SerialNum flushedSerial, - uint64_t approxDiskBytes) { + uint64_t approxDiskBytes, + double replay_operation_cost) { IFlushHandler::SP handler = createAndGetHandler(handlerName); IFlushTarget::SP target = std::make_shared<SimpleFlushTarget>(targetName, targetType, flushedSerial, - approxDiskBytes); + approxDiskBytes, + replay_operation_cost); _result.push_back(std::make_shared<FlushContext>(handler, target, 0)); return *this; } ContextsBuilder &add(const vespalib::string &handlerName, const vespalib::string &targetName, SerialNum flushedSerial, - uint64_t approxDiskBytes) { - return add(handlerName, targetName, IFlushTarget::Type::FLUSH, flushedSerial, approxDiskBytes); + uint64_t approxDiskBytes, + double replay_operation_cost = 0.0) { + return add(handlerName, targetName, IFlushTarget::Type::FLUSH, flushedSerial, approxDiskBytes, replay_operation_cost); } ContextsBuilder &add(const vespalib::string &targetName, SerialNum flushedSerial, - uint64_t approxDiskBytes) { - return add("handler1", targetName, IFlushTarget::Type::FLUSH, flushedSerial, approxDiskBytes); + uint64_t approxDiskBytes, + double replay_operation_cost = 0.0) { + return add("handler1", targetName, IFlushTarget::Type::FLUSH, flushedSerial, approxDiskBytes, replay_operation_cost); } ContextsBuilder &addGC(const vespalib::string &targetName, SerialNum flushedSerial, - uint64_t approxDiskBytes) { - return add("handler1", targetName, IFlushTarget::Type::GC, flushedSerial, approxDiskBytes); + uint64_t approxDiskBytes, + double replay_operation_cost = 0.0) { + return add("handler1", targetName, IFlushTarget::Type::GC, flushedSerial, approxDiskBytes, replay_operation_cost); } FlushContext::List build() const { return _result; } }; @@ -99,6 +104,7 @@ class CandidatesBuilder private: const FlushContext::List *_sortedFlushContexts; size_t _numCandidates; + mutable std::vector<FlushTargetCandidate> _candidates; flushengine::TlsStats _tlsStats; Config _cfg; @@ -106,6 +112,7 @@ public: CandidatesBuilder(const FlushContext::List &sortedFlushContexts) : _sortedFlushContexts(&sortedFlushContexts), _numCandidates(sortedFlushContexts.size()), + _candidates(), _tlsStats(1000, 11, 110), _cfg(2.0, 3.0, 4.0) {} @@ -125,8 +132,16 @@ public: replayEndSerial); return *this; } + void setup_candidates() const { + _candidates.clear(); + _candidates.reserve(_sortedFlushContexts->size()); + for (const auto &flush_context : *_sortedFlushContexts) { + _candidates.emplace_back(flush_context, _tlsStats.getLastSerial(), _cfg); + } + } FlushTargetCandidates build() const { - return FlushTargetCandidates(*_sortedFlushContexts, + setup_candidates(); + return FlushTargetCandidates(_candidates, _numCandidates, _tlsStats, _cfg); @@ -196,9 +211,12 @@ struct FlushStrategyFixture { flushengine::TlsStatsMap _tlsStatsMap; PrepareRestartFlushStrategy strategy; - FlushStrategyFixture() + FlushStrategyFixture(const Config &config) : _tlsStatsMap(defaultTransactionLogStats()), - strategy(DEFAULT_CFG) + strategy(config) + {} + FlushStrategyFixture() + : FlushStrategyFixture(DEFAULT_CFG) {} FlushContext::List getFlushTargets(const FlushContext::List &targetList, const flushengine::TlsStatsMap &tlsStatsMap) const { @@ -297,6 +315,12 @@ TEST_F("require that flush targets for different flush handlers are treated inde TEST_DO(assertFlushContexts("[foo,baz,quz]", targets)); } +TEST_F("require that expensive to replay target is flushed", FlushStrategyFixture(Config(2.0, 1.0, 4.0))) +{ + FlushContext::List targets = f.getFlushTargets(ContextsBuilder(). + add("foo", 10, 249).add("bar", 60, 150).add("baz", 60, 150, 12.0).build(), f._tlsStatsMap); + TEST_DO(assertFlushContexts("[foo,baz]", targets)); +} TEST_MAIN() { diff --git a/searchcore/src/tests/proton/initializer/task_runner_test.cpp b/searchcore/src/tests/proton/initializer/task_runner_test.cpp index 0363da8d083..f351b25cfca 100644 --- a/searchcore/src/tests/proton/initializer/task_runner_test.cpp +++ b/searchcore/src/tests/proton/initializer/task_runner_test.cpp @@ -36,14 +36,17 @@ class NamedTask : public InitializerTask protected: vespalib::string _name; TestLog &_log; + size_t _transient_memory_usage; public: - NamedTask(const vespalib::string &name, TestLog &log) + NamedTask(const vespalib::string &name, TestLog &log, size_t transient_memory_usage = 0) : _name(name), - _log(log) + _log(log), + _transient_memory_usage(transient_memory_usage) { } virtual void run() override { _log.append(_name); } + size_t get_transient_memory_usage() const override { return _transient_memory_usage; } }; @@ -79,6 +82,22 @@ struct TestJob { B->addDependency(D); return TestJob(std::move(log), std::move(C)); } + + static TestJob setupResourceUsingTasks() + { + auto log = std::make_unique<TestLog>(); + auto task_a = std::make_shared<NamedTask>("A", *log, 0); + auto task_b = std::make_shared<NamedTask>("B", *log, 10); + auto task_c = std::make_shared<NamedTask>("C", *log, 2); + auto task_d = std::make_shared<NamedTask>("D", *log, 5); + auto task_e = std::make_shared<NamedTask>("E", *log, 0); + task_e->addDependency(task_a); + task_e->addDependency(task_b); + task_e->addDependency(task_c); + task_e->addDependency(task_d); + return TestJob(std::move(log), std::move(task_e)); + } + }; TestJob::TestJob(TestLog::UP log, InitializerTask::SP root) @@ -138,6 +157,13 @@ TEST_F("multiple threads, dag graph", Fixture(10)) LOG(info, "dabc=%d, dbac=%d", dabc_count, dbac_count); } +TEST_F("single thread with resource using tasks", Fixture(1)) +{ + auto job = TestJob::setupResourceUsingTasks(); + f.run(job._root); + EXPECT_EQUAL("BDCAE", job._log->result()); +} + TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 00a319db394..9d5b67af81c 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -363,10 +363,10 @@ struct MyWorld { return docsum_matcher->get_rank_features(); } - MatchingElements::UP get_matching_elements(const DocsumRequest &req, const StructFieldMapper &mapper) { + MatchingElements::UP get_matching_elements(const DocsumRequest &req, const MatchingElementsFields &fields) { Matcher::SP matcher = createMatcher(); auto docsum_matcher = matcher->create_docsum_matcher(req, searchContext, attributeContext, *sessionManager); - return docsum_matcher->get_matching_elements(mapper); + return docsum_matcher->get_matching_elements(fields); } }; @@ -922,10 +922,10 @@ TEST("require that docsum matcher can extract matching elements from same elemen world.basicSetup(); world.add_same_element_results("foo", "bar"); auto request = world.create_docsum_request(make_same_element_stack_dump("foo", "bar"), {20}); - StructFieldMapper mapper; - mapper.add_mapping("my", "my.a1"); - mapper.add_mapping("my", "my.f1"); - auto result = world.get_matching_elements(*request, mapper); + MatchingElementsFields fields; + fields.add_mapping("my", "my.a1"); + fields.add_mapping("my", "my.f1"); + auto result = world.get_matching_elements(*request, fields); const auto &list = result->get_matching_elements(20, "my"); ASSERT_EQUAL(list.size(), 1u); EXPECT_EQUAL(list[0], 2u); @@ -936,10 +936,10 @@ TEST("require that docsum matcher can extract matching elements from single attr world.basicSetup(); world.add_same_element_results("foo", "bar"); auto request = world.create_docsum_request(make_simple_stack_dump("my.a1", "foo"), {20}); - StructFieldMapper mapper; - mapper.add_mapping("my", "my.a1"); - mapper.add_mapping("my", "my.f1"); - auto result = world.get_matching_elements(*request, mapper); + MatchingElementsFields fields; + fields.add_mapping("my", "my.a1"); + fields.add_mapping("my", "my.f1"); + auto result = world.get_matching_elements(*request, fields); const auto &list = result->get_matching_elements(20, "my"); ASSERT_EQUAL(list.size(), 2u); EXPECT_EQUAL(list[0], 2u); diff --git a/searchcore/src/tests/proton/matching/querynodes_test.cpp b/searchcore/src/tests/proton/matching/querynodes_test.cpp index 5d01753dfb6..c1247b630a3 100644 --- a/searchcore/src/tests/proton/matching/querynodes_test.cpp +++ b/searchcore/src/tests/proton/matching/querynodes_test.cpp @@ -48,6 +48,9 @@ using search::query::QueryBuilder; using search::queryeval::AndNotSearch; using search::queryeval::AndSearch; using search::queryeval::Blueprint; +using search::queryeval::ChildrenIterators; +using search::queryeval::ElementIteratorWrapper; +using search::queryeval::ElementIterator; using search::queryeval::EmptySearch; using search::queryeval::FakeRequestContext; using search::queryeval::FakeResult; @@ -102,12 +105,12 @@ public: explicit Create(bool strict = true) : _strict(strict) {} Create &add(SearchIterator *s) { - _children.push_back(s); + _children.emplace_back(s); return *this; } - operator SearchIterator *() const { - return SearchType::create(_children, _strict); + operator SearchIterator *() { + return SearchType::create(std::move(_children), _strict).release(); } }; typedef Create<OrSearch> MyOr; @@ -141,9 +144,11 @@ public: return *this; } - operator SearchIterator *() const { + operator SearchIterator *() { return SourceBlenderSearch::create( - ISourceSelectorDummy::makeDummyIterator(), _children, _strict); + ISourceSelectorDummy::makeDummyIterator(), + _children, + _strict).release(); } }; @@ -243,20 +248,20 @@ SearchIterator *getLeaf(const string &fld, const string &tag) { template <> SearchIterator *getLeaf<Phrase>(const string &fld, const string &tag) { SimplePhraseSearch::Children children; - children.push_back(getTerm(phrase_term1, fld, tag)); - children.push_back(getTerm(phrase_term2, fld, tag)); + children.emplace_back(getTerm(phrase_term1, fld, tag)); + children.emplace_back(getTerm(phrase_term2, fld, tag)); static TermFieldMatchData tmd; TermFieldMatchDataArray tfmda; tfmda.add(&tmd).add(&tmd); vector<uint32_t> eval_order(2); - return new SimplePhraseSearch(children, MatchData::UP(), tfmda, eval_order, tmd, true); + return new SimplePhraseSearch(std::move(children), MatchData::UP(), tfmda, eval_order, tmd, true); } template <typename NearType> SearchIterator *getNearParent(SearchIterator *a, SearchIterator *b) { typename NearType::Children children; - children.push_back(a); - children.push_back(b); + children.emplace_back(a); + children.emplace_back(b); TermFieldMatchDataArray data; static TermFieldMatchData tmd; // we only check how many term/field combinations @@ -264,15 +269,15 @@ SearchIterator *getNearParent(SearchIterator *a, SearchIterator *b) { // two terms searching in (two index fields + two attribute fields) data.add(&tmd).add(&tmd).add(&tmd).add(&tmd) .add(&tmd).add(&tmd).add(&tmd).add(&tmd); - return new NearType(children, data, distance, true); + return new NearType(std::move(children), data, distance, true); } template <typename SearchType> SearchIterator *getSimpleParent(SearchIterator *a, SearchIterator *b) { typename SearchType::Children children; - children.push_back(a); - children.push_back(b); - return SearchType::create(children, true); + children.emplace_back(a); + children.emplace_back(b); + return SearchType::create(std::move(children), true).release(); } template <typename T> @@ -290,16 +295,14 @@ SearchIterator *getParent<ONear>(SearchIterator *a, SearchIterator *b) { template <> SearchIterator *getParent<SameElement>(SearchIterator *a, SearchIterator *b) { - std::vector<SearchIterator::UP> children; - children.emplace_back(a); - children.emplace_back(b); - TermFieldMatchDataArray data; static TermFieldMatchData tmd; + std::vector<ElementIterator::UP> children; + children.emplace_back(std::make_unique<ElementIteratorWrapper>(SearchIterator::UP(a), tmd)); + children.emplace_back(std::make_unique<ElementIteratorWrapper>(SearchIterator::UP(b), tmd)); // we only check how many term/field combinations // are below the SameElement parent: // two terms searching in one index field - data.add(&tmd).add(&tmd); - return new SameElementSearch(nullptr, std::move(children), data, true); + return new SameElementSearch(nullptr, std::move(children), true); } template <> diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 2927457a6a5..d6a3fea4735 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -686,6 +686,7 @@ TEST_F("require that get returns the first document found", SimpleFixture) { EXPECT_EQUAL(tstamp1, result.getTimestamp()); ASSERT_TRUE(result.hasDocument()); EXPECT_EQUAL(*doc1, result.getDocument()); + EXPECT_FALSE(result.is_tombstone()); } TEST_F("require that createIterator does", SimpleFixture) { diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index a210698d5b6..fd60781d868 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -250,6 +250,9 @@ summary.log.chunk.skipcrconread bool default=false ## Max size per summary file. summary.log.maxfilesize long default=1000000000 +## Max number of lid entries per file +summary.log.maxnumlids int default=40000000 + ## Max disk bloat factor. This will trigger compacting. summary.log.maxdiskbloatfactor double default=0.1 @@ -396,7 +399,7 @@ lidspacecompaction.removebatchblockrate double default=0.5 ## When considering compaction, if the current observed rate of remove operations ## is higher than the given block rate, the lid space compaction job is blocked. ## It is considered again at the next regular interval (see above). -lidspacecompaction.removeblockrate double default=100000.0 +lidspacecompaction.removeblockrate double default=100.0 ## This is the maximum value visibilitydelay you can have. ## A to higher value here will cost more memory while not improving too much. diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp index 5c9321a6ff3..55e9ce16f70 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp @@ -20,7 +20,7 @@ GroupingContext::deserialize(const char *groupSpec, uint32_t groupSpecLen) uint32_t numGroupings = 0; nis >> numGroupings; for (size_t i = 0; i < numGroupings; i++) { - GroupingPtr grouping(new search::aggregation::Grouping); + auto grouping = std::make_shared<search::aggregation::Grouping>(); grouping->deserialize(nis); grouping->setClock(&_clock); grouping->setTimeOfDoom(_timeOfDoom); diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp index 2c2a5ceacff..e336d5b25c9 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp @@ -41,7 +41,7 @@ GroupingSession::init(GroupingContext & groupingContext, const IAttributeContext GroupingPtr g(sessionList[i]); // Make internal copy of those we want to keep for another pass if (!_sessionId.empty() && g->getLastLevel() < g->levels().size()) { - GroupingPtr gp(new Grouping(*g)); + auto gp = std::make_shared<Grouping>(*g); gp->setLastLevel(gp->levels().size()); _groupingMap[gp->getId()] = gp; g = gp; @@ -62,7 +62,7 @@ GroupingSession::prepareThreadContextCreation(size_t num_threads) GroupingContext::UP GroupingSession::createThreadContext(size_t thread_id, const IAttributeContext &attrCtx) { - GroupingContext::UP ctx(new GroupingContext(*_mgrContext)); + auto ctx = std::make_unique<GroupingContext>(*_mgrContext); if (thread_id == 0) { GroupingContext::GroupingList &groupingList = _mgrContext->getGroupingList(); for (size_t i = 0; i < groupingList.size(); ++i) { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt index 550d0a1c4c4..c20f279503f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(searchcore_attribute STATIC attribute_aspect_delayer.cpp attribute_collection_spec_factory.cpp attribute_collection_spec.cpp + attribute_config_inspector.cpp attribute_directory.cpp attribute_factory.cpp attribute_initializer.cpp @@ -13,6 +14,7 @@ vespa_add_library(searchcore_attribute STATIC attribute_manager_initializer.cpp attribute_populator.cpp attribute_spec.cpp + attribute_transient_memory_calculator.cpp attribute_type_matcher.cpp attribute_usage_filter.cpp attribute_usage_sampler_context.cpp diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp new file mode 100644 index 00000000000..d950b5e8ed3 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_config_inspector.h" +#include <vespa/searchlib/attribute/configconverter.h> +#include <vespa/config-attributes.h> +#include <vespa/vespalib/stllike/hash_map.hpp> + +using search::attribute::ConfigConverter; + +namespace proton { + +AttributeConfigInspector::AttributeConfigInspector(const AttributesConfig& config) + : _hash() +{ + for (auto& attr : config.attribute) { + auto res = _hash.insert(std::make_pair(attr.name, ConfigConverter::convert(attr))); + assert(res.second); + } +} + +AttributeConfigInspector::~AttributeConfigInspector() = default; + +const search::attribute::Config* +AttributeConfigInspector::get_config(const vespalib::string& name) const +{ + auto itr = _hash.find(name); + return (itr != _hash.end()) ? &itr->second : nullptr; +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h new file mode 100644 index 00000000000..8952b34c6ff --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/searchcommon/attribute/config.h> + +namespace vespa::config::search::internal { class InternalAttributesType; } + +namespace proton { + +/* + * Class used to find attribute config given attribute name based on config + * from config server. + */ +class AttributeConfigInspector { + vespalib::hash_map<vespalib::string, search::attribute::Config> _hash; +public: + using AttributesConfig = const vespa::config::search::internal::InternalAttributesType; + AttributeConfigInspector(const AttributesConfig& config); + ~AttributeConfigInspector(); + const search::attribute::Config* get_config(const vespalib::string& name) const; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index 88e4d7a2191..368e4ce12f2 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -4,6 +4,7 @@ #include "attributedisklayout.h" #include "attribute_directory.h" #include "i_attribute_factory.h" +#include "attribute_transient_memory_calculator.h" #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -12,8 +13,6 @@ #include <vespa/searchlib/util/fileutil.h> #include <vespa/searchlib/attribute/attribute_header.h> #include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/loadedenumvalue.h> -#include <vespa/searchlib/attribute/loadedvalue.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> @@ -263,29 +262,8 @@ size_t AttributeInitializer::get_transient_memory_usage() const { if (_header_ok) { - auto &header = *_header; - if (_spec.getConfig().fastSearch()) { - if (header.getEnumerated()) { - return sizeof(search::attribute::LoadedEnumAttribute) * header.get_total_value_count(); - } else { - switch (_spec.getConfig().basicType().type()) { - case BasicType::Type::INT8: - return sizeof(search::attribute::LoadedValue<int8_t>) * header.get_total_value_count(); - case BasicType::Type::INT16: - return sizeof(search::attribute::LoadedValue<int16_t>) * header.get_total_value_count(); - case BasicType::Type::INT32: - return sizeof(search::attribute::LoadedValue<int32_t>) * header.get_total_value_count(); - case BasicType::Type::INT64: - return sizeof(search::attribute::LoadedValue<int64_t>) * header.get_total_value_count(); - case BasicType::Type::FLOAT: - return sizeof(search::attribute::LoadedValue<float>) * header.get_total_value_count(); - case BasicType::Type::DOUBLE: - return sizeof(search::attribute::LoadedValue<double>) * header.get_total_value_count(); - default: - return 0u; - } - } - } + AttributeTransientMemoryCalculator get_transient_memory_usage; + return get_transient_memory_usage(*_header, _spec.getConfig()); } return 0u; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp index 435cd479305..9198cfdafab 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp @@ -42,6 +42,9 @@ public: _result.add(result); } } + size_t get_transient_memory_usage() const override { + return _initializer->get_transient_memory_usage(); + } }; class AttributeManagerInitializerTask : public vespalib::Executor::Task diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.cpp new file mode 100644 index 00000000000..37706328149 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.cpp @@ -0,0 +1,63 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_transient_memory_calculator.h" +#include <vespa/searchlib/attribute/attribute_header.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchlib/attribute/loadedenumvalue.h> +#include <vespa/searchlib/attribute/loadedvalue.h> +#include <vespa/searchlib/attribute/attributevector.h> + +using search::attribute::BasicType; + +namespace proton { + +namespace { + +size_t +get_transient_memory_usage(bool old_enumerated, + const search::attribute::Config& new_config, + uint64_t total_value_count) +{ + if (new_config.fastSearch()) { + if (old_enumerated) { + return sizeof(search::attribute::LoadedEnumAttribute) * total_value_count; + } else { + switch (new_config.basicType().type()) { + case BasicType::Type::INT8: + return sizeof(search::attribute::LoadedValue<int8_t>) * total_value_count; + case BasicType::Type::INT16: + return sizeof(search::attribute::LoadedValue<int16_t>) * total_value_count; + case BasicType::Type::INT32: + return sizeof(search::attribute::LoadedValue<int32_t>) * total_value_count; + case BasicType::Type::INT64: + return sizeof(search::attribute::LoadedValue<int64_t>) * total_value_count; + case BasicType::Type::FLOAT: + return sizeof(search::attribute::LoadedValue<float>) * total_value_count; + case BasicType::Type::DOUBLE: + return sizeof(search::attribute::LoadedValue<double>) * total_value_count; + default: + ; + } + } + } + return 0u; +} + +} +size_t +AttributeTransientMemoryCalculator::operator()(const search::AttributeVector& attribute_vector, + const search::attribute::Config& new_config) const +{ + uint64_t total_value_count = attribute_vector.getStatus().getNumValues(); + bool old_enumerated = attribute_vector.getEnumeratedSave(); + return get_transient_memory_usage(old_enumerated, new_config, total_value_count); +} + +size_t +AttributeTransientMemoryCalculator::operator()(const search::attribute::AttributeHeader& old_header, + const search::attribute::Config& new_config) const +{ + return get_transient_memory_usage(old_header.getEnumerated(), new_config, old_header.get_total_value_count()); +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.h new file mode 100644 index 00000000000..34ab8f02768 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.h @@ -0,0 +1,34 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstddef> +#include <cstdint> + +namespace search { class AttributeVector; } + +namespace search::attribute { + +class AttributeHeader; +class Config; + +} + +namespace proton { + +/** + * Class to calculate transient memory during load of attribute vector + * in the future based on current attribute vector and new config. + */ +class AttributeTransientMemoryCalculator +{ +public: + AttributeTransientMemoryCalculator() = default; + ~AttributeTransientMemoryCalculator() = default; + size_t operator()(const search::AttributeVector& attribute_vector, + const search::attribute::Config& new_config) const; + size_t operator()(const search::attribute::AttributeHeader& old_header, + const search::attribute::Config& new_config) const; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp index 39235fb8d00..43c9e52315b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp @@ -2,28 +2,35 @@ #include "attribute_usage_sampler_context.h" #include "attribute_usage_filter.h" +#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h> namespace proton { -AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter &filter) +AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter &filter, std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider) : _usage(), + _transient_memory_usage(0u), _lock(), - _filter(filter) + _filter(filter), + _attribute_config_inspector(std::move(attribute_config_inspector)), + _transient_memory_usage_provider(std::move(transient_memory_usage_provider)) { } AttributeUsageSamplerContext::~AttributeUsageSamplerContext() { _filter.setAttributeStats(_usage); + _transient_memory_usage_provider->set_transient_memory_usage(_transient_memory_usage); } void AttributeUsageSamplerContext::merge(const search::AddressSpaceUsage &usage, + size_t transient_memory_usage, const vespalib::string &attributeName, const vespalib::string &subDbName) { Guard guard(_lock); _usage.merge(usage, attributeName, subDbName); + _transient_memory_usage = std::max(_transient_memory_usage, transient_memory_usage); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h index 21e4ff6a90f..6f33ac483c1 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h @@ -8,10 +8,13 @@ namespace proton { class AttributeUsageFilter; +class AttributeConfigInspector; +class TransientMemoryUsageProvider; /* - * Context for sampling attribute usage stats. When instance is - * destroyed, the aggregated stats is passed on to attribute usage filter. + * Context for sampling attribute usage stats and transient memory usage. + * When instance is destroyed, the aggregated stats is passed on to + * attribute usage filter and the transient memory usage provider. */ class AttributeUsageSamplerContext { @@ -19,16 +22,22 @@ class AttributeUsageSamplerContext using Guard = std::lock_guard<Mutex>; AttributeUsageStats _usage; + size_t _transient_memory_usage; Mutex _lock; AttributeUsageFilter &_filter; + std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector; + std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; public: - AttributeUsageSamplerContext(AttributeUsageFilter &filter); + AttributeUsageSamplerContext(AttributeUsageFilter &filter, std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider); ~AttributeUsageSamplerContext(); void merge(const search::AddressSpaceUsage &usage, + size_t transient_memory_usage, const vespalib::string &attributeName, const vespalib::string &subDbName); + const AttributeConfigInspector& get_attribute_config_inspector() const { return *_attribute_config_inspector; } const AttributeUsageStats & getUsage() const { return _usage; } + size_t get_transient_memory_usage() const { return _transient_memory_usage; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp index a1a8409e93b..685433cd5c7 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp @@ -2,8 +2,12 @@ #include "attribute_usage_sampler_functor.h" #include "attribute_usage_sampler_context.h" +#include "attribute_config_inspector.h" +#include "attribute_transient_memory_calculator.h" #include <vespa/searchlib/attribute/attributevector.h> +using search::attribute::BasicType; + namespace proton { AttributeUsageSamplerFunctor::AttributeUsageSamplerFunctor( @@ -23,7 +27,14 @@ AttributeUsageSamplerFunctor::operator()(const search::attribute::IAttributeVect const auto & attributeVector = dynamic_cast<const search::AttributeVector &>(iAttributeVector); search::AddressSpaceUsage usage = attributeVector.getAddressSpaceUsage(); vespalib::string attributeName = attributeVector.getName(); - _samplerContext->merge(usage, attributeName, _subDbName); + auto& old_config = attributeVector.getConfig(); + auto* current_config = _samplerContext->get_attribute_config_inspector().get_config(attributeName); + if (current_config == nullptr) { + current_config = &old_config; + } + AttributeTransientMemoryCalculator get_transient_memory_usage; + size_t transient_memory_usage = get_transient_memory_usage(attributeVector, *current_config); + _samplerContext->merge(usage, transient_memory_usage, attributeName, _subDbName); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 33b9d162163..8f19d5c203b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -12,26 +12,48 @@ #include <vespa/searchcore/proton/common/attribute_updater.h> #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/imported_attribute_vector.h> +#include <vespa/searchlib/tensor/prepare_result.h> #include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <future> #include <vespa/log/log.h> LOG_SETUP(".proton.attribute.attribute_writer"); using namespace document; using namespace search; + +using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId; using search::attribute::ImportedAttributeVector; +using search::tensor::PrepareResult; using vespalib::ISequencedTaskExecutor; -using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId; namespace proton { using LidVector = LidVectorContext::LidVector; +namespace { + +bool +use_two_phase_put_for_attribute(const AttributeVector& attr) +{ + const auto& cfg = attr.getConfig(); + if (cfg.basicType() == search::attribute::BasicType::Type::TENSOR && + cfg.hnsw_index_params().has_value() && + cfg.hnsw_index_params().value().allow_multi_threaded_indexing()) + { + return true; + } + return false; +} + +} + AttributeWriter::WriteField::WriteField(AttributeVector &attribute) : _fieldPath(), _attribute(attribute), - _structFieldAttribute(false) + _structFieldAttribute(false), + _use_two_phase_put(use_two_phase_put_for_attribute(attribute)) { const vespalib::string &name = attribute.getName(); _structFieldAttribute = attribute::isStructFieldAttribute(name); @@ -57,11 +79,11 @@ AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType) AttributeWriter::WriteContext::WriteContext(ExecutorId executorId) : _executorId(executorId), _fields(), - _hasStructFieldAttribute(false) + _hasStructFieldAttribute(false), + _use_two_phase_put(false) { } - AttributeWriter::WriteContext::WriteContext(WriteContext &&rhs) noexcept = default; AttributeWriter::WriteContext::~WriteContext() = default; @@ -75,6 +97,13 @@ AttributeWriter::WriteContext::add(AttributeVector &attr) if (_fields.back().isStructFieldAttribute()) { _hasStructFieldAttribute = true; } + if (_fields.back().use_two_phase_put()) { + // Only support for one field per context when this is true. + assert(_fields.size() == 1); + _use_two_phase_put = true; + } else { + assert(!_use_two_phase_put); + } } void @@ -113,6 +142,27 @@ applyPutToAttribute(SerialNum serialNum, const FieldValue::UP &fieldValue, Docum } void +complete_put_to_attribute(SerialNum serial_num, + uint32_t docid, + AttributeVector& attr, + const FieldValue::SP& field_value, + std::future<std::unique_ptr<PrepareResult>>& result_future, + bool immediate_commit, + AttributeWriter::OnWriteDoneType) +{ + ensureLidSpace(serial_num, docid, attr); + if (field_value.get()) { + auto result = result_future.get(); + AttributeUpdater::complete_set_value(attr, docid, *field_value, std::move(result)); + } else { + attr.clearDoc(docid); + } + if (immediate_commit) { + attr.commit(serial_num, serial_num); + } +} + +void applyRemoveToAttribute(SerialNum serialNum, DocumentIdT lid, bool immediateCommit, AttributeVector &attr, AttributeWriter::OnWriteDoneType) { @@ -148,7 +198,6 @@ applyReplayDone(uint32_t docIdLimit, AttributeVector &attr) attr.shrinkLidSpace(); } - void applyHeartBeat(SerialNum serialNum, AttributeVector &attr) { @@ -166,7 +215,6 @@ applyCommit(SerialNum serialNum, AttributeWriter::OnWriteDoneType , AttributeVec } } - void applyCompactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum, AttributeVector &attr) { @@ -208,7 +256,6 @@ struct BatchUpdateTask : public vespalib::Executor::Task { } } - SerialNum _serialNum; DocumentIdT _lid; bool _immediateCommit; @@ -221,6 +268,7 @@ class FieldContext vespalib::string _name; ExecutorId _executorId; AttributeVector *_attr; + bool _use_two_phase_put; public: FieldContext(ISequencedTaskExecutor &writer, AttributeVector *attr); @@ -228,13 +276,14 @@ public: bool operator<(const FieldContext &rhs) const; ExecutorId getExecutorId() const { return _executorId; } AttributeVector *getAttribute() const { return _attr; } + bool use_two_phase_put() const { return _use_two_phase_put; } }; - FieldContext::FieldContext(ISequencedTaskExecutor &writer, AttributeVector *attr) : _name(attr->getName()), _executorId(writer.getExecutorId(attr->getNamePrefix())), - _attr(attr) + _attr(attr), + _use_two_phase_put(use_two_phase_put_for_attribute(*attr)) { } @@ -303,6 +352,100 @@ PutTask::run() } } + +class PreparePutTask : public vespalib::Executor::Task { +private: + const SerialNum _serial_num; + const uint32_t _docid; + AttributeVector& _attr; + FieldValue::SP _field_value; + std::promise<std::unique_ptr<PrepareResult>> _result_promise; + +public: + PreparePutTask(SerialNum serial_num_in, + uint32_t docid_in, + const AttributeWriter::WriteField& field, + std::shared_ptr<DocumentFieldExtractor> field_extractor); + ~PreparePutTask() override; + void run() override; + SerialNum serial_num() const { return _serial_num; } + uint32_t docid() const { return _docid; } + AttributeVector& attr() { return _attr; } + FieldValue::SP field_value() { return _field_value; } + std::future<std::unique_ptr<PrepareResult>> result_future() { + return _result_promise.get_future(); + } +}; + +PreparePutTask::PreparePutTask(SerialNum serial_num_in, + uint32_t docid_in, + const AttributeWriter::WriteField& field, + std::shared_ptr<DocumentFieldExtractor> field_extractor) + : _serial_num(serial_num_in), + _docid(docid_in), + _attr(field.getAttribute()), + _field_value(), + _result_promise() +{ + // Note: No need to store the field extractor as we are not extracting struct fields. + auto value = field_extractor->getFieldValue(field.getFieldPath()); + _field_value.reset(value.release()); +} + +PreparePutTask::~PreparePutTask() = default; + +void +PreparePutTask::run() +{ + if (_attr.getStatus().getLastSyncToken() < _serial_num) { + if (_field_value.get()) { + _result_promise.set_value(AttributeUpdater::prepare_set_value(_attr, _docid, *_field_value)); + } + } +} + +class CompletePutTask : public vespalib::Executor::Task { +private: + const SerialNum _serial_num; + const uint32_t _docid; + AttributeVector& _attr; + FieldValue::SP _field_value; + std::future<std::unique_ptr<PrepareResult>> _result_future; + const bool _immediate_commit; + std::remove_reference_t<AttributeWriter::OnWriteDoneType> _on_write_done; + +public: + CompletePutTask(PreparePutTask& prepare_task, + bool immediate_commit, + AttributeWriter::OnWriteDoneType on_write_done); + ~CompletePutTask() override; + void run() override; +}; + +CompletePutTask::CompletePutTask(PreparePutTask& prepare_task, + bool immediate_commit, + AttributeWriter::OnWriteDoneType on_write_done) + : _serial_num(prepare_task.serial_num()), + _docid(prepare_task.docid()), + _attr(prepare_task.attr()), + _field_value(prepare_task.field_value()), + _result_future(prepare_task.result_future()), + _immediate_commit(immediate_commit), + _on_write_done(on_write_done) +{ +} + +CompletePutTask::~CompletePutTask() = default; + +void +CompletePutTask::run() +{ + if (_attr.getStatus().getLastSyncToken() < _serial_num) { + complete_put_to_attribute(_serial_num, _docid, _attr, _field_value, _result_future, + _immediate_commit, _on_write_done); + } +} + class RemoveTask : public vespalib::Executor::Task { const AttributeWriter::WriteContext &_wc; @@ -316,7 +459,6 @@ public: void run() override; }; - RemoveTask::RemoveTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, uint32_t lid, bool immediateCommit, AttributeWriter::OnWriteDoneType onWriteDone) : _wc(wc), _serialNum(serialNum), @@ -419,13 +561,22 @@ AttributeWriter::setupWriteContexts() fieldContexts.emplace_back(_attributeFieldWriter, attr); } std::sort(fieldContexts.begin(), fieldContexts.end()); - for (auto &fc : fieldContexts) { + for (const auto& fc : fieldContexts) { + if (fc.use_two_phase_put()) { + continue; + } if (_writeContexts.empty() || (_writeContexts.back().getExecutorId() != fc.getExecutorId())) { _writeContexts.emplace_back(fc.getExecutorId()); } _writeContexts.back().add(*fc.getAttribute()); } + for (const auto& fc : fieldContexts) { + if (fc.use_two_phase_put()) { + _writeContexts.emplace_back(fc.getExecutorId()); + _writeContexts.back().add(*fc.getAttribute()); + } + } for (const auto &wc : _writeContexts) { if (wc.hasStructFieldAttribute()) { _hasStructFieldAttribute = true; @@ -452,9 +603,19 @@ AttributeWriter::internalPut(SerialNum serialNum, const Document &doc, DocumentI } auto extractor = std::make_shared<DocumentFieldExtractor>(doc); for (const auto &wc : _writeContexts) { - if (allAttributes || wc.hasStructFieldAttribute()) { - auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, immediateCommit, allAttributes, onWriteDone); - _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask)); + if (wc.use_two_phase_put()) { + assert(wc.getFields().size() == 1); + auto prepare_task = std::make_unique<PreparePutTask>(serialNum, lid, wc.getFields()[0], extractor); + auto complete_task = std::make_unique<CompletePutTask>(*prepare_task, immediateCommit, onWriteDone); + // We use the local docid to create an executor id to round-robin between the threads. + _attributeFieldWriter.executeTask(_attributeFieldWriter.getExecutorId(lid), std::move(prepare_task)); + _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(complete_task)); + } else { + if (allAttributes || wc.hasStructFieldAttribute()) { + auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, immediateCommit, allAttributes, + onWriteDone); + _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask)); + } } } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index 4a9726dd113..726379220e3 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -19,20 +19,23 @@ namespace proton { class AttributeWriter : public IAttributeWriter { private: - typedef search::AttributeVector AttributeVector; - typedef document::FieldPath FieldPath; - typedef document::DataType DataType; - typedef document::DocumentType DocumentType; - typedef document::FieldValue FieldValue; + using AttributeVector = search::AttributeVector; + using FieldPath = document::FieldPath; + using DataType = document::DataType; + using DocumentType = document::DocumentType; + using FieldValue = document::FieldValue; const IAttributeManager::SP _mgr; vespalib::ISequencedTaskExecutor &_attributeFieldWriter; using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId; public: - class WriteField - { + /** + * Represents an attribute vector for a field and details about how to write to it. + */ + class WriteField { FieldPath _fieldPath; AttributeVector &_attribute; bool _structFieldAttribute; // in array/map of struct + bool _use_two_phase_put; public: WriteField(AttributeVector &attribute); ~WriteField(); @@ -40,12 +43,18 @@ public: const FieldPath &getFieldPath() const { return _fieldPath; } void buildFieldPath(const DocumentType &docType); bool isStructFieldAttribute() const { return _structFieldAttribute; } + bool use_two_phase_put() const { return _use_two_phase_put; } }; - class WriteContext - { + + /** + * Represents a set of fields (as attributes) that are handled by the same write thread. + */ + class WriteContext { ExecutorId _executorId; std::vector<WriteField> _fields; bool _hasStructFieldAttribute; + // When this is true, the context only contains a single field. + bool _use_two_phase_put; public: WriteContext(ExecutorId executorId); WriteContext(WriteContext &&rhs) noexcept; @@ -56,6 +65,7 @@ public: ExecutorId getExecutorId() const { return _executorId; } const std::vector<WriteField> &getFields() const { return _fields; } bool hasStructFieldAttribute() const { return _hasStructFieldAttribute; } + bool use_two_phase_put() const { return _use_two_phase_put; } }; private: using AttrWithId = std::pair<search::AttributeVector *, ExecutorId>; @@ -103,6 +113,11 @@ public: void onReplayDone(uint32_t docIdLimit) override; bool hasStructFieldAttribute() const override; + + // Should only be used for unit testing. + const std::vector<WriteContext>& get_write_contexts() const { + return _writeContexts; + } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index 0a61ec8d882..2b5f4b028dc 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -165,9 +165,15 @@ FlushableAttribute::FlushableAttribute(const AttributeVectorSP attr, _fileHeaderContext(fileHeaderContext), _attributeFieldWriter(attributeFieldWriter), _hwInfo(hwInfo), - _attrDir(attrDir) + _attrDir(attrDir), + _replay_operation_cost(0.0) { _lastStats.setPathElementsToLog(8); + auto &config = attr->getConfig(); + if (config.basicType() == search::attribute::BasicType::Type::TENSOR && + config.tensorType().is_tensor() && config.tensorType().is_dense() && config.hnsw_index_params().has_value()) { + _replay_operation_cost = 100.0; // replaying operations to hnsw index is 100 times more expensive than reading from tls + } } @@ -236,4 +242,10 @@ FlushableAttribute::getApproxBytesToWriteToDisk() const return _attr->getEstimatedSaveByteSize(); } +double +FlushableAttribute::get_replay_operation_cost() const +{ + return _replay_operation_cost; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h index 8d807c153c0..a759bcce26e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h @@ -38,6 +38,7 @@ private: vespalib::ISequencedTaskExecutor &_attributeFieldWriter; HwInfo _hwInfo; std::shared_ptr<AttributeDirectory> _attrDir; + double _replay_operation_cost; Task::UP internalInitFlush(SerialNum currentSerial); @@ -71,6 +72,7 @@ public: virtual Task::UP initFlush(SerialNum currentSerial) override; virtual FlushStats getLastFlushStats() const override { return _lastStats; } virtual uint64_t getApproxBytesToWriteToDisk() const override; + virtual double get_replay_operation_cost() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index f3303f36199..4270006e301 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -20,8 +20,13 @@ vespa_add_library(searchcore_pcommon STATIC selectpruner.cpp state_reporter_utils.cpp statusreport.cpp + transient_memory_usage_provider.cpp DEPENDS searchcore_proton_metrics searchcore_fconfig ${VESPA_STDCXX_FS_LIB} ) + +if(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.2" OR VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8") + set_source_files_properties(hw_info_sampler.cpp PROPERTIES COMPILE_FLAGS -DRHEL_8_2_KLUDGE) +endif() diff --git a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp index 8fd47c17acb..d7cf6caff28 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp @@ -31,6 +31,7 @@ LOG_SETUP(".proton.common.attribute_updater"); using namespace document; using vespalib::make_string; +using search::tensor::PrepareResult; using search::tensor::TensorAttribute; using search::attribute::ReferenceAttribute; @@ -471,27 +472,33 @@ AttributeUpdater::updateValue(StringAttribute & vec, uint32_t lid, const FieldVa } } +namespace { + +template <typename ExpFieldValueType> void -AttributeUpdater::updateValue(PredicateAttribute &vec, uint32_t lid, const FieldValue &val) +validate_field_value_type(const FieldValue& val, const vespalib::string& attr_type, const vespalib::string& value_type) { - if (!val.inherits(PredicateFieldValue::classId)) { + if (!val.inherits(ExpFieldValueType::classId)) { throw UpdateException( - make_string("PredicateAttribute must be updated with " - "PredicateFieldValues.")); + make_string("%s must be updated with %s, but was '%s'", + attr_type.c_str(), value_type.c_str(), val.toString(false).c_str())); } +} + +} + +void +AttributeUpdater::updateValue(PredicateAttribute &vec, uint32_t lid, const FieldValue &val) +{ + validate_field_value_type<PredicateFieldValue>(val, "PredicateAttribute", "PredicateFieldValue"); vec.updateValue(lid, static_cast<const PredicateFieldValue &>(val)); } void AttributeUpdater::updateValue(TensorAttribute &vec, uint32_t lid, const FieldValue &val) { - if (!val.inherits(TensorFieldValue::classId)) { - throw UpdateException( - make_string("TensorAttribute must be updated with " - "TensorFieldValues.")); - } - const auto &tensor = static_cast<const TensorFieldValue &>(val). - getAsTensorPtr(); + validate_field_value_type<TensorFieldValue>(val, "TensorAttribute", "TensorFieldValue"); + const auto &tensor = static_cast<const TensorFieldValue &>(val).getAsTensorPtr(); if (tensor) { vec.setTensor(lid, *tensor); } else { @@ -506,7 +513,7 @@ AttributeUpdater::updateValue(ReferenceAttribute &vec, uint32_t lid, const Field vec.clearDoc(lid); throw UpdateException( make_string("ReferenceAttribute must be updated with " - "ReferenceFieldValues.")); + "ReferenceFieldValue, but was '%s'", val.toString(false).c_str())); } const auto &reffv = static_cast<const ReferenceFieldValue &>(val); if (reffv.hasValidDocumentId()) { @@ -516,4 +523,57 @@ AttributeUpdater::updateValue(ReferenceAttribute &vec, uint32_t lid, const Field } } +namespace { + +void +validate_tensor_attribute_type(AttributeVector& attr) +{ + const auto& info = attr.getClass(); + if (!info.inherits(TensorAttribute::classId)) { + throw UpdateException( + make_string("Expected attribute vector '%s' to be a TensorAttribute, but was '%s'", + attr.getName().c_str(), info.name())); + } +} + +std::unique_ptr<PrepareResult> +prepare_set_tensor(TensorAttribute& attr, uint32_t docid, const FieldValue& val) +{ + validate_field_value_type<TensorFieldValue>(val, "TensorAttribute", "TensorFieldValue"); + const auto& tensor = static_cast<const TensorFieldValue&>(val).getAsTensorPtr(); + if (tensor) { + return attr.prepare_set_tensor(docid, *tensor); + } + return std::unique_ptr<PrepareResult>(); +} + +void +complete_set_tensor(TensorAttribute& attr, uint32_t docid, const FieldValue& val, std::unique_ptr<PrepareResult> prepare_result) +{ + validate_field_value_type<TensorFieldValue>(val, "TensorAttribute", "TensorFieldValue"); + const auto& tensor = static_cast<const TensorFieldValue&>(val).getAsTensorPtr(); + if (tensor) { + attr.complete_set_tensor(docid, *tensor, std::move(prepare_result)); + } else { + attr.clearDoc(docid); + } +} + +} + +std::unique_ptr<PrepareResult> +AttributeUpdater::prepare_set_value(AttributeVector& attr, uint32_t docid, const FieldValue& val) +{ + validate_tensor_attribute_type(attr); + return prepare_set_tensor(static_cast<TensorAttribute&>(attr), docid, val); +} + +void +AttributeUpdater::complete_set_value(AttributeVector& attr, uint32_t docid, const FieldValue& val, + std::unique_ptr<PrepareResult> prepare_result) +{ + validate_tensor_attribute_type(attr); + complete_set_tensor(static_cast<TensorAttribute&>(attr), docid, val, std::move(prepare_result)); +} + } // namespace search diff --git a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h index 01be6299692..32d14f6dd5a 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h +++ b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.h @@ -10,7 +10,10 @@ namespace search { class PredicateAttribute; -namespace tensor { class TensorAttribute; } +namespace tensor { + class PrepareResult; + class TensorAttribute; +} namespace attribute {class ReferenceAttribute; } VESPA_DEFINE_EXCEPTION(UpdateException, vespalib::Exception); @@ -20,14 +23,18 @@ VESPA_DEFINE_EXCEPTION(UpdateException, vespalib::Exception); */ class AttributeUpdater { using Field = document::Field; - using FieldValue = document::FieldValue; using FieldUpdate = document::FieldUpdate; + using FieldValue = document::FieldValue; using ValueUpdate = document::ValueUpdate; public: static void handleUpdate(AttributeVector & vec, uint32_t lid, const FieldUpdate & upd); static void handleValue(AttributeVector & vec, uint32_t lid, const FieldValue & val); + static std::unique_ptr<tensor::PrepareResult> prepare_set_value(AttributeVector& attr, uint32_t docid, const FieldValue& val); + static void complete_set_value(AttributeVector& attr, uint32_t docid, const FieldValue& val, + std::unique_ptr<tensor::PrepareResult> prepare_result); + private: template <typename V> static void handleUpdate(V & vec, uint32_t lid, const ValueUpdate & upd); diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp index b381ee9122f..cdec0b440c4 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp @@ -157,3 +157,16 @@ HwInfoSampler::sampleDiskWriteSpeed(const vespalib::string &path, const Config & } } + +#ifdef RHEL_8_2_KLUDGE + +// Kludge to avoid unresolved symbols with gcc-toolset-9 on RHEL 8.2 +#include <codecvt> + +namespace std { + +template class codecvt_utf8<wchar_t>; + +} + +#endif diff --git a/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h new file mode 100644 index 00000000000..cd87150195c --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/i_transient_memory_usage_provider.h @@ -0,0 +1,20 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstddef> + +namespace proton { + +/* + * Interface class providing transient memory usage, e.g. extra memory needed + * for loading or saving an attribute vector. It provides an aggregated view + * over several components (e.g. all attribute vectors for a document type). + */ +class ITransientMemoryUsageProvider { +public: + virtual ~ITransientMemoryUsageProvider() = default; + virtual size_t get_transient_memory_usage() const = 0; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp new file mode 100644 index 00000000000..0752af29a0a --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.cpp @@ -0,0 +1,27 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "transient_memory_usage_provider.h" + +namespace proton { + +TransientMemoryUsageProvider::TransientMemoryUsageProvider() + : ITransientMemoryUsageProvider(), + _transient_memory_usage(0u) +{ +} + +TransientMemoryUsageProvider::~TransientMemoryUsageProvider() = default; + +size_t +TransientMemoryUsageProvider::get_transient_memory_usage() const +{ + return _transient_memory_usage.load(std::memory_order_relaxed); +} + +void +TransientMemoryUsageProvider::set_transient_memory_usage(size_t transient_memory_usage) +{ + _transient_memory_usage.store(transient_memory_usage, std::memory_order_relaxed); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h new file mode 100644 index 00000000000..8cda4278ad8 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/transient_memory_usage_provider.h @@ -0,0 +1,25 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_transient_memory_usage_provider.h" + +#include <atomic> + +namespace proton { + +/* + * Class providing transient memory usage, e.g. extra memory needed + * for loading or saving an attribute vector. It provides an aggregated view + * over several components (e.g. all attribute vectors for a document type). + */ +class TransientMemoryUsageProvider : public ITransientMemoryUsageProvider { + std::atomic<size_t> _transient_memory_usage; +public: + TransientMemoryUsageProvider(); + virtual ~TransientMemoryUsageProvider(); + size_t get_transient_memory_usage() const override; + void set_transient_memory_usage(size_t transient_memory_usage); +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index 872b4b27584..0869fc175a7 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -64,7 +64,7 @@ DocsumContext::initState() DocsumReply::UP DocsumContext::createReply() { - DocsumReply::UP reply(new DocsumReply()); + auto reply = std::make_unique<DocsumReply>(); search::RawBuf buf(4096); _docsumWriter.InitState(_attrMgr, &_docsumState); reply->docsums.resize(_docsumState._docsumcnt); @@ -214,10 +214,10 @@ DocsumContext::ParseLocation(search::docsummary::GetDocsumsState *state) } std::unique_ptr<MatchingElements> -DocsumContext::fill_matching_elements(const StructFieldMapper &struct_field_mapper) +DocsumContext::fill_matching_elements(const MatchingElementsFields &fields) { if (_matcher) { - return _matcher->get_matching_elements(_request, _searchCtx, _attrCtx, _sessionMgr, struct_field_mapper); + return _matcher->get_matching_elements(_request, _searchCtx, _attrCtx, _sessionMgr, fields); } return std::make_unique<MatchingElements>(); } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h index 467add7face..1624048828f 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h @@ -53,7 +53,7 @@ public: void FillSummaryFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env) override; void FillRankFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env) override; void ParseLocation(search::docsummary::GetDocsumsState * state) override; - std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::StructFieldMapper &struct_field_mapper) override; + std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields &fields) override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp index ecc0569cccd..4c76361cf8e 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp @@ -27,7 +27,7 @@ FieldCache::FieldCache(const ResultClass &resClass, const Field &field = docType.getField(fieldName); LOG(debug, "Caching Field instance for field '%s': %s.%u", fieldName.c_str(), field.getName().data(), field.getId()); - _cache.push_back(Field::CSP(new Field(field))); + _cache.push_back(std::make_shared<const Field>(field)); } else { _cache.push_back(Field::CSP()); } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.cpp index b97eba64f4c..25fe684b949 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcacherepo.cpp @@ -12,17 +12,17 @@ namespace proton { FieldCacheRepo::FieldCacheRepo() : _repo(), - _defaultCache(new FieldCache()) + _defaultCache(std::make_shared<const FieldCache>()) { } FieldCacheRepo::FieldCacheRepo(const ResultConfig &resConfig, const DocumentType &docType) : _repo(), - _defaultCache(new FieldCache()) + _defaultCache(std::make_shared<const FieldCache>()) { for (ResultConfig::const_iterator it(resConfig.begin()), mt(resConfig.end()); it != mt; it++) { - FieldCache::CSP cache(new FieldCache(*it, docType)); + auto cache = std::make_shared<const FieldCache>(*it, docType); vespalib::string className(it->GetClassName()); LOG(debug, "Adding field cache for summary class '%s' to repo", className.c_str()); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 96f69179482..ea71cafb73a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -788,14 +788,13 @@ DocumentMetaStore::getLidUsageStats() const Blueprint::UP DocumentMetaStore::createWhiteListBlueprint() const { - return _lidAlloc.createWhiteListBlueprint(getCommittedDocIdLimit()); + return _lidAlloc.createWhiteListBlueprint(); } AttributeVector::SearchContext::UP DocumentMetaStore::getSearch(std::unique_ptr<search::QueryTermSimple> qTerm, const SearchContextParams &) const { - return AttributeVector::SearchContext::UP - (new documentmetastore::SearchContext(std::move(qTerm), *this)); + return std::make_unique<documentmetastore::SearchContext>(std::move(qTerm), *this); } DocumentMetaStore::ConstIterator @@ -1006,7 +1005,7 @@ void DocumentMetaStore::holdUnblockShrinkLidSpace() { assert(_shrinkLidSpaceBlockers > 0); - GenerationHeldBase::UP hold(new ShrinkBlockHeld(*this)); + auto hold = std::make_unique<ShrinkBlockHeld>(*this); getGenerationHolder().hold(std::move(hold)); incGeneration(); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp index b6486607821..959bdfa9baf 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp @@ -15,7 +15,7 @@ DocumentMetaStoreContext::DocumentMetaStoreContext(BucketDBOwner::SP bucketDB, const vespalib::string &name, const search::GrowStrategy &grow, const DocumentMetaStore::IGidCompare::SP &gidCompare) : - _metaStoreAttr(new DocumentMetaStore(bucketDB, name, grow, gidCompare)), + _metaStoreAttr(std::make_shared<DocumentMetaStore>(bucketDB, name, grow, gidCompare)), _metaStore(std::dynamic_pointer_cast<IDocumentMetaStore>(_metaStoreAttr)) { } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 525f4f7a267..61662f621f1 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -195,30 +195,20 @@ class WhiteListBlueprint : public SimpleLeafBlueprint { private: const search::GrowableBitVector &_activeLids; - const uint32_t _docIdLimit; mutable std::mutex _lock; mutable std::vector<search::fef::TermFieldMatchData *> _matchDataVector; SearchIterator::UP - createLeafSearch(const TermFieldMatchDataArray &tfmda, - bool strict) const override + createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { assert(tfmda.size() == 0); (void) tfmda; - search::fef::TermFieldMatchData *tfmd = - new search::fef::TermFieldMatchData; - { - std::lock_guard<std::mutex> lock(_lock); - _matchDataVector.push_back(tfmd); - } - return search::BitVectorIterator::create(&_activeLids, _docIdLimit, *tfmd, strict); + return createFilterSearch(strict, FilterConstraint::UPPER_BOUND); } - public: - WhiteListBlueprint(const search::GrowableBitVector &activeLids, uint32_t docIdLimit) + WhiteListBlueprint(const search::GrowableBitVector &activeLids) : SimpleLeafBlueprint(FieldSpecBaseList()), _activeLids(activeLids), - _docIdLimit(docIdLimit), _matchDataVector() { setEstimate(HitEstimate(_activeLids.size(), false)); @@ -226,6 +216,15 @@ public: bool isWhiteList() const override { return true; } + SearchIterator::UP createFilterSearch(bool strict, FilterConstraint) const override { + auto tfmd = new search::fef::TermFieldMatchData; + { + std::lock_guard<std::mutex> lock(_lock); + _matchDataVector.push_back(tfmd); + } + return search::BitVectorIterator::create(&_activeLids, get_docid_limit(), *tfmd, strict); + } + ~WhiteListBlueprint() { for (auto matchData : _matchDataVector) { delete matchData; @@ -236,9 +235,9 @@ public: } Blueprint::UP -LidAllocator::createWhiteListBlueprint(uint32_t docIdLimit) const +LidAllocator::createWhiteListBlueprint() const { - return std::make_unique<WhiteListBlueprint>(_activeLids.getBitVector(), docIdLimit); + return std::make_unique<WhiteListBlueprint>(_activeLids.getBitVector()); } void diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h index ad41c6a8224..ccf9ef4513e 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h @@ -35,8 +35,7 @@ public: DocId getFreeLid(DocId lidLimit); DocId peekFreeLid(DocId lidLimit); - void ensureSpace(uint32_t newSize, - uint32_t newCapacity); + void ensureSpace(uint32_t newSize, uint32_t newCapacity); void registerLid(DocId lid) { _usedLids.setBit(lid); } void unregisterLid(DocId lid); size_t getUsedLidsSize() const; @@ -48,7 +47,7 @@ public: generation_t currentGeneration); bool holdLidOK(DocId lid, DocId lidLimit) const; void constructFreeList(DocId lidLimit); - search::queryeval::Blueprint::UP createWhiteListBlueprint(uint32_t docIdLimit) const; + search::queryeval::Blueprint::UP createWhiteListBlueprint() const; void updateActiveLids(DocId lid, bool active); void clearDocs(DocId lidLow, DocId lidLimit); void shrinkLidSpace(DocId committedDocIdLimit); diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp index 69f754cd594..26586da7784 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp @@ -27,7 +27,7 @@ PruneRemovedDocumentsOperation(DocumentIdT docIdLimit, uint32_t subDbId) : RemoveDocumentsOperation(FeedOperation::PRUNE_REMOVED_DOCUMENTS), _subDbId(subDbId) { - LidVectorContext::SP lidsToRemove(new LidVectorContext(docIdLimit)); + auto lidsToRemove = std::make_shared<LidVectorContext>(docIdLimit); setLidsToRemove(lidsToRemove); } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp index ef482a19ca3..14abade84b5 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.cpp @@ -33,7 +33,7 @@ RemoveDocumentsOperation::deserializeLidsToRemove(vespalib::nbostream &is) for (i = 0; i < mapSize; ++i) { uint32_t subDbId; is >> subDbId; - LidVectorContext::SP lidsToRemove(new LidVectorContext()); + auto lidsToRemove = std::make_shared<LidVectorContext>(); lidsToRemove->deserialize(is); setLidsToRemove(subDbId, lidsToRemove); } diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt index ecd90d8a992..340007f4513 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_library(searchcore_flushengine STATIC flushcontext.cpp flushengine.cpp flush_engine_explorer.cpp + flush_target_candidate.cpp flush_target_candidates.cpp flushtargetproxy.cpp flushtask.cpp diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.cpp new file mode 100644 index 00000000000..6be6d31372a --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.cpp @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "flush_target_candidate.h" +#include "flushcontext.h" + +namespace proton { + +FlushTargetCandidate::FlushTargetCandidate(std::shared_ptr<FlushContext> flush_context, search::SerialNum current_serial, const Config &cfg) + : _flush_context(std::move(flush_context)), + _replay_operation_cost(_flush_context->getTarget()->get_replay_operation_cost() * cfg.tlsReplayOperationCost), + _flushed_serial(_flush_context->getTarget()->getFlushedSerialNum()), + _current_serial(current_serial), + _replay_cost(_replay_operation_cost * (_current_serial - _flushed_serial)), + _approx_bytes_to_write_to_disk(_flush_context->getTarget()->getApproxBytesToWriteToDisk()), + _write_cost(_approx_bytes_to_write_to_disk * cfg.flushTargetWriteCost), + _always_flush(_replay_cost >= _write_cost) +{ +} + +FlushTargetCandidate::~FlushTargetCandidate() = default; + +} diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.h b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.h new file mode 100644 index 00000000000..5920fff6942 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidate.h @@ -0,0 +1,37 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "prepare_restart_flush_strategy.h" +#include <memory> +#include <vespa/searchlib/common/serialnum.h> + +namespace proton { + +class FlushContext; + +/** + * Class describing a flush target candidate for the prepare restart flush strategy. + */ +class FlushTargetCandidate +{ + std::shared_ptr<FlushContext> _flush_context; + double _replay_operation_cost; + search::SerialNum _flushed_serial; + search::SerialNum _current_serial; + double _replay_cost; + uint64_t _approx_bytes_to_write_to_disk; + double _write_cost; + bool _always_flush; + + using Config = PrepareRestartFlushStrategy::Config; +public: + FlushTargetCandidate(std::shared_ptr<FlushContext> flush_context, search::SerialNum current_serial, const Config &cfg); + ~FlushTargetCandidate(); + const std::shared_ptr<FlushContext> &get_flush_context() const { return _flush_context; } + search::SerialNum get_flushed_serial() const { return _flushed_serial; } + double get_write_cost() const { return _write_cost; } + bool get_always_flush() const { return _always_flush; } +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp index 0051c209ef9..f71da453559 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "flush_target_candidates.h" +#include "flush_target_candidate.h" #include "tls_stats.h" namespace proton { @@ -13,17 +14,17 @@ using TlsReplayCost = FlushTargetCandidates::TlsReplayCost; namespace { SerialNum -calculateReplayStartSerial(const FlushContext::List &sortedFlushContexts, - size_t numCandidates, +calculateReplayStartSerial(vespalib::ConstArrayRef<FlushTargetCandidate> candidates, + size_t num_candidates, const flushengine::TlsStats &tlsStats) { - if (numCandidates == 0) { + if (num_candidates == 0) { return tlsStats.getFirstSerial(); } - if (numCandidates == sortedFlushContexts.size()) { + if (num_candidates == candidates.size()) { return tlsStats.getLastSerial() + 1; } - return sortedFlushContexts[numCandidates]->getTarget()->getFlushedSerialNum() + 1; + return candidates[num_candidates].get_flushed_serial() + 1; } TlsReplayCost @@ -44,43 +45,44 @@ calculateTlsReplayCost(const flushengine::TlsStats &tlsStats, } double -calculateFlushTargetsWriteCost(const FlushContext::List &sortedFlushContexts, - size_t numCandidates, - const Config &cfg) +calculateFlushTargetsWriteCost(vespalib::ConstArrayRef<FlushTargetCandidate> candidates, + size_t num_candidates) { double result = 0; - for (size_t i = 0; i < numCandidates; ++i) { - const auto &flushContext = sortedFlushContexts[i]; - result += (flushContext->getTarget()->getApproxBytesToWriteToDisk() * - cfg.flushTargetWriteCost); + for (size_t i = 0; i < num_candidates; ++i) { + result += candidates[i].get_write_cost(); } return result; } } -FlushTargetCandidates::FlushTargetCandidates(const FlushContext::List &sortedFlushContexts, - size_t numCandidates, +FlushTargetCandidates::FlushTargetCandidates(vespalib::ConstArrayRef<FlushTargetCandidate> candidates, + size_t num_candidates, const flushengine::TlsStats &tlsStats, const Config &cfg) - : _sortedFlushContexts(&sortedFlushContexts), - _numCandidates(numCandidates), + : _candidates(candidates), + _num_candidates(std::min(num_candidates, _candidates.size())), _tlsReplayCost(calculateTlsReplayCost(tlsStats, cfg, - calculateReplayStartSerial(sortedFlushContexts, - numCandidates, + calculateReplayStartSerial(_candidates, + _num_candidates, tlsStats))), - _flushTargetsWriteCost(calculateFlushTargetsWriteCost(sortedFlushContexts, - numCandidates, - cfg)) + _flushTargetsWriteCost(calculateFlushTargetsWriteCost(_candidates, + _num_candidates)) { } FlushContext::List FlushTargetCandidates::getCandidates() const { - FlushContext::List result(_sortedFlushContexts->begin(), - _sortedFlushContexts->begin() + _numCandidates); + FlushContext::List result; + result.reserve(_num_candidates); + for (const auto &candidate : _candidates) { + if (result.size() < _num_candidates || candidate.get_always_flush()) { + result.emplace_back(candidate.get_flush_context()); + } + } return result; } diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h index ea09989de31..2979173331c 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_target_candidates.h @@ -2,11 +2,14 @@ #pragma once #include "prepare_restart_flush_strategy.h" +#include <vespa/vespalib/util/arrayref.h> namespace proton { namespace flushengine { class TlsStats; } +class FlushTargetCandidate; + /** * A set of flush targets that are candidates to be flushed. * @@ -27,8 +30,8 @@ public: double totalCost() const { return bytesCost + operationsCost; } }; private: - const FlushContext::List *_sortedFlushContexts; // NOTE: ownership is handled outside - size_t _numCandidates; + vespalib::ConstArrayRef<FlushTargetCandidate> _candidates; // NOTE: ownership is handled outside + size_t _num_candidates; TlsReplayCost _tlsReplayCost; double _flushTargetsWriteCost; @@ -37,8 +40,8 @@ private: public: using UP = std::unique_ptr<FlushTargetCandidates>; - FlushTargetCandidates(const FlushContext::List &sortedFlushContexts, - size_t numCandidates, + FlushTargetCandidates(vespalib::ConstArrayRef<FlushTargetCandidate> candidates, + size_t num_candidates, const flushengine::TlsStats &tlsStats, const Config &cfg); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp index 6cfb8cb6c3d..21f9c8465b0 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/prepare_restart_flush_strategy.cpp @@ -2,6 +2,7 @@ #include "prepare_restart_flush_strategy.h" #include "flush_target_candidates.h" +#include "flush_target_candidate.h" #include "tls_stats_map.h" #include <sstream> #include <algorithm> @@ -70,17 +71,15 @@ flatten(const FlushContextsMap &flushContextsPerHandler) } void -sortByOldestFlushedSerialNumber(FlushContext::List &flushContexts) +sortByOldestFlushedSerialNumber(std::vector<FlushTargetCandidate>& candidates) { - std::sort(flushContexts.begin(), flushContexts.end(), - [](const auto &lhs, const auto &rhs) { - if (lhs->getTarget()->getFlushedSerialNum() == - rhs->getTarget()->getFlushedSerialNum()) { - return lhs->getName() < rhs->getName(); - } - return lhs->getTarget()->getFlushedSerialNum() < - rhs->getTarget()->getFlushedSerialNum(); - }); + std::sort(candidates.begin(), candidates.end(), + [](const auto &lhs, const auto &rhs) { + if (lhs.get_flushed_serial() == rhs.get_flushed_serial()) { + return lhs.get_flush_context()->getName() < rhs.get_flush_context()->getName(); + } + return lhs.get_flushed_serial() < rhs.get_flushed_serial(); + }); } vespalib::string @@ -103,12 +102,16 @@ findBestTargetsToFlush(const FlushContext::List &unsortedFlushContexts, const flushengine::TlsStats &tlsStats, const Config &cfg) { - FlushContext::List sortedFlushContexts = unsortedFlushContexts; - sortByOldestFlushedSerialNumber(sortedFlushContexts); + std::vector<FlushTargetCandidate> candidates; + candidates.reserve(unsortedFlushContexts.size()); + for (const auto &flush_context : unsortedFlushContexts) { + candidates.emplace_back(flush_context, tlsStats.getLastSerial(), cfg); + } + sortByOldestFlushedSerialNumber(candidates); - FlushTargetCandidates bestSet(sortedFlushContexts, 0, tlsStats, cfg); - for (size_t numCandidates = 1; numCandidates <= sortedFlushContexts.size(); ++numCandidates) { - FlushTargetCandidates nextSet(sortedFlushContexts, numCandidates, tlsStats, cfg); + FlushTargetCandidates bestSet(candidates, 0, tlsStats, cfg); + for (size_t numCandidates = 1; numCandidates <= candidates.size(); ++numCandidates) { + FlushTargetCandidates nextSet(candidates, numCandidates, tlsStats, cfg); LOG(debug, "findBestTargetsToFlush(): Created candidate set: " "flushTargets=[%s], tlsReplayBytesCost=%f, tlsReplayOperationsCost=%f, flushTargetsWriteCost=%f, totalCost=%f", toString(nextSet.getCandidates()).c_str(), diff --git a/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.cpp b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.cpp index 4f14f709d29..ff423a2c9ef 100644 --- a/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.cpp +++ b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.cpp @@ -18,4 +18,10 @@ InitializerTask::addDependency(SP dependency) _dependencies.emplace_back(std::move(dependency)); } +size_t +InitializerTask::get_transient_memory_usage() const +{ + return 0u; +} + } diff --git a/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.h b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.h index ecf98b86fc4..669a55844d6 100644 --- a/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.h +++ b/searchcore/src/vespa/searchcore/proton/initializer/initializer_task.h @@ -31,6 +31,7 @@ public: void setDone() { _state = State::DONE; } void addDependency(SP dependency); virtual void run() = 0; + virtual size_t get_transient_memory_usage() const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp index 86c2b525113..16348dbace6 100644 --- a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp +++ b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp @@ -110,6 +110,7 @@ TaskRunner::pollTask(Context::SP context) TaskList readyTasks; TaskSet checked; getReadyTasks(context->rootTask(), readyTasks, checked); + std::sort(readyTasks.begin(), readyTasks.end(), [](const auto &a, const auto &b) -> bool { return a->get_transient_memory_usage() > b->get_transient_memory_usage(); }); internalRunTasks(readyTasks, context); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp index c8c5a3a427b..7149bf9ab7f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp @@ -32,7 +32,7 @@ struct Mixer { Blueprint::UP mix(Blueprint::UP indexes) { if (attributes.get() == 0) { if (indexes.get() == 0) { - return Blueprint::UP(new EmptyBlueprint()); + return std::make_unique<EmptyBlueprint>(); } return Blueprint::UP(std::move(indexes)); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp index a9f87cb249e..c8d0a572cd9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.cpp @@ -19,7 +19,7 @@ LOG_SETUP(".proton.matching.docsum_matcher"); using search::FeatureSet; using search::MatchingElements; -using search::StructFieldMapper; +using search::MatchingElementsFields; using search::fef::FeatureResolver; using search::fef::RankProgram; using search::queryeval::AndNotBlueprint; @@ -100,13 +100,13 @@ void find_matching_elements(const std::vector<uint32_t> &docs, const SameElement for (uint32_t i = 0; i < docs.size(); ++i) { search->find_matching_elements(docs[i], matches); if (!matches.empty()) { - result.add_matching_elements(docs[i], same_element.struct_field_name(), matches); + result.add_matching_elements(docs[i], same_element.field_name(), matches); matches.clear(); } } } -void find_matching_elements(const std::vector<uint32_t> &docs, const vespalib::string &struct_field_name, const AttrSearchCtx &attr_ctx, MatchingElements &result) { +void find_matching_elements(const std::vector<uint32_t> &docs, const vespalib::string &field_name, const AttrSearchCtx &attr_ctx, MatchingElements &result) { int32_t weight = 0; std::vector<uint32_t> matches; for (uint32_t i = 0; i < docs.size(); ++i) { @@ -114,26 +114,26 @@ void find_matching_elements(const std::vector<uint32_t> &docs, const vespalib::s matches.push_back(id); } if (!matches.empty()) { - result.add_matching_elements(docs[i], struct_field_name, matches); + result.add_matching_elements(docs[i], field_name, matches); matches.clear(); } } } -void find_matching_elements(const StructFieldMapper &mapper, const std::vector<uint32_t> &docs, const Blueprint &bp, MatchingElements &result) { +void find_matching_elements(const MatchingElementsFields &fields, const std::vector<uint32_t> &docs, const Blueprint &bp, MatchingElements &result) { if (auto same_element = as<SameElementBlueprint>(bp)) { - if (mapper.is_struct_field(same_element->struct_field_name())) { + if (fields.has_field(same_element->field_name())) { find_matching_elements(docs, *same_element, result); } } else if (const AttrSearchCtx *attr_ctx = bp.get_attribute_search_context()) { - if (mapper.is_struct_subfield(attr_ctx->attributeName())) { - find_matching_elements(docs, mapper.get_struct_field(attr_ctx->attributeName()), *attr_ctx, result); + if (fields.has_struct_field(attr_ctx->attributeName())) { + find_matching_elements(docs, fields.get_enclosing_field(attr_ctx->attributeName()), *attr_ctx, result); } } else if (auto and_not = as<AndNotBlueprint>(bp)) { - find_matching_elements(mapper, docs, and_not->getChild(0), result); + find_matching_elements(fields, docs, and_not->getChild(0), result); } else if (auto intermediate = as<IntermediateBlueprint>(bp)) { for (size_t i = 0; i < intermediate->childCnt(); ++i) { - find_matching_elements(mapper, docs, intermediate->getChild(i), result); + find_matching_elements(fields, docs, intermediate->getChild(i), result); } } } @@ -189,12 +189,12 @@ DocsumMatcher::get_rank_features() const } MatchingElements::UP -DocsumMatcher::get_matching_elements(const StructFieldMapper &field_mapper) const +DocsumMatcher::get_matching_elements(const MatchingElementsFields &fields) const { auto result = std::make_unique<MatchingElements>(); - if (_mtf && !field_mapper.empty()) { + if (_mtf && !fields.empty()) { if (const Blueprint *root = _mtf->query().peekRoot()) { - find_matching_elements(field_mapper, _docs, *root, *result); + find_matching_elements(fields, _docs, *root, *result); } } return result; diff --git a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h index 7fdfbc1d2ba..2e2250c476c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/docsum_matcher.h @@ -3,8 +3,8 @@ #pragma once #include <vespa/searchlib/common/featureset.h> -#include <vespa/searchlib/common/struct_field_mapper.h> #include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vector> #include <memory> @@ -22,7 +22,7 @@ class DocsumMatcher { private: using FeatureSet = search::FeatureSet; - using StructFieldMapper = search::StructFieldMapper; + using MatchingElementsFields = search::MatchingElementsFields; using MatchingElements = search::MatchingElements; std::shared_ptr<SearchSession> _from_session; @@ -40,7 +40,7 @@ public: FeatureSet::UP get_summary_features() const; FeatureSet::UP get_rank_features() const; - MatchingElements::UP get_matching_elements(const StructFieldMapper &field_mapper) const; + MatchingElements::UP get_matching_elements(const MatchingElementsFields &fields) const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.cpp b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.cpp index 5758b9a796f..316cf92597e 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.cpp @@ -7,8 +7,8 @@ namespace proton::matching { FakeSearchContext::FakeSearchContext(size_t initialNumDocs) : _clock(), _doom(_clock, vespalib::steady_time()), - _selector(new search::FixedSourceSelector(0, "fs", initialNumDocs)), - _indexes(new IndexCollection(_selector)), + _selector(std::make_shared<search::FixedSourceSelector>(0, "fs", initialNumDocs)), + _indexes(std::make_shared<IndexCollection>(_selector)), _attrSearchable(), _docIdLimit(initialNumDocs) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h index 5a4ccc892b3..d23dd2ac551 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h +++ b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h @@ -35,7 +35,7 @@ public: ~FakeSearchContext(); FakeSearchContext &addIdx(uint32_t id) { - _indexes->append(id, IndexSearchable::SP(new FakeIndexSearchable())); + _indexes->append(id, std::make_shared<FakeIndexSearchable>()); return *this; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index f7fce994bd1..f19b416b92f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -185,6 +185,8 @@ MatchToolsFactory(QueryLimiter & queryLimiter, _query.optimize(); trace.addEvent(4, "MTF: Fetch Postings"); _query.fetchPostings(); + trace.addEvent(5, "MTF: Handle Global Filters"); + _query.handle_global_filters(searchContext.getDocIdLimit()); _query.freeze(); trace.addEvent(5, "MTF: prepareSharedState"); _rankSetup.prepareSharedState(_queryEnv, _queryEnv.getObjectStore()); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 630ac66f4f1..93b4f63060f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -27,7 +27,7 @@ using namespace search::grouping; using search::DocumentMetaData; using search::LidUsageStats; using search::FeatureSet; -using search::StructFieldMapper; +using search::MatchingElementsFields; using search::MatchingElements; using search::attribute::IAttributeContext; using search::fef::MatchDataLayout; @@ -336,10 +336,10 @@ Matcher::getRankFeatures(const DocsumRequest & req, ISearchContext & searchCtx, MatchingElements::UP Matcher::get_matching_elements(const DocsumRequest &req, ISearchContext &search_ctx, IAttributeContext &attr_ctx, SessionManager &session_manager, - const StructFieldMapper &field_mapper) + const MatchingElementsFields &fields) { auto docsum_matcher = create_docsum_matcher(req, search_ctx, attr_ctx, session_manager); - return docsum_matcher->get_matching_elements(field_mapper); + return docsum_matcher->get_matching_elements(fields); } DocsumMatcher::UP diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.h b/searchcore/src/vespa/searchcore/proton/matching/matcher.h index 14562ffd6ca..243fdad63ae 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.h @@ -12,7 +12,7 @@ #include <vespa/searchcommon/attribute/i_attribute_functor.h> #include <vespa/searchlib/fef/blueprintfactory.h> #include <vespa/searchlib/common/featureset.h> -#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/resultset.h> #include <vespa/searchlib/queryeval/blueprint.h> @@ -54,8 +54,8 @@ private: using SearchRequest = search::engine::SearchRequest; using Properties = search::fef::Properties; using my_clock = std::chrono::steady_clock; - using StructFieldMapper = search::StructFieldMapper; using MatchingElements = search::MatchingElements; + using MatchingElementsFields = search::MatchingElementsFields; IndexEnvironment _indexEnv; search::fef::BlueprintFactory _blueprintFactory; std::shared_ptr<search::fef::RankSetup> _rankSetup; @@ -162,13 +162,13 @@ public: * @param search_ctx abstract view of searchable data * @param attr_ctx abstract view of attribute data * @param session_manager multilevel grouping session and query cache - * @param field_mapper knows which fields to collect information - * about and how they relate to each other + * @param fields knows which fields to collect information + * about and how they relate to each other * @return matching elements **/ MatchingElements::UP get_matching_elements(const DocsumRequest &req, ISearchContext &search_ctx, IAttributeContext &attr_ctx, SessionManager &session_manager, - const StructFieldMapper &field_mapper); + const MatchingElementsFields &fields); DocsumMatcher::UP create_docsum_matcher(const DocsumRequest &req, ISearchContext &search_ctx, IAttributeContext &attr_ctx, SessionManager &session_manager); diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 26d35998dd3..5213a2b9230 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -189,13 +189,6 @@ void Query::optimize() { _blueprint = Blueprint::optimize(std::move(_blueprint)); - if (_blueprint->getState().want_global_filter()) { - // XXX we need to somehow compute a real global filter - std::shared_ptr<search::BitVector> empty_global_filter; - _blueprint->set_global_filter(empty_global_filter); - // optimized order may change after accounting for global filter: - _blueprint = Blueprint::optimize(std::move(_blueprint)); - } LOG(debug, "optimized blueprint:\n%s\n", _blueprint->asString().c_str()); } @@ -206,6 +199,26 @@ Query::fetchPostings() } void +Query::handle_global_filters(uint32_t docid_limit) +{ + using search::queryeval::GlobalFilter; + if (_blueprint->getState().want_global_filter()) { + auto constraint = Blueprint::FilterConstraint::UPPER_BOUND; + bool strict = true; + auto filter_iterator = _blueprint->createFilterSearch(strict, constraint); + filter_iterator->initRange(1, docid_limit); + auto white_list = filter_iterator->get_hits(1); + auto global_filter = GlobalFilter::create(std::move(white_list)); + _blueprint->set_global_filter(*global_filter); + // optimized order may change after accounting for global filter: + _blueprint = Blueprint::optimize(std::move(_blueprint)); + LOG(debug, "blueprint after handle_global_filters:\n%s\n", _blueprint->asString().c_str()); + // strictness may change if optimized order changed: + fetchPostings(); + } +} + +void Query::freeze() { _blueprint->freeze(); diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index 11aade1c59b..3ed6229830d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -89,6 +89,7 @@ public: **/ void optimize(); void fetchPostings(); + void handle_global_filters(uint32_t docidLimit); void freeze(); /** diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp index 8b3257ff47b..18475f06fc2 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_job_trackers.cpp @@ -17,14 +17,14 @@ namespace proton { DocumentDBJobTrackers::DocumentDBJobTrackers() : _lock(), _now(std::chrono::steady_clock::now()), - _attributeFlush(new JobTracker(_now, _lock)), - _memoryIndexFlush(new JobTracker(_now, _lock)), - _diskIndexFusion(new JobTracker(_now, _lock)), - _documentStoreFlush(new JobTracker(_now, _lock)), - _documentStoreCompact(new JobTracker(_now, _lock)), - _bucketMove(new JobTracker(_now, _lock)), - _lidSpaceCompact(new JobTracker(_now, _lock)), - _removedDocumentsPrune(new JobTracker(_now, _lock)) + _attributeFlush(std::make_shared<JobTracker>(_now, _lock)), + _memoryIndexFlush(std::make_shared<JobTracker>(_now, _lock)), + _diskIndexFusion(std::make_shared<JobTracker>(_now, _lock)), + _documentStoreFlush(std::make_shared<JobTracker>(_now, _lock)), + _documentStoreCompact(std::make_shared<JobTracker>(_now, _lock)), + _bucketMove(std::make_shared<JobTracker>(_now, _lock)), + _lidSpaceCompact(std::make_shared<JobTracker>(_now, _lock)), + _removedDocumentsPrune(std::make_shared<JobTracker>(_now, _lock)) { } @@ -36,7 +36,7 @@ IFlushTarget::SP trackFlushTarget(const IJobTracker::SP &tracker, const IFlushTarget::SP &target) { - return IFlushTarget::SP(new JobTrackedFlushTarget(tracker, target)); + return std::make_shared<JobTrackedFlushTarget>(tracker, target); } } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp index 9eebe5010d2..14d76645fc7 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp @@ -25,7 +25,7 @@ JobTrackedFlushTarget::initFlush(SerialNum currentSerial) FlushTask::UP targetTask = _target->initFlush(currentSerial); _tracker->end(); if (targetTask.get() != nullptr) { - return FlushTask::UP(new JobTrackedFlushTask(_tracker, std::move(targetTask))); + return std::make_unique<JobTrackedFlushTask>(_tracker, std::move(targetTask)); } return FlushTask::UP(); } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp index 43e4e4d3f75..c890592b411 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp @@ -10,6 +10,7 @@ ResourceUsageMetrics::ResourceUsageMetrics(metrics::MetricSet *parent) diskUtilization("disk_utilization", {}, "The relative amount of disk used compared to the disk resource limit", this), memory("memory", {}, "The relative amount of memory used by this process (value in the range [0, 1])", this), memoryUtilization("memory_utilization", {}, "The relative amount of memory used compared to the memory resource limit", this), + transient_memory("transient_memory", {}, "The relative amount of transient memory needed for load (value in the range [0, 1])", this), memoryMappings("memory_mappings", {}, "The number of mapped memory areas", this), openFileDescriptors("open_file_descriptors", {}, "The number of open files", this), feedingBlocked("feeding_blocked", {}, "Whether feeding is blocked due to resource limits being reached (value is either 0 or 1)", this) diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h index 497a0fe2ba0..8f7036ee832 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h @@ -15,6 +15,7 @@ struct ResourceUsageMetrics : metrics::MetricSet metrics::DoubleValueMetric diskUtilization; metrics::DoubleValueMetric memory; metrics::DoubleValueMetric memoryUtilization; + metrics::DoubleValueMetric transient_memory; metrics::LongValueMetric memoryMappings; metrics::LongValueMetric openFileDescriptors; metrics::LongValueMetric feedingBlocked; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp index a6e8ac54e86..85644c2111a 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp @@ -33,7 +33,7 @@ TransLogServerMetrics::considerAddDomains(const DomainStats &stats) for (const auto &elem : stats) { const vespalib::string &documentType = elem.first; if (_domainMetrics.find(documentType) == _domainMetrics.end()) { - _domainMetrics[documentType] = DomainMetrics::UP(new DomainMetrics(_parent, documentType)); + _domainMetrics[documentType] = std::make_unique<DomainMetrics>(_parent, documentType); } } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 91ccac6fce1..1dc5199557a 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -434,7 +434,7 @@ PersistenceEngine::get(const Bucket& b, const document::FieldSet& fields, const search::DocumentMetaData meta = retriever.getDocumentMetaData(did); if (meta.timestamp != 0 && meta.bucketId == b.getBucketId()) { if (meta.removed) { - return GetResult(); + return GetResult::make_for_tombstone(meta.timestamp); } document::Document::UP doc = retriever.getDocument(meta.lid); if (!doc || doc->getId().getGlobalId() != meta.gid) { diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp index 9b1c66780cd..e939b07c26b 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp @@ -101,9 +101,8 @@ getFieldsToPopulate(const ARIConfig &newCfg, attrCfg.basicType().asString(), toStr(populateField)); if (populateField) { - fieldsToPopulate.push_back(IReprocessingRewriter::SP - (new DocumentFieldPopulator(name, - guard.getSP(), subDbName))); + fieldsToPopulate.push_back(std::make_shared<DocumentFieldPopulator> + (name, guard.getSP(), subDbName)); } } return fieldsToPopulate; diff --git a/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h b/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h index fe0820b9a39..3333041fb91 100644 --- a/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h +++ b/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h @@ -19,7 +19,7 @@ struct AttributeWriterFactory : public IAttributeWriterFactory const AttributeWriter &oldAdapter = dynamic_cast<const AttributeWriter &>(*old.get()); const proton::IAttributeManager::SP &oldMgr = oldAdapter.getAttributeManager(); proton::IAttributeManager::SP newMgr = oldMgr->create(attrSpec); - return IAttributeWriter::SP(new AttributeWriter(newMgr)); + return std::make_shared<AttributeWriter>(newMgr); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp index edbf0c631a5..baf89a55ae3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp @@ -78,7 +78,7 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) if (snapshot.isChanged<ProtonConfig>(_configId, currentGen)) { LOG(spam, "Proton config is changed"); std::unique_ptr<ProtonConfig> protonConfig = snapshot.getConfig<ProtonConfig>(_configId); - TuneFileDocumentDB::SP tuneFileDocumentDB(new TuneFileDocumentDB); + auto tuneFileDocumentDB = std::make_shared<TuneFileDocumentDB>(); TuneFileDocumentDB &tune = *tuneFileDocumentDB; ProtonConfig &conf = *protonConfig; tune._index._indexing._write.setFromConfig<ProtonConfig::Indexing::Write>(conf.indexing.write.io); diff --git a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp index fa0e0f33469..eb3c8ce6aa4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/clusterstatehandler.cpp @@ -64,7 +64,7 @@ ClusterStateHandler::performSetClusterState(const ClusterState *calc, (calc->nodeInitializing() ? "true" : "false"), _changedHandlers.size()); if (!_changedHandlers.empty()) { - IBucketStateCalculator::SP newCalc(new ClusterStateAdapter(*calc)); + auto newCalc = std::make_shared<ClusterStateAdapter>(*calc); typedef std::vector<IClusterStateChangedHandler *> Chv; Chv &chs(_changedHandlers); for (Chv::const_iterator it = chs.begin(), ite = chs.end(); it != ite; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index 3191d82bf11..4855577e712 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -156,6 +156,7 @@ DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) _hwInfo(hwInfo), _memoryStats(), _diskUsedSizeBytes(), + _transient_memory_usage(0u), _config(), _state(), _acceptWrite(true), @@ -182,6 +183,13 @@ DiskMemUsageFilter::setDiskUsedSize(uint64_t diskUsedSizeBytes) } void +DiskMemUsageFilter::set_transient_memory_usage(size_t transient_memory_usage) +{ + Guard guard(_lock); + _transient_memory_usage = transient_memory_usage; +} + +void DiskMemUsageFilter::setConfig(Config config_in) { Guard guard(_lock); @@ -203,6 +211,20 @@ DiskMemUsageFilter::getDiskUsedSize() const return _diskUsedSizeBytes; } +size_t +DiskMemUsageFilter::get_transient_memory_usage() const +{ + Guard guard(_lock); + return _transient_memory_usage; +} + +double +DiskMemUsageFilter::get_relative_transient_memory_usage() const +{ + Guard guard(_lock); + return static_cast<double>(_transient_memory_usage) / _hwInfo.memory().sizeBytes(); +} + DiskMemUsageFilter::Config DiskMemUsageFilter::getConfig() const { @@ -230,7 +252,6 @@ DiskMemUsageFilter::getAcceptState() const return _state; } - void DiskMemUsageFilter::addDiskMemUsageListener(IDiskMemUsageListener *listener) { diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h index afb8c3d2c7a..6d6fad257ac 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h @@ -43,6 +43,7 @@ private: HwInfo _hwInfo; vespalib::ProcessMemoryStats _memoryStats; uint64_t _diskUsedSizeBytes; + size_t _transient_memory_usage; Config _config; State _state; std::atomic<bool> _acceptWrite; @@ -59,9 +60,12 @@ public: ~DiskMemUsageFilter() override; void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in); void setDiskUsedSize(uint64_t diskUsedSizeBytes); + void set_transient_memory_usage(size_t transient_memory_usage); void setConfig(Config config); vespalib::ProcessMemoryStats getMemoryStats() const; uint64_t getDiskUsedSize() const; + size_t get_transient_memory_usage() const; + double get_relative_transient_memory_usage() const; Config getConfig() const; const HwInfo &getHwInfo() const { return _hwInfo; } DiskMemUsageState usageState() const; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp index a3fb1244c45..f7f505cd754 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp @@ -3,6 +3,7 @@ #include "disk_mem_usage_sampler.h" #include <vespa/vespalib/util/scheduledexecutor.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/searchcore/proton/common/i_transient_memory_usage_provider.h> #include <filesystem> using vespalib::makeLambdaTask; @@ -14,7 +15,9 @@ DiskMemUsageSampler::DiskMemUsageSampler(const std::string &path_in, const Confi _path(path_in), _sampleInterval(60s), _lastSampleTime(vespalib::steady_clock::now()), - _periodicTimer() + _periodicTimer(), + _lock(), + _transient_memory_usage_providers() { setConfig(config); } @@ -49,6 +52,7 @@ DiskMemUsageSampler::sampleUsage() { sampleMemoryUsage(); sampleDiskUsage(); + sample_transient_memory_usage(); } namespace { @@ -118,4 +122,37 @@ DiskMemUsageSampler::sampleMemoryUsage() _filter.setMemoryStats(vespalib::ProcessMemoryStats::create()); } +void +DiskMemUsageSampler::sample_transient_memory_usage() +{ + size_t max_transient_memory_usage = 0; + { + std::lock_guard<std::mutex> guard(_lock); + for (auto provider : _transient_memory_usage_providers) { + auto transient_memory_usage = provider->get_transient_memory_usage(); + max_transient_memory_usage = std::max(max_transient_memory_usage, transient_memory_usage); + } + } + _filter.set_transient_memory_usage(max_transient_memory_usage); +} + +void +DiskMemUsageSampler::add_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider) +{ + std::lock_guard<std::mutex> guard(_lock); + _transient_memory_usage_providers.push_back(provider); +} + +void +DiskMemUsageSampler::remove_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider) +{ + std::lock_guard<std::mutex> guard(_lock); + for (auto itr = _transient_memory_usage_providers.begin(); itr != _transient_memory_usage_providers.end(); ++itr) { + if (*itr == provider) { + _transient_memory_usage_providers.erase(itr); + break; + } + } +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h index bbfc95ff82c..38e80a6cec1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h @@ -9,6 +9,8 @@ namespace vespalib { class ScheduledExecutor; } namespace proton { +class ITransientMemoryUsageProvider; + /* * Class to sample disk and memory usage used for filtering write operations. */ @@ -18,10 +20,13 @@ class DiskMemUsageSampler { vespalib::duration _sampleInterval; vespalib::steady_time _lastSampleTime; std::unique_ptr<vespalib::ScheduledExecutor> _periodicTimer; + std::mutex _lock; + std::vector<std::shared_ptr<const ITransientMemoryUsageProvider>> _transient_memory_usage_providers; void sampleUsage(); void sampleDiskUsage(); void sampleMemoryUsage(); + void sample_transient_memory_usage(); public: struct Config { DiskMemUsageFilter::Config filterConfig; @@ -55,6 +60,8 @@ public: const DiskMemUsageFilter &writeFilter() const { return _filter; } IDiskMemUsageNotifier ¬ifier() { return _filter; } + void add_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider); + void remove_transient_memory_usage_provider(std::shared_ptr<const ITransientMemoryUsageProvider> provider); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp index a4e8b6752b0..24965db5919 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp @@ -54,7 +54,7 @@ DocumentDBLidSpaceCompactionConfig::DocumentDBLidSpaceCompactionConfig() _allowedLidBloat(1000000000), _allowedLidBloatFactor(1.0), _remove_batch_block_rate(0.5), - _remove_block_rate(100000), + _remove_block_rate(100), _disabled(false), _maxDocsToScan(10000) { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 55f95ce0518..7567aa9c322 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -13,10 +13,12 @@ #include "maintenance_jobs_injector.h" #include "reconfig_params.h" #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchcore/proton/common/statusreport.h> +#include <vespa/searchcore/proton/common/transient_memory_usage_provider.h> #include <vespa/searchcore/proton/docsummary/isummarymanager.h> #include <vespa/searchcore/proton/feedoperation/noopoperation.h> #include <vespa/searchcore/proton/index/index_writer.h> @@ -163,6 +165,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _state(), _dmUsageForwarder(_writeService.master()), _writeFilter(), + _transient_memory_usage_provider(std::make_shared<TransientMemoryUsageProvider>()), _feedHandler(_writeService, tlsSpec, docTypeName, _state, *this, _writeFilter, *this, tlsDirectWriter), _subDBs(*this, *this, _feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext, metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, @@ -929,7 +932,7 @@ DocumentDB::hasDocument(const document::DocumentId &id) } void -DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) +DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector) { // Called by executor thread _maintenanceController.killJobs(); @@ -954,6 +957,8 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) _visibility, // ICommitable _subDBs.getReadySubDB()->getAttributeManager(), _subDBs.getNotReadySubDB()->getAttributeManager(), + std::move(attribute_config_inspector), + _transient_memory_usage_provider, _writeFilter); } @@ -963,18 +968,21 @@ DocumentDB::performStartMaintenance() // Called by executor thread // Only start once, after replay done - DocumentDBMaintenanceConfig::SP maintenanceConfig; + std::shared_ptr<DocumentDBConfig> activeConfig; { lock_guard guard(_configMutex); if (_state.getClosed()) return; - assert(_activeConfigSnapshot); - maintenanceConfig = _activeConfigSnapshot->getMaintenanceConfigSP(); + activeConfig = _activeConfigSnapshot; } + assert(activeConfig); if (_maintenanceController.getStopping()) { return; } - injectMaintenanceJobs(*maintenanceConfig); + auto maintenanceConfig = activeConfig->getMaintenanceConfigSP(); + const auto &attributes_config = activeConfig->getAttributesConfig(); + auto attribute_config_inspector = std::make_unique<AttributeConfigInspector>(attributes_config); + injectMaintenanceJobs(*maintenanceConfig, std::move(attribute_config_inspector)); _maintenanceController.start(maintenanceConfig); } @@ -992,10 +1000,12 @@ DocumentDB::forwardMaintenanceConfig() assert(activeConfig); DocumentDBMaintenanceConfig::SP maintenanceConfig(activeConfig->getMaintenanceConfigSP()); + const auto &attributes_config = activeConfig->getAttributesConfig(); + auto attribute_config_inspector = std::make_unique<AttributeConfigInspector>(attributes_config); if (!_state.getClosed()) { if (_maintenanceController.getStarted() && !_maintenanceController.getStopping()) { - injectMaintenanceJobs(*maintenanceConfig); + injectMaintenanceJobs(*maintenanceConfig, std::move(attribute_config_inspector)); } _maintenanceController.newConfig(maintenanceConfig); } @@ -1086,4 +1096,10 @@ DocumentDB::getDistributionKey() const return _owner.getDistributionKey(); } +std::shared_ptr<const ITransientMemoryUsageProvider> +DocumentDB::transient_memory_usage_provider() +{ + return _transient_memory_usage_provider; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 917d753683a..33dfa85b628 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -44,10 +44,13 @@ namespace search { namespace vespa::config::search::core::internal { class InternalProtonType; } namespace proton { +class AttributeConfigInspector; class IDocumentDBOwner; +class ITransientMemoryUsageProvider; struct MetricsWireService; class StatusReport; class ExecutorThreadingServiceStats; +class TransientMemoryUsageProvider; namespace matching { class SessionManager; } @@ -132,6 +135,7 @@ private: DDBState _state; DiskMemUsageForwarder _dmUsageForwarder; AttributeUsageFilter _writeFilter; + std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; FeedHandler _feedHandler; DocumentSubDBCollection _subDBs; @@ -412,7 +416,7 @@ public: /** * Implements IFeedHandlerOwner **/ - void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config); + void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector); void performStartMaintenance(); void stopMaintenance(); void forwardMaintenanceConfig(); @@ -435,6 +439,7 @@ public: void enterOnlineState(); void waitForOnlineState(); IDiskMemUsageListener *diskMemUsageListener() { return &_dmUsageForwarder; } + std::shared_ptr<const ITransientMemoryUsageProvider> transient_memory_usage_provider(); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index d1d88434bd6..68e65acb87d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -192,6 +192,7 @@ deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::M WriteableFileChunk::Config fileConfig(deriveCompression(chunk.compression), chunk.maxbytes); LogDataStore::Config logConfig; logConfig.setMaxFileSize(log.maxfilesize) + .setMaxNumLids(log.maxnumlids) .setMaxDiskBloatFactor(std::min(flush.diskbloatfactor, flush.each.diskbloatfactor)) .setMaxBucketSpread(log.maxbucketspread).setMinFileSizeFactor(log.minfilesizefactor) .compactCompression(deriveCompression(log.compact.compression)) diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index 483497eb008..d69f0365fc0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -9,6 +9,7 @@ #include "prune_session_cache_job.h" #include "pruneremoveddocumentsjob.h" #include "sample_attribute_usage_job.h" +#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> using vespalib::system_clock; @@ -99,6 +100,8 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, ICommitable &commit, IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, + std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, + std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique<HeartBeatJob>(hbHandler, config.getHeartBeatConfig())); controller.registerJobInDefaultPool(std::make_unique<PruneSessionCacheJob>(scPruner, config.getSessionCachePruneInterval())); @@ -123,7 +126,9 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, notReadyAttributeManager, attributeUsageFilter, docTypeName, - config.getAttributeUsageSampleInterval())); + config.getAttributeUsageSampleInterval(), + std::move(attribute_config_inspector), + transient_memory_usage_provider)); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h index 94203e144dc..f0859335919 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h @@ -12,6 +12,7 @@ namespace proton { +class AttributeConfigInspector; class IPruneRemovedDocumentsHandler; struct IDocumentMoveHandler; class IBucketModifiedHandler; @@ -21,6 +22,7 @@ struct IBucketStateCalculator; struct IAttributeManager; class AttributeUsageFilter; class IDiskMemUsageNotifier; +class TransientMemoryUsageProvider; namespace bucketdb { class IBucketCreateNotifier; } /** @@ -53,6 +55,8 @@ struct MaintenanceJobsInjector ICommitable & commit, IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, + std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, + std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider, AttributeUsageFilter &attributeUsageFilter); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp index 7ba9b971715..61b37a47d09 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp @@ -58,7 +58,6 @@ MatchView::getMatcher(const vespalib::string & rankProfile) const return retval; } - MatchContext::UP MatchView::createContext() const { IAttributeContext::UP attrCtx = _attrMgr->createContext(); @@ -66,7 +65,6 @@ MatchView::createContext() const { return std::make_unique<MatchContext>(std::move(attrCtx), std::move(searchCtx)); } - std::unique_ptr<SearchReply> MatchView::match(std::shared_ptr<const ISearchHandler> searchHandler, const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const @@ -74,13 +72,12 @@ MatchView::match(std::shared_ptr<const ISearchHandler> searchHandler, const Sear Matcher::SP matcher = getMatcher(req.ranking); SearchSession::OwnershipBundle owned_objects; owned_objects.search_handler = std::move(searchHandler); + owned_objects.readGuard = _metaStore->getReadGuard(); owned_objects.context = createContext(); - owned_objects.readGuard = _metaStore->getReadGuard();; MatchContext *ctx = owned_objects.context.get(); const search::IDocumentMetaStore & dms = owned_objects.readGuard->get(); return matcher->match(req, threadBundle, ctx->getSearchContext(), ctx->getAttributeContext(), *_sessionMgr, dms, std::move(owned_objects)); } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index c886b371064..962ee65c10d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -212,6 +212,7 @@ Proton::Proton(const config::ConfigUri & configUri, _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout), _warmupExecutor(), _sharedExecutor(), + _compile_cache_executor_binding(), _queryLimiter(), _clock(0.001), _threadPool(128 * 1024), @@ -301,7 +302,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _warmupExecutor = std::make_unique<vespalib::ThreadStackExecutor>(4, 128*1024, index_warmup_executor); const size_t sharedThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu()); - _sharedExecutor = std::make_unique<vespalib::BlockingThreadStackExecutor>(sharedThreads, 128*1024, sharedThreads*16, proton_shared_executor); + _sharedExecutor = std::make_shared<vespalib::BlockingThreadStackExecutor>(sharedThreads, 128*1024, sharedThreads*16, proton_shared_executor); + _compile_cache_executor_binding = vespalib::eval::CompileCache::bind(_sharedExecutor); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(protonConfig.initialize.threads, 128 * 1024, initialize_executor); @@ -454,6 +456,7 @@ Proton::~Proton() _persistenceEngine.reset(); _tls.reset(); _warmupExecutor.reset(); + _compile_cache_executor_binding.reset(); _sharedExecutor.reset(); _clock.stop(); LOG(debug, "Explicit destructor done"); @@ -606,6 +609,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, auto flushHandler = std::make_shared<FlushHandlerProxy>(ret); _flushEngine->putFlushHandler(docTypeName, flushHandler); _diskMemUsageSampler->notifier().addDiskMemUsageListener(ret->diskMemUsageListener()); + _diskMemUsageSampler->add_transient_memory_usage_provider(ret->transient_memory_usage_provider()); return ret; } @@ -643,6 +647,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName) _metricsEngine->removeMetricsHook(old->getMetricsUpdateHook()); _metricsEngine->removeDocumentDBMetrics(old->getMetrics()); _diskMemUsageSampler->notifier().removeDiskMemUsageListener(old->diskMemUsageListener()); + _diskMemUsageSampler->remove_transient_memory_usage_provider(old->transient_memory_usage_provider()); // Caller should have removed & drained relevant timer tasks old->close(); } @@ -710,6 +715,7 @@ Proton::updateMetrics(const vespalib::MonitorGuard &) metrics.resourceUsage.diskUtilization.set(usageState.diskState().utilization()); metrics.resourceUsage.memory.set(usageState.memoryState().usage()); metrics.resourceUsage.memoryUtilization.set(usageState.memoryState().utilization()); + metrics.resourceUsage.transient_memory.set(usageFilter.get_relative_transient_memory_usage()); metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount()); metrics.resourceUsage.openFileDescriptors.set(FastOS_File::count_open_files()); metrics.resourceUsage.feedingBlocked.set((usageFilter.acceptWriteOperation() ? 0.0 : 1.0)); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 4c9d4c77cc4..d5c1a8b7b78 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -26,6 +26,7 @@ #include <vespa/vespalib/net/json_handler_repo.h> #include <vespa/vespalib/net/state_explorer.h> #include <vespa/vespalib/util/varholder.h> +#include <vespa/eval/eval/llvm/compile_cache.h> #include <mutex> #include <shared_mutex> @@ -111,7 +112,8 @@ private: ProtonConfigurer _protonConfigurer; ProtonConfigFetcher _protonConfigFetcher; std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor; - std::unique_ptr<vespalib::ThreadStackExecutorBase> _sharedExecutor; + std::shared_ptr<vespalib::ThreadStackExecutorBase> _sharedExecutor; + vespalib::eval::CompileCache::ExecutorBinding::UP _compile_cache_executor_binding; matching::QueryLimiter _queryLimiter; vespalib::Clock _clock; FastOS_ThreadPool _threadPool; diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp index 6f1fd8b90e0..83c268fc755 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp @@ -47,6 +47,8 @@ ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool memory.setDouble("utilization", usageState.memoryState().utilization()); memory.setLong("physicalMemory", _usageFilter.getHwInfo().memory().sizeBytes()); convertMemoryStatsToSlime(_usageFilter.getMemoryStats(), memory.setObject("stats")); + size_t transient_memory = _usageFilter.get_transient_memory_usage(); + memory.setLong("transient", transient_memory); } else { object.setDouble("disk", usageState.diskState().usage()); object.setDouble("memory", usageState.memoryState().usage()); diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp index 1f5f29c7708..a6497966891 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp @@ -2,6 +2,7 @@ #include "sample_attribute_usage_job.h" #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> +#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h> #include <vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h> @@ -13,11 +14,15 @@ SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, AttributeUsageFilter &attributeUsageFilter, const vespalib::string &docTypeName, - vespalib::duration interval) + vespalib::duration interval, + std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, + std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider) : IMaintenanceJob("sample_attribute_usage." + docTypeName, vespalib::duration::zero(), interval), _readyAttributeManager(readyAttributeManager), _notReadyAttributeManager(notReadyAttributeManager), - _attributeUsageFilter(attributeUsageFilter) + _attributeUsageFilter(attributeUsageFilter), + _attribute_config_inspector(std::move(attribute_config_inspector)), + _transient_memory_usage_provider(std::move(transient_memory_usage_provider)) { } @@ -26,7 +31,7 @@ SampleAttributeUsageJob::~SampleAttributeUsageJob() = default; bool SampleAttributeUsageJob::run() { - auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter); + auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter, _attribute_config_inspector, _transient_memory_usage_provider); _readyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "ready")); _notReadyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "notready")); return true; diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h index 72a0bf1a665..6efbfb67c4c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h @@ -7,7 +7,9 @@ namespace proton { struct IAttributeManager; +class AttributeConfigInspector; class AttributeUsageFilter; +class TransientMemoryUsageProvider; /** * Class used to sample attribute resource usage and pass aggregated @@ -21,12 +23,16 @@ class SampleAttributeUsageJob : public IMaintenanceJob IAttributeManagerSP _readyAttributeManager; IAttributeManagerSP _notReadyAttributeManager; AttributeUsageFilter &_attributeUsageFilter; + std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector; + std::shared_ptr<TransientMemoryUsageProvider> _transient_memory_usage_provider; public: SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, AttributeUsageFilter &attributeUsageFilter, const vespalib::string &docTypeName, - vespalib::duration interval); + vespalib::duration interval, + std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, + std::shared_ptr<TransientMemoryUsageProvider> transient_memory_usage_provider); ~SampleAttributeUsageJob() override; bool run() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index 9ef038b7325..e169f51ef9d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -9,6 +9,7 @@ #include <vespa/searchcore/proton/common/indexschema_inspector.h> #include <vespa/searchcore/proton/reference/i_document_db_reference_resolver.h> #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h> +#include <vespa/eval/eval/llvm/compile_cache.h> using namespace vespa::config::search; using namespace config; @@ -87,6 +88,9 @@ void SearchableDocSubDBConfigurer::reconfigureSearchView(MatchView::SP matchView) { SearchView::SP curr = _searchView.get(); + // make sure the initial search does not spend time waiting for + // expression compilation completion during rank program setup. + vespalib::eval::CompileCache::wait_pending(); _searchView.set(SearchView::create(curr->getSummarySetup(), std::move(matchView))); } diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index 3e49bb449ff..8e5d3018532 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -11,16 +11,26 @@ namespace proton::test { class MockAttributeManager : public IAttributeManager { private: search::attribute::test::MockAttributeManager _mock; + std::vector<search::AttributeVector*> _writables; std::unique_ptr<ImportedAttributesRepo> _importedAttributes; + vespalib::ISequencedTaskExecutor* _writer; public: MockAttributeManager() : _mock(), - _importedAttributes() + _writables(), + _importedAttributes(), + _writer() {} - void addAttribute(const vespalib::string &name, const search::AttributeVector::SP &attr) { + search::AttributeVector::SP addAttribute(const vespalib::string &name, const search::AttributeVector::SP &attr) { _mock.addAttribute(name, attr); + _writables.push_back(attr.get()); + return attr; + } + + void set_writer(vespalib::ISequencedTaskExecutor& writer) { + _writer = &writer; } search::AttributeGuard::UP getAttribute(const vespalib::string &name) const override { @@ -56,13 +66,18 @@ public: HDR_ABORT("should not be reached"); } vespalib::ISequencedTaskExecutor &getAttributeFieldWriter() const override { - HDR_ABORT("should not be reached"); - } - search::AttributeVector *getWritableAttribute(const vespalib::string &) const override { + assert(_writer != nullptr); + return *_writer; + } + search::AttributeVector *getWritableAttribute(const vespalib::string &name) const override { + auto attr = getAttribute(name); + if (attr) { + return attr->get(); + } return nullptr; } const std::vector<search::AttributeVector *> &getWritableAttributes() const override { - HDR_ABORT("should not be reached"); + return _writables; } void asyncForEachAttribute(std::shared_ptr<IConstAttributeFunctor>) const override { } diff --git a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h b/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h index 03d9ba8d55c..10ed19a7244 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h +++ b/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h @@ -137,6 +137,11 @@ public: virtual uint64_t getApproxBytesToWriteToDisk() const = 0; /** + * Return cost of replaying a feed operation relative to cost of reading a feed operation from tls. + */ + virtual double get_replay_operation_cost() const { return 0.0; } + + /** * Returns the last serial number for the transaction applied to * target before it was flushed to disk. The transaction log can * not be pruned beyond this. diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 19cad2f3905..c5dd468e4fd 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -94,17 +94,19 @@ vespa_define_module( src/tests/attribute/save_target src/tests/attribute/searchable src/tests/attribute/searchcontext + src/tests/attribute/searchcontextelementiterator src/tests/attribute/sourceselector src/tests/attribute/stringattribute src/tests/attribute/tensorattribute src/tests/bitcompression/expgolomb src/tests/bitvector + src/tests/btree src/tests/bytecomplens src/tests/common/bitvector src/tests/common/location src/tests/common/matching_elements + src/tests/common/matching_elements_fields src/tests/common/resultset - src/tests/common/struct_field_mapper src/tests/common/summaryfeatures src/tests/diskindex/bitvector src/tests/diskindex/diskindex diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index 1cb314165cd..3728b87c6df 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -278,14 +278,25 @@ AttributeManagerTest::testConfigConvert() AttributeVector::Config out = ConfigConverter::convert(a); EXPECT_EQUAL("tensor(x[5])", out.tensorType().to_spec()); } + { // distance metric (default) + CACA a; + auto out = ConfigConverter::convert(a); + EXPECT_TRUE(out.distance_metric() == DistanceMetric::Euclidean); + } + { // distance metric (explicit) + CACA a; + a.distancemetric = AttributesConfig::Attribute::Distancemetric::GEODEGREES; + auto out = ConfigConverter::convert(a); + EXPECT_TRUE(out.distance_metric() == DistanceMetric::GeoDegrees); + } { // hnsw index params (enabled) - auto dm_in = AttributesConfig::Attribute::Index::Hnsw::Distancemetric::ANGULAR; - auto dm_out = search::attribute::DistanceMetric::Angular; + auto dm_in = AttributesConfig::Attribute::Distancemetric::ANGULAR; + auto dm_out = DistanceMetric::Angular; CACA a; + a.distancemetric = dm_in; a.index.hnsw.enabled = true; a.index.hnsw.maxlinkspernode = 32; a.index.hnsw.neighborstoexploreatinsert = 300; - a.index.hnsw.distancemetric = dm_in; auto out = ConfigConverter::convert(a); EXPECT_TRUE(out.hnsw_index_params().has_value()); EXPECT_EQUAL(32u, out.hnsw_index_params().value().max_links_per_node()); diff --git a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp index 665803b3057..5e633dcc97d 100644 --- a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp @@ -12,14 +12,15 @@ #include <vespa/searchlib/attribute/singlenumericattribute.h> #include <vespa/searchlib/attribute/singlenumericattribute.hpp> #include <vespa/searchlib/attribute/singlenumericpostattribute.hpp> +#include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/query/tree/location.h> #include <vespa/searchlib/query/tree/point.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> +#include <vespa/searchlib/queryeval/filter_wrapper.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/nearest_neighbor_blueprint.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> -#include <vespa/searchlib/fef/matchdata.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> @@ -41,10 +42,11 @@ using search::query::SimpleStringTerm; using search::query::Weight; using search::queryeval::Blueprint; using search::queryeval::EmptyBlueprint; +using search::queryeval::FakeRequestContext; using search::queryeval::FieldSpec; +using search::queryeval::FilterWrapper; using search::queryeval::NearestNeighborBlueprint; using search::queryeval::SearchIterator; -using search::queryeval::FakeRequestContext; using std::string; using std::vector; using vespalib::eval::TensorSpec; @@ -56,7 +58,6 @@ using namespace search; namespace { const string field = "field"; -const int32_t weight = 1; class MyAttributeManager : public IAttributeManager { AttributeVector::SP _attribute_vector; @@ -67,10 +68,6 @@ public: _attribute_vector(std::move(rhs._attribute_vector)) { } - MyAttributeManager(AttributeVector *attr) - : _attribute_vector(attr) - { - } MyAttributeManager(AttributeVector::SP attr) : _attribute_vector(std::move(attr)) @@ -141,54 +138,89 @@ search_for_term(const string &term, IAttributeManager &attribute_manager) return ret; } -template <typename T> -struct AttributeVectorTypeFinder { - typedef SingleStringExtAttribute Type; - static void add(Type & a, const T & v) { a.add(v, weight); } -}; +template <typename ChildType, typename ParentType> +ChildType& +downcast(ParentType& parent) +{ + auto* result = dynamic_cast<ChildType*>(&parent); + assert(result != nullptr); + return *result; +} -template <> -struct AttributeVectorTypeFinder<int64_t> { - typedef search::SingleValueNumericAttribute<search::IntegerAttributeTemplate<int64_t> > Type; - static void add(Type & a, int64_t v) { a.set(a.getNumDocs()-1, v); a.commit(); } +struct StringAttributeFiller { + using ValueType = vespalib::string; + static void add(AttributeVector& attr, const vespalib::string& value) { + auto& real = downcast<StringAttribute>(attr); + real.update(attr.getNumDocs() - 1, value); + real.commit(); + } }; -struct FastSearchLongAttribute { - typedef search::SingleValueNumericPostingAttribute< search::EnumAttribute<search::IntegerAttributeTemplate<int64_t> > > Type; - static void add(Type & a, int64_t v) { a.update(a.getNumDocs()-1, v); a.commit(); } +struct IntegerAttributeFiller { + using ValueType = int64_t; + static void add(AttributeVector& attr, int64_t value) { + auto& real = downcast<IntegerAttribute>(attr); + real.update(attr.getNumDocs() - 1, value); + real.commit(); + } }; -template <typename AT, typename T> -MyAttributeManager -fill(typename AT::Type * attr, T value) +template <typename FillerType> +void +fill(AttributeVector& attr, typename FillerType::ValueType value) { AttributeVector::DocId docid; - attr->addDoc(docid); - attr->addDoc(docid); - attr->addDoc(docid); + attr.addDoc(docid); + attr.addDoc(docid); + attr.addDoc(docid); assert(DOCID_LIMIT-1 == docid); - AT::add(*attr, value); - return MyAttributeManager(attr); + FillerType::add(attr, value); +} + +AttributeVector::SP +make_string_attribute(const std::string& value) +{ + Config cfg(BasicType::STRING, CollectionType::SINGLE); + auto attr = AttributeFactory::createAttribute(field, cfg); + fill<StringAttributeFiller>(*attr, value); + return attr; +} + +AttributeVector::SP +make_int_attribute(int64_t value) +{ + Config cfg(BasicType::INT32, CollectionType::SINGLE); + auto attr = AttributeFactory::createAttribute(field, cfg); + fill<IntegerAttributeFiller>(*attr, value); + return attr; +} + +AttributeVector::SP +make_fast_search_long_attribute(int64_t value) +{ + Config cfg(BasicType::fromType(int64_t()), CollectionType::SINGLE); + cfg.setFastSearch(true); + auto attr = AttributeFactory::createAttribute(field, cfg); + fill<IntegerAttributeFiller>(*attr, value); + return attr; } -template <typename T> MyAttributeManager -makeAttributeManager(T value) +makeAttributeManager(const std::string& value) { - typedef AttributeVectorTypeFinder<T> AT; - typedef typename AT::Type AttributeVectorType; - AttributeVectorType *attr = new AttributeVectorType(field); - return fill<AT, T>(attr, value); + return MyAttributeManager(make_string_attribute(value)); +} + +MyAttributeManager +makeAttributeManager(int64_t value) +{ + return MyAttributeManager(make_int_attribute(value)); } MyAttributeManager makeFastSearchLongAttribute(int64_t value) { - typedef FastSearchLongAttribute::Type AttributeVectorType; - Config cfg(BasicType::fromType(int64_t()), CollectionType::SINGLE); - cfg.setFastSearch(true); - AttributeVectorType *attr = new AttributeVectorType(field, cfg); - return fill<FastSearchLongAttribute, int64_t>(attr, value); + return MyAttributeManager(make_fast_search_long_attribute(value)); } } // namespace @@ -265,16 +297,7 @@ make_int_attribute(const vespalib::string& name) return AttributeFactory::createAttribute(name, cfg); } -template <typename BlueprintType> -const BlueprintType& -as_type(const Blueprint& blueprint) -{ - const auto* result = dynamic_cast<const BlueprintType*>(&blueprint); - assert(result != nullptr); - return *result; -} - -class NearestNeighborFixture { +class BlueprintFactoryFixture { public: MyAttributeManager mgr; vespalib::string attr_name; @@ -282,7 +305,7 @@ public: FakeRequestContext request_ctx; AttributeBlueprintFactory source; - NearestNeighborFixture(AttributeVector::SP attr) + BlueprintFactoryFixture(AttributeVector::SP attr) : mgr(attr), attr_name(attr->getName()), attr_ctx(mgr), @@ -290,13 +313,28 @@ public: source() { } + ~BlueprintFactoryFixture() {} + Blueprint::UP create_blueprint(const Node& term) { + auto result = source.createBlueprint(request_ctx, FieldSpec(attr_name, 0, 0), term); + result->fetchPostings(queryeval::ExecuteInfo::TRUE); + result->setDocIdLimit(DOCID_LIMIT); + return result; + } +}; + +class NearestNeighborFixture : public BlueprintFactoryFixture { +public: + NearestNeighborFixture(AttributeVector::SP attr) + : BlueprintFactoryFixture(std::move(attr)) + { + } ~NearestNeighborFixture() {} void set_query_tensor(const TensorSpec& tensor_spec) { request_ctx.set_query_tensor("query_tensor", tensor_spec); } Blueprint::UP create_blueprint() { query::NearestNeighborTerm term("query_tensor", attr_name, 0, Weight(0), 7, true, 33); - return source.createBlueprint(request_ctx, FieldSpec(attr_name, 0, 0), term); + return BlueprintFactoryFixture::create_blueprint(term); } }; @@ -309,7 +347,7 @@ expect_nearest_neighbor_blueprint(const vespalib::string& attribute_tensor_type_ f.set_query_tensor(query_tensor); auto result = f.create_blueprint(); - const auto& nearest = as_type<NearestNeighborBlueprint>(*result); + const auto& nearest = downcast<const NearestNeighborBlueprint>(*result); EXPECT_EQ(attribute_tensor_type_spec, nearest.get_attribute_tensor().getTensorType().to_spec()); EXPECT_EQ(converted_query_tensor, DefaultTensorEngine::ref().to_spec(nearest.get_query_tensor())); EXPECT_EQ(7u, nearest.get_target_num_hits()); @@ -359,4 +397,17 @@ TEST(AttributeBlueprintTest, empty_blueprint_is_created_when_nearest_neighbor_te expect_empty_blueprint(make_tensor_attribute(field, "tensor(x[2])"), dense_x_3); // tensor types are not same size } +TEST(AttributeBlueprintTest, attribute_field_blueprint_wraps_filter_search_iterator) +{ + BlueprintFactoryFixture f(make_string_attribute("foo")); + SimpleStringTerm term("foo", field, 0, Weight(0)); + auto blueprint = f.create_blueprint(term); + + auto itr = blueprint->createFilterSearch(true, Blueprint::FilterConstraint::UPPER_BOUND); + auto& wrapper = downcast<FilterWrapper>(*itr); + wrapper.initRange(1, 3); + EXPECT_FALSE(wrapper.seek(1)); + EXPECT_TRUE(wrapper.seek(2)); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp index 416ddb5fbc0..ffae7824668 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp @@ -1,12 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributeiterators.h> -#include <vespa/searchlib/attribute/attributevector.hpp> -#include <vespa/searchlib/attribute/elementiterator.h> +#include <vespa/searchlib/attribute/searchcontextelementiterator.h> #include <vespa/searchlib/attribute/flagattribute.h> -#include <vespa/searchlib/attribute/multistringattribute.h> #include <vespa/searchlib/attribute/singleboolattribute.h> #include <vespa/searchlib/attribute/singlenumericattribute.h> #include <vespa/searchlib/attribute/singlestringattribute.h> @@ -22,6 +19,7 @@ #include <vespa/searchlib/test/searchiteratorverifier.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/compress.h> +#include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/log/log.h> LOG_SETUP("searchcontext_test"); @@ -628,29 +626,24 @@ void SearchContextTest::testSearch(const ConfigMap & cfgs) { template<typename T, typename A> class Verifier : public search::test::SearchIteratorVerifier { public: - Verifier(const std::vector<T> & keys, const vespalib::string & keyAsString, const vespalib::string & name, - const Config & cfg, bool withElementId); + Verifier(const std::vector<T> & keys, const vespalib::string & keyAsString, + const vespalib::string & name, const Config & cfg); ~Verifier() override; SearchIterator::UP create(bool strict) const override { _sc->fetchPostings(queryeval::ExecuteInfo::create(strict, 1.0)); - auto search = _sc->createIterator(&_dummy, strict); - if (_withElementId) { - search = std::make_unique<attribute::ElementIterator>(std::move(search), *_sc, _dummy); - } - return search; + return _sc->createIterator(&_dummy, strict); } private: mutable TermFieldMatchData _dummy; - const bool _withElementId; AttributePtr _attribute; SearchContextPtr _sc; }; template<typename T, typename A> -Verifier<T, A>::Verifier(const std::vector<T> & keys, const vespalib::string & keyAsString, const vespalib::string & name, - const Config & cfg, bool withElementId) - : _withElementId(withElementId), +Verifier<T, A>::Verifier(const std::vector<T> & keys, const vespalib::string & keyAsString, + const vespalib::string & name, const Config & cfg) + : _dummy(), _attribute(AttributeFactory::createAttribute(name + "-initrange", cfg)), _sc() { @@ -670,22 +663,18 @@ Verifier<T, A>::~Verifier() = default; template<typename T, typename A> void SearchContextTest::testSearchIterator(const std::vector<T> & keys, const vespalib::string &keyAsString, const ConfigMap &cfgs) { - - for (bool withElementId : {false, true} ) { - for (const auto & cfg : cfgs) { - { - Verifier<T, A> verifier(keys, keyAsString, cfg.first, cfg.second, withElementId); - verifier.verify(); - } - { - Config withFilter(cfg.second); - withFilter.setIsFilter(true); - Verifier<T, A> verifier(keys, keyAsString, cfg.first + "-filter", withFilter, withElementId); - verifier.verify(); - } + for (const auto & cfg : cfgs) { + { + Verifier<T, A> verifier(keys, keyAsString, cfg.first, cfg.second); + verifier.verify(); + } + { + Config withFilter(cfg.second); + withFilter.setIsFilter(true); + Verifier<T, A> verifier(keys, keyAsString, cfg.first + "-filter", withFilter); + verifier.verify(); } } - } void SearchContextTest::testSearchIteratorConformance() { @@ -976,11 +965,13 @@ SearchContextTest::testSearchIteratorUnpacking(const AttributePtr & attr, Search pos.setElementWeight(100); md.appendPosition(pos); - SearchBasePtr sb = sc.createIterator(&md, strict); + SearchBasePtr sbp = sc.createIterator(&md, strict); + SearchIterator & search = *sbp; + queryeval::ElementIterator::UP elemIt; if (withElementId) { - sb = std::make_unique<attribute::ElementIterator>(std::move(sb), sc, md); + elemIt = std::make_unique<attribute::SearchContextElementIterator>(std::move(sbp), sc); } - sb->initFullRange(); + search.initFullRange(); std::vector<int32_t> weights(3); if (attr->getCollectionType() == CollectionType::SINGLE || @@ -1000,41 +991,40 @@ SearchContextTest::testSearchIteratorUnpacking(const AttributePtr & attr, Search } // unpack and check weights - sb->unpack(1); - EXPECT_EQUAL(sb->getDocId(), 1u); + search.unpack(1); + EXPECT_EQUAL(search.getDocId(), 1u); EXPECT_EQUAL(md.getDocId(), 1u); EXPECT_EQUAL(md.getWeight(), weights[0]); - sb->unpack(2); - EXPECT_EQUAL(sb->getDocId(), 2u); + search.unpack(2); + EXPECT_EQUAL(search.getDocId(), 2u); EXPECT_EQUAL(md.getDocId(), 2u); if (withElementId && attr->hasMultiValue() && !attr->hasWeightedSetType()) { - EXPECT_EQUAL(2, md.end()- md.begin()); - EXPECT_EQUAL(md.begin()[0].getElementId(), 0u); - EXPECT_EQUAL(md.begin()[0].getElementWeight(), 1); - EXPECT_EQUAL(md.begin()[1].getElementId(), 1u); - EXPECT_EQUAL(md.begin()[1].getElementWeight(), 1); + std::vector<uint32_t> elems; + elemIt->getElementIds(2, elems); + ASSERT_EQUAL(2u, elems.size()); + EXPECT_EQUAL(0u,elems[0]); + EXPECT_EQUAL(1u,elems[1]); } else { EXPECT_EQUAL(md.getWeight(), weights[1]); } - sb->unpack(3); - EXPECT_EQUAL(sb->getDocId(), 3u); + search.unpack(3); + EXPECT_EQUAL(search.getDocId(), 3u); EXPECT_EQUAL(md.getDocId(), 3u); if (withElementId && attr->hasMultiValue() && !attr->hasWeightedSetType()) { - EXPECT_EQUAL(3, md.end()- md.begin()); - EXPECT_EQUAL(md.begin()[0].getElementId(), 0u); - EXPECT_EQUAL(md.begin()[0].getElementWeight(), 1); - EXPECT_EQUAL(md.begin()[1].getElementId(), 1u); - EXPECT_EQUAL(md.begin()[1].getElementWeight(), 1); - EXPECT_EQUAL(md.begin()[2].getElementId(), 2u); - EXPECT_EQUAL(md.begin()[2].getElementWeight(), 1); + std::vector<uint32_t> elems; + elemIt->getElementIds(3, elems); + ASSERT_EQUAL(3u, elems.size()); + EXPECT_EQUAL(0u,elems[0]); + EXPECT_EQUAL(1u,elems[1]); + EXPECT_EQUAL(2u,elems[2]); } else { EXPECT_EQUAL(md.getWeight(), weights[2]); } if (extra) { - sb->unpack(4); - EXPECT_EQUAL(sb->getDocId(), 4u); + search.unpack(4); + EXPECT_EQUAL(search.getDocId(), 4u); EXPECT_EQUAL(md.getDocId(), 4u); EXPECT_EQUAL(md.getWeight(), 1); } diff --git a/searchlib/src/tests/attribute/searchcontextelementiterator/CMakeLists.txt b/searchlib/src/tests/attribute/searchcontextelementiterator/CMakeLists.txt new file mode 100644 index 00000000000..1a27b446d30 --- /dev/null +++ b/searchlib/src/tests/attribute/searchcontextelementiterator/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_attribute_searchcontextelementiterator_test_app TEST + SOURCES + searchcontextelementiterator_test.cpp + DEPENDS + searchlib + gtest +) +vespa_add_test(NAME searchlib_attribute_searchcontextelementiterator_test_app COMMAND searchlib_attribute_searchcontextelementiterator_test_app) diff --git a/searchlib/src/tests/attribute/searchcontextelementiterator/searchcontextelementiterator_test.cpp b/searchlib/src/tests/attribute/searchcontextelementiterator/searchcontextelementiterator_test.cpp new file mode 100644 index 00000000000..22fd409320c --- /dev/null +++ b/searchlib/src/tests/attribute/searchcontextelementiterator/searchcontextelementiterator_test.cpp @@ -0,0 +1,128 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/attribute/searchcontextelementiterator.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/queryeval/fake_search.h> +#include <vespa/searchlib/query/query_term_simple.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> +# + +#include <vespa/vespalib/gtest/gtest.h> + +#include <vespa/log/log.h> +LOG_SETUP("searchcontextelementiterator_test"); + +using namespace search::attribute; +using namespace search; + +namespace { + +AttributeVector::SP +createAndFillAttribute() { + AttributeFactory factory; + AttributeVector::SP attribute = factory.createAttribute("mva", Config(BasicType::INT32, CollectionType::ARRAY)); + attribute->addDocs(6); + IntegerAttribute & ia = dynamic_cast<IntegerAttribute &>(*attribute); + ia.append(1, 3, 1); + for (int v : {1,2,3,1,2,3}) { + ia.append(2, v, 1); + } + for (int v : {1,2,3,4,5,1,2,3,4,5,6}) { + ia.append(4, v, 1); + } + ia.append(5, 5, 1); + attribute->commit(); + return attribute; +} + +queryeval::FakeResult +createResult() { + queryeval::FakeResult result; + result.doc(2).elem(0).pos(7).pos(9) + .elem(3).pos(1); + result.doc(4).elem(0).pos(2) + .elem(5).pos(1).pos(2).pos(3); + return result; +} + +void +verifySeek(queryeval::ElementIterator & elemIt) { + elemIt.initFullRange(); + EXPECT_FALSE(elemIt.seek(1)); + EXPECT_TRUE(elemIt.seek(2)); + EXPECT_FALSE(elemIt.seek(3)); + EXPECT_TRUE(elemIt.seek(4)); + EXPECT_FALSE(elemIt.seek(5)); +} + +void +verifyGetElementIds(queryeval::ElementIterator & elemIt, const std::vector<std::vector<uint32_t>> & expectedALL) { + elemIt.initFullRange(); + std::vector<uint32_t> elems; + for (uint32_t docId : {1,2,3,4,5}) { + const auto & expected = expectedALL[docId]; + elems.clear(); + EXPECT_EQ(expected.empty(), !elemIt.seek(docId)); + assert(expected.empty() != elemIt.seek(docId)); + if (elemIt.seek(docId)) { + elemIt.getElementIds(docId, elems); + EXPECT_EQ(expected.size(), elems.size()); + EXPECT_EQ(expected, elems); + } + } +} + +void +verifyMergeElementIds(queryeval::ElementIterator & elemIt, std::vector<uint32_t> initial, const std::vector<std::vector<uint32_t>> & expectedALL) { + elemIt.initFullRange(); + std::vector<uint32_t> elems; + for (uint32_t docId : {1,2,3,4,5}) { + const auto & expected = expectedALL[docId]; + elems = initial; + if (elemIt.seek(docId)) { + elemIt.mergeElementIds(docId, elems); + EXPECT_EQ(expected.size(), elems.size()); + EXPECT_EQ(expected, elems); + } + } +} + +void +verifyElementIterator(queryeval::ElementIterator & elemIt) { + verifySeek(elemIt); + std::vector<std::vector<uint32_t>> expectedALL = {{}, {}, {0, 3}, {}, {0, 5}, {}}; + std::vector<std::vector<uint32_t>> expectedNONE = {{}, {}, {}, {}, {}, {}}; + std::vector<std::vector<uint32_t>> expectedSOME = {{}, {}, {3}, {}, {5}, {}}; + verifyGetElementIds(elemIt, expectedALL); + verifyMergeElementIds(elemIt, {0,1,2,3,4,5}, expectedALL); + verifyMergeElementIds(elemIt, {}, expectedNONE); + verifyMergeElementIds(elemIt, {1,3,4,5}, expectedSOME); +} + +} + +TEST(ElementIteratorTest, require_that_searchcontext) +{ + AttributeVector::SP attribute = createAndFillAttribute(); + fef::TermFieldMatchData tfmd; + + SearchContextParams params; + ISearchContext::UP sc = attribute->createSearchContext(std::make_unique<QueryTermSimple>("1", QueryTermSimple::SearchTerm::WORD), params); + SearchContextElementIterator elemIt(sc->createIterator(&tfmd, false), *sc); + verifyElementIterator(elemIt); +} + +TEST(ElementIteratorTest, require_that_non_searchcontext) +{ + fef::TermFieldMatchData tfmd; + fef::TermFieldMatchDataArray tfmda; + tfmda.add(&tfmd); + queryeval::FakeResult result = createResult(); + auto search = std::make_unique<queryeval::FakeSearch>("","","", result, tfmda); + queryeval::ElementIteratorWrapper wrapper(std::move(search), tfmd); + verifyElementIterator(wrapper); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 18a6a5a8188..7a4c6c9e56a 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -7,6 +7,7 @@ #include <vespa/fastos/file.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/queryeval/nearest_neighbor_blueprint.h> #include <vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/doc_vector_access.h> @@ -32,6 +33,8 @@ using search::AttributeGuard; using search::AttributeVector; using search::attribute::DistanceMetric; using search::attribute::HnswIndexParams; +using search::queryeval::NearestNeighborBlueprint; +using search::queryeval::GlobalFilter; using search::tensor::DefaultNearestNeighborIndexFactory; using search::tensor::DenseTensorAttribute; using search::tensor::DocVectorAccess; @@ -46,6 +49,7 @@ using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; using vespalib::tensor::DefaultTensorEngine; using vespalib::tensor::DenseTensor; +using vespalib::tensor::DenseTensorView; using vespalib::tensor::Tensor; using DoubleVector = std::vector<double>; @@ -154,6 +158,15 @@ public: auto vector = _vectors.get_vector(docid).typify<double>(); _adds.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); } + std::unique_ptr<search::tensor::PrepareResult> prepare_add_document(uint32_t, + vespalib::tensor::TypedCells, + vespalib::GenerationHandler::Guard) const override { + return std::unique_ptr<search::tensor::PrepareResult>(); + } + void complete_add_document(uint32_t docid, + std::unique_ptr<search::tensor::PrepareResult>) override { + add_document(docid); + } void remove_document(uint32_t docid) override { auto vector = _vectors.get_vector(docid).typify<double>(); _removes.emplace_back(docid, DoubleVector(vector.begin(), vector.end())); @@ -186,6 +199,16 @@ public: (void) explore_k; return std::vector<Neighbor>(); } + std::vector<Neighbor> find_top_k_with_filter(uint32_t k, vespalib::tensor::TypedCells vector, + const search::BitVector& filter, uint32_t explore_k) const override + { + (void) k; + (void) vector; + (void) explore_k; + (void) filter; + return std::vector<Neighbor>(); + } + const search::tensor::DistanceFunction *distance_function() const override { return nullptr; } }; @@ -238,6 +261,7 @@ struct Fixture { _useDenseTensorAttribute(useDenseTensorAttribute) { if (enable_hnsw_index) { + _cfg.set_distance_metric(DistanceMetric::Euclidean); _cfg.set_hnsw_index_params(HnswIndexParams(4, 20, DistanceMetric::Euclidean)); } setup(); @@ -777,5 +801,80 @@ TEST_F("Nearest neighbor index type is added to attribute file header", DenseTen EXPECT_EQUAL("hnsw", header.getTag("nearest_neighbor_index").asString()); } -TEST_MAIN() { TEST_RUN_ALL(); } +class NearestNeighborBlueprintFixture : public DenseTensorAttributeMockIndex { +public: + using QueryTensor = DenseTensor<double>; + + NearestNeighborBlueprintFixture() { + set_tensor(1, vec_2d(1, 1)); + set_tensor(2, vec_2d(2, 2)); + set_tensor(3, vec_2d(3, 3)); + set_tensor(4, vec_2d(4, 4)); + set_tensor(5, vec_2d(5, 5)); + set_tensor(6, vec_2d(6, 6)); + set_tensor(7, vec_2d(7, 7)); + set_tensor(8, vec_2d(8, 8)); + set_tensor(9, vec_2d(9, 9)); + set_tensor(10, vec_2d(0, 0)); + } + + std::unique_ptr<QueryTensor> createDenseTensor(const TensorSpec &spec) { + auto value = DefaultTensorEngine::ref().from_spec(spec); + QueryTensor *tensor = dynamic_cast<QueryTensor *>(value.get()); + ASSERT_TRUE(tensor != nullptr); + value.release(); + return std::unique_ptr<QueryTensor>(tensor); + } + + std::unique_ptr<NearestNeighborBlueprint> make_blueprint() { + search::queryeval::FieldSpec field("foo", 0, 0); + auto bp = std::make_unique<NearestNeighborBlueprint>( + field, + as_dense_tensor(), + createDenseTensor(vec_2d(17, 42)), + 3, true, 5); + EXPECT_EQUAL(11u, bp->getState().estimate().estHits); + EXPECT_TRUE(bp->may_approximate()); + return bp; + } +}; + +TEST_F("NN blueprint handles empty filter", NearestNeighborBlueprintFixture) +{ + auto bp = f.make_blueprint(); + auto empty_filter = GlobalFilter::create(); + bp->set_global_filter(*empty_filter); + EXPECT_EQUAL(3u, bp->getState().estimate().estHits); + EXPECT_TRUE(bp->may_approximate()); +} +TEST_F("NN blueprint handles strong filter", NearestNeighborBlueprintFixture) +{ + auto bp = f.make_blueprint(); + auto filter = search::BitVector::create(11); + filter->setBit(3); + filter->invalidateCachedCount(); + auto strong_filter = GlobalFilter::create(std::move(filter)); + bp->set_global_filter(*strong_filter); + EXPECT_EQUAL(1u, bp->getState().estimate().estHits); + EXPECT_TRUE(bp->may_approximate()); +} + +TEST_F("NN blueprint handles weak filter", NearestNeighborBlueprintFixture) +{ + auto bp = f.make_blueprint(); + auto filter = search::BitVector::create(11); + filter->setBit(1); + filter->setBit(3); + filter->setBit(5); + filter->setBit(7); + filter->setBit(9); + filter->setBit(11); + filter->invalidateCachedCount(); + auto weak_filter = GlobalFilter::create(std::move(filter)); + bp->set_global_filter(*weak_filter); + EXPECT_EQUAL(3u, bp->getState().estimate().estHits); + EXPECT_TRUE(bp->may_approximate()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/btree/.gitignore b/searchlib/src/tests/btree/.gitignore new file mode 100644 index 00000000000..ec4090e3658 --- /dev/null +++ b/searchlib/src/tests/btree/.gitignore @@ -0,0 +1 @@ +searchlib_scanspeed_app diff --git a/searchlib/src/tests/btree/CMakeLists.txt b/searchlib/src/tests/btree/CMakeLists.txt new file mode 100644 index 00000000000..ff396144c52 --- /dev/null +++ b/searchlib/src/tests/btree/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_scanspeed_app + SOURCES + scanspeed.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_scanspeed_app COMMAND vespalib_scanspeed_app BENCHMARK) diff --git a/searchlib/src/tests/btree/scanspeed.cpp b/searchlib/src/tests/btree/scanspeed.cpp new file mode 100644 index 00000000000..1474edd6b0b --- /dev/null +++ b/searchlib/src/tests/btree/scanspeed.cpp @@ -0,0 +1,181 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/btree/btreeroot.h> +#include <vespa/vespalib/btree/btreebuilder.h> +#include <vespa/vespalib/btree/btreenodeallocator.h> +#include <vespa/vespalib/btree/btree.h> +#include <vespa/vespalib/btree/btreestore.h> +#include <vespa/vespalib/btree/btreenodeallocator.hpp> +#include <vespa/vespalib/btree/btreenode.hpp> +#include <vespa/vespalib/btree/btreenodestore.hpp> +#include <vespa/vespalib/btree/btreeiterator.hpp> +#include <vespa/vespalib/btree/btreeroot.hpp> +#include <vespa/vespalib/btree/btreebuilder.hpp> +#include <vespa/vespalib/btree/btree.hpp> +#include <vespa/vespalib/btree/btreestore.hpp> +#include <vespa/vespalib/util/time.h> +#include <vespa/searchlib/common/bitvector.h> + +#include <vespa/fastos/app.h> + +using vespalib::btree::BTree; +using vespalib::btree::BTreeNode; +using vespalib::btree::BTreeTraits; + +enum class ScanMethod +{ + ITERATOR, + FUNCTOR +}; + +class ScanSpeed : public FastOS_Application +{ + template <typename Traits> + void work_loop(ScanMethod scan_method); + int Main() override; +}; + + +namespace { + +const char *scan_method_name(ScanMethod scan_method) +{ + switch (scan_method) { + case ScanMethod::ITERATOR: + return "iterator"; + default: + return "functor"; + } +} + +class ScanOnce { +public: + virtual ~ScanOnce() = default; + virtual void operator()(search::BitVector &bv) = 0; +}; + +template <typename Tree> +class ScanTree : public ScanOnce { +protected: + const Tree &_tree; + int _startval; + int _endval; +public: + ScanTree(const Tree &tree, int startval, int endval) + : _tree(tree), + _startval(startval), + _endval(endval) + { + } + ~ScanTree() override { } +}; + +template <typename Tree> +class ScanWithIterator : public ScanTree<Tree> { +public: + ScanWithIterator(const Tree &tree, int startval, int endval) + : ScanTree<Tree>(tree, startval, endval) + { + } + ~ScanWithIterator() override = default; + void operator()(search::BitVector &bv) override; +}; + +template <typename Tree> +void +ScanWithIterator<Tree>::operator()(search::BitVector &bv) +{ + using ConstIterator = typename Tree::ConstIterator; + ConstIterator itr(BTreeNode::Ref(), this->_tree.getAllocator()); + itr.lower_bound(this->_tree.getRoot(), this->_startval); + while (itr.valid() && itr.getKey() < this->_endval) { + bv.setBit(itr.getKey()); + ++itr; + } +} + +template <typename Tree> +class ScanWithFunctor : public ScanTree<Tree> { + +public: + ScanWithFunctor(const Tree &tree, int startval, int endval) + : ScanTree<Tree>(tree, startval, endval) + { + } + ~ScanWithFunctor() override = default; + void operator()(search::BitVector &bv) override; +}; + +template <typename Tree> +void +ScanWithFunctor<Tree>::operator()(search::BitVector &bv) +{ + using ConstIterator = typename Tree::ConstIterator; + ConstIterator start(BTreeNode::Ref(), this->_tree.getAllocator()); + ConstIterator end(BTreeNode::Ref(), this->_tree.getAllocator()); + start.lower_bound(this->_tree.getRoot(), this->_startval); + end.lower_bound(this->_tree.getRoot(), this->_endval); + start.foreach_key_range(end, [&](int key) { bv.setBit(key); } ); +} + +} + +template <typename Traits> +void +ScanSpeed::work_loop(ScanMethod scan_method) +{ + vespalib::GenerationHandler g; + using Tree = BTree<int, int, vespalib::btree::NoAggregated, std::less<int>, Traits>; + using Builder = typename Tree::Builder; + Tree tree; + Builder builder(tree.getAllocator()); + size_t numEntries = 1000000; + size_t numInnerLoops = 1000; + for (size_t i = 0; i < numEntries; ++i) { + builder.insert(i, 0); + } + tree.assign(builder); + assert(numEntries == tree.size()); + assert(tree.isValid()); + std::unique_ptr<ScanOnce> scan_once; + if (scan_method == ScanMethod::ITERATOR) { + scan_once = std::make_unique<ScanWithIterator<Tree>>(tree, 4, numEntries - 4); + } else { + scan_once = std::make_unique<ScanWithFunctor<Tree>>(tree, 4, numEntries - 4); + } + auto bv = search::BitVector::create(numEntries); + vespalib::Timer timer; + for (size_t innerl = 0; innerl < numInnerLoops; ++innerl) { + (*scan_once)(*bv); + } + double used = vespalib::to_s(timer.elapsed()); + printf("Elapsed time for scanning %ld entries is %8.5f, " + "scanmethod=%s, fanout=%u,%u\n", + numEntries * numInnerLoops, + used, + scan_method_name(scan_method), + static_cast<int>(Traits::LEAF_SLOTS), + static_cast<int>(Traits::INTERNAL_SLOTS)); + fflush(stdout); +} + + +int +ScanSpeed::Main() +{ + using SmallTraits = BTreeTraits<4, 4, 31, false>; + using DefTraits = vespalib::btree::BTreeDefaultTraits; + using LargeTraits = BTreeTraits<32, 16, 10, true>; + using HugeTraits = BTreeTraits<64, 16, 10, true>; + work_loop<SmallTraits>(ScanMethod::ITERATOR); + work_loop<DefTraits>(ScanMethod::ITERATOR); + work_loop<LargeTraits>(ScanMethod::ITERATOR); + work_loop<HugeTraits>(ScanMethod::ITERATOR); + work_loop<SmallTraits>(ScanMethod::FUNCTOR); + work_loop<DefTraits>(ScanMethod::FUNCTOR); + work_loop<LargeTraits>(ScanMethod::FUNCTOR); + work_loop<HugeTraits>(ScanMethod::FUNCTOR); + return 0; +} + +FASTOS_MAIN(ScanSpeed); diff --git a/searchlib/src/tests/common/matching_elements_fields/CMakeLists.txt b/searchlib/src/tests/common/matching_elements_fields/CMakeLists.txt new file mode 100644 index 00000000000..3d6f338315a --- /dev/null +++ b/searchlib/src/tests/common/matching_elements_fields/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_common_matching_elements_fields_test_app TEST + SOURCES + matching_elements_fields_test.cpp + DEPENDS + searchlib + gtest +) +vespa_add_test(NAME searchlib_common_matching_elements_fields_test_app COMMAND searchlib_common_matching_elements_fields_test_app) diff --git a/searchlib/src/tests/common/matching_elements_fields/matching_elements_fields_test.cpp b/searchlib/src/tests/common/matching_elements_fields/matching_elements_fields_test.cpp new file mode 100644 index 00000000000..766be30f8f1 --- /dev/null +++ b/searchlib/src/tests/common/matching_elements_fields/matching_elements_fields_test.cpp @@ -0,0 +1,55 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/searchlib/common/matching_elements_fields.h> + +using namespace search; + +namespace { + +std::string str(const vespalib::string &s) { return std::string(s.data(), s.size()); } + +} + +struct MatchingElementsFieldsTest : ::testing::Test { + MatchingElementsFields fields; + MatchingElementsFieldsTest() : fields() { + fields.add_mapping("foo", "foo.a"); + fields.add_mapping("foo", "foo.b"); + fields.add_mapping("bar", "bar.x"); + fields.add_mapping("bar", "bar.y"); + fields.add_field("baz"); + } + ~MatchingElementsFieldsTest() = default; +}; + +TEST_F(MatchingElementsFieldsTest, require_that_field_can_be_identified) { + EXPECT_TRUE(fields.has_field("foo")); + EXPECT_TRUE(fields.has_field("bar")); + EXPECT_TRUE(fields.has_field("baz")); + EXPECT_TRUE(!fields.has_field("foo.a")); + EXPECT_TRUE(!fields.has_field("bar.x")); + EXPECT_TRUE(!fields.has_field("bogus")); +} + +TEST_F(MatchingElementsFieldsTest, require_that_struct_field_can_be_identified) { + EXPECT_TRUE(!fields.has_struct_field("foo")); + EXPECT_TRUE(!fields.has_struct_field("bar")); + EXPECT_TRUE(!fields.has_struct_field("baz")); + EXPECT_TRUE(fields.has_struct_field("foo.a")); + EXPECT_TRUE(fields.has_struct_field("bar.x")); + EXPECT_TRUE(!fields.has_struct_field("bogus")); +} + +TEST_F(MatchingElementsFieldsTest, require_that_struct_field_maps_to_enclosing_field_name) { + EXPECT_EQ(str(fields.get_enclosing_field("foo.a")), str("foo")); + EXPECT_EQ(str(fields.get_enclosing_field("foo.b")), str("foo")); + EXPECT_EQ(str(fields.get_enclosing_field("bar.x")), str("bar")); + EXPECT_EQ(str(fields.get_enclosing_field("bar.y")), str("bar")); +} + +TEST_F(MatchingElementsFieldsTest, require_that_nonexisting_struct_field_maps_to_empty_string) { + EXPECT_EQ(str(fields.get_enclosing_field("bogus")), str("")); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/common/struct_field_mapper/CMakeLists.txt b/searchlib/src/tests/common/struct_field_mapper/CMakeLists.txt deleted file mode 100644 index f5712d22989..00000000000 --- a/searchlib/src/tests/common/struct_field_mapper/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_common_struct_field_mapper_test_app TEST - SOURCES - struct_field_mapper_test.cpp - DEPENDS - searchlib - gtest -) -vespa_add_test(NAME searchlib_common_struct_field_mapper_test_app COMMAND searchlib_common_struct_field_mapper_test_app) diff --git a/searchlib/src/tests/common/struct_field_mapper/struct_field_mapper_test.cpp b/searchlib/src/tests/common/struct_field_mapper/struct_field_mapper_test.cpp deleted file mode 100644 index c5368111859..00000000000 --- a/searchlib/src/tests/common/struct_field_mapper/struct_field_mapper_test.cpp +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/gtest/gtest.h> -#include <vespa/searchlib/common/struct_field_mapper.h> - -using namespace search; - -namespace { - -std::string str(const vespalib::string &s) { return std::string(s.data(), s.size()); } - -} - -struct StructFieldMapperTest : ::testing::Test { - StructFieldMapper mapper; - StructFieldMapperTest() : mapper() { - mapper.add_mapping("foo", "foo.a"); - mapper.add_mapping("foo", "foo.b"); - mapper.add_mapping("bar", "bar.x"); - mapper.add_mapping("bar", "bar.y"); - } - ~StructFieldMapperTest() = default; -}; - -TEST_F(StructFieldMapperTest, require_that_struct_field_can_be_identified) { - EXPECT_TRUE(mapper.is_struct_field("foo")); - EXPECT_TRUE(mapper.is_struct_field("bar")); - EXPECT_TRUE(!mapper.is_struct_field("foo.a")); - EXPECT_TRUE(!mapper.is_struct_field("bar.x")); - EXPECT_TRUE(!mapper.is_struct_field("bogus")); -} - -TEST_F(StructFieldMapperTest, require_that_struct_subfield_can_be_identified) { - EXPECT_TRUE(!mapper.is_struct_subfield("foo")); - EXPECT_TRUE(!mapper.is_struct_subfield("bar")); - EXPECT_TRUE(mapper.is_struct_subfield("foo.a")); - EXPECT_TRUE(mapper.is_struct_subfield("bar.x")); - EXPECT_TRUE(!mapper.is_struct_subfield("bogus")); -} - -TEST_F(StructFieldMapperTest, require_that_struct_subfield_maps_to_enclosing_struct_field_name) { - EXPECT_EQ(str(mapper.get_struct_field("foo.a")), str("foo")); - EXPECT_EQ(str(mapper.get_struct_field("foo.b")), str("foo")); - EXPECT_EQ(str(mapper.get_struct_field("bar.x")), str("bar")); - EXPECT_EQ(str(mapper.get_struct_field("bar.y")), str("bar")); -} - -TEST_F(StructFieldMapperTest, require_that_nonexisting_struct_subfield_maps_to_empty_string) { - EXPECT_EQ(str(mapper.get_struct_field("bogus")), str("")); -} - -GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index 31c21723cd0..70c23dc4191 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -147,6 +147,7 @@ struct WriteFixture : public FixtureBase { void updateLidMap(uint32_t docIdLimit) { vespalib::LockGuard guard(updateLock); chunk.updateLidMap(guard, lidObserver, serialNum, docIdLimit); + serialNum = chunk.getSerialNum(); } }; @@ -180,6 +181,30 @@ TEST("require that docIdLimit is written to and read from idx file header") } } +TEST("require that numlids are updated") { + { + WriteFixture f("tmp", 1000, false); + f.updateLidMap(1000); + EXPECT_EQUAL(0u, f.chunk.getNumLids()); + f.append(1); + EXPECT_EQUAL(1u, f.chunk.getNumLids()); + f.append(2); + f.append(3); + EXPECT_EQUAL(3u, f.chunk.getNumLids()); + f.append(3); + EXPECT_EQUAL(4u, f.chunk.getNumLids()); + f.flush(); + } + { + WriteFixture f("tmp", 1000, true); + EXPECT_EQUAL(0u, f.chunk.getNumLids()); + f.updateLidMap(1000); + EXPECT_EQUAL(4u, f.chunk.getNumLids()); + f.append(7); + EXPECT_EQUAL(5u, f.chunk.getNumLids()); + } +} + template <typename FixtureType> void assertUpdateLidMap(FixtureType &f) diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index c383358db9e..9a19a795076 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -22,6 +22,7 @@ using namespace search::docstore; using namespace search; using namespace vespalib::alloc; using search::index::DummyFileHeaderContext; +using search::test::DirectoryHandler; class MyTlSyncer : public transactionlog::SyncProxy { SerialNum _syncedTo; @@ -202,15 +203,9 @@ TEST("test that DirectIOPadding works accordng to spec") { } #endif -TEST("testGrowing") { - FastOS_File::EmptyAndRemoveDirectory("growing"); - EXPECT_TRUE(FastOS_File::MakeDirectory("growing")); - LogDataStore::Config config; //(100000, 0.1, 3.0, 0.2, 8, true, CompressionConfig::LZ4, - // WriteableFileChunk::Config(CompressionConfig(CompressionConfig::LZ4, 9, 60), 1000)); - config.setMaxFileSize(100000).setMaxDiskBloatFactor(0.1).setMaxBucketSpread(3.0).setMinFileSizeFactor(0.2) - .compactCompression({CompressionConfig::LZ4}) - .setFileConfig({{CompressionConfig::LZ4, 9, 60}, 1000}); - vespalib::ThreadStackExecutor executor(8, 128*1024); +void verifyGrowing(const LogDataStore::Config & config, uint32_t minFiles, uint32_t maxFiles) { + DirectoryHandler tmpDir("growing"); + vespalib::ThreadStackExecutor executor(4, 128*1024); DummyFileHeaderContext fileHeaderContext; MyTlSyncer tlSyncer; { @@ -243,30 +238,32 @@ TEST("testGrowing") { datastore.compact(30000); datastore.remove(31000, 0); checkStats(datastore, 31000, 30000); + EXPECT_LESS_EQUAL(minFiles, datastore.getAllActiveFiles().size()); + EXPECT_GREATER_EQUAL(maxFiles, datastore.getAllActiveFiles().size()); } { LogDataStore datastore(executor, "growing", config, GrowStrategy(), TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); checkStats(datastore, 30000, 30000); + EXPECT_LESS_EQUAL(minFiles, datastore.getAllActiveFiles().size()); + EXPECT_GREATER_EQUAL(maxFiles, datastore.getAllActiveFiles().size()); } - - FastOS_File::EmptyAndRemoveDirectory("growing"); +} +TEST("testGrowingChunkedBySize") { + LogDataStore::Config config; + config.setMaxFileSize(100000).setMaxDiskBloatFactor(0.1).setMaxBucketSpread(3.0).setMinFileSizeFactor(0.2) + .compactCompression({CompressionConfig::LZ4}) + .setFileConfig({{CompressionConfig::LZ4, 9, 60}, 1000}); + verifyGrowing(config, 40, 120); } -class TmpDirectory { -public: - TmpDirectory(const vespalib::string & dir) : _dir(dir) - { - FastOS_File::EmptyAndRemoveDirectory(_dir.c_str()); - ASSERT_TRUE(FastOS_File::MakeDirectory(_dir.c_str())); - } - ~TmpDirectory() { - FastOS_File::EmptyAndRemoveDirectory(_dir.c_str()); - } - const vespalib::string & getDir() const { return _dir; } -private: - vespalib::string _dir; -}; +TEST("testGrowingChunkedByNumLids") { + LogDataStore::Config config; + config.setMaxNumLids(1000).setMaxDiskBloatFactor(0.1).setMaxBucketSpread(3.0).setMinFileSizeFactor(0.2) + .compactCompression({CompressionConfig::LZ4}) + .setFileConfig({{CompressionConfig::LZ4, 9, 60}, 1000}); + verifyGrowing(config,10, 10); +} void fetchAndTest(IDataStore & datastore, uint32_t lid, const void *a, size_t sz) { @@ -349,7 +346,7 @@ public: ~VisitStore(); IDataStore & getStore() { return _datastore; } private: - TmpDirectory _myDir; + DirectoryHandler _myDir; LogDataStore::Config _config; DummyFileHeaderContext _fileHeaderContext; vespalib::ThreadStackExecutor _executor; @@ -357,8 +354,7 @@ private: LogDataStore _datastore; }; -VisitStore::~VisitStore() { -} +VisitStore::~VisitStore() =default; TEST("test visit cache does not cache empty ones and is able to access some backing store.") { const char * A7 = "aAaAaAa"; @@ -500,7 +496,7 @@ private: vespalib::hash_set<uint32_t> _actual; bool _allowVisitCaching; }; - TmpDirectory _myDir; + DirectoryHandler _myDir; document::DocumentTypeRepo _repo; LogDocumentStore::Config _config; DummyFileHeaderContext _fileHeaderContext; @@ -756,7 +752,7 @@ TEST("requireThatSyncTokenIsUpdatedAfterFlush") { } TEST("requireThatFlushTimeIsAvailableAfterFlush") { - TmpDirectory testDir("flushtime"); + DirectoryHandler testDir("flushtime"); vespalib::system_time before(vespalib::system_clock::now()); DummyFileHeaderContext fileHeaderContext; LogDataStore::Config config; diff --git a/searchlib/src/tests/fef/fef_test.cpp b/searchlib/src/tests/fef/fef_test.cpp index 4d1163de6ee..50f8a770618 100644 --- a/searchlib/src/tests/fef/fef_test.cpp +++ b/searchlib/src/tests/fef/fef_test.cpp @@ -67,12 +67,11 @@ TEST("test TermFieldMatchDataAppend") EXPECT_EQUAL(1u, tmd.capacity()); tmd.appendPosition(pos); EXPECT_EQUAL(2u, tmd.size()); - EXPECT_EQUAL(2u, tmd.capacity()); + EXPECT_EQUAL(42u, tmd.capacity()); uint32_t resizeCount(0); const TermFieldMatchDataPosition * prev = tmd.begin(); for (size_t i(2); i < std::numeric_limits<uint16_t>::max(); i++) { EXPECT_EQUAL(i, tmd.size()); - EXPECT_EQUAL(std::min(size_t(std::numeric_limits<uint16_t>::max()), vespalib::roundUp2inN(i)), tmd.capacity()); tmd.appendPosition(pos); const TermFieldMatchDataPosition * cur = tmd.begin(); if (cur != prev) { @@ -80,13 +79,15 @@ TEST("test TermFieldMatchDataAppend") resizeCount++; } } - EXPECT_EQUAL(15u, resizeCount); - EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.size()); - EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.capacity()); - tmd.appendPosition(pos); - EXPECT_EQUAL(prev, tmd.begin()); + EXPECT_EQUAL(11u, resizeCount); EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.size()); EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.capacity()); + for (size_t i(0); i < 10; i++) { + tmd.appendPosition(pos); + EXPECT_EQUAL(prev, tmd.begin()); + EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.size()); + EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.capacity()); + } } TEST("verify size of essential fef classes") { diff --git a/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp b/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp index 419b5261510..f264a3ab949 100644 --- a/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp +++ b/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/phrasesplitter.h> +#include <vespa/searchlib/fef/phrase_splitter_query_env.h> #include <vespa/searchlib/fef/test/queryenvironment.h> #include <iomanip> #include <iostream> @@ -42,7 +43,8 @@ Benchmark::run(size_t numRuns, size_t numPositions) tmd->appendPosition(TermFieldMatchDataPosition(0, i, 0, numPositions)); } - PhraseSplitter ps(qe, 0); + PhraseSplitterQueryEnv ps_query_env(qe, 0); + PhraseSplitter ps(ps_query_env); std::cout << "Start benchmark with numRuns(" << numRuns << ") and numPositions(" << numPositions << ")" << std::endl; diff --git a/searchlib/src/tests/fef/phrasesplitter/phrasesplitter_test.cpp b/searchlib/src/tests/fef/phrasesplitter/phrasesplitter_test.cpp index 1a7c4ccc467..73fe22c2799 100644 --- a/searchlib/src/tests/fef/phrasesplitter/phrasesplitter_test.cpp +++ b/searchlib/src/tests/fef/phrasesplitter/phrasesplitter_test.cpp @@ -5,6 +5,7 @@ LOG_SETUP("phrasesplitter_test"); #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/phrasesplitter.h> +#include <vespa/searchlib/fef/phrase_splitter_query_env.h> #include <vespa/searchlib/fef/test/queryenvironment.h> namespace search { @@ -84,12 +85,13 @@ PhraseSplitterTest::testSplitter() terms.push_back(SimpleTermData()); terms.back().addField(0).setHandle(mdl.allocTermField(0)); MatchData::UP md = mdl.createMatchData(); - PhraseSplitter ps(qe, 0); - ASSERT_TRUE(ps.getNumTerms() == 1); + PhraseSplitterQueryEnv ps_query_env(qe, 0); + PhraseSplitter ps(ps_query_env); + ASSERT_TRUE(ps.get_query_env().getNumTerms() == 1); ps.bind_match_data(*md); ps.update(); // check that nothing is served from the splitter - EXPECT_EQUAL(ps.getTerm(0), &terms[0]); + EXPECT_EQUAL(ps.get_query_env().getTerm(0), &terms[0]); TermFieldHandle handle = terms[0].lookupField(0)->getHandle(); EXPECT_EQUAL(ps.resolveTermField(handle), md->resolveTermField(handle)); } @@ -103,14 +105,15 @@ PhraseSplitterTest::testSplitter() terms.back().addField(0).setHandle(mdl.allocTermField(0)); terms.back().addField(7).setHandle(mdl.allocTermField(7)); MatchData::UP md = mdl.createMatchData(); - PhraseSplitter ps(qe, 7); - ASSERT_TRUE(ps.getNumTerms() == 3); + PhraseSplitterQueryEnv ps_query_env(qe, 7); + PhraseSplitter ps(ps_query_env); + ASSERT_TRUE(ps.get_query_env().getNumTerms() == 3); ps.bind_match_data(*md); ps.update(); // check that all is served from the splitter for (size_t i = 0; i < 3; ++i) { // fprintf(stderr, "checking term %d\n", (int)i); - const ITermData *td = ps.getTerm(i); + const ITermData *td = ps.get_query_env().getTerm(i); EXPECT_NOT_EQUAL(td, &terms[0]); EXPECT_NOT_EQUAL(td->lookupField(7), (ITermFieldData *)0); EXPECT_EQUAL(td->lookupField(0), (ITermFieldData *)0); @@ -135,15 +138,16 @@ PhraseSplitterTest::testSplitter() } terms[1].setPhraseLength(3); MatchData::UP md = mdl.createMatchData(); - PhraseSplitter ps(qe, 4); - ASSERT_TRUE(ps.getNumTerms() == 5); + PhraseSplitterQueryEnv ps_query_env(qe, 4); + PhraseSplitter ps(ps_query_env); + ASSERT_TRUE(ps.get_query_env().getNumTerms() == 5); ps.bind_match_data(*md); ps.update(); { // first term // fprintf(stderr, "first term\n"); - EXPECT_EQUAL(ps.getTerm(0), &terms[0]); - TEST_DO(assertTermData(ps.getTerm(0), 0, 1, 4, 0)); - TEST_DO(assertTermData(ps.getTerm(0), 0, 1, 7, 1)); + EXPECT_EQUAL(ps.get_query_env().getTerm(0), &terms[0]); + TEST_DO(assertTermData(ps.get_query_env().getTerm(0), 0, 1, 4, 0)); + TEST_DO(assertTermData(ps.get_query_env().getTerm(0), 0, 1, 7, 1)); TermFieldHandle handle = terms[0].lookupField(4)->getHandle(); EXPECT_EQUAL(ps.resolveTermField(handle), md->resolveTermField(handle)); @@ -152,7 +156,7 @@ PhraseSplitterTest::testSplitter() } for (size_t i = 0; i < 3; ++i) { // phrase // fprintf(stderr, "phrase term %zd\n", i); - const ITermData *td = ps.getTerm(i + 1); + const ITermData *td = ps.get_query_env().getTerm(i + 1); EXPECT_NOT_EQUAL(td, &terms[1]); TEST_DO(assertTermData(td, 1, 1, 4, i + 11)); // skipHandles == 11 EXPECT_EQUAL(td->lookupField(7), (ITermFieldData *)0); @@ -161,9 +165,9 @@ PhraseSplitterTest::testSplitter() } { // last term // fprintf(stderr, "last term\n"); - EXPECT_EQUAL(ps.getTerm(4), &terms[2]); - TEST_DO(assertTermData(ps.getTerm(4), 2, 1, 4, 4)); - TEST_DO(assertTermData(ps.getTerm(4), 2, 1, 7, 5)); + EXPECT_EQUAL(ps.get_query_env().getTerm(4), &terms[2]); + TEST_DO(assertTermData(ps.get_query_env().getTerm(4), 2, 1, 4, 4)); + TEST_DO(assertTermData(ps.get_query_env().getTerm(4), 2, 1, 7, 5)); // fprintf(stderr, "inspect term %p #f %zd\n", &terms[2], terms[2].numFields()); fflush(stderr); @@ -189,8 +193,9 @@ PhraseSplitterTest::testSplitterUpdate() terms[0].setPhraseLength(2); terms[2].setPhraseLength(2); MatchData::UP md = mdl.createMatchData(); - PhraseSplitter ps(qe, 0); - ASSERT_TRUE(ps.getNumTerms() == 5); + PhraseSplitterQueryEnv ps_query_env(qe, 0); + PhraseSplitter ps(ps_query_env); + ASSERT_TRUE(ps.get_query_env().getNumTerms() == 5); { // first phrase TermFieldMatchData * tmd = md->resolveTermField(terms[0].lookupField(0)->getHandle()); tmd->appendPosition(TermFieldMatchDataPosition(0, 10, 0, 1000)); @@ -206,19 +211,19 @@ PhraseSplitterTest::testSplitterUpdate() ps.bind_match_data(*md); ps.update(); for (size_t i = 0; i < 2; ++i) { // first phrase - const TermFieldMatchData * tmd = ps.resolveTermField(ps.getTerm(i)->lookupField(0)->getHandle()); + const TermFieldMatchData * tmd = ps.resolveTermField(ps.get_query_env().getTerm(i)->lookupField(0)->getHandle()); TermFieldMatchData::PositionsIterator itr = tmd->begin(); EXPECT_EQUAL((itr++)->getPosition(), 10 + i); ASSERT_TRUE(itr == tmd->end()); } { // first term - TermFieldMatchData * tmd = md->resolveTermField(ps.getTerm(2)->lookupField(0)->getHandle()); + TermFieldMatchData * tmd = md->resolveTermField(ps.get_query_env().getTerm(2)->lookupField(0)->getHandle()); TermFieldMatchData::PositionsIterator itr = tmd->begin(); EXPECT_EQUAL((itr++)->getPosition(), 20u); ASSERT_TRUE(itr == tmd->end()); } for (size_t i = 0; i < 2; ++i) { // second phrase - const TermFieldMatchData * tmd = ps.resolveTermField(ps.getTerm(i + 3)->lookupField(0)->getHandle()); + const TermFieldMatchData * tmd = ps.resolveTermField(ps.get_query_env().getTerm(i + 3)->lookupField(0)->getHandle()); TermFieldMatchData::PositionsIterator itr = tmd->begin(); EXPECT_EQUAL((itr++)->getPosition(), 30 + i); ASSERT_TRUE(itr == tmd->end()); diff --git a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp index a75b91cb78a..3de34047ecd 100644 --- a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp +++ b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp @@ -215,11 +215,11 @@ TEST("append positions") { tfmd.appendPosition(pos); tfmd.appendPosition(pos2); EXPECT_EQUAL(2u, tfmd.size()); - EXPECT_EQUAL(2u, tfmd.capacity()); + EXPECT_EQUAL(42u, tfmd.capacity()); TermFieldMatchDataPosition pos3(0x31020304, 0x30203040, 0x31223344, 0x32345678); tfmd.appendPosition(pos3); EXPECT_EQUAL(3u, tfmd.size()); - EXPECT_EQUAL(4u, tfmd.capacity()); + EXPECT_EQUAL(42u, tfmd.capacity()); EXPECT_EQUAL(0x01020304u, tfmd.begin()->getElementId()); EXPECT_EQUAL(0x10203040u, tfmd.begin()->getPosition()); EXPECT_EQUAL(0x11223344, tfmd.begin()->getElementWeight()); diff --git a/searchlib/src/tests/fef/termmatchdatamerger/CMakeLists.txt b/searchlib/src/tests/fef/termmatchdatamerger/CMakeLists.txt index 4eb00cacf38..d01b3f84436 100644 --- a/searchlib/src/tests/fef/termmatchdatamerger/CMakeLists.txt +++ b/searchlib/src/tests/fef/termmatchdatamerger/CMakeLists.txt @@ -1,8 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_termmatchdatamerger_test_app TEST SOURCES termmatchdatamerger_test.cpp DEPENDS searchlib + GTest::GTest ) vespa_add_test(NAME searchlib_termmatchdatamerger_test_app COMMAND searchlib_termmatchdatamerger_test_app) diff --git a/searchlib/src/tests/fef/termmatchdatamerger/termmatchdatamerger_test.cpp b/searchlib/src/tests/fef/termmatchdatamerger/termmatchdatamerger_test.cpp index a8670f43a6b..eb04c34b595 100644 --- a/searchlib/src/tests/fef/termmatchdatamerger/termmatchdatamerger_test.cpp +++ b/searchlib/src/tests/fef/termmatchdatamerger/termmatchdatamerger_test.cpp @@ -1,11 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("termmatchdatamerger_test"); -#include <vespa/vespalib/testkit/testapp.h> - #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/fef/termmatchdatamerger.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace search::fef; @@ -21,19 +18,7 @@ TermFieldMatchDataPosition make_pos(uint32_t pos) } // namespace <unnamed> -class Test : public vespalib::TestApp -{ -public: - void testMergeEmptyInput(); - void testMergeSimple(); - void testMergeMultifield(); - void testMergeDuplicates(); - void testMergeFieldLength(); - int Main() override; -}; - -void -Test::testMergeEmptyInput() +TEST(TermMatchDataMergerTest, merge_empty_input) { TermFieldMatchData out; TermFieldMatchDataArray output; @@ -48,12 +33,11 @@ Test::testMergeEmptyInput() uint32_t docid = 5; in.reset(docid); merger.merge(docid); - EXPECT_EQUAL(docid, out.getDocId()); + EXPECT_EQ(docid, out.getDocId()); EXPECT_TRUE(out.begin() == out.end()); } -void -Test::testMergeSimple() +TEST(TermMatchDataMergerTest, merge_simple) { TermFieldMatchData a; TermFieldMatchData b; @@ -86,26 +70,26 @@ Test::testMergeSimple() merger.merge(docid); - EXPECT_EQUAL(docid, out.getDocId()); - EXPECT_EQUAL(8u, out.end() - out.begin()); - - EXPECT_EQUAL( 5u, out.begin()[0].getPosition()); - EXPECT_EQUAL( 7u, out.begin()[1].getPosition()); - EXPECT_EQUAL(10u, out.begin()[2].getPosition()); - EXPECT_EQUAL(15u, out.begin()[3].getPosition()); - EXPECT_EQUAL(20u, out.begin()[4].getPosition()); - EXPECT_EQUAL(22u, out.begin()[5].getPosition()); - EXPECT_EQUAL(27u, out.begin()[6].getPosition()); - EXPECT_EQUAL(28u, out.begin()[7].getPosition()); - - EXPECT_EQUAL(0.25, out.begin()[0].getMatchExactness()); - EXPECT_EQUAL( 0.5, out.begin()[1].getMatchExactness()); - EXPECT_EQUAL( 1.5, out.begin()[2].getMatchExactness()); - EXPECT_EQUAL( 1.0, out.begin()[3].getMatchExactness()); - EXPECT_EQUAL( 4.0, out.begin()[4].getMatchExactness()); - EXPECT_EQUAL(0.75, out.begin()[5].getMatchExactness()); - EXPECT_EQUAL( 3.0, out.begin()[6].getMatchExactness()); - EXPECT_EQUAL( 7.5, out.begin()[7].getMatchExactness()); + EXPECT_EQ(docid, out.getDocId()); + EXPECT_EQ(8u, out.end() - out.begin()); + + EXPECT_EQ( 5u, out.begin()[0].getPosition()); + EXPECT_EQ( 7u, out.begin()[1].getPosition()); + EXPECT_EQ(10u, out.begin()[2].getPosition()); + EXPECT_EQ(15u, out.begin()[3].getPosition()); + EXPECT_EQ(20u, out.begin()[4].getPosition()); + EXPECT_EQ(22u, out.begin()[5].getPosition()); + EXPECT_EQ(27u, out.begin()[6].getPosition()); + EXPECT_EQ(28u, out.begin()[7].getPosition()); + + EXPECT_EQ(0.25, out.begin()[0].getMatchExactness()); + EXPECT_EQ( 0.5, out.begin()[1].getMatchExactness()); + EXPECT_EQ( 1.5, out.begin()[2].getMatchExactness()); + EXPECT_EQ( 1.0, out.begin()[3].getMatchExactness()); + EXPECT_EQ( 4.0, out.begin()[4].getMatchExactness()); + EXPECT_EQ(0.75, out.begin()[5].getMatchExactness()); + EXPECT_EQ( 3.0, out.begin()[6].getMatchExactness()); + EXPECT_EQ( 7.5, out.begin()[7].getMatchExactness()); // one stale input @@ -117,24 +101,22 @@ Test::testMergeSimple() merger.merge(docid); - EXPECT_EQUAL(docid, out.getDocId()); - EXPECT_EQUAL(3u, out.end() - out.begin()); + EXPECT_EQ(docid, out.getDocId()); + EXPECT_EQ(3u, out.end() - out.begin()); - EXPECT_EQUAL( 5u, out.begin()[0].getPosition()); - EXPECT_EQUAL(10u, out.begin()[1].getPosition()); - EXPECT_EQUAL(15u, out.begin()[2].getPosition()); + EXPECT_EQ( 5u, out.begin()[0].getPosition()); + EXPECT_EQ(10u, out.begin()[1].getPosition()); + EXPECT_EQ(15u, out.begin()[2].getPosition()); // both inputs are stale docid = 15; merger.merge(docid); - EXPECT_NOT_EQUAL(docid, out.getDocId()); + EXPECT_NE(docid, out.getDocId()); } - -void -Test::testMergeMultifield() +TEST(TermMatchDataMergerTest, merge_multiple_fields) { TermFieldMatchData a; TermFieldMatchData b; @@ -174,30 +156,29 @@ Test::testMergeMultifield() merger.merge(docid); - EXPECT_EQUAL(docid, out1.getDocId()); - EXPECT_EQUAL(docid, out2.getDocId()); - EXPECT_NOT_EQUAL(docid, out3.getDocId()); + EXPECT_EQ(docid, out1.getDocId()); + EXPECT_EQ(docid, out2.getDocId()); + EXPECT_NE(docid, out3.getDocId()); - EXPECT_EQUAL(2u, out1.end() - out1.begin()); - EXPECT_EQUAL(3u, out2.end() - out2.begin()); + EXPECT_EQ(2u, out1.end() - out1.begin()); + EXPECT_EQ(3u, out2.end() - out2.begin()); - EXPECT_EQUAL( 5u, out1.begin()[0].getPosition()); - EXPECT_EQUAL(15u, out1.begin()[1].getPosition()); + EXPECT_EQ( 5u, out1.begin()[0].getPosition()); + EXPECT_EQ(15u, out1.begin()[1].getPosition()); - EXPECT_EQUAL( 5u, out2.begin()[0].getPosition()); - EXPECT_EQUAL( 7u, out2.begin()[1].getPosition()); - EXPECT_EQUAL(20u, out2.begin()[2].getPosition()); + EXPECT_EQ( 5u, out2.begin()[0].getPosition()); + EXPECT_EQ( 7u, out2.begin()[1].getPosition()); + EXPECT_EQ(20u, out2.begin()[2].getPosition()); - EXPECT_EQUAL(1.0, out1.begin()[0].getMatchExactness()); - EXPECT_EQUAL(1.0, out1.begin()[1].getMatchExactness()); + EXPECT_EQ(1.0, out1.begin()[0].getMatchExactness()); + EXPECT_EQ(1.0, out1.begin()[1].getMatchExactness()); - EXPECT_EQUAL(1.5, out2.begin()[0].getMatchExactness()); - EXPECT_EQUAL(0.5, out2.begin()[1].getMatchExactness()); - EXPECT_EQUAL(1.5, out2.begin()[2].getMatchExactness()); + EXPECT_EQ(1.5, out2.begin()[0].getMatchExactness()); + EXPECT_EQ(0.5, out2.begin()[1].getMatchExactness()); + EXPECT_EQ(1.5, out2.begin()[2].getMatchExactness()); } -void -Test::testMergeDuplicates() +TEST(TermMatchDataMergerTest, merge_duplicates) { TermFieldMatchData a; TermFieldMatchData b; @@ -225,23 +206,22 @@ Test::testMergeDuplicates() merger.merge(docid); - EXPECT_EQUAL(docid, out.getDocId()); - EXPECT_EQUAL(5u, out.end() - out.begin()); - - EXPECT_EQUAL( 3u, out.begin()[0].getPosition()); - EXPECT_EQUAL(1.5, out.begin()[0].getMatchExactness()); - EXPECT_EQUAL( 5u, out.begin()[1].getPosition()); - EXPECT_EQUAL(0.5, out.begin()[1].getMatchExactness()); - EXPECT_EQUAL(10u, out.begin()[2].getPosition()); - EXPECT_EQUAL(1.5, out.begin()[2].getMatchExactness()); - EXPECT_EQUAL(15u, out.begin()[3].getPosition()); - EXPECT_EQUAL(1.5, out.begin()[3].getMatchExactness()); - EXPECT_EQUAL(17u, out.begin()[4].getPosition()); - EXPECT_EQUAL(1.5, out.begin()[4].getMatchExactness()); + EXPECT_EQ(docid, out.getDocId()); + EXPECT_EQ(5u, out.end() - out.begin()); + + EXPECT_EQ( 3u, out.begin()[0].getPosition()); + EXPECT_EQ(1.5, out.begin()[0].getMatchExactness()); + EXPECT_EQ( 5u, out.begin()[1].getPosition()); + EXPECT_EQ(0.5, out.begin()[1].getMatchExactness()); + EXPECT_EQ(10u, out.begin()[2].getPosition()); + EXPECT_EQ(1.5, out.begin()[2].getMatchExactness()); + EXPECT_EQ(15u, out.begin()[3].getPosition()); + EXPECT_EQ(1.5, out.begin()[3].getMatchExactness()); + EXPECT_EQ(17u, out.begin()[4].getPosition()); + EXPECT_EQ(1.5, out.begin()[4].getMatchExactness()); } -void -Test::testMergeFieldLength() +TEST(TermMatchDataMergerTest, merge_max_element_length) { TermFieldMatchData a; TermFieldMatchData b; @@ -261,20 +241,93 @@ Test::testMergeFieldLength() b.appendPosition(make_pos(2)); merger.merge(docid); - EXPECT_EQUAL(docid, out.getDocId()); - EXPECT_EQUAL(1000u, out.getIterator().getFieldLength()); + EXPECT_EQ(docid, out.getDocId()); + EXPECT_EQ(1000u, out.getIterator().getFieldLength()); +} + +class TermMatchDataMergerTest2 : public ::testing::Test +{ +protected: + TermFieldMatchData a; + TermFieldMatchData b; + MDMIs input; + TermFieldMatchData out; + TermFieldMatchDataArray output; + TermMatchDataMerger merger; + + TermMatchDataMergerTest2() + : a(), + b(), + input({{&a, 0.5},{&b, 1.5}}), + out(), + output(), + merger(input, output.add(&out)) + { + } +}; + +TEST_F(TermMatchDataMergerTest2, merge_no_normal_features) +{ + out.setNeedNormalFeatures(false); + + uint32_t docid = 5; + + a.reset(docid); + a.appendPosition(make_pos(5)); + + b.reset(docid); + b.appendPosition(make_pos(3)); + + merger.merge(docid); + EXPECT_EQ(docid, out.getDocId()); + EXPECT_EQ(0u, out.size()); +} + +TEST_F(TermMatchDataMergerTest2, merge_interleaved_features) +{ + out.setNeedNormalFeatures(false); + out.setNeedInterleavedFeatures(true); + + uint32_t docid = 5; + + a.reset(docid); + a.setNumOccs(1); + a.setFieldLength(30); + + b.reset(docid); + b.setNumOccs(1); + b.setFieldLength(35); + + merger.merge(docid); + EXPECT_EQ(docid, out.getDocId()); + EXPECT_EQ(2u, out.getNumOccs()); + EXPECT_EQ(35u, out.getFieldLength()); } -int -Test::Main() +TEST_F(TermMatchDataMergerTest2, merge_interleaved_features_with_detected_duplicate) { - TEST_INIT("termmatchdatamerger_test"); - testMergeEmptyInput(); - testMergeSimple(); - testMergeMultifield(); - testMergeDuplicates(); - testMergeFieldLength(); - TEST_DONE(); + out.setNeedNormalFeatures(true); + out.setNeedInterleavedFeatures(true); + + uint32_t docid = 5; + + a.reset(docid); + a.setNumOccs(1); + a.setFieldLength(30); + a.appendPosition(make_pos(5)); + + b.reset(docid); + b.setNumOccs(1); + b.setFieldLength(30); + b.appendPosition(make_pos(5)); + + merger.merge(docid); + EXPECT_EQ(docid, out.getDocId()); + EXPECT_EQ(1u, out.end() - out.begin()); + EXPECT_EQ( 5u, out.begin()[0].getPosition()); + EXPECT_EQ(1.5, out.begin()[0].getMatchExactness()); + EXPECT_EQ(1u, out.getNumOccs()); + EXPECT_EQ(30u, out.getFieldLength()); } -TEST_APPHOOK(Test); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/hitcollector/hitcollector_test.cpp b/searchlib/src/tests/hitcollector/hitcollector_test.cpp index 31a24d2a8f1..2274314c7da 100644 --- a/searchlib/src/tests/hitcollector/hitcollector_test.cpp +++ b/searchlib/src/tests/hitcollector/hitcollector_test.cpp @@ -55,7 +55,7 @@ void checkResult(const ResultSet & rs, const std::vector<RankedHit> & exp) for (uint32_t i = 0; i < exp.size(); ++i) { EXPECT_EQUAL(rh[i]._docId, exp[i]._docId); - EXPECT_EQUAL(rh[i]._rankValue, exp[i]._rankValue); + EXPECT_EQUAL(rh[i]._rankValue + 1.0, exp[i]._rankValue + 1.0); } } else { ASSERT_TRUE(rs.getArray() == nullptr); @@ -328,7 +328,7 @@ TEST("testScaling") { finalScores[3] = 300; finalScores[4] = 400; - testScaling(initScores, std::move(finalScores), exp); + TEST_DO(testScaling(initScores, std::move(finalScores), exp)); } { // scale down and adjust up exp[0]._rankValue = 200; // scaled @@ -342,7 +342,7 @@ TEST("testScaling") { finalScores[3] = 500; finalScores[4] = 600; - testScaling(initScores, std::move(finalScores), exp); + TEST_DO(testScaling(initScores, std::move(finalScores), exp)); } { // scale up and adjust down @@ -357,7 +357,7 @@ TEST("testScaling") { finalScores[3] = 3250; finalScores[4] = 4500; - testScaling(initScores, std::move(finalScores), exp); + TEST_DO(testScaling(initScores, std::move(finalScores), exp)); } { // minimal scale (second phase range = 0 (4 - 4) -> 1) exp[0]._rankValue = 1; // scaled @@ -371,7 +371,7 @@ TEST("testScaling") { finalScores[3] = 4; finalScores[4] = 4; - testScaling(initScores, std::move(finalScores), exp); + TEST_DO(testScaling(initScores, std::move(finalScores), exp)); } { // minimal scale (first phase range = 0 (4000 - 4000) -> 1) std::vector<feature_t> is(initScores); @@ -387,7 +387,7 @@ TEST("testScaling") { finalScores[3] = 400; finalScores[4] = 500; - testScaling(is, std::move(finalScores), exp); + TEST_DO(testScaling(is, std::move(finalScores), exp)); } } diff --git a/searchlib/src/tests/nativerank/nativerank.cpp b/searchlib/src/tests/nativerank/nativerank.cpp index 8028cb9940d..b28e385b597 100644 --- a/searchlib/src/tests/nativerank/nativerank.cpp +++ b/searchlib/src/tests/nativerank/nativerank.cpp @@ -170,7 +170,8 @@ Test::testNativeFieldMatch() f.firstOccTable = &t; f.numOccTable = &t; p.vector.push_back(f); - NativeFieldMatchExecutor nfme(ft.getQueryEnv(), p); + NativeFieldMatchExecutorSharedState nfmess(ft.getQueryEnv(), p); + NativeFieldMatchExecutor nfme(nfmess); EXPECT_EQUAL(p.minFieldLength, 6u); EXPECT_EQUAL(nfme.getFirstOccBoost(0, 0, 4), 0); EXPECT_EQUAL(nfme.getFirstOccBoost(0, 1, 4), 1); @@ -544,12 +545,12 @@ Test::testNativeProximity() env.getProperties().add("vespa.term.2.connexity", "0.6"); { NativeProximityExecutor::FieldSetup setup(0); - NativeProximityExecutor::TermPairVector & pairs = setup.pairs; - NativeProximityExecutor::generateTermPairs(env, terms, 0, setup); + NativeProximityExecutorSharedState::TermPairVector & pairs = setup.pairs; + NativeProximityExecutorSharedState::generateTermPairs(env, terms, 0, setup); EXPECT_EQUAL(pairs.size(), 0u); - NativeProximityExecutor::generateTermPairs(env, terms, 1, setup); + NativeProximityExecutorSharedState::generateTermPairs(env, terms, 1, setup); EXPECT_EQUAL(pairs.size(), 0u); - NativeProximityExecutor::generateTermPairs(env, terms, 2, setup); + NativeProximityExecutorSharedState::generateTermPairs(env, terms, 2, setup); EXPECT_EQUAL(pairs.size(), 2u); EXPECT_TRUE(pairs[0].first.termData() == &a); EXPECT_TRUE(pairs[0].second.termData() == &b); @@ -562,7 +563,7 @@ Test::testNativeProximity() pairs.clear(); setup.divisor = 0; - NativeProximityExecutor::generateTermPairs(env, terms, 3, setup); + NativeProximityExecutorSharedState::generateTermPairs(env, terms, 3, setup); EXPECT_EQUAL(pairs.size(), 3u); EXPECT_TRUE(pairs[0].first.termData() == &a); EXPECT_TRUE(pairs[0].second.termData() == &b); @@ -581,7 +582,7 @@ Test::testNativeProximity() b.setWeight(search::query::Weight(0)); // test that (ab) is filtered away - NativeProximityExecutor::generateTermPairs(env, terms, 2, setup); + NativeProximityExecutorSharedState::generateTermPairs(env, terms, 2, setup); EXPECT_EQUAL(pairs.size(), 1u); EXPECT_TRUE(pairs[0].first.termData() == &b); EXPECT_TRUE(pairs[0].second.termData() == &c); diff --git a/searchlib/src/tests/query/streaming_query_large_test.cpp b/searchlib/src/tests/query/streaming_query_large_test.cpp index e2a49311aac..28401e9d685 100644 --- a/searchlib/src/tests/query/streaming_query_large_test.cpp +++ b/searchlib/src/tests/query/streaming_query_large_test.cpp @@ -10,6 +10,14 @@ using namespace search; using namespace search::query; using namespace search::streaming; +#ifndef __SANITIZE_ADDRESS__ +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define __SANITIZE_ADDRESS__ +#endif +#endif +#endif + namespace { void setMaxStackSize(rlim_t maxStackSize) @@ -27,7 +35,11 @@ void setMaxStackSize(rlim_t maxStackSize) // a stack overflow if the stack usage increases. TEST("testveryLongQueryResultingInBug6850778") { const uint32_t NUMITEMS=20000; +#ifdef __SANITIZE_ADDRESS__ + setMaxStackSize(12 * 1024 * 1024); +#else setMaxStackSize(4 * 1024 * 1024); +#endif QueryBuilder<SimpleQueryNodeTypes> builder; for (uint32_t i=0; i <= NUMITEMS; i++) { builder.addAnd(2); diff --git a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp index 6625a4a09ce..0672e51378e 100644 --- a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp @@ -39,11 +39,11 @@ public: return true; } - virtual SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + SearchIterator::UP + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("or", subSearches, &md, strict)); + return SearchIterator::UP(new MySearch("or", std::move(subSearches), &md, strict)); } static MyOr& create() { return *(new MyOr()); } @@ -56,11 +56,11 @@ class OtherOr : public OrBlueprint { private: public: - virtual SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + SearchIterator::UP + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("or", subSearches, &md, strict)); + return SearchIterator::UP(new MySearch("or", std::move(subSearches), &md, strict)); } static OtherOr& create() { return *(new OtherOr()); } @@ -86,11 +86,11 @@ public: return (i == 0); } - virtual SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + SearchIterator::UP + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("and", subSearches, &md, strict)); + return SearchIterator::UP(new MySearch("and", std::move(subSearches), &md, strict)); } static MyAnd& create() { return *(new MyAnd()); } @@ -103,11 +103,11 @@ class OtherAnd : public AndBlueprint { private: public: - virtual SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + SearchIterator::UP + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("and", subSearches, &md, strict)); + return SearchIterator::UP(new MySearch("and", std::move(subSearches), &md, strict)); } static OtherAnd& create() { return *(new OtherAnd()); } @@ -118,11 +118,11 @@ public: class OtherAndNot : public AndNotBlueprint { public: - virtual SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + SearchIterator::UP + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("andnot", subSearches, &md, strict)); + return SearchIterator::UP(new MySearch("andnot", std::move(subSearches), &md, strict)); } static OtherAndNot& create() { return *(new OtherAndNot()); } diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index 4f3d5443ab8..eb6e49747a1 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -70,8 +70,9 @@ TEST("test AndNot Blueprint") { EXPECT_EQUAL(false, a.getState().want_global_filter()); a.addChild(ap(MyLeafSpec(20).addField(1, 1).want_global_filter().create())); EXPECT_EQUAL(true, a.getState().want_global_filter()); - std::shared_ptr<BitVector> empty_global_filter; - a.set_global_filter(empty_global_filter); + auto empty_global_filter = GlobalFilter::create(); + EXPECT_FALSE(empty_global_filter->has_filter()); + a.set_global_filter(*empty_global_filter); EXPECT_EQUAL(false, got_global_filter(a.getChild(0))); EXPECT_EQUAL(true, got_global_filter(a.getChild(1))); } @@ -145,8 +146,8 @@ TEST("test And Blueprint") { EXPECT_EQUAL(false, a.getState().want_global_filter()); a.addChild(ap(MyLeafSpec(20).addField(1, 1).want_global_filter().create())); EXPECT_EQUAL(true, a.getState().want_global_filter()); - std::shared_ptr<BitVector> empty_global_filter; - a.set_global_filter(empty_global_filter); + auto empty_global_filter = GlobalFilter::create(); + a.set_global_filter(*empty_global_filter); EXPECT_EQUAL(false, got_global_filter(a.getChild(0))); EXPECT_EQUAL(true, got_global_filter(a.getChild(1))); } @@ -225,8 +226,8 @@ TEST("test Or Blueprint") { EXPECT_EQUAL(false, o.getState().want_global_filter()); o.addChild(ap(MyLeafSpec(20).addField(1, 1).want_global_filter().create())); EXPECT_EQUAL(true, o.getState().want_global_filter()); - std::shared_ptr<BitVector> empty_global_filter; - o.set_global_filter(empty_global_filter); + auto empty_global_filter = GlobalFilter::create(); + o.set_global_filter(*empty_global_filter); EXPECT_EQUAL(false, got_global_filter(o.getChild(0))); EXPECT_EQUAL(true, got_global_filter(o.getChild(o.childCnt() - 1))); } @@ -380,8 +381,8 @@ TEST("test Rank Blueprint") { EXPECT_EQUAL(false, a.getState().want_global_filter()); a.addChild(ap(MyLeafSpec(20).addField(1, 1).want_global_filter().create())); EXPECT_EQUAL(true, a.getState().want_global_filter()); - std::shared_ptr<BitVector> empty_global_filter; - a.set_global_filter(empty_global_filter); + auto empty_global_filter = GlobalFilter::create(); + a.set_global_filter(*empty_global_filter); EXPECT_EQUAL(false, got_global_filter(a.getChild(0))); EXPECT_EQUAL(true, got_global_filter(a.getChild(1))); } @@ -1290,7 +1291,7 @@ TEST("require that children does not optimize when parents refuse them to") { } } -TEST("require_that_unpack_optimization_is_overruled_by_equiv") { +TEST("require_that_unpack_optimization_is_not_overruled_by_equiv") { FieldSpecBaseList fields; fields.add(FieldSpecBase(1, 1)); fields.add(FieldSpecBase(2, 2)); @@ -1321,7 +1322,7 @@ TEST("require_that_unpack_optimization_is_overruled_by_equiv") { EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); - EXPECT_EQUAL("search::queryeval::OrLikeSearch<true, search::queryeval::(anonymous namespace)::FullUnpack>", + EXPECT_EQUAL("search::queryeval::OrLikeSearch<true, search::queryeval::(anonymous namespace)::SelectiveUnpack>", e.getChildren()[0]->getClassName()); } @@ -1331,7 +1332,7 @@ TEST("require_that_unpack_optimization_is_overruled_by_equiv") { EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); - EXPECT_EQUAL("search::queryeval::OrLikeSearch<true, search::queryeval::(anonymous namespace)::FullUnpack>", + EXPECT_EQUAL("search::queryeval::OrLikeSearch<true, search::queryeval::NoUnpack>", e.getChildren()[0]->getClassName()); } } diff --git a/searchlib/src/tests/queryeval/blueprint/mysearch.h b/searchlib/src/tests/queryeval/blueprint/mysearch.h index f0400fb1d90..012e19f26f5 100644 --- a/searchlib/src/tests/queryeval/blueprint/mysearch.h +++ b/searchlib/src/tests/queryeval/blueprint/mysearch.h @@ -9,11 +9,9 @@ namespace search::queryeval { //----------------------------------------------------------------------------- -class MySearch : public SearchIterator +class MySearch : public MultiSearch { public: - typedef MultiSearch::Children Children; - typedef std::vector<SearchIterator::UP> MyChildren; typedef search::fef::TermFieldMatchDataArray TFMDA; typedef search::fef::MatchData MatchData; @@ -21,7 +19,6 @@ private: vespalib::string _tag; bool _isLeaf; bool _isStrict; - MyChildren _children; TFMDA _match; MatchData *_md; @@ -33,21 +30,18 @@ protected: public: MySearch(const std::string &tag, bool leaf, bool strict) - : _tag(tag), _isLeaf(leaf), _isStrict(strict), _children(), + : _tag(tag), _isLeaf(leaf), _isStrict(strict), _match(), _md(0) {} MySearch(const std::string &tag, const TFMDA &tfmda, bool strict) - : _tag(tag), _isLeaf(true), _isStrict(strict), _children(), + : _tag(tag), _isLeaf(true), _isStrict(strict), _match(tfmda), _md(0) {} - MySearch(const std::string &tag, const Children &children, + MySearch(const std::string &tag, Children children, MatchData *md, bool strict) - : _tag(tag), _isLeaf(false), _isStrict(strict), _children(), - _match(), _md(md) { - for (size_t i(0); i < children.size(); i++) { - _children.emplace_back(children[i]); - } - } + : MultiSearch(std::move(children)), + _tag(tag), _isLeaf(false), _isStrict(strict), + _match(), _md(md) {} MySearch &add(SearchIterator *search) { _children.emplace_back(search); @@ -98,7 +92,7 @@ public: visit(visitor, "_tag", _tag); visit(visitor, "_isLeaf", _isLeaf); visit(visitor, "_isStrict", _isStrict); - visit(visitor, "_children", _children); + MultiSearch::visitMembers(visitor); visit(visitor, "_handles", _handles); } @@ -132,7 +126,7 @@ public: set_cost_tier(value); return *this; } - void set_global_filter(std::shared_ptr<BitVector>) override { + void set_global_filter(const GlobalFilter &) override { _got_global_filter = true; } bool got_global_filter() const { return _got_global_filter; } diff --git a/searchlib/src/tests/queryeval/equiv/CMakeLists.txt b/searchlib/src/tests/queryeval/equiv/CMakeLists.txt index 368f12e34a5..246b2ce0426 100644 --- a/searchlib/src/tests/queryeval/equiv/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/equiv/CMakeLists.txt @@ -1,8 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_equiv_test_app TEST SOURCES equiv_test.cpp DEPENDS searchlib + GTest::GTest ) vespa_add_test(NAME searchlib_equiv_test_app COMMAND searchlib_equiv_test_app) diff --git a/searchlib/src/tests/queryeval/equiv/equiv_test.cpp b/searchlib/src/tests/queryeval/equiv/equiv_test.cpp index ecd1c8cd218..412130ecaab 100644 --- a/searchlib/src/tests/queryeval/equiv/equiv_test.cpp +++ b/searchlib/src/tests/queryeval/equiv/equiv_test.cpp @@ -1,11 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("equiv_test"); -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> #include <vespa/searchlib/queryeval/equiv_blueprint.h> #include <vespa/searchlib/fef/matchdatalayout.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace search::queryeval; using search::fef::MatchData; @@ -14,22 +14,28 @@ using search::fef::TermFieldHandle; using search::fef::TermFieldMatchData; using search::fef::FieldPositionsIterator; -class Test : public vespalib::TestApp { -public: - void testEquiv(); - int Main() override; +class EquivTest : public ::testing::Test { +protected: + EquivTest(); + ~EquivTest(); + + void test_equiv(bool strict, bool unpack_normal_features, bool unpack_interleaved_features); }; +EquivTest::EquivTest() = default; + +EquivTest::~EquivTest() = default; + void -Test::testEquiv() +EquivTest::test_equiv(bool strict, bool unpack_normal_features, bool unpack_interleaved_features) { FakeResult a; FakeResult b; FakeResult c; - a.doc(5).pos(1); - b.doc(5).pos(2); - c.doc(5).pos(3).doc(10).pos(4); + a.doc(5).pos(1).len(30).field_length(30).num_occs(1); + b.doc(5).pos(2).len(30).field_length(30).num_occs(1); + c.doc(5).pos(3).len(30).field_length(30).num_occs(1).doc(10).pos(4).len(35).field_length(35).num_occs(1); MatchDataLayout subLayout; TermFieldHandle fbh11 = subLayout.allocTermField(1); @@ -39,91 +45,151 @@ Test::testEquiv() FieldSpecBaseList fields; fields.add(FieldSpecBase(1, 1)); fields.add(FieldSpecBase(2, 2)); - EquivBlueprint *eq_b = new EquivBlueprint(fields, subLayout); - - eq_b->addTerm(Blueprint::UP(new FakeBlueprint(FieldSpec("foo", 1, fbh11), a)), 1.0); - eq_b->addTerm(Blueprint::UP(new FakeBlueprint(FieldSpec("bar", 2, fbh21), b)), 1.0); - eq_b->addTerm(Blueprint::UP(new FakeBlueprint(FieldSpec("bar", 2, fbh22), c)), 1.0); - - Blueprint::UP bp(eq_b); - for (int i = 0; i <= 1; ++i) { - bool strict = (i == 0); - TEST_STATE(strict ? "strict" : "non-strict"); - MatchData::UP md = MatchData::makeTestInstance(100, 10); - bp->fetchPostings(ExecuteInfo::create(strict)); - SearchIterator::UP search = bp->createSearch(*md, strict); - search->initFullRange(); - - EXPECT_TRUE(!search->seek(3)); - if (!strict) { - EXPECT_EQUAL(SearchIterator::beginId(), search->getDocId()); - EXPECT_TRUE(search->seek(5u)); - } - EXPECT_EQUAL(5u, search->getDocId()); - { // test doc 5 results - search->unpack(5u); - { - TermFieldMatchData &data = *md->resolveTermField(1); - EXPECT_EQUAL(1u, data.getFieldId()); - EXPECT_EQUAL(5u, data.getDocId()); - FieldPositionsIterator itr = data.getIterator(); - EXPECT_EQUAL(1u, itr.size()); + auto bp = std::make_unique<EquivBlueprint>(fields, subLayout); + + bp->addTerm(std::make_unique<FakeBlueprint>(FieldSpec("foo", 1, fbh11), a), 1.0); + bp->addTerm(std::make_unique<FakeBlueprint>(FieldSpec("bar", 2, fbh21), b), 1.0); + bp->addTerm(std::make_unique<FakeBlueprint>(FieldSpec("bar", 2, fbh22), c), 1.0); + + MatchData::UP md = MatchData::makeTestInstance(100, 10); + for (uint32_t field_id = 1; field_id <= 2; ++field_id) { + TermFieldMatchData &data = *md->resolveTermField(field_id); + data.setNeedNormalFeatures(unpack_normal_features); + data.setNeedInterleavedFeatures(unpack_interleaved_features); + } + bp->fetchPostings(ExecuteInfo::create(strict)); + SearchIterator::UP search = bp->createSearch(*md, strict); + search->initFullRange(); + + EXPECT_TRUE(!search->seek(3)); + if (!strict) { + EXPECT_EQ(SearchIterator::beginId(), search->getDocId()); + EXPECT_TRUE(search->seek(5u)); + } + EXPECT_EQ(5u, search->getDocId()); + { // test doc 5 results + search->unpack(5u); + { + TermFieldMatchData &data = *md->resolveTermField(1); + EXPECT_EQ(1u, data.getFieldId()); + EXPECT_EQ(5u, data.getDocId()); + FieldPositionsIterator itr = data.getIterator(); + if (unpack_normal_features) { + EXPECT_EQ(1u, itr.size()); ASSERT_TRUE(itr.valid()); - EXPECT_EQUAL(1u, itr.getPosition()); + EXPECT_EQ(1u, itr.getPosition()); itr.next(); - EXPECT_TRUE(!itr.valid()); } - { - TermFieldMatchData &data = *md->resolveTermField(2); - EXPECT_EQUAL(2u, data.getFieldId()); - EXPECT_EQUAL(5u, data.getDocId()); - FieldPositionsIterator itr = data.getIterator(); - EXPECT_EQUAL(2u, itr.size()); + EXPECT_TRUE(!itr.valid()); + if (unpack_interleaved_features) { + EXPECT_EQ(1u, data.getNumOccs()); + EXPECT_EQ(30u, data.getFieldLength()); + } else { + EXPECT_EQ(0u, data.getNumOccs()); + EXPECT_EQ(0u, data.getFieldLength()); + } + } + { + TermFieldMatchData &data = *md->resolveTermField(2); + EXPECT_EQ(2u, data.getFieldId()); + EXPECT_EQ(5u, data.getDocId()); + FieldPositionsIterator itr = data.getIterator(); + if (unpack_normal_features) { + EXPECT_EQ(2u, itr.size()); ASSERT_TRUE(itr.valid()); - EXPECT_EQUAL(2u, itr.getPosition()); + EXPECT_EQ(2u, itr.getPosition()); itr.next(); ASSERT_TRUE(itr.valid()); - EXPECT_EQUAL(3u, itr.getPosition()); + EXPECT_EQ(3u, itr.getPosition()); itr.next(); - EXPECT_TRUE(!itr.valid()); + } + EXPECT_TRUE(!itr.valid()); + if (unpack_interleaved_features) { + EXPECT_EQ(2u, data.getNumOccs()); + EXPECT_EQ(30u, data.getFieldLength()); + } else { + EXPECT_EQ(0u, data.getNumOccs()); + EXPECT_EQ(0u, data.getFieldLength()); } } - EXPECT_TRUE(!search->seek(7)); - if (!strict) { - EXPECT_EQUAL(5u, search->getDocId()); - EXPECT_TRUE(search->seek(10u)); - } - EXPECT_EQUAL(10u, search->getDocId()); - { // test doc 10 results - search->unpack(10u); - EXPECT_EQUAL(5u, md->resolveTermField(1)->getDocId()); // no match - { - TermFieldMatchData &data = *md->resolveTermField(2); - EXPECT_EQUAL(2u, data.getFieldId()); - EXPECT_EQUAL(10u, data.getDocId()); - FieldPositionsIterator itr = data.getIterator(); - EXPECT_EQUAL(1u, itr.size()); + } + EXPECT_TRUE(!search->seek(7)); + if (!strict) { + EXPECT_EQ(5u, search->getDocId()); + EXPECT_TRUE(search->seek(10u)); + } + EXPECT_EQ(10u, search->getDocId()); + { // test doc 10 results + search->unpack(10u); + EXPECT_EQ(5u, md->resolveTermField(1)->getDocId()); // no match + { + TermFieldMatchData &data = *md->resolveTermField(2); + EXPECT_EQ(2u, data.getFieldId()); + EXPECT_EQ(10u, data.getDocId()); + FieldPositionsIterator itr = data.getIterator(); + if (unpack_normal_features) { + EXPECT_EQ(1u, itr.size()); ASSERT_TRUE(itr.valid()); - EXPECT_EQUAL(4u, itr.getPosition()); + EXPECT_EQ(4u, itr.getPosition()); itr.next(); - EXPECT_TRUE(!itr.valid()); + } + EXPECT_TRUE(!itr.valid()); + if (unpack_interleaved_features) { + EXPECT_EQ(1u, data.getNumOccs()); + EXPECT_EQ(35u, data.getFieldLength()); + } else { + EXPECT_EQ(0u, data.getNumOccs()); + EXPECT_EQ(0u, data.getFieldLength()); } } - EXPECT_TRUE(!search->seek(13)); - if (strict) { - EXPECT_TRUE(search->isAtEnd()); - } else { - EXPECT_EQUAL(10u, search->getDocId()); - } } + EXPECT_TRUE(!search->seek(13)); + if (strict) { + EXPECT_TRUE(search->isAtEnd()); + } else { + EXPECT_EQ(10u, search->getDocId()); + } +} + + +TEST_F(EquivTest, nonstrict) +{ + test_equiv(false, true, false); +} + +TEST_F(EquivTest, strict) +{ + test_equiv(true, true, false); +} + +TEST_F(EquivTest, nonstrict_no_normal_no_interleaved) +{ + test_equiv(false, false, false); +} + +TEST_F(EquivTest, strict_no_normal_no_interleaved) +{ + test_equiv(true, false, false); +} + +TEST_F(EquivTest, nonstrict_no_normal_interleaved) +{ + test_equiv(false, false, true); +} + +TEST_F(EquivTest, strict_no_normal_interleaved) +{ + test_equiv(true, false, true); +} + +TEST_F(EquivTest, nonstrict_normal_interleaved) +{ + test_equiv(false, true, true); } -int -Test::Main() +TEST_F(EquivTest, strict_normal_interleaved) { - TEST_INIT("equiv_test"); - testEquiv(); - TEST_DONE(); + test_equiv(true, true, true); } -TEST_APPHOOK(Test); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/queryeval/monitoring_search_iterator/monitoring_search_iterator_test.cpp b/searchlib/src/tests/queryeval/monitoring_search_iterator/monitoring_search_iterator_test.cpp index e863cbe7a73..0e0840d9013 100644 --- a/searchlib/src/tests/queryeval/monitoring_search_iterator/monitoring_search_iterator_test.cpp +++ b/searchlib/src/tests/queryeval/monitoring_search_iterator/monitoring_search_iterator_test.cpp @@ -77,16 +77,18 @@ struct TreeFixture : _itr() { MultiSearch::Children children; - children.push_back(new MonitoringSearchIterator("child1", - SearchIterator::UP - (new SimpleSearch(SimpleResult().addHit(2).addHit(4).addHit(6))), - false)); - children.push_back(new MonitoringSearchIterator("child2", - SearchIterator::UP - (new SimpleSearch(SimpleResult().addHit(3).addHit(4).addHit(5))), + children.emplace_back( + new MonitoringSearchIterator("child1", + SearchIterator::UP + (new SimpleSearch(SimpleResult().addHit(2).addHit(4).addHit(6))), + false)); + children.emplace_back( + new MonitoringSearchIterator("child2", + SearchIterator::UP + (new SimpleSearch(SimpleResult().addHit(3).addHit(4).addHit(5))), false)); _itr.reset(new MonitoringSearchIterator("and", - SearchIterator::UP(AndSearch::create(children, true)), + SearchIterator::UP(AndSearch::create(std::move(children), true)), false)); _res.search(*_itr); } diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp index 6e0baf3b2e8..e1d0e034a89 100644 --- a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp +++ b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp @@ -98,9 +98,9 @@ Test::testSearch(bool strict) TermFieldMatchData tfmd; MultiSearch::Children andd; for (size_t i(0); i < _bvs.size(); i++) { - andd.push_back(BitVectorIterator::create(_bvs[i].get(), tfmd, strict, false).release()); + andd.push_back(BitVectorIterator::create(_bvs[i].get(), tfmd, strict, false)); } - SearchIterator::UP s(T::create(andd, strict)); + SearchIterator::UP s = T::create(std::move(andd), strict); if (_optimize) { LOG(info, "Optimizing iterator"); s = MultiBitVectorIteratorBase::optimize(std::move(s)); diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp index 12c38d3fe02..565c013a2fe 100644 --- a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp +++ b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp @@ -34,6 +34,7 @@ public: void testAndWith(bool invert); void testEndGuard(bool invert); void testIteratorConformance(); + void testUnpackOfOr(); template<typename T> void testThatOptimizePreservesUnpack(); template <typename T> @@ -44,6 +45,7 @@ public: void testSearch(bool strict, bool invert); int Main() override; private: + void verifyUnpackOfOr(const UnpackInfo & unpackInfo); void verifySelectiveUnpack(SearchIterator & s, const TermFieldMatchData * tfmd); void searchAndCompare(SearchIterator::UP s, uint32_t docIdLimit); void setup(); @@ -58,6 +60,8 @@ private: for (int i = 0; i < 3; ++i) { if (_bvs_inverted[i]->testBit(1)) { _bvs[i]->clearBit(1); + } else { + _bvs[i]->setBit(1); } } } @@ -121,10 +125,10 @@ Test::testAndWith(bool invert) TermFieldMatchData tfmd; { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(AndSearch::create(children, false)); + SearchIterator::UP s = AndSearch::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); s->initFullRange(); @@ -133,10 +137,10 @@ Test::testAndWith(bool invert) H lastHits2F = seekNoReset(*s, 130, _bvs[0]->size()); children.clear(); - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); - children.push_back(createIter(2, invert, tfmd, false).release()); - s.reset(AndSearch::create(children, false)); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); + children.push_back(createIter(2, invert, tfmd, false)); + s = AndSearch::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); s->initFullRange(); H firstHits3 = seekNoReset(*s, 1, 130); @@ -193,12 +197,12 @@ Test::testBug7163266() MultiSearch::Children children; UnpackInfo unpackInfo; for (size_t i(0); i < 28; i++) { - children.push_back(new TrueSearch(tfmd[2])); + children.emplace_back(new TrueSearch(tfmd[2])); unpackInfo.add(i); } - children.push_back(createIter(0, false, tfmd[0], false).release()); - children.push_back(createIter(1, false, tfmd[1], false).release()); - SearchIterator::UP s(AndSearch::create(children, false, unpackInfo)); + children.push_back(createIter(0, false, tfmd[0], false)); + children.push_back(createIter(1, false, tfmd[1], false)); + SearchIterator::UP s = AndSearch::create(std::move(children), false, unpackInfo); const MultiSearch * ms = dynamic_cast<const MultiSearch *>(s.get()); EXPECT_TRUE(ms != nullptr); EXPECT_EQUAL(30u, ms->getChildren().size()); @@ -229,14 +233,14 @@ Test::testThatOptimizePreservesUnpack() _bvs[1]->setBit(1); _bvs[2]->setBit(1); MultiSearch::Children children; - children.push_back(createIter(0, false, tfmd[0], false).release()); - children.push_back(createIter(1, false, tfmd[1], false).release()); - children.push_back(new TrueSearch(tfmd[2])); - children.push_back(createIter(2, false, tfmd[3], false).release()); + children.push_back(createIter(0, false, tfmd[0], false)); + children.push_back(createIter(1, false, tfmd[1], false)); + children.emplace_back(new TrueSearch(tfmd[2])); + children.push_back(createIter(2, false, tfmd[3], false)); UnpackInfo unpackInfo; unpackInfo.add(1); unpackInfo.add(2); - SearchIterator::UP s(T::create(children, false, unpackInfo)); + SearchIterator::UP s = T::create(std::move(children), false, unpackInfo); s->initFullRange(); const MultiSearch * ms = dynamic_cast<const MultiSearch *>(s.get()); EXPECT_TRUE(ms != nullptr); @@ -253,6 +257,63 @@ Test::testThatOptimizePreservesUnpack() fixup_bitvectors(); } +void verifyOrUnpack(SearchIterator & s, TermFieldMatchData tfmd[3]) { + s.initFullRange(); + s.seek(1); + for (size_t i = 0; i < 3; i++) { + EXPECT_EQUAL(0u, tfmd[i].getDocId()); + } + s.unpack(1); + EXPECT_EQUAL(0u, tfmd[0].getDocId()); + EXPECT_EQUAL(1u, tfmd[1].getDocId()); + EXPECT_EQUAL(0u, tfmd[2].getDocId()); +} + +void +Test::testUnpackOfOr() { + _bvs[0]->clearBit(1); + _bvs[1]->setBit(1); + _bvs[2]->clearBit(1); + UnpackInfo all; + all.forceAll(); + verifyUnpackOfOr(all); + + UnpackInfo unpackInfo; + unpackInfo.add(1); + unpackInfo.add(2); + verifyUnpackOfOr(unpackInfo); + + fixup_bitvectors(); +} + +void +Test::verifyUnpackOfOr(const UnpackInfo &unpackInfo) +{ + TermFieldMatchData tfmdA[3]; + MultiSearch::Children children; + children.push_back(createIter(0, false, tfmdA[0], false)); + children.push_back(createIter(1, false, tfmdA[1], false)); + children.push_back(createIter(2, false, tfmdA[2], false)); + SearchIterator::UP s = OrSearch::create(std::move(children), false, unpackInfo); + verifyOrUnpack(*s, tfmdA); + + for (auto & tfmd : tfmdA) { + tfmd.resetOnlyDocId(0); + } + + const MultiSearch * ms = dynamic_cast<const MultiSearch *>(s.get()); + EXPECT_TRUE(ms != nullptr); + EXPECT_EQUAL(3u, ms->getChildren().size()); + + s = MultiBitVectorIteratorBase::optimize(std::move(s)); + s->initFullRange(); + ms = dynamic_cast<const MultiBitVectorIteratorBase *>(s.get()); + EXPECT_TRUE(ms != nullptr); + EXPECT_EQUAL(3u, ms->getChildren().size()); + verifyOrUnpack(*s, tfmdA); + +} + void Test::verifySelectiveUnpack(SearchIterator & s, const TermFieldMatchData * tfmd) { @@ -292,23 +353,23 @@ Test::testSearch(bool strict, bool invert) uint32_t docIdLimit(_bvs[0]->size()); { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, strict).release()); - SearchIterator::UP s(T::create(children, strict)); + children.push_back(createIter(0, invert, tfmd, strict)); + SearchIterator::UP s = T::create(std::move(children), strict); searchAndCompare(std::move(s), docIdLimit); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, strict).release()); - children.push_back(createIter(1, invert, tfmd, strict).release()); - SearchIterator::UP s(T::create(children, strict)); + children.push_back(createIter(0, invert, tfmd, strict)); + children.push_back(createIter(1, invert, tfmd, strict)); + SearchIterator::UP s = T::create(std::move(children), strict); searchAndCompare(std::move(s), docIdLimit); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, strict).release()); - children.push_back(createIter(1, invert, tfmd, strict).release()); - children.push_back(createIter(2, invert, tfmd, strict).release()); - SearchIterator::UP s(T::create(children, strict)); + children.push_back(createIter(0, invert, tfmd, strict)); + children.push_back(createIter(1, invert, tfmd, strict)); + children.push_back(createIter(2, invert, tfmd, strict)); + SearchIterator::UP s = T::create(std::move(children), strict); searchAndCompare(std::move(s), docIdLimit); } } @@ -321,79 +382,79 @@ Test::testOptimizeCommon(bool isAnd, bool invert) { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(1u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[0]) != nullptr); + EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[0].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, false)); + children.emplace_back(new EmptySearch()); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[0]) != nullptr); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[0].get()) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(new EmptySearch()); - children.push_back(createIter(0, invert, tfmd, false).release()); + children.emplace_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0]) != nullptr); - EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0].get()) != nullptr); + EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[1].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(new EmptySearch()); - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.emplace_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0]) != nullptr); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0].get()) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[1].get()) != nullptr); EXPECT_TRUE(Trinary::False == m.getChildren()[1]->is_strict()); } { MultiSearch::Children children; - children.push_back(new EmptySearch()); - children.push_back(createIter(0, invert, tfmd, true).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.emplace_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, true)); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0]) != nullptr); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0].get()) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[1].get()) != nullptr); EXPECT_TRUE(Trinary::True == m.getChildren()[1]->is_strict()); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); SearchIterator::UP filter(s->andWith(createIter(2, invert, tfmd, false), 9)); @@ -404,9 +465,9 @@ Test::testOptimizeCommon(bool isAnd, bool invert) } children.clear(); - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); - s.reset(T::create(children, true)); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); + s = T::create(std::move(children), true); s = MultiBitVectorIteratorBase::optimize(std::move(s)); filter = s->andWith(createIter(2, invert, tfmd, false), 9); @@ -426,10 +487,10 @@ Test::testOptimizeAndOr(bool invert) { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(s.get()) != nullptr); @@ -437,67 +498,67 @@ Test::testOptimizeAndOr(bool invert) } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(new EmptySearch()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, false)); + children.emplace_back(new EmptySearch()); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0]) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0].get()) != nullptr); EXPECT_TRUE(Trinary::False == m.getChildren()[0]->is_strict()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); - children.push_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); + children.emplace_back(new EmptySearch()); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0]) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0].get()) != nullptr); EXPECT_TRUE(Trinary::False == m.getChildren()[0]->is_strict()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, true).release()); - children.push_back(new EmptySearch()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, true)); + children.emplace_back(new EmptySearch()); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0]) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0].get()) != nullptr); EXPECT_TRUE(Trinary::True == m.getChildren()[0]->is_strict()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, true).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); - children.push_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, true)); + children.push_back(createIter(1, invert, tfmd, false)); + children.emplace_back(new EmptySearch()); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0]) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0].get()) != nullptr); EXPECT_TRUE(Trinary::True == m.getChildren()[0]->is_strict()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1].get()) != nullptr); } } @@ -508,9 +569,9 @@ Test::testEndGuard(bool invert) TermFieldMatchData tfmd; MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, true).release()); - children.push_back(createIter(1, invert, tfmd, true).release()); - SearchIterator::UP s(T::create(children, false)); + children.push_back(createIter(0, invert, tfmd, true)); + children.push_back(createIter(1, invert, tfmd, true)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); s->initFullRange(); EXPECT_TRUE(s); @@ -557,11 +618,11 @@ SearchIterator::UP Verifier::create(bool strict) const { MultiSearch::Children bvs; for (const auto & bv : _bvs) { - bvs.push_back(BitVectorIterator::create(bv.get(), getDocIdLimit(), _tfmd, strict, false).release()); + bvs.push_back(BitVectorIterator::create(bv.get(), getDocIdLimit(), _tfmd, strict, false)); } - SearchIterator::UP iter(_is_and ? AndSearch::create(bvs, strict) : OrSearch::create(bvs, strict)); + SearchIterator::UP iter(_is_and ? AndSearch::create(std::move(bvs), strict) : OrSearch::create(std::move(bvs), strict)); auto mbvit = MultiBitVectorIteratorBase::optimize(std::move(iter)); - EXPECT_TRUE((bvs.size() < 2) || (dynamic_cast<const MultiBitVectorIteratorBase *>(mbvit.get()) != nullptr)); + EXPECT_TRUE((_bvs.size() < 2) || (dynamic_cast<const MultiBitVectorIteratorBase *>(mbvit.get()) != nullptr)); EXPECT_EQUAL(strict, Trinary::True == mbvit->is_strict()); return mbvit; } @@ -584,6 +645,8 @@ Test::Main() testThatOptimizePreservesUnpack<OrSearch>(); testThatOptimizePreservesUnpack<AndSearch>(); TEST_FLUSH(); + testUnpackOfOr(); + TEST_FLUSH(); testEndGuard(false); testEndGuard(true); TEST_FLUSH(); diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index 5d933cb1285..e8c83b8548a 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -6,6 +6,7 @@ #include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/tensor/tensor.h> +#include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/feature.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/queryeval/nearest_neighbor_iterator.h> @@ -23,6 +24,7 @@ LOG_SETUP("nearest_neighbor_test"); using search::feature_t; using search::tensor::DenseTensorAttribute; using search::AttributeVector; +using search::BitVector; using vespalib::eval::ValueType; using CellType = vespalib::eval::ValueType::CellType; using vespalib::eval::TensorSpec; @@ -65,13 +67,15 @@ struct Fixture vespalib::string _typeSpec; std::shared_ptr<DenseTensorAttribute> _tensorAttr; std::shared_ptr<AttributeVector> _attr; + std::unique_ptr<BitVector> _global_filter; Fixture(const vespalib::string &typeSpec) : _cfg(BasicType::TENSOR, CollectionType::SINGLE), _name("test"), _typeSpec(typeSpec), _tensorAttr(), - _attr() + _attr(), + _global_filter() { _cfg.setTensorType(ValueType::from_spec(typeSpec)); _tensorAttr = makeAttr(); @@ -93,6 +97,15 @@ struct Fixture } } + void setFilter(std::vector<uint32_t> docids) { + uint32_t sz = _attr->getNumDocs(); + _global_filter = BitVector::create(sz); + for (uint32_t id : docids) { + EXPECT_LESS(id, sz); + _global_filter->setBit(id); + } + } + void setTensor(uint32_t docId, const Tensor &tensor) { ensureSpace(docId); _tensorAttr->setTensor(docId, tensor); @@ -119,7 +132,8 @@ SimpleResult find_matches(Fixture &env, const DenseTensorView &qtv) { auto &tfmd = *(md->resolveTermField(0)); auto &attr = *(env._tensorAttr); NearestNeighborDistanceHeap dh(2); - auto search = NearestNeighborIterator::create(strict, tfmd, qtv, attr, dh, env.dist_fun()); + const BitVector *filter = env._global_filter.get(); + auto search = NearestNeighborIterator::create(strict, tfmd, qtv, attr, dh, filter, env.dist_fun()); if (strict) { return SimpleResult().searchStrict(*search, attr.getNumDocs()); } else { @@ -158,13 +172,45 @@ TEST("require that NearestNeighborIterator returns expected results") { TEST_DO(verify_iterator_returns_expected_results(denseSpecFloat, denseSpecFloat)); } +void +verify_iterator_returns_filtered_results(const vespalib::string& attribute_tensor_type_spec, + const vespalib::string& query_tensor_type_spec) +{ + Fixture fixture(attribute_tensor_type_spec); + fixture.ensureSpace(6); + fixture.setFilter({1,3,4}); + fixture.setTensor(1, 3.0, 4.0); + fixture.setTensor(2, 6.0, 8.0); + fixture.setTensor(3, 5.0, 12.0); + fixture.setTensor(4, 4.0, 3.0); + fixture.setTensor(5, 8.0, 6.0); + fixture.setTensor(6, 4.0, 3.0); + auto nullTensor = createTensor(query_tensor_type_spec, 0.0, 0.0); + SimpleResult result = find_matches<true>(fixture, *nullTensor); + SimpleResult nullExpect({1,3,4}); + EXPECT_EQUAL(result, nullExpect); + result = find_matches<false>(fixture, *nullTensor); + EXPECT_EQUAL(result, nullExpect); + auto farTensor = createTensor(query_tensor_type_spec, 9.0, 9.0); + SimpleResult farExpect({1,3,4}); + result = find_matches<true>(fixture, *farTensor); + EXPECT_EQUAL(result, farExpect); + result = find_matches<false>(fixture, *farTensor); + EXPECT_EQUAL(result, farExpect); +} + +TEST("require that NearestNeighborIterator returns filtered results") { + TEST_DO(verify_iterator_returns_filtered_results(denseSpecDouble, denseSpecDouble)); + TEST_DO(verify_iterator_returns_filtered_results(denseSpecFloat, denseSpecFloat)); +} + template <bool strict> std::vector<feature_t> get_rawscores(Fixture &env, const DenseTensorView &qtv) { auto md = MatchData::makeTestInstance(2, 2); auto &tfmd = *(md->resolveTermField(0)); auto &attr = *(env._tensorAttr); NearestNeighborDistanceHeap dh(2); - auto search = NearestNeighborIterator::create(strict, tfmd, qtv, attr, dh, env.dist_fun()); + auto search = NearestNeighborIterator::create(strict, tfmd, qtv, attr, dh, nullptr, env.dist_fun()); uint32_t limit = attr.getNumDocs(); uint32_t docid = 1; search->initRange(docid, limit); diff --git a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp index 7926a518317..9761b0da2d7 100644 --- a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp +++ b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp @@ -89,14 +89,17 @@ struct WandTestSpec : public WandSpec WandTestSpec(uint32_t scoresToTrack, uint32_t scoresAdjustFrequency = 1, score_t scoreThreshold = 0, double thresholdBoostFactor = 1); ~WandTestSpec(); - SearchIterator *create() { + SearchIterator::UP create() { MatchData::UP childrenMatchData = createMatchData(); MatchData *tmp = childrenMatchData.get(); - return new TrackedSearch("PWAND", getHistory(), ParallelWeakAndSearch::create(getTerms(tmp), - matchParams, - RankParams(rootMatchData, - std::move(childrenMatchData)), - true)); + return SearchIterator::UP( + new TrackedSearch("PWAND", getHistory(), + ParallelWeakAndSearch::create( + getTerms(tmp), + matchParams, + RankParams(rootMatchData, + std::move(childrenMatchData)), + true))); } }; diff --git a/searchlib/src/tests/queryeval/queryeval.cpp b/searchlib/src/tests/queryeval/queryeval.cpp index 5601baa9113..48b91607ab1 100644 --- a/searchlib/src/tests/queryeval/queryeval.cpp +++ b/searchlib/src/tests/queryeval/queryeval.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/regex/regex.h> #include <vespa/searchlib/test/initrange.h> #include <vespa/searchlib/queryeval/andnotsearch.h> #include <vespa/searchlib/queryeval/andsearch.h> @@ -34,6 +35,9 @@ using search::test::InitRangeVerifier; //----------------------------------------------------------------------------- +constexpr auto lower_bound = Blueprint::FilterConstraint::LOWER_BOUND; +constexpr auto upper_bound = Blueprint::FilterConstraint::UPPER_BOUND; + template <typename T, typename V=std::vector<T> > class Collect { @@ -52,11 +56,13 @@ SearchIterator *simple(const std::string &tag) { return &((new SimpleSearch(SimpleResult()))->tag(tag)); } -Collect<SearchIterator*, MultiSearch::Children> search2(const std::string &t1, const std::string &t2) { - return Collect<SearchIterator*, MultiSearch::Children>().add(simple(t1)).add(simple(t2)); +MultiSearch::Children search2(const std::string &t1, const std::string &t2) { + MultiSearch::Children children; + children.emplace_back(simple(t1)); + children.emplace_back(simple(t2)); + return children; } - class ISourceSelectorDummy : public ISourceSelector { static SourceStore _sourceStoreDummy; @@ -91,9 +97,9 @@ void testMultiSearch(SearchIterator & search) { TEST("test that OR.andWith is a NOOP") { TermFieldMatchData tfmd; MultiSearch::Children ch; - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - SearchIterator::UP search(OrSearch::create(ch, true)); + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + SearchIterator::UP search(OrSearch::create(std::move(ch), true)); auto filter = std::make_unique<TrueSearch>(tfmd); EXPECT_TRUE(search->andWith(std::move(filter), 1)); @@ -102,9 +108,9 @@ TEST("test that OR.andWith is a NOOP") { TEST("test that non-strict AND.andWith is a NOOP") { TermFieldMatchData tfmd; MultiSearch::Children ch; - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - SearchIterator::UP search(AndSearch::create(ch, false)); + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + SearchIterator::UP search(AndSearch::create(std::move(ch), false)); SearchIterator::UP filter = std::make_unique<TrueSearch>(tfmd); filter = search->andWith(std::move(filter), 8); EXPECT_TRUE(filter); @@ -112,10 +118,10 @@ TEST("test that non-strict AND.andWith is a NOOP") { TEST("test that strict AND.andWith steals filter and places it correctly based on estimate") { TermFieldMatchData tfmd; - MultiSearch::Children ch; - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - SearchIterator::UP search(AndSearch::create(ch, true)); + std::vector<SearchIterator *> ch; + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + SearchIterator::UP search(AndSearch::create({ch[0], ch[1]}, true)); static_cast<AndSearch &>(*search).estimate(7); auto filter = std::make_unique<TrueSearch>(tfmd); SearchIterator * filterP = filter.get(); @@ -123,18 +129,18 @@ TEST("test that strict AND.andWith steals filter and places it correctly based o EXPECT_TRUE(nullptr == search->andWith(std::move(filter), 8).get()); const MultiSearch::Children & andChildren = static_cast<MultiSearch &>(*search).getChildren(); EXPECT_EQUAL(3u, andChildren.size()); - EXPECT_EQUAL(ch[0], andChildren[0]); - EXPECT_EQUAL(filterP, andChildren[1]); - EXPECT_EQUAL(ch[1], andChildren[2]); + EXPECT_EQUAL(ch[0], andChildren[0].get()); + EXPECT_EQUAL(filterP, andChildren[1].get()); + EXPECT_EQUAL(ch[1], andChildren[2].get()); auto filter2 = std::make_unique<TrueSearch>(tfmd); SearchIterator * filter2P = filter2.get(); EXPECT_TRUE(nullptr == search->andWith(std::move(filter2), 6).get()); EXPECT_EQUAL(4u, andChildren.size()); - EXPECT_EQUAL(filter2P, andChildren[0]); - EXPECT_EQUAL(ch[0], andChildren[1]); - EXPECT_EQUAL(filterP, andChildren[2]); - EXPECT_EQUAL(ch[1], andChildren[3]); + EXPECT_EQUAL(filter2P, andChildren[0].get()); + EXPECT_EQUAL(ch[0], andChildren[1].get()); + EXPECT_EQUAL(filterP, andChildren[2].get()); + EXPECT_EQUAL(ch[1], andChildren[3].get()); } class NonStrictTrueSearch : public TrueSearch @@ -146,73 +152,58 @@ public: TEST("test that strict AND.andWith does not place non-strict iterator first") { TermFieldMatchData tfmd; - MultiSearch::Children ch; - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - SearchIterator::UP search(AndSearch::create(ch, true)); + std::vector<SearchIterator *> ch; + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + SearchIterator::UP search(AndSearch::create({ch[0], ch[1]}, true)); static_cast<AndSearch &>(*search).estimate(7); auto filter = std::make_unique<NonStrictTrueSearch>(tfmd); SearchIterator * filterP = filter.get(); EXPECT_TRUE(nullptr == search->andWith(std::move(filter), 6).get()); const MultiSearch::Children & andChildren = static_cast<MultiSearch &>(*search).getChildren(); EXPECT_EQUAL(3u, andChildren.size()); - EXPECT_EQUAL(ch[0], andChildren[0]); - EXPECT_EQUAL(filterP, andChildren[1]); - EXPECT_EQUAL(ch[1], andChildren[2]); + EXPECT_EQUAL(ch[0], andChildren[0].get()); + EXPECT_EQUAL(filterP, andChildren[1].get()); + EXPECT_EQUAL(ch[1], andChildren[2].get()); } TEST("test that strict rank search forwards to its greedy first child") { TermFieldMatchData tfmd; - SearchIterator::UP search( - RankSearch::create( - Collect<SearchIterator*, MultiSearch::Children>() - .add(AndSearch::create(search2("a", "b"), true)) - .add(new TrueSearch(tfmd)), - true) - ); + SearchIterator::UP search = RankSearch::create({ AndSearch::create(search2("a", "b"), true), new TrueSearch(tfmd) }, true); auto filter = std::make_unique<TrueSearch>(tfmd); EXPECT_TRUE(nullptr == search->andWith(std::move(filter), 8).get()); } TEST("test that non-strict rank search does NOT forward to its greedy first child") { TermFieldMatchData tfmd; - SearchIterator::UP search( - RankSearch::create( - Collect<SearchIterator*, MultiSearch::Children>() - .add(AndSearch::create(search2("a", "b"), true)) - .add(new TrueSearch(tfmd)), - false) - ); + SearchIterator::UP search = RankSearch::create({ AndSearch::create(search2("a", "b"), true), new TrueSearch(tfmd) }, false); auto filter = std::make_unique<TrueSearch>(tfmd); EXPECT_TRUE(nullptr != search->andWith(std::move(filter), 8).get()); } TEST("test that strict andnot search forwards to its greedy first child") { TermFieldMatchData tfmd; - SearchIterator::UP search( - AndNotSearch::create( - Collect<SearchIterator*, MultiSearch::Children>() - .add(AndSearch::create(search2("a", "b"), true)) - .add(new TrueSearch(tfmd)), - true) - ); + SearchIterator::UP search = AndNotSearch::create({ AndSearch::create(search2("a", "b"), true), new TrueSearch(tfmd) }, true); auto filter = std::make_unique<TrueSearch>(tfmd); EXPECT_TRUE(nullptr == search->andWith(std::move(filter), 8).get()); } TEST("test that non-strict andnot search does NOT forward to its greedy first child") { TermFieldMatchData tfmd; - SearchIterator::UP search( - AndNotSearch::create( - Collect<SearchIterator*, MultiSearch::Children>() - .add(AndSearch::create(search2("a", "b"), true)) - .add(new TrueSearch(tfmd)), - false) - ); + SearchIterator::UP search = AndNotSearch::create({ AndSearch::create(search2("a", "b"), true), new TrueSearch(tfmd) }, false); auto filter = std::make_unique<TrueSearch>(tfmd); EXPECT_TRUE(nullptr != search->andWith(std::move(filter), 8).get()); } +void expect_match(std::string input, std::string regexp) { + using vespalib::Regex; + Regex pattern = Regex::from_pattern(regexp, Regex::Options::DotMatchesNewline); + if (! EXPECT_TRUE(pattern.partial_match(input))) { + fprintf(stderr, "no match for pattern: >>>%s<<< in input:\n>>>\n%s\n<<<\n", + regexp.c_str(), input.c_str()); + } +} + TEST("testAnd") { SimpleResult a; SimpleResult b; @@ -232,8 +223,19 @@ TEST("testAnd") { res.search(*and_ab); SimpleResult expect; expect.addHit(5).addHit(30); + EXPECT_EQUAL(res, expect); + SearchIterator::UP filter_ab = and_b->createFilterSearch(true, upper_bound); + SimpleResult filter_res; + filter_res.search(*filter_ab); EXPECT_EQUAL(res, expect); + std::string dump = filter_ab->asString(); + expect_match(dump, "upper"); + expect_match(dump, "AndSearchStrict.*NoUnpack.*SimpleSearch.*upper.*SimpleSearch.*upper"); + filter_ab = and_b->createFilterSearch(false, lower_bound); + dump = filter_ab->asString(); + expect_match(dump, "lower"); + expect_match(dump, "AndSearchNoStrict.*NoUnpack.*SimpleSearch.*lower.*SimpleSearch.*lower"); } TEST("mutisearch and initRange") { @@ -257,16 +259,27 @@ TEST("testOr") { res.search(*or_ab); SimpleResult expect; expect.addHit(5).addHit(10).addHit(17).addHit(30); + EXPECT_EQUAL(res, expect); + SearchIterator::UP filter_ab = or_b->createFilterSearch(true, upper_bound); + SimpleResult filter_res; + filter_res.search(*filter_ab); EXPECT_EQUAL(res, expect); + std::string dump = filter_ab->asString(); + expect_match(dump, "upper"); + expect_match(dump, "OrLikeSearch.true.*NoUnpack.*SimpleSearch.*upper.*SimpleSearch.*upper"); + filter_ab = or_b->createFilterSearch(false, lower_bound); + dump = filter_ab->asString(); + expect_match(dump, "lower"); + expect_match(dump, "OrLikeSearch.false.*NoUnpack.*SimpleSearch.*lower.*SimpleSearch.*lower"); } { TermFieldMatchData tfmd; MultiSearch::Children ch; - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - SearchIterator::UP orSearch(OrSearch::create(ch, true)); + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + SearchIterator::UP orSearch(OrSearch::create(std::move(ch), true)); testMultiSearch(*orSearch); } } @@ -274,8 +287,8 @@ TEST("testOr") { class TestInsertRemoveSearch : public MultiSearch { public: - TestInsertRemoveSearch(const MultiSearch::Children & children) : - MultiSearch(children), + TestInsertRemoveSearch(ChildrenIterators children) : + MultiSearch(std::move(children)), _accumRemove(0), _accumInsert(0) { } @@ -292,31 +305,31 @@ struct MultiSearchRemoveTest { }; TEST("testMultiSearch") { - MultiSearch::Children children; - children.push_back(new EmptySearch()); - children.push_back(new EmptySearch()); - children.push_back(new EmptySearch()); - TestInsertRemoveSearch ms(children); + std::vector<SearchIterator *> orig; + orig.emplace_back(new EmptySearch()); + orig.emplace_back(new EmptySearch()); + orig.emplace_back(new EmptySearch()); + TestInsertRemoveSearch ms({orig[0], orig[1], orig[2]}); EXPECT_EQUAL(3u, ms.getChildren().size()); - EXPECT_EQUAL(children[0], ms.getChildren()[0]); - EXPECT_EQUAL(children[1], ms.getChildren()[1]); - EXPECT_EQUAL(children[2], ms.getChildren()[2]); + EXPECT_EQUAL(orig[0], ms.getChildren()[0].get()); + EXPECT_EQUAL(orig[1], ms.getChildren()[1].get()); + EXPECT_EQUAL(orig[2], ms.getChildren()[2].get()); EXPECT_EQUAL(0u, ms._accumInsert); EXPECT_EQUAL(0u, ms._accumRemove); - EXPECT_EQUAL(children[1], MultiSearchRemoveTest::remove(ms, 1).get()); + EXPECT_EQUAL(orig[1], MultiSearchRemoveTest::remove(ms, 1).get()); EXPECT_EQUAL(2u, ms.getChildren().size()); - EXPECT_EQUAL(children[0], ms.getChildren()[0]); - EXPECT_EQUAL(children[2], ms.getChildren()[1]); + EXPECT_EQUAL(orig[0], ms.getChildren()[0].get()); + EXPECT_EQUAL(orig[2], ms.getChildren()[1].get()); EXPECT_EQUAL(0u, ms._accumInsert); EXPECT_EQUAL(1u, ms._accumRemove); - children.push_back(new EmptySearch()); - ms.insert(1, SearchIterator::UP(children.back())); + orig.emplace_back(new EmptySearch()); + ms.insert(1, SearchIterator::UP(orig.back())); EXPECT_EQUAL(3u, ms.getChildren().size()); - EXPECT_EQUAL(children[0], ms.getChildren()[0]); - EXPECT_EQUAL(children[3], ms.getChildren()[1]); - EXPECT_EQUAL(children[2], ms.getChildren()[2]); + EXPECT_EQUAL(orig[0], ms.getChildren()[0].get()); + EXPECT_EQUAL(orig[3], ms.getChildren()[1].get()); + EXPECT_EQUAL(orig[2], ms.getChildren()[2].get()); EXPECT_EQUAL(1u, ms._accumInsert); EXPECT_EQUAL(1u, ms._accumRemove); } @@ -371,8 +384,19 @@ TEST("testAndNot") { res.search(*andnot_ab); SimpleResult expect; expect.addHit(10); + EXPECT_EQUAL(res, expect); + SearchIterator::UP filter_ab = andnot_b->createFilterSearch(true, upper_bound); + SimpleResult filter_res; + filter_res.search(*filter_ab); EXPECT_EQUAL(res, expect); + std::string dump = filter_ab->asString(); + expect_match(dump, "upper"); + expect_match(dump, "AndNotSearch.*SimpleSearch.*<strict,upper>.*SimpleSearch.*<nostrict,lower>"); + filter_ab = andnot_b->createFilterSearch(false, lower_bound); + dump = filter_ab->asString(); + expect_match(dump, "lower"); + expect_match(dump, "AndNotSearch.*SimpleSearch.*<nostrict,lower>.*SimpleSearch.*<nostrict,upper>"); } { SimpleResult a; @@ -581,23 +605,19 @@ TEST("testDump") { #ifdef __clang__ #pragma clang diagnostic pop #endif - SearchIterator::UP search( - AndSearch::create( - Collect<SearchIterator*, MultiSearch::Children>() - .add(AndNotSearch::create(search2("+", "-"), true)) - .add(AndSearch::create(search2("and_a", "and_b"), true)) - .add(new BooleanMatchIteratorWrapper(SearchIterator::UP(simple("wrapped")), TermFieldMatchDataArray())) - .add(new NearSearch(search2("near_a", "near_b"), - TermFieldMatchDataArray(), - 5u, true)) - .add(new ONearSearch(search2("onear_a", "onear_b"), - TermFieldMatchDataArray(), 10, true)) - .add(OrSearch::create(search2("or_a", "or_b"), false)) - .add(RankSearch::create(search2("rank_a", "rank_b"),false)) - .add(SourceBlenderSearch::create(selector(), Collect<Source, SourceBlenderSearch::Children>() - .add(Source(simple("blend_a"), 2)) - .add(Source(simple("blend_b"), 4)), true)) - , true)); + + SearchIterator::UP search = AndSearch::create( { + AndNotSearch::create(search2("+", "-"), true), + AndSearch::create(search2("and_a", "and_b"), true), + new BooleanMatchIteratorWrapper(SearchIterator::UP(simple("wrapped")), TermFieldMatchDataArray()), + new NearSearch(search2("near_a", "near_b"), TermFieldMatchDataArray(), 5u, true), + new ONearSearch(search2("onear_a", "onear_b"), TermFieldMatchDataArray(), 10, true), + OrSearch::create(search2("or_a", "or_b"), false), + RankSearch::create(search2("rank_a", "rank_b"),false), + SourceBlenderSearch::create(selector(), Collect<Source, SourceBlenderSearch::Children>() + .add(Source(simple("blend_a"), 2)) + .add(Source(simple("blend_b"), 4)), + true) }, true); vespalib::string sas = search->asString(); EXPECT_TRUE(sas.size() > 50); vespalib::Slime slime; @@ -800,26 +820,26 @@ TEST("test InitRangeVerifier") { TEST("Test multisearch and andsearchstrict iterators adheres to initRange") { InitRangeVerifier ir; - ir.verify( AndSearch::create({ ir.createIterator(ir.getExpectedDocIds(), false).release(), - ir.createFullIterator().release() }, false)); + ir.verify( AndSearch::create({ ir.createIterator(ir.getExpectedDocIds(), false), + ir.createFullIterator() }, false)); - ir.verify( AndSearch::create({ ir.createIterator(ir.getExpectedDocIds(), true).release(), - ir.createFullIterator().release() }, true)); + ir.verify( AndSearch::create({ ir.createIterator(ir.getExpectedDocIds(), true), + ir.createFullIterator() }, true)); } TEST("Test andnotsearchstrict iterators adheres to initRange") { InitRangeVerifier ir; - TEST_DO(ir.verify( AndNotSearch::create({ir.createIterator(ir.getExpectedDocIds(), false).release(), - ir.createEmptyIterator().release() }, false))); - TEST_DO(ir.verify( AndNotSearch::create({ir.createIterator(ir.getExpectedDocIds(), true).release(), - ir.createEmptyIterator().release() }, true))); + TEST_DO(ir.verify( AndNotSearch::create({ir.createIterator(ir.getExpectedDocIds(), false), + ir.createEmptyIterator() }, false))); + TEST_DO(ir.verify( AndNotSearch::create({ir.createIterator(ir.getExpectedDocIds(), true), + ir.createEmptyIterator() }, true))); auto inverted = InitRangeVerifier::invert(ir.getExpectedDocIds(), ir.getDocIdLimit()); - TEST_DO(ir.verify( AndNotSearch::create({ir.createFullIterator().release(), - ir.createIterator(inverted, false).release() }, false))); - TEST_DO(ir.verify( AndNotSearch::create({ir.createFullIterator().release(), - ir.createIterator(inverted, false).release() }, true))); + TEST_DO(ir.verify( AndNotSearch::create({ir.createFullIterator(), + ir.createIterator(inverted, false) }, false))); + TEST_DO(ir.verify( AndNotSearch::create({ir.createFullIterator(), + ir.createIterator(inverted, false) }, true))); } diff --git a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp index 378e16480b8..db177b98b84 100644 --- a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp +++ b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp @@ -8,12 +8,12 @@ #include <vespa/searchlib/queryeval/same_element_search.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchcommon/attribute/i_search_context.h> -#include <vespa/searchlib/attribute/elementiterator.h> +#include <vespa/searchlib/attribute/searchcontextelementiterator.h> #include <vespa/vespalib/test/insertion_operators.h> using namespace search::fef; using namespace search::queryeval; -using search::attribute::ElementIterator; +using search::attribute::SearchContextElementIterator; void verify_elements(SameElementSearch &se, uint32_t docid, const std::initializer_list<uint32_t> list) { std::vector<uint32_t> expect(list); @@ -99,11 +99,11 @@ TEST("require that strict iterator seeks to next hit") { auto search = bp->createSearch(*md, true); search->initRange(1, 1000); EXPECT_LESS(search->getDocId(), 1u); - EXPECT_TRUE(!search->seek(1)); + EXPECT_FALSE(search->seek(1)); EXPECT_EQUAL(search->getDocId(), 7u); EXPECT_TRUE(search->seek(9)); EXPECT_EQUAL(search->getDocId(), 9u); - EXPECT_TRUE(!search->seek(10)); + EXPECT_FALSE(search->seek(10)); EXPECT_TRUE(search->isAtEnd()); } @@ -134,8 +134,8 @@ TEST("require that attribute iterators are wrapped for element unpacking") { SameElementSearch *se = dynamic_cast<SameElementSearch*>(search.get()); ASSERT_TRUE(se != nullptr); ASSERT_EQUAL(se->children().size(), 2u); - EXPECT_TRUE(dynamic_cast<ElementIterator*>(se->children()[0].get()) != nullptr); - EXPECT_TRUE(dynamic_cast<ElementIterator*>(se->children()[1].get()) != nullptr); + EXPECT_TRUE(dynamic_cast<SearchContextElementIterator*>(se->children()[0].get()) != nullptr); + EXPECT_TRUE(dynamic_cast<SearchContextElementIterator*>(se->children()[1].get()) != nullptr); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp index dfe2e2edbd9..d76f13afdca 100644 --- a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp +++ b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp @@ -170,10 +170,12 @@ public: } SimplePhraseSearch::Children children; for (size_t i = 0; i < _children.size(); ++i) { - children.push_back(_children[i]->createSearch(*_md, _strict).release()); + children.push_back(_children[i]->createSearch(*_md, _strict)); } - search = std::make_unique<SimplePhraseSearch>(children, MatchData::UP(), childMatch, _order, - *_md->resolveTermField(phrase_handle), _strict); + search = std::make_unique<SimplePhraseSearch>(std::move(children), + MatchData::UP(), childMatch, _order, + *_md->resolveTermField(phrase_handle), + _strict); } search->initFullRange(); return search.release(); diff --git a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp index 1cf39183206..0de0815731b 100644 --- a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp +++ b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp @@ -139,19 +139,19 @@ constexpr vespalib::duration max_time = 1000s; struct ChildFactory { ChildFactory() {} virtual std::string name() const = 0; - virtual SearchIterator *createChild(uint32_t idx, uint32_t limit) const = 0; + virtual SearchIterator::UP createChild(uint32_t idx, uint32_t limit) const = 0; virtual ~ChildFactory() {} }; struct SparseVectorFactory { virtual std::string name() const = 0; - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const = 0; + virtual SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const = 0; virtual ~SparseVectorFactory() {} }; struct FilterStrategy { virtual std::string name() const = 0; - virtual SearchIterator *createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const = 0; + virtual SearchIterator::UP createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const = 0; virtual ~FilterStrategy() {} }; @@ -184,8 +184,8 @@ struct ModSearchFactory : ChildFactory { virtual std::string name() const override { return vespalib::make_string("ModSearch(%u)", bias); } - virtual SearchIterator *createChild(uint32_t idx, uint32_t limit) const override { - return new ModSearch(bias + idx, limit); + SearchIterator::UP createChild(uint32_t idx, uint32_t limit) const override { + return SearchIterator::UP(new ModSearch(bias + idx, limit)); } }; @@ -197,7 +197,7 @@ struct VespaWandFactory : SparseVectorFactory { virtual std::string name() const override { return vespalib::make_string("VespaWand(%u)", n); } - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { wand::Terms terms; for (size_t i = 0; i < childCnt; ++i) { terms.push_back(wand::Term(childFactory.createChild(i, limit), default_weight, limit / (i + 1))); @@ -212,12 +212,12 @@ struct RiseWandFactory : SparseVectorFactory { virtual std::string name() const override { return vespalib::make_string("RiseWand(%u)", n); } - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { wand::Terms terms; for (size_t i = 0; i < childCnt; ++i) { terms.push_back(wand::Term(childFactory.createChild(i, limit), default_weight, limit / (i + 1))); } - return new rise::TermFrequencyRiseWand(terms, n); + return SearchIterator::UP(new rise::TermFrequencyRiseWand(terms, n)); } }; @@ -226,11 +226,12 @@ struct WeightedSetFactory : SparseVectorFactory { virtual std::string name() const override { return vespalib::make_string("WeightedSet"); } - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { - std::vector<SearchIterator*> terms; + SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + std::vector<SearchIterator *> terms; std::vector<int32_t> weights; for (size_t i = 0; i < childCnt; ++i) { - terms.push_back(childFactory.createChild(i, limit)); + // TODO: pass ownership with unique_ptr + terms.push_back(childFactory.createChild(i, limit).release()); weights.push_back(default_weight); } return WeightedSetTermSearch::create(terms, tfmd, weights, MatchData::UP(nullptr)); @@ -242,22 +243,22 @@ struct DotProductFactory : SparseVectorFactory { virtual std::string name() const override { return vespalib::make_string("DotProduct"); } - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { MatchDataLayout layout; std::vector<TermFieldHandle> handles; for (size_t i = 0; i < childCnt; ++i) { handles.push_back(layout.allocTermField(0)); } - std::vector<SearchIterator*> terms; + std::vector<SearchIterator *> terms; std::vector<TermFieldMatchData*> childMatch; std::vector<int32_t> weights; MatchData::UP md = layout.createMatchData(); for (size_t i = 0; i < childCnt; ++i) { - terms.push_back(childFactory.createChild(i, limit)); + terms.push_back(childFactory.createChild(i, limit).release()); childMatch.push_back(md->resolveTermField(handles[i])); weights.push_back(default_weight); } - return DotProductSearch::create(terms, tfmd, childMatch, weights, std::move(md)).release(); + return DotProductSearch::create(terms, tfmd, childMatch, weights, std::move(md)); } }; @@ -265,12 +266,12 @@ struct OrFactory : SparseVectorFactory { virtual std::string name() const override { return vespalib::make_string("Or"); } - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { OrSearch::Children children; for (size_t i = 0; i < childCnt; ++i) { children.push_back(childFactory.createChild(i, limit)); } - return OrSearch::create(children, true); + return OrSearch::create(std::move(children), true); } }; @@ -280,7 +281,7 @@ struct NoFilterStrategy : FilterStrategy { virtual std::string name() const override { return vespalib::make_string("NoFilter"); } - virtual SearchIterator *createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { return vectorFactory.createSparseVector(childFactory, childCnt, limit); } }; @@ -289,11 +290,11 @@ struct PositiveFilterBeforeStrategy : FilterStrategy { virtual std::string name() const override { return vespalib::make_string("PositiveBefore"); } - virtual SearchIterator *createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { AndSearch::Children children; - children.push_back(new ModSearch(2, limit)); // <- 50% hits (hardcoded) + children.emplace_back(new ModSearch(2, limit)); // <- 50% hits (hardcoded) children.push_back(vectorFactory.createSparseVector(childFactory, childCnt, limit)); - return AndSearch::create(children, true); + return AndSearch::create(std::move(children), true); } }; @@ -301,11 +302,11 @@ struct NegativeFilterAfterStrategy : FilterStrategy { virtual std::string name() const override { return vespalib::make_string("NegativeAfter"); } - virtual SearchIterator *createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { AndNotSearch::Children children; children.push_back(vectorFactory.createSparseVector(childFactory, childCnt, limit)); - children.push_back(new ModSearch(2, limit)); // <- 50% hits (hardcoded) - return AndNotSearch::create(children, true); + children.emplace_back(new ModSearch(2, limit)); // <- 50% hits (hardcoded) + return AndNotSearch::create(std::move(children), true); } }; diff --git a/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp b/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp index 8d70daa4ca8..dee1bdb0b9a 100644 --- a/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp +++ b/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp @@ -120,32 +120,32 @@ SearchIterator *TERM(std::initializer_list<uint32_t> hits, bool strict) { return new MyTerm(hits, strict); } -SearchIterator *ANDNOT(std::initializer_list<SearchIterator *> children, bool strict) { - return AndNotSearch::create(children, strict); +SearchIterator::UP ANDNOT(ChildrenIterators children, bool strict) { + return AndNotSearch::create(std::move(children), strict); } -SearchIterator *AND(std::initializer_list<SearchIterator *> children, bool strict) { - return AndSearch::create(children, strict); +SearchIterator::UP AND(ChildrenIterators children, bool strict) { + return AndSearch::create(std::move(children), strict); } -SearchIterator *ANDz(std::initializer_list<SearchIterator *> children, bool strict) { - return AndSearch::create(children, strict, no_unpack()); +SearchIterator::UP ANDz(ChildrenIterators children, bool strict) { + return AndSearch::create(std::move(children), strict, no_unpack()); } -SearchIterator *ANDs(std::initializer_list<SearchIterator *> children, bool strict) { - return AndSearch::create(children, strict, selective_unpack()); +SearchIterator::UP ANDs(ChildrenIterators children, bool strict) { + return AndSearch::create(std::move(children), strict, selective_unpack()); } -SearchIterator *OR(std::initializer_list<SearchIterator *> children, bool strict) { - return OrSearch::create(children, strict); +SearchIterator::UP OR(ChildrenIterators children, bool strict) { + return OrSearch::create(std::move(children), strict); } -SearchIterator *ORz(std::initializer_list<SearchIterator *> children, bool strict) { - return OrSearch::create(children, strict, no_unpack()); +SearchIterator::UP ORz(ChildrenIterators children, bool strict) { + return OrSearch::create(std::move(children), strict, no_unpack()); } -SearchIterator *ORs(std::initializer_list<SearchIterator *> children, bool strict) { - return OrSearch::create(children, strict, selective_unpack()); +SearchIterator::UP ORs(ChildrenIterators children, bool strict) { + return OrSearch::create(std::move(children), strict, selective_unpack()); } //----------------------------------------------------------------------------- @@ -156,22 +156,22 @@ std::unique_ptr<T> UP(T *t) { return std::unique_ptr<T>(t); } //----------------------------------------------------------------------------- SearchIterator::UP make_search(bool strict) { - return UP(AND({OR({TERM({2,7}, true), - TERM({4,8}, true), - TERM({5,6,9}, true)}, true), - OR({TERM({1,4,7}, false), - TERM({2,5,8}, true), - TERM({3,6}, false)}, false), - OR({TERM({1,2,3}, false), - TERM({4,6}, false), - TERM({8,9}, false)}, false)}, strict)); + return AND({OR({ TERM({2,7}, true), + TERM({4,8}, true), + TERM({5,6,9}, true) }, true), + OR({ TERM({1,4,7}, false), + TERM({2,5,8}, true), + TERM({3,6}, false) }, false), + OR({ TERM({1,2,3}, false), + TERM({4,6}, false), + TERM({8,9}, false) }, false)}, strict); } SearchIterator::UP make_filter_search(bool strict) { - return UP(ANDNOT({TERM({1,2,3,4,5,6,7,8,9}, true), - TERM({1,9}, false), - TERM({3,7}, true), - TERM({5}, false)}, strict)); + return ANDNOT({ TERM({1,2,3,4,5,6,7,8,9}, true), + TERM({1,9}, false), + TERM({3,7}, true), + TERM({5}, false) }, strict); } void add_if_inside(uint32_t docid, uint32_t begin, uint32_t end, std::vector<uint32_t> &expect) { @@ -262,7 +262,7 @@ TEST("require that termwise filter search produces appropriate results") { } TEST("require that termwise ANDNOT with single term works") { - TEST_DO(verify({2,3,4}, *make_termwise(UP(ANDNOT({TERM({1,2,3,4,5}, true)}, true)), true), 2, 5)); + TEST_DO(verify({2,3,4}, *make_termwise(ANDNOT({ TERM({1,2,3,4,5}, true) }, true), true), 2, 5)); } TEST("require that pseudo term is rewindable") { @@ -361,20 +361,20 @@ TEST("require that match data keeps track of the termwise limit") { //----------------------------------------------------------------------------- TEST("require that terwise test search string dump is detailed enough") { - EXPECT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(), - make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString()); + EXPECT_EQUAL(make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString(), + make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString()); - EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(), - make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, false), TERM({3}, true)}, true)), true)->asString()); + EXPECT_NOT_EQUAL(make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString(), + make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, false), TERM({3}, true) }, true), true)->asString()); - EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(), - make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, false)), true)->asString()); + EXPECT_NOT_EQUAL(make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString(), + make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, false), true)->asString()); - EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(), - make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), false)->asString()); + EXPECT_NOT_EQUAL(make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString(), + make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), false)->asString()); - EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(), - make_termwise(UP(OR({TERM({1,2,3}, true), TERM({3}, true), TERM({2,3}, true)}, true)), true)->asString()); + EXPECT_NOT_EQUAL(make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString(), + make_termwise(OR({ TERM({1,2,3}, true), TERM({3}, true), TERM({2,3}, true) }, true), true)->asString()); } //----------------------------------------------------------------------------- @@ -389,7 +389,7 @@ TEST("require that basic termwise evaluation works") { my_or.addChild(UP(new MyBlueprint({2}, true, 2))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - make_termwise(UP(OR({TERM({1}, strict), TERM({2}, strict)}, strict)), strict)->asString()); + make_termwise(OR({ TERM({1}, strict), TERM({2}, strict) }, strict), strict)->asString()); } } @@ -434,7 +434,9 @@ TEST("require that termwise evaluation can be multi-level, but not duplicated") my_or.addChild(std::move(child)); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - make_termwise(UP(OR({TERM({1}, strict), ORz({TERM({2}, strict), TERM({3}, strict)}, strict)}, strict)), strict)->asString()); + make_termwise(OR({ TERM({1}, strict), + ORz({ TERM({2}, strict), TERM({3}, strict) }, strict) }, + strict), strict)->asString()); } } @@ -450,7 +452,7 @@ TEST("require that OR can be completely termwise") { my_or.addChild(UP(new MyBlueprint({2}, true, 2))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - make_termwise(UP(OR({TERM({1}, strict), TERM({2}, strict)}, strict)), strict)->asString()); + make_termwise(OR({ TERM({1}, strict), TERM({2}, strict) }, strict), strict)->asString()); } } @@ -465,7 +467,8 @@ TEST("require that OR can be partially termwise") { my_or.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - UP(ORs({make_termwise(UP(OR({TERM({1}, strict), TERM({3}, strict)}, strict)), strict).release(), TERM({2}, strict)}, strict))->asString()); + ORs({ make_termwise(OR({ TERM({1}, strict), TERM({3}, strict) }, strict), strict), + TERM({2}, strict) }, strict)->asString()); } } @@ -480,7 +483,9 @@ TEST("require that OR puts termwise subquery at the right place") { my_or.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - UP(ORs({TERM({1}, strict), make_termwise(UP(OR({TERM({2}, strict), TERM({3}, strict)}, strict)), strict).release()}, strict))->asString()); + ORs({ TERM({1}, strict), + make_termwise(OR({ TERM({2}, strict), TERM({3}, strict) }, strict), + strict) }, strict)->asString()); } } @@ -496,7 +501,10 @@ TEST("require that OR can use termwise eval also when having non-termwise childr my_or.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - UP(ORz({TERM({1}, strict), make_termwise(UP(OR({TERM({2}, strict), TERM({3}, strict)}, strict)), strict).release()}, strict))->asString()); + ORz({ TERM({1}, strict), + make_termwise(OR({ TERM({2}, strict), TERM({3}, strict) }, strict), + strict)}, + strict)->asString()); } } @@ -512,7 +520,7 @@ TEST("require that AND can be completely termwise") { my_and.addChild(UP(new MyBlueprint({2}, true, 2))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(), - make_termwise(UP(AND({TERM({1}, strict), TERM({2}, false)}, strict)), strict)->asString()); + make_termwise(AND({ TERM({1}, strict), TERM({2}, false) }, strict), strict)->asString()); } } @@ -527,7 +535,10 @@ TEST("require that AND can be partially termwise") { my_and.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(), - UP(ANDs({make_termwise(UP(AND({TERM({1}, strict), TERM({3}, false)}, strict)), strict).release(), TERM({2}, false)}, strict))->asString()); + ANDs({ make_termwise(AND({ TERM({1}, strict), TERM({3}, false) }, + strict), + strict), + TERM({2}, false) }, strict)->asString()); } } @@ -542,7 +553,9 @@ TEST("require that AND puts termwise subquery at the right place") { my_and.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(), - UP(ANDs({TERM({1}, strict), make_termwise(UP(AND({TERM({2}, false), TERM({3}, false)}, false)), false).release()}, strict))->asString()); + ANDs({ TERM({1}, strict), + make_termwise(AND({ TERM({2}, false), TERM({3}, false) }, false), + false) }, strict)->asString()); } } @@ -558,7 +571,9 @@ TEST("require that AND can use termwise eval also when having non-termwise child my_and.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(), - UP(ANDz({TERM({1}, strict), make_termwise(UP(AND({TERM({2}, false), TERM({3}, false)}, false)), false).release()}, strict))->asString()); + ANDz({ TERM({1}, strict), + make_termwise(AND({ TERM({2}, false), TERM({3}, false) }, false), + false) }, strict)->asString()); } } @@ -573,7 +588,8 @@ TEST("require that ANDNOT can be completely termwise") { my_andnot.addChild(UP(new MyBlueprint({2}, true, 2))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_andnot.createSearch(*md, strict)->asString(), - make_termwise(UP(ANDNOT({TERM({1}, strict), TERM({2}, false)}, strict)), strict)->asString()); + make_termwise(ANDNOT({ TERM({1}, strict), TERM({2}, false) }, + strict), strict)->asString()); } } @@ -586,7 +602,9 @@ TEST("require that ANDNOT can be partially termwise") { my_andnot.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_andnot.createSearch(*md, strict)->asString(), - UP(ANDNOT({TERM({1}, strict), make_termwise(UP(OR({TERM({2}, false), TERM({3}, false)}, false)), false).release()}, strict))->asString()); + ANDNOT({ TERM({1}, strict), + make_termwise(OR({ TERM({2}, false), TERM({3}, false) }, false), + false) }, strict)->asString()); } } @@ -600,7 +618,9 @@ TEST("require that ANDNOT can be partially termwise with first child being termw my_andnot.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_andnot.createSearch(*md, strict)->asString(), - UP(ANDNOT({make_termwise(UP(ANDNOT({TERM({1}, strict), TERM({3}, false)}, strict)), strict).release(), TERM({2}, false)}, strict))->asString()); + ANDNOT({ make_termwise(ANDNOT({ TERM({1}, strict), TERM({3}, false) }, strict), + strict), + TERM({2}, false) }, strict)->asString()); } } @@ -613,13 +633,13 @@ TEST("require that termwise blueprint helper calculates unpack info correctly") my_or.addChild(UP(new MyBlueprint({3}, true, 3))); my_or.addChild(UP(new MyBlueprint({4}, true, 4))); // ranked my_or.addChild(UP(new MyBlueprint({5}, true, 5))); - MultiSearch::Children dummy_searches(5, nullptr); + MultiSearch::Children dummy_searches(5); UnpackInfo unpack; // non-termwise unpack info unpack.add(1); unpack.add(3); - TermwiseBlueprintHelper helper(my_or, dummy_searches, unpack); - EXPECT_EQUAL(helper.children.size(), 3u); - EXPECT_EQUAL(helper.termwise.size(), 2u); + TermwiseBlueprintHelper helper(my_or, std::move(dummy_searches), unpack); + EXPECT_EQUAL(helper.get_result().size(), 3u); + EXPECT_EQUAL(helper.get_termwise_children().size(), 2u); EXPECT_EQUAL(helper.first_termwise, 2u); EXPECT_TRUE(!helper.termwise_unpack.needUnpack(0)); EXPECT_TRUE(helper.termwise_unpack.needUnpack(1)); diff --git a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp index 95e4cc8ea79..f5aafab87c3 100644 --- a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp +++ b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp @@ -179,9 +179,9 @@ struct FilterFactory : WandFactory { virtual std::string name() const override { return make_string("Filter (mod=%u) [%s]", n, factory.name().c_str()); } virtual SearchIterator::UP create(const wand::Terms &terms) override { AndNotSearch::Children children; - children.push_back(factory.create(terms).release()); - children.push_back(new ModSearch(stats, n, search::endDocId, n, NULL)); - return SearchIterator::UP(AndNotSearch::create(children, true)); + children.push_back(factory.create(terms)); + children.emplace_back(new ModSearch(stats, n, search::endDocId, n, NULL)); + return AndNotSearch::create(std::move(children), true); } }; diff --git a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp index 85d2f3f4a37..8514a221230 100644 --- a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp +++ b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp @@ -123,7 +123,7 @@ struct MockFixture { mock = new MockSearch(initial); children.push_back(mock); weights.push_back(1); - search.reset(WeightedSetTermSearch::create(children, tfmd, weights, MatchData::UP(nullptr))); + search = WeightedSetTermSearch::create(children, tfmd, weights, MatchData::UP(nullptr)); } }; diff --git a/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt index 04c7312a63f..b6a87502fdf 100644 --- a/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt +++ b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt @@ -7,3 +7,11 @@ vespa_add_executable(searchlib_hnsw_index_test_app TEST gtest ) vespa_add_test(NAME searchlib_hnsw_index_test_app COMMAND searchlib_hnsw_index_test_app) + +vespa_add_executable(mt_stress_hnsw_app TEST + SOURCES + stress_hnsw_mt.cpp + DEPENDS + searchlib + gtest +) diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 7d38be7db4a..7dc0efc106d 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -1,6 +1,7 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/eval/tensor/dense/typed_cells.h> +#include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/tensor/distance_functions.h> #include <vespa/searchlib/tensor/doc_vector_access.h> #include <vespa/searchlib/tensor/hnsw_index.h> @@ -19,6 +20,7 @@ using vespalib::MemoryUsage; using namespace search::tensor; using namespace vespalib::slime; using vespalib::Slime; +using search::BitVector; template <typename FloatType> @@ -56,12 +58,14 @@ using HnswIndexUP = std::unique_ptr<HnswIndex>; class HnswIndexTest : public ::testing::Test { public: FloatVectors vectors; + std::unique_ptr<BitVector> global_filter; LevelGenerator* level_generator; GenerationHandler gen_handler; HnswIndexUP index; HnswIndexTest() : vectors(), + global_filter(), level_generator(), gen_handler(), index() @@ -95,6 +99,14 @@ public: gen_handler.updateFirstUsedGeneration(); index->trim_hold_lists(gen_handler.getFirstUsedGeneration()); } + void set_filter(std::vector<uint32_t> docids) { + uint32_t sz = 10; + global_filter = BitVector::create(sz); + for (uint32_t id : docids) { + EXPECT_LT(id, sz); + global_filter->setBit(id); + } + } GenerationHandler::Guard take_read_guard() { return gen_handler.takeGuard(); } @@ -122,7 +134,7 @@ public: void expect_top_3(uint32_t docid, std::vector<uint32_t> exp_hits) { uint32_t k = 3; auto qv = vectors.get_vector(docid); - auto rv = index->top_k_candidates(qv, k).peek(); + auto rv = index->top_k_candidates(qv, k, global_filter.get()).peek(); std::sort(rv.begin(), rv.end(), LesserDistance()); size_t idx = 0; for (const auto & hit : rv) { @@ -197,6 +209,15 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_level_0_graph_with_simple_select_ne expect_top_3(7, {7, 3, 2}); expect_top_3(8, {4, 3, 1}); expect_top_3(9, {7, 3, 2}); + + set_filter({2,3,4,6}); + expect_top_3(2, {2, 3}); + expect_top_3(4, {4, 3}); + expect_top_3(5, {6, 2}); + expect_top_3(6, {6, 2}); + expect_top_3(7, {3, 2}); + expect_top_3(8, {4, 3}); + expect_top_3(9, {3, 2}); } TEST_F(HnswIndexTest, 2d_vectors_inserted_and_removed) @@ -500,15 +521,31 @@ TEST_F(HnswIndexTest, shrink_called_heuristic) TEST(LevelGeneratorTest, gives_various_levels) { InvLogLevelGenerator generator(4); - EXPECT_EQ(2u, generator.max_level()); - EXPECT_EQ(1u, generator.max_level()); - EXPECT_EQ(0u, generator.max_level()); - EXPECT_EQ(1u, generator.max_level()); - EXPECT_EQ(0u, generator.max_level()); - EXPECT_EQ(1u, generator.max_level()); - EXPECT_EQ(0u, generator.max_level()); - EXPECT_EQ(0u, generator.max_level()); - EXPECT_EQ(0u, generator.max_level()); + std::vector<uint32_t> got_levels(16); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 2, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0 + })); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + })); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0 + })); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1 + })); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 2 + })); + for (auto & v : got_levels) { v = generator.max_level(); } + EXPECT_EQ(got_levels, std::vector<uint32_t>({ + 0, 1, 1, 0, 3, 1, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0 + })); uint32_t left = 1000000; std::vector<uint32_t> hist; @@ -528,4 +565,60 @@ TEST(LevelGeneratorTest, gives_various_levels) EXPECT_TRUE(hist.size() < 14); } +class TwoPhaseTest : public HnswIndexTest { +public: + TwoPhaseTest() : HnswIndexTest() { + init(true); + vectors.set(4, {1, 3}).set(5, {13, 3}).set(6, {7, 13}) + .set(1, {3, 7}).set(2, {7, 1}).set(3, {11, 7}) + .set(7, {6, 5}).set(8, {5, 5}).set(9, {6, 6}); + } + using UP = std::unique_ptr<PrepareResult>; + UP prepare_add(uint32_t docid, uint32_t max_level = 0) { + level_generator->level = max_level; + vespalib::GenerationHandler::Guard dummy; + auto vector = vectors.get_vector(docid); + return index->prepare_add_document(docid, vector, dummy); + } + void complete_add(uint32_t docid, UP up) { + index->complete_add_document(docid, std::move(up)); + commit(); + } +}; + +TEST_F(TwoPhaseTest, two_phase_add) +{ + add_document(1); + add_document(2); + add_document(3); + expect_entry_point(1, 0); + add_document(4, 1); + add_document(5, 1); + add_document(6, 2); + expect_entry_point(6, 2); + + expect_level_0(1, {2,4,6}); + expect_level_0(2, {1,3,4,5}); + expect_level_0(3, {2,5,6}); + + expect_levels(4, {{1,2}, {5,6}}); + expect_levels(5, {{2,3}, {4,6}}); + expect_levels(6, {{1,3}, {4,5}, {}}); + + auto up = prepare_add(7, 1); + // simulate things happening while 7 is in progress: + add_document(8); // added + remove_document(1); // removed + remove_document(5); + vectors.set(5, {8, 14}); // updated and moved + add_document(5, 2); + add_document(9, 1); // added + complete_add(7, std::move(up)); + + // 1 filtered out because it was removed + // 5 filtered out because it was updated + expect_levels(7, {{2}, {4}}); +} + + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp new file mode 100644 index 00000000000..4dec9550f6f --- /dev/null +++ b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp @@ -0,0 +1,348 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <stdio.h> +#include <unistd.h> +#include <chrono> +#include <cstdlib> +#include <future> +#include <vector> + +#include <vespa/eval/tensor/dense/typed_cells.h> +#include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchlib/tensor/distance_functions.h> +#include <vespa/searchlib/tensor/doc_vector_access.h> +#include <vespa/searchlib/tensor/hnsw_index.h> +#include <vespa/searchlib/tensor/inv_log_level_generator.h> +#include <vespa/searchlib/tensor/random_level_generator.h> +#include <vespa/vespalib/data/input.h> +#include <vespa/vespalib/data/memory_input.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/blockingthreadstackexecutor.h> +#include <vespa/vespalib/util/generationhandler.h> +#include <vespa/vespalib/util/lambdatask.h> + +#include <vespa/log/log.h> +LOG_SETUP("stress_hnsw_mt"); + +using namespace search::tensor; +using namespace vespalib::slime; +using search::BitVector; +using vespalib::GenerationHandler; +using vespalib::MemoryUsage; +using vespalib::Slime; + +#define NUM_DIMS 128 +#define NUM_POSSIBLE_V 1000000 +#define NUM_POSSIBLE_DOCS 30000 +#define NUM_OPS 1000000 + +class RndGen { +private: + std::mt19937_64 urng; + std::uniform_real_distribution<double> uf; +public: + RndGen() : urng(0x1234deadbeef5678uLL), uf(0.0, 1.0) {} + + double nextUniform() { + return uf(urng); + } +}; + +using ConstVectorRef = vespalib::ConstArrayRef<float>; + +struct MallocPointVector { + float v[NUM_DIMS]; + operator ConstVectorRef() const { return ConstVectorRef(v, NUM_DIMS); } +}; +static MallocPointVector *aligned_alloc_pv(size_t num) { + size_t num_bytes = num * sizeof(MallocPointVector); + double mega_bytes = num_bytes / (1024.0*1024.0); + fprintf(stderr, "allocate %.2f MB of vectors\n", mega_bytes); + char *mem = (char *)malloc(num_bytes + 512); + mem += 512; + size_t val = (size_t)mem; + size_t unalign = val % 512; + mem -= unalign; + return reinterpret_cast<MallocPointVector *>(mem); +} + +void read_vector_file(MallocPointVector *p) { + std::string data_set = "sift"; + std::string data_dir = "."; + char *home = getenv("HOME"); + if (home) { + data_dir = home; + data_dir += "/" + data_set; + } + std::string fn = data_dir + "/" + data_set + "_base.fvecs"; + int fd = open(fn.c_str(), O_RDONLY); + if (fd < 0) { + perror(fn.c_str()); + exit(1); + } + int d; + size_t rv; + fprintf(stderr, "reading %u vectors from %s\n", NUM_POSSIBLE_V, fn.c_str()); + for (uint32_t i = 0; i < NUM_POSSIBLE_V; ++i) { + rv = read(fd, &d, 4); + ASSERT_EQ(rv, 4u); + ASSERT_EQ(d, NUM_DIMS); + rv = read(fd, &p[i].v, NUM_DIMS*sizeof(float)); + ASSERT_EQ(rv, sizeof(MallocPointVector)); + } + close(fd); + fprintf(stderr, "reading %u vectors OK\n", NUM_POSSIBLE_V); +} + +class MyDocVectorStore : public DocVectorAccess { +private: + MallocPointVector *_vectors; +public: + MyDocVectorStore() { + _vectors = aligned_alloc_pv(NUM_POSSIBLE_DOCS); + } + MyDocVectorStore& set(uint32_t docid, ConstVectorRef vec) { + assert(docid < NUM_POSSIBLE_DOCS); + memcpy(&_vectors[docid], vec.cbegin(), sizeof(MallocPointVector)); + return *this; + } + vespalib::tensor::TypedCells get_vector(uint32_t docid) const override { + assert(docid < NUM_POSSIBLE_DOCS); + ConstVectorRef ref(_vectors[docid]); + return vespalib::tensor::TypedCells(ref); + } +}; + +using FloatSqEuclideanDistance = SquaredEuclideanDistance<float>; +using HnswIndexUP = std::unique_ptr<HnswIndex>; + +class Stressor : public ::testing::Test { +private: + struct LoadedVectors { + MallocPointVector *pv_storage; + void load() { + pv_storage = aligned_alloc_pv(size()); + read_vector_file(pv_storage); + } + size_t size() const { return NUM_POSSIBLE_V; } + vespalib::ConstArrayRef<float> operator[] (size_t i) { + return pv_storage[i]; + } + } loaded_vectors; +public: + BitVector::UP in_progress; + std::mutex in_progress_lock; + BitVector::UP existing_ids; + RndGen rng; + MyDocVectorStore vectors; + GenerationHandler gen_handler; + HnswIndexUP index; + vespalib::BlockingThreadStackExecutor multi_prepare_workers; + vespalib::BlockingThreadStackExecutor write_thread; + + using PrepUP = std::unique_ptr<PrepareResult>; + using ReadGuard = GenerationHandler::Guard; + using PrepareFuture = std::future<PrepUP>; + + // union of data required by tasks + struct TaskBase : vespalib::Executor::Task { + Stressor &parent; + uint32_t docid; + ConstVectorRef vec; + PrepareFuture prepare_future; + ReadGuard read_guard; + + TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, PrepareFuture f, ReadGuard g) + : parent(p), docid(d), vec(v), prepare_future(std::move(f)), read_guard(g) + {} + TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, ReadGuard g) // prepare add + : TaskBase(p, d, v, PrepareFuture(), g) {} + TaskBase(Stressor &p, uint32_t d, ConstVectorRef v, PrepareFuture r) // complete add+update + : TaskBase(p, d, v, std::move(r), ReadGuard()) {} + TaskBase(Stressor &p, uint32_t d) // complete remove + : TaskBase(p, d, ConstVectorRef(), PrepareFuture(), ReadGuard()) {} + + ~TaskBase() {} + }; + + struct PrepareAddTask : TaskBase { + using TaskBase::TaskBase; + std::promise<PrepUP> result_promise; + auto get_result_future() { + return result_promise.get_future(); + } + void run() override { + auto v = vespalib::tensor::TypedCells(vec); + auto up = parent.index->prepare_add_document(docid, v, read_guard); + result_promise.set_value(std::move(up)); + } + }; + + struct CompleteAddTask : TaskBase { + using TaskBase::TaskBase; + void run() override { + auto prepare_result = prepare_future.get(); + parent.vectors.set(docid, vec); + parent.index->complete_add_document(docid, std::move(prepare_result)); + parent.existing_ids->setBit(docid); + parent.commit(docid); + } + }; + + struct CompleteRemoveTask : TaskBase { + using TaskBase::TaskBase; + void run() override { + parent.index->remove_document(docid); + parent.existing_ids->clearBit(docid); + parent.commit(docid); + } + }; + + struct CompleteUpdateTask : TaskBase { + using TaskBase::TaskBase; + void run() override { + auto prepare_result = prepare_future.get(); + parent.index->remove_document(docid); + parent.vectors.set(docid, vec); + parent.index->complete_add_document(docid, std::move(prepare_result)); + EXPECT_EQ(parent.existing_ids->testBit(docid), true); + parent.commit(docid); + } + }; + + Stressor() + : loaded_vectors(), + in_progress(BitVector::create(NUM_POSSIBLE_DOCS)), + existing_ids(BitVector::create(NUM_POSSIBLE_DOCS)), + rng(), + vectors(), + gen_handler(), + index(), + multi_prepare_workers(10, 128*1024, 50), + write_thread(1, 128*1024, 500) + { + loaded_vectors.load(); + } + + ~Stressor() {} + + void init() { + uint32_t m = 16; + index = std::make_unique<HnswIndex>(vectors, std::make_unique<FloatSqEuclideanDistance>(), + std::make_unique<InvLogLevelGenerator>(m), + HnswIndex::Config(2*m, m, 200, true)); + } + size_t get_rnd(size_t size) { + return rng.nextUniform() * size; + } + void add_document(uint32_t docid) { + size_t vec_num = get_rnd(loaded_vectors.size()); + ConstVectorRef vec = loaded_vectors[vec_num]; + auto guard = take_read_guard(); + auto prepare_task = std::make_unique<PrepareAddTask>(*this, docid, vec, guard); + auto complete_task = std::make_unique<CompleteAddTask>(*this, docid, vec, prepare_task->get_result_future()); + auto r = multi_prepare_workers.execute(std::move(prepare_task)); + ASSERT_EQ(r.get(), nullptr); + r = write_thread.execute(std::move(complete_task)); + ASSERT_EQ(r.get(), nullptr); + } + void remove_document(uint32_t docid) { + auto task = std::make_unique<CompleteRemoveTask>(*this, docid); + auto r = write_thread.execute(std::move(task)); + ASSERT_EQ(r.get(), nullptr); + } + void update_document(uint32_t docid) { + size_t vec_num = get_rnd(loaded_vectors.size()); + ConstVectorRef vec = loaded_vectors[vec_num]; + auto guard = take_read_guard(); + auto prepare_task = std::make_unique<PrepareAddTask>(*this, docid, vec, guard); + auto complete_task = std::make_unique<CompleteUpdateTask>(*this, docid, vec, prepare_task->get_result_future()); + auto r = multi_prepare_workers.execute(std::move(prepare_task)); + ASSERT_EQ(r.get(), nullptr); + r = write_thread.execute(std::move(complete_task)); + ASSERT_EQ(r.get(), nullptr); + } + void commit(uint32_t docid) { + index->transfer_hold_lists(gen_handler.getCurrentGeneration()); + gen_handler.incGeneration(); + gen_handler.updateFirstUsedGeneration(); + index->trim_hold_lists(gen_handler.getFirstUsedGeneration()); + std::lock_guard<std::mutex> guard(in_progress_lock); + in_progress->clearBit(docid); + // printf("commit: %u\n", docid); + } + void gen_operation() { + uint32_t docid = get_rnd(NUM_POSSIBLE_DOCS); + { + std::lock_guard<std::mutex> guard(in_progress_lock); + while (in_progress->testBit(docid)) { + docid = get_rnd(NUM_POSSIBLE_DOCS); + } + in_progress->setBit(docid); + } + if (existing_ids->testBit(docid)) { + if (get_rnd(100) < 70) { + // printf("start remove op: %u\n", docid); + remove_document(docid); + } else { + // printf("start update op: %u\n", docid); + update_document(docid); + } + } else { + // printf("start add op: %u\n", docid); + add_document(docid); + } + } + GenerationHandler::Guard take_read_guard() { + return gen_handler.takeGuard(); + } + MemoryUsage memory_usage() const { + return index->memory_usage(); + } + uint32_t count_in_progress() { + std::lock_guard<std::mutex> guard(in_progress_lock); + in_progress->invalidateCachedCount(); + return in_progress->countTrueBits(); + } + std::string json_state() { + Slime actualSlime; + SlimeInserter inserter(actualSlime); + index->get_state(inserter); + vespalib::SimpleBuffer buf; + vespalib::slime::JsonFormat::encode(actualSlime, buf, false); + return buf.get().make_string(); + } +}; + + +TEST_F(Stressor, stress) +{ + init(); + for (int i = 0; i < NUM_OPS; ++i) { + gen_operation(); + if (i % 1000 == 0) { + uint32_t cnt = count_in_progress(); + fprintf(stderr, "generating operations %d / %d; in progress: %u ops\n", + i, NUM_OPS, cnt); + auto r = write_thread.execute(vespalib::makeLambdaTask([&]() { + EXPECT_TRUE(index->check_link_symmetry()); + })); + EXPECT_EQ(r.get(), nullptr); + } + } + fprintf(stderr, "waiting for queued operations...\n"); + multi_prepare_workers.sync(); + write_thread.sync(); + EXPECT_EQ(count_in_progress(), 0); + EXPECT_TRUE(index->check_link_symmetry()); + fprintf(stderr, "HNSW index state after test:\n%s\n", json_state().c_str()); + existing_ids->invalidateCachedCount(); + fprintf(stderr, "Expected valid nodes: %u\n", existing_ids->countTrueBits()); + fprintf(stderr, "all done.\n"); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp index 7aa78fbe06b..2db6437664e 100644 --- a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp @@ -37,7 +37,7 @@ using V = std::vector<uint32_t>; void populate(HnswGraph &graph) { // no 0 graph.make_node_for_document(1, 1); - graph.make_node_for_document(2, 2); + auto er = graph.make_node_for_document(2, 2); // no 3 graph.make_node_for_document(4, 2); graph.make_node_for_document(5, 0); @@ -49,7 +49,7 @@ void populate(HnswGraph &graph) { graph.set_link_array(6, 0, V{1, 2, 4}); graph.set_link_array(2, 1, V{4}); graph.set_link_array(4, 1, V{2}); - graph.set_entry_node(2, 1); + graph.set_entry_node({2, er, 1}); } void modify(HnswGraph &graph) { @@ -63,7 +63,7 @@ void modify(HnswGraph &graph) { graph.set_link_array(4, 1, V{7}); graph.set_link_array(7, 1, V{4}); - graph.set_entry_node(4, 1); + graph.set_entry_node({4, graph.get_node_ref(4), 1}); } @@ -110,8 +110,9 @@ public: void expect_copy_as_populated() const { EXPECT_EQ(copy.size(), 7); - EXPECT_EQ(copy.entry_docid, 2); - EXPECT_EQ(copy.entry_level, 1); + auto entry = copy.get_entry_node(); + EXPECT_EQ(entry.docid, 2); + EXPECT_EQ(entry.level, 1); expect_empty_d(0); expect_empty_d(3); diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index b9e3af8c729..6ab70e11fc0 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -34,7 +34,7 @@ vespa_add_library(searchlib_attribute OBJECT defines.cpp diversity.cpp dociditerator.cpp - elementiterator.cpp + searchcontextelementiterator.cpp enumattribute.cpp enumattributesaver.cpp enumcomparator.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 14d33914d05..74a23db8b95 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -18,6 +18,8 @@ #include <vespa/searchlib/queryeval/dot_product_blueprint.h> #include <vespa/searchlib/queryeval/dot_product_search.h> #include <vespa/searchlib/queryeval/emptysearch.h> +#include <vespa/searchlib/queryeval/field_spec.hpp> +#include <vespa/searchlib/queryeval/filter_wrapper.h> #include <vespa/searchlib/queryeval/get_weight_from_node.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> @@ -28,7 +30,6 @@ #include <vespa/searchlib/queryeval/wand/parallel_weak_and_search.h> #include <vespa/searchlib/queryeval/weighted_set_term_blueprint.h> #include <vespa/searchlib/queryeval/weighted_set_term_search.h> -#include <vespa/searchlib/queryeval/field_spec.hpp> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/vespalib/util/regexp.h> #include <vespa/vespalib/util/stringfmt.h> @@ -62,6 +63,7 @@ using search::queryeval::CreateBlueprintVisitorHelper; using search::queryeval::DotProductBlueprint; using search::queryeval::FieldSpec; using search::queryeval::FieldSpecBaseList; +using search::queryeval::FilterWrapper; using search::queryeval::IRequestContext; using search::queryeval::NoUnpack; using search::queryeval::OrLikeSearch; @@ -126,21 +128,25 @@ public: .diversityCutoffStrict(diversityCutoffStrict)) {} - SearchIterator::UP - createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { assert(tfmda.size() == 1); return _search_context->createIterator(tfmda[0], strict); } - SearchIterator::UP - createSearch(fef::MatchData &md, bool strict) const override { + SearchIterator::UP createSearch(fef::MatchData &md, bool strict) const override { const State &state = getState(); assert(state.numFields() == 1); return _search_context->createIterator(state.field(0).resolve(md), strict); } - void - fetchPostings(const queryeval::ExecuteInfo &execInfo) override { + SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override { + (void) constraint; // We provide an iterator with exact results, so no need to take constraint into consideration. + auto wrapper = std::make_unique<FilterWrapper>(getState().numFields()); + wrapper->wrap(createLeafSearch(wrapper->tfmda(), strict)); + return wrapper; + } + + void fetchPostings(const queryeval::ExecuteInfo &execInfo) override { _search_context->fetchPostings(execInfo); } @@ -163,7 +169,8 @@ AttributeFieldBlueprint::visitMembers(vespalib::ObjectVisitor &visitor) const template <bool is_strict> struct LocationPreFilterIterator : public OrLikeSearch<is_strict, NoUnpack> { - LocationPreFilterIterator(const std::vector<SearchIterator *> &children) : OrLikeSearch<is_strict, NoUnpack>(children, NoUnpack()) {} + LocationPreFilterIterator(OrSearch::Children children) + : OrLikeSearch<is_strict, NoUnpack>(std::move(children), NoUnpack()) {} void doUnpack(uint32_t) override {} }; @@ -209,14 +216,14 @@ public: SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { - std::vector<SearchIterator *> children; + OrSearch::Children children; for (auto it(_rangeSearches.begin()), mt(_rangeSearches.end()); it != mt; it++) { - children.push_back((*it)->createIterator(tfmda[0], strict).release()); + children.push_back((*it)->createIterator(tfmda[0], strict)); } if (strict) { - return std::make_unique<LocationPreFilterIterator<true>>(children); + return std::make_unique<LocationPreFilterIterator<true>>(std::move(children)); } else { - return std::make_unique<LocationPreFilterIterator<false>>(children); + return std::make_unique<LocationPreFilterIterator<false>>(std::move(children)); } } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp index af3fbca6943..121cb736471 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp @@ -162,6 +162,7 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr auto child_tfmd = match_data->resolveTermField(handle); std::vector<queryeval::SearchIterator*> children(_contexts.size()); for (size_t i = 0; i < _contexts.size(); ++i) { + // TODO: pass ownership with unique_ptr children[i] = _contexts[i]->createIterator(child_tfmd, true).release(); } return queryeval::SearchIterator::UP(queryeval::WeightedSetTermSearch::create(children, tfmd, _weights, std::move(match_data))); diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp index 17df4628606..d1226a07703 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp @@ -110,12 +110,67 @@ AttributePostingListIteratorT<PL>::doSeek(uint32_t docId) } } +namespace { + +template <typename> struct is_tree_iterator; + +template <typename P> +struct is_tree_iterator<DocIdIterator<P>> { + static constexpr bool value = false; +}; + +template <typename P> +struct is_tree_iterator<DocIdMinMaxIterator<P>> { + static constexpr bool value = false; +}; + +template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT> +struct is_tree_iterator<vespalib::btree::BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>> { + static constexpr bool value = true; +}; + +template <typename PL> +inline constexpr bool is_tree_iterator_v = is_tree_iterator<PL>::value; + +template <typename PL> +void get_hits_helper(BitVector& result, PL& iterator, uint32_t end_id) +{ + auto end_itr = iterator; + if (end_itr.valid() && end_itr.getKey() < end_id) { + end_itr.seek(end_id); + } + iterator.foreach_key_range(end_itr, [&](uint32_t key) { result.setBit(key); }); + iterator = end_itr; +} + +template <typename PL> +void or_hits_helper(BitVector& result, PL& iterator, uint32_t end_id) +{ + auto end_itr = iterator; + if (end_itr.valid() && end_itr.getKey() < end_id) { + end_itr.seek(end_id); + } + iterator.foreach_key_range(end_itr, [&](uint32_t key) + { + if (!result.testBit(key)) { + result.setBit(key); + } + }); + iterator = end_itr; +} + +} + template <typename PL> std::unique_ptr<BitVector> AttributePostingListIteratorT<PL>::get_hits(uint32_t begin_id) { BitVector::UP result(BitVector::create(begin_id, getEndId())); - for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { - result->setBit(_iterator.getKey()); + if constexpr (is_tree_iterator_v<PL>) { + get_hits_helper(*result, _iterator, getEndId()); + } else { + for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { + result->setBit(_iterator.getKey()); + } } result->invalidateCachedCount(); return result; @@ -125,9 +180,13 @@ template <typename PL> void AttributePostingListIteratorT<PL>::or_hits_into(BitVector & result, uint32_t begin_id) { (void) begin_id; - for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { - if ( ! result.testBit(_iterator.getKey()) ) { - result.setBit(_iterator.getKey()); + if constexpr (is_tree_iterator_v<PL>) { + or_hits_helper(result, _iterator, getEndId()); + } else { + for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { + if ( ! result.testBit(_iterator.getKey()) ) { + result.setBit(_iterator.getKey()); + } } } result.invalidateCachedCount(); @@ -143,8 +202,12 @@ template <typename PL> std::unique_ptr<BitVector> FilterAttributePostingListIteratorT<PL>::get_hits(uint32_t begin_id) { BitVector::UP result(BitVector::create(begin_id, getEndId())); - for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { - result->setBit(_iterator.getKey()); + if constexpr (is_tree_iterator_v<PL>) { + get_hits_helper(*result, _iterator, getEndId()); + } else { + for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { + result->setBit(_iterator.getKey()); + } } result->invalidateCachedCount(); return result; @@ -154,9 +217,13 @@ template <typename PL> void FilterAttributePostingListIteratorT<PL>::or_hits_into(BitVector & result, uint32_t begin_id) { (void) begin_id; - for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { - if ( ! result.testBit(_iterator.getKey()) ) { - result.setBit(_iterator.getKey()); + if constexpr (is_tree_iterator_v<PL>) { + or_hits_helper(result, _iterator, getEndId()); + } else { + for (; _iterator.valid() && _iterator.getKey() < getEndId(); ++_iterator) { + if ( ! result.testBit(_iterator.getKey()) ) { + result.setBit(_iterator.getKey()); + } } } result.invalidateCachedCount(); diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index c573f0b4210..f6c39b9570d 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -73,10 +73,9 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg) predicateParams.setBounds(cfg.lowerbound, cfg.upperbound); predicateParams.setDensePostingListThreshold(cfg.densepostinglistthreshold); retval.setPredicateParams(predicateParams); - if (cfg.index.hnsw.enabled) { - using CfgDm = AttributesConfig::Attribute::Index::Hnsw::Distancemetric; - DistanceMetric dm; - switch (cfg.index.hnsw.distancemetric) { + using CfgDm = AttributesConfig::Attribute::Distancemetric; + DistanceMetric dm(DistanceMetric::Euclidean); + switch (cfg.distancemetric) { case CfgDm::EUCLIDEAN: dm = DistanceMetric::Euclidean; break; @@ -86,7 +85,9 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg) case CfgDm::GEODEGREES: dm = DistanceMetric::GeoDegrees; break; - } + } + retval.set_distance_metric(dm); + if (cfg.index.hnsw.enabled) { retval.set_hnsw_index_params(HnswIndexParams(cfg.index.hnsw.maxlinkspernode, cfg.index.hnsw.neighborstoexploreatinsert, dm)); diff --git a/searchlib/src/vespa/searchlib/attribute/elementiterator.cpp b/searchlib/src/vespa/searchlib/attribute/elementiterator.cpp deleted file mode 100644 index 3be4c478376..00000000000 --- a/searchlib/src/vespa/searchlib/attribute/elementiterator.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "elementiterator.h" -#include <vespa/searchcommon/attribute/i_search_context.h> -#include <vespa/searchlib/fef/termfieldmatchdata.h> - -using search::fef::TermFieldMatchDataPosition; - -namespace search::attribute { - -void -ElementIterator::doSeek(uint32_t docid) { - _search->doSeek(docid); - setDocId(_search->getDocId()); -} - -void -ElementIterator::doUnpack(uint32_t docid) { - _tfmd.reset(docid); - int32_t weight(0); - for (int32_t id = _searchContext.find(docid, 0, weight); id >= 0; id = _searchContext.find(docid, id+1, weight)) { - _tfmd.appendPosition(TermFieldMatchDataPosition(id, 0, weight, 1)); - } -} - -vespalib::Trinary -ElementIterator::is_strict() const { - return _search->is_strict(); -} - -void -ElementIterator::initRange(uint32_t beginid, uint32_t endid) { - SearchIterator::initRange(beginid, endid); - _search->initRange(beginid, endid); - setDocId(_search->getDocId()); -} - -ElementIterator::ElementIterator(SearchIterator::UP search, const ISearchContext & sc, fef::TermFieldMatchData & tfmd) - : _search(std::move(search)), - _searchContext(sc), - _tfmd(tfmd) -{ -} - -ElementIterator::~ElementIterator() = default; - -} diff --git a/searchlib/src/vespa/searchlib/attribute/elementiterator.h b/searchlib/src/vespa/searchlib/attribute/elementiterator.h deleted file mode 100644 index 232139751b6..00000000000 --- a/searchlib/src/vespa/searchlib/attribute/elementiterator.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/searchlib/queryeval/searchiterator.h> - -namespace search::fef { class TermFieldMatchData; } -namespace search::attribute { - -class ISearchContext; - -class ElementIterator : public queryeval::SearchIterator -{ -private: - SearchIterator::UP _search; - const ISearchContext & _searchContext; - fef::TermFieldMatchData & _tfmd; - - void doSeek(uint32_t docid) override; - void doUnpack(uint32_t docid) override; - Trinary is_strict() const override; - void initRange(uint32_t beginid, uint32_t endid) override; -public: - ElementIterator(SearchIterator::UP search, const ISearchContext & sc, fef::TermFieldMatchData & tfmd); - ~ElementIterator(); -}; - -} diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index b527f89b224..58a220922fd 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -19,9 +19,7 @@ EnumAttribute(const vespalib::string &baseFileName, } template <typename B> -EnumAttribute<B>::~EnumAttribute() -{ -} +EnumAttribute<B>::~EnumAttribute() = default; template <typename B> void EnumAttribute<B>::load_enum_store(LoadedVector& loaded) diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 42738b44461..11a923a2b80 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -17,6 +17,7 @@ #include <vespa/vespalib/datastore/unique_store_string_allocator.h> #include <vespa/vespalib/util/buffer.h> #include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/stllike/allocator.h> #include <vespa/vespalib/util/stringfmt.h> #include <cmath> @@ -114,8 +115,8 @@ public: private: AllocatorType& _allocator; vespalib::datastore::IUniqueStoreDictionary& _dict; - std::vector<EntryRef> _refs; - std::vector<uint32_t> _payloads; + std::vector<EntryRef, vespalib::allocator_large<EntryRef>> _refs; + std::vector<uint32_t, vespalib::allocator_large<uint32_t>> _payloads; public: NonEnumeratedLoader(AllocatorType& allocator, vespalib::datastore::IUniqueStoreDictionary& dict) diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index 5f49111f77b..bc0d965bcc1 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -44,7 +44,6 @@ ImportedSearchContext::ImportedSearchContext( _target_search_context(_target_attribute.createSearchContext(std::move(term), params)), _targetLids(_reference_attribute.getTargetLids()), _merger(_reference_attribute.getCommittedDocIdLimit()), - _fetchPostingsDone(false), _params(params) { } @@ -239,15 +238,11 @@ ImportedSearchContext::considerAddSearchCacheEntry() } void ImportedSearchContext::fetchPostings(const queryeval::ExecuteInfo &execInfo) { - assert(!_fetchPostingsDone); - _fetchPostingsDone = true; if (!_searchCacheLookup) { _target_search_context->fetchPostings(execInfo); - if (execInfo.isStrict() - || (_target_attribute.getIsFastSearch() && execInfo.hitRate() > 0.01)) - { - makeMergedPostings(_target_attribute.getIsFilter()); - considerAddSearchCacheEntry(); + if (!_merger.merge_done() && (execInfo.isStrict() || (_target_attribute.getIsFastSearch() && execInfo.hitRate() > 0.01))) { + makeMergedPostings(_target_attribute.getIsFilter()); + considerAddSearchCacheEntry(); } } } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index 1c73ac6c8c2..4c3b6a89a14 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -37,7 +37,6 @@ class ImportedSearchContext : public ISearchContext { std::unique_ptr<ISearchContext> _target_search_context; TargetLids _targetLids; PostingListMerger<int32_t> _merger; - bool _fetchPostingsDone; SearchContextParams _params; uint32_t getTargetLid(uint32_t lid) const { diff --git a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h index 6a10ba73951..1c2e6583ad9 100644 --- a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h +++ b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.h @@ -62,6 +62,8 @@ public: { if (__builtin_expect(key < limit, true)) { bv.setBit(key); } }); } + bool merge_done() const { return hasArray() || hasBitVector(); } + // Until diversity handling has been rewritten PostingVector &getWritableArray() { return _array; } StartVector &getWritableStartPos() { return _startPos; } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 69450acd98d..8cd7a7064f6 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -107,7 +107,6 @@ protected: * Synthetic posting lists for range search, in array or bitvector form */ PostingListMerger<DataT> _merger; - bool _fetchPostingsDone; static const long MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION = 100; static const long MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 20; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index 4cd8db9010a..09e5a9da5bc 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -25,8 +25,7 @@ PostingListSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uin uint32_t minBvDocFreq, bool useBitVector, const ISearchContext &searchContext) : PostingListSearchContext(dictionary, docIdLimit, numValues, hasWeight, esb, minBvDocFreq, useBitVector, searchContext), _postingList(postingList), - _merger(docIdLimit), - _fetchPostingsDone(false) + _merger(docIdLimit) { } @@ -116,22 +115,18 @@ template <typename DataT> void PostingListSearchContextT<DataT>::fetchPostings(const queryeval::ExecuteInfo & execInfo) { - if (_fetchPostingsDone) return; - - _fetchPostingsDone = true; - - if (_uniqueValues < 2u) return; - - if (execInfo.isStrict() && !fallbackToFiltering()) { - size_t sum(countHits()); - if (sum < _docIdLimit / 64) { - _merger.reserveArray(_uniqueValues, sum); - fillArray(); - } else { - _merger.allocBitVector(); - fillBitVector(); + if (!_merger.merge_done() && _uniqueValues >= 2u) { + if (execInfo.isStrict() && !fallbackToFiltering()) { + size_t sum(countHits()); + if (sum < _docIdLimit / 64) { + _merger.reserveArray(_uniqueValues, sum); + fillArray(); + } else { + _merger.allocBitVector(); + fillBitVector(); + } + _merger.merge(); } - _merger.merge(); } } @@ -141,12 +136,12 @@ void PostingListSearchContextT<DataT>::diversify(bool forward, size_t wanted_hits, const IAttributeVector &diversity_attr, size_t max_per_group, size_t cutoff_groups, bool cutoff_strict) { - assert(!_fetchPostingsDone); - _fetchPostingsDone = true; - _merger.reserveArray(128, wanted_hits); - diversity::diversify(forward, _lowerDictItr, _upperDictItr, _postingList, wanted_hits, diversity_attr, - max_per_group, cutoff_groups, cutoff_strict, _merger.getWritableArray(), _merger.getWritableStartPos()); - _merger.merge(); + if (!_merger.merge_done()) { + _merger.reserveArray(128, wanted_hits); + diversity::diversify(forward, _lowerDictItr, _upperDictItr, _postingList, wanted_hits, diversity_attr, + max_per_group, cutoff_groups, cutoff_strict, _merger.getWritableArray(), _merger.getWritableStartPos()); + _merger.merge(); + } } @@ -155,7 +150,6 @@ SearchIterator::UP PostingListSearchContextT<DataT>:: createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) { - assert(_fetchPostingsDone); if (_uniqueValues == 0u) { return std::make_unique<EmptySearch>(); } diff --git a/searchlib/src/vespa/searchlib/attribute/searchcontextelementiterator.cpp b/searchlib/src/vespa/searchlib/attribute/searchcontextelementiterator.cpp new file mode 100644 index 00000000000..87c09186746 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/searchcontextelementiterator.cpp @@ -0,0 +1,42 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "searchcontextelementiterator.h" +#include <vespa/searchcommon/attribute/i_search_context.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> + +using search::fef::TermFieldMatchDataPosition; + +namespace search::attribute { + +void +SearchContextElementIterator::getElementIds(uint32_t docId, std::vector<uint32_t> & elementIds) { + int32_t weight(0); + for (int32_t id = _searchContext.find(docId, 0, weight); id >= 0; id = _searchContext.find(docId, id+1, weight)) { + elementIds.push_back(id); + } +} +void +SearchContextElementIterator::mergeElementIds(uint32_t docId, std::vector<uint32_t> & elementIds) { + size_t toKeep(0); + int32_t id(-1); + int32_t weight(0); + for (int32_t candidate : elementIds) { + if (candidate > id) { + id = _searchContext.find(docId, candidate, weight); + if (id < 0) break; + } + if (id == candidate) { + elementIds[toKeep++] = candidate; + } + } + elementIds.resize(toKeep); +} + +SearchContextElementIterator::SearchContextElementIterator(queryeval::SearchIterator::UP search, const ISearchContext & sc) + : ElementIterator(std::move(search)), + _searchContext(sc) +{} + +SearchContextElementIterator::~SearchContextElementIterator() = default; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/searchcontextelementiterator.h b/searchlib/src/vespa/searchlib/attribute/searchcontextelementiterator.h new file mode 100644 index 00000000000..c6c5aac1cd2 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/searchcontextelementiterator.h @@ -0,0 +1,23 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/queryeval/elementiterator.h> + +namespace search::fef { class TermFieldMatchData; } +namespace search::attribute { + +class ISearchContext; + +class SearchContextElementIterator : public queryeval::ElementIterator +{ +private: + const ISearchContext & _searchContext; +public: + void getElementIds(uint32_t docId, std::vector<uint32_t> & elementIds) override; + void mergeElementIds(uint32_t docId, std::vector<uint32_t> & elementIds) override; + SearchContextElementIterator(queryeval::SearchIterator::UP search, const ISearchContext & sc); + ~SearchContextElementIterator() override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt index 36264a2035b..5d30260a169 100644 --- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt @@ -18,6 +18,7 @@ vespa_add_library(searchlib_common OBJECT locationiterators.cpp mapnames.cpp matching_elements.cpp + matching_elements_fields.cpp packets.cpp partialbitvector.cpp resultset.cpp @@ -26,7 +27,6 @@ vespa_add_library(searchlib_common OBJECT sortdata.cpp sortresults.cpp sortspec.cpp - struct_field_mapper.cpp threaded_compactable_lid_space.cpp tunefileinfo.cpp DEPENDS diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp index 3add7d1a328..659e3718a13 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp @@ -17,6 +17,28 @@ size_t computeCapacity(size_t capacity, size_t allocatedBytes) { return possibleCapacity; } +// This is to ensure that we only read size and capacity once during copy +// to ensure that they do not change unexpectedly under our feet due to resizing in different thread. +std::pair<BitVector::Index, BitVector::Index> +extract_size_size(const BitVector & bv) { + BitVector::Index size = bv.size(); + return std::pair<BitVector::Index, BitVector::Index>(size, size); +} + +std::pair<BitVector::Index, BitVector::Index> +extract_size_capacity(const AllocatedBitVector & bv) { + BitVector::Index size = bv.size(); + BitVector::Index capacity = bv.capacity(); + while (capacity < size) { + // Since size and capacity might be changed in another thread we need + // this fallback to avoid inconsistency during shrink. + std::atomic_thread_fence(std::memory_order_seq_cst); + size = bv.size(); + capacity = bv.capacity(); + } + return std::pair<BitVector::Index, BitVector::Index>(size, capacity); +} + } AllocatedBitVector::AllocatedBitVector(Index numberOfElements) : @@ -56,21 +78,21 @@ AllocatedBitVector::AllocatedBitVector(Index numberOfElements, Index capacityBit } AllocatedBitVector::AllocatedBitVector(const AllocatedBitVector & rhs) : - AllocatedBitVector(rhs, rhs.capacity()) + AllocatedBitVector(rhs, extract_size_capacity(rhs)) { } AllocatedBitVector::AllocatedBitVector(const BitVector & rhs) : - AllocatedBitVector(rhs, rhs.size()) + AllocatedBitVector(rhs, extract_size_size(rhs)) { } -AllocatedBitVector::AllocatedBitVector(const BitVector & rhs, Index capacity_) : +AllocatedBitVector::AllocatedBitVector(const BitVector & rhs, std::pair<Index, Index> size_capacity) : BitVector(), - _capacityBits(capacity_), - _alloc(allocatePaddedAndAligned(0, rhs.size(), capacity_)) + _capacityBits(size_capacity.second), + _alloc(allocatePaddedAndAligned(0, size_capacity.first, size_capacity.second)) { _capacityBits = computeCapacity(_capacityBits, _alloc.size()); - memcpy(_alloc.get(), rhs.getStart(), rhs.sizeBytes()); - init(_alloc.get(), 0, rhs.size()); + memcpy(_alloc.get(), rhs.getStart(), numBytes(size_capacity.first - rhs.getStartIndex())); + init(_alloc.get(), 0, size_capacity.first); setBit(size()); updateCount(); } diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h index c52c52354a1..5a7d2e634ea 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h @@ -73,7 +73,7 @@ private: BitVector::swap(rhs); } - AllocatedBitVector(const BitVector &other, Index capacity); + AllocatedBitVector(const BitVector &other, std::pair<Index, Index> size_capacity); /** * Prepare for potential reuse where new value might be filled in by diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index 96234e373dc..0a33e23de72 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -167,7 +167,7 @@ BitVector::countInterval(Range range_in) const ++endw; } if (startw < endw) { - res += IAccelrated::getAccelrator().populationCount(bitValues + startw, endw - startw); + res += IAccelrated::getAccelerator().populationCount(bitValues + startw, endw - startw); } if (partialEnd) { res += Optimized::popCount(bitValues[endw] & ~endBits(last)); @@ -185,13 +185,13 @@ BitVector::orWith(const BitVector & right) if (right.size() > 0) { ssize_t commonBytes = numActiveBytes(getStartIndex(), right.size()) - sizeof(Word); if (commonBytes > 0) { - IAccelrated::getAccelrator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); + IAccelrated::getAccelerator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); } Index last(right.size() - 1); getWordIndex(last)[0] |= (right.getWordIndex(last)[0] & ~endBits(last)); } } else { - IAccelrated::getAccelrator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes()); + IAccelrated::getAccelerator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes()); } repairEnds(); invalidateCachedCount(); @@ -216,7 +216,7 @@ BitVector::andWith(const BitVector & right) verifyInclusiveStart(*this, right); uint32_t commonBytes = std::min(getActiveBytes(), numActiveBytes(getStartIndex(), right.size())); - IAccelrated::getAccelrator().andBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); + IAccelrated::getAccelerator().andBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); if (right.size() < size()) { clearInterval(right.size(), size()); } @@ -235,13 +235,13 @@ BitVector::andNotWith(const BitVector& right) if (right.size() > 0) { ssize_t commonBytes = numActiveBytes(getStartIndex(), right.size()) - sizeof(Word); if (commonBytes > 0) { - IAccelrated::getAccelrator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); + IAccelrated::getAccelerator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); } Index last(right.size() - 1); getWordIndex(last)[0] &= ~(right.getWordIndex(last)[0] & ~endBits(last)); } } else { - IAccelrated::getAccelrator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes()); + IAccelrated::getAccelerator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes()); } repairEnds(); @@ -250,7 +250,7 @@ BitVector::andNotWith(const BitVector& right) void BitVector::notSelf() { - IAccelrated::getAccelrator().notBit(getActiveStart(), getActiveBytes()); + IAccelrated::getAccelerator().notBit(getActiveStart(), getActiveBytes()); setGuardBit(); invalidateCachedCount(); } diff --git a/searchlib/src/vespa/searchlib/common/bitvectoriterator.cpp b/searchlib/src/vespa/searchlib/common/bitvectoriterator.cpp index 9c17dfa6f74..18cece7b8b0 100644 --- a/searchlib/src/vespa/searchlib/common/bitvectoriterator.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvectoriterator.cpp @@ -2,7 +2,6 @@ #include "bitvectoriterator.h" #include <vespa/searchlib/queryeval/emptysearch.h> -#include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/objects/visit.h> @@ -39,12 +38,6 @@ BitVectorIterator::visitMembers(vespalib::ObjectVisitor &visitor) const visit(visitor, "termfieldmatchdata.docid", _tfmd.getDocId()); } -void -BitVectorIterator::doUnpack(uint32_t docId) -{ - _tfmd.resetOnlyDocId(docId); -} - template<bool inverse> class BitVectorIteratorT : public BitVectorIterator { public: diff --git a/searchlib/src/vespa/searchlib/common/bitvectoriterator.h b/searchlib/src/vespa/searchlib/common/bitvectoriterator.h index 6200837d449..8b1112de97e 100644 --- a/searchlib/src/vespa/searchlib/common/bitvectoriterator.h +++ b/searchlib/src/vespa/searchlib/common/bitvectoriterator.h @@ -4,11 +4,11 @@ #include "bitvector.h" #include <vespa/searchlib/queryeval/searchiterator.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> namespace search { namespace fef { class TermFieldMatchDataArray; } -namespace fef { class TermFieldMatchData; } class BitVectorIterator : public queryeval::SearchIterator { @@ -20,7 +20,9 @@ protected: const BitVector & _bv; private: void visitMembers(vespalib::ObjectVisitor &visitor) const override; - void doUnpack(uint32_t docId) override; + void doUnpack(uint32_t docId) override final { + _tfmd.resetOnlyDocId(docId); + } bool isBitVector() const override { return true; } fef::TermFieldMatchData &_tfmd; public: diff --git a/searchlib/src/vespa/searchlib/common/matching_elements.cpp b/searchlib/src/vespa/searchlib/common/matching_elements.cpp index 1a4653e267b..87049f7d843 100644 --- a/searchlib/src/vespa/searchlib/common/matching_elements.cpp +++ b/searchlib/src/vespa/searchlib/common/matching_elements.cpp @@ -9,19 +9,19 @@ MatchingElements::MatchingElements() = default; MatchingElements::~MatchingElements() = default; void -MatchingElements::add_matching_elements(uint32_t docid, const vespalib::string &struct_field_name, const std::vector<uint32_t> &elements) +MatchingElements::add_matching_elements(uint32_t docid, const vespalib::string &field_name, const std::vector<uint32_t> &elements) { - auto &list = _map[key_t(docid, struct_field_name)]; + auto &list = _map[key_t(docid, field_name)]; std::vector<uint32_t> new_list; std::set_union(list.begin(), list.end(), elements.begin(), elements.end(), std::back_inserter(new_list)); list = std::move(new_list); } const std::vector<uint32_t> & -MatchingElements::get_matching_elements(uint32_t docid, const vespalib::string &struct_field_name) const +MatchingElements::get_matching_elements(uint32_t docid, const vespalib::string &field_name) const { static const std::vector<uint32_t> empty; - auto res = _map.find(key_t(docid, struct_field_name)); + auto res = _map.find(key_t(docid, field_name)); if (res == _map.end()) { return empty; } diff --git a/searchlib/src/vespa/searchlib/common/matching_elements.h b/searchlib/src/vespa/searchlib/common/matching_elements.h index b31b258ea4c..60fe7167bfd 100644 --- a/searchlib/src/vespa/searchlib/common/matching_elements.h +++ b/searchlib/src/vespa/searchlib/common/matching_elements.h @@ -9,8 +9,8 @@ namespace search { /** - * Keeps track of which elements matched the query for a set of struct - * fields across multiple documents. + * Keeps track of which elements matched the query for a set of fields + * across multiple documents. **/ class MatchingElements { @@ -26,8 +26,8 @@ public: using UP = std::unique_ptr<MatchingElements>; - void add_matching_elements(uint32_t docid, const vespalib::string &struct_field_name, const std::vector<uint32_t> &elements); - const std::vector<uint32_t> &get_matching_elements(uint32_t docid, const vespalib::string &struct_field_name) const; + void add_matching_elements(uint32_t docid, const vespalib::string &field_name, const std::vector<uint32_t> &elements); + const std::vector<uint32_t> &get_matching_elements(uint32_t docid, const vespalib::string &field_name) const; }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/matching_elements_fields.cpp b/searchlib/src/vespa/searchlib/common/matching_elements_fields.cpp new file mode 100644 index 00000000000..b36e1eb333e --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/matching_elements_fields.cpp @@ -0,0 +1,10 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "matching_elements_fields.h" + +namespace search { + +MatchingElementsFields::MatchingElementsFields() = default; +MatchingElementsFields::~MatchingElementsFields() = default; + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/common/matching_elements_fields.h b/searchlib/src/vespa/searchlib/common/matching_elements_fields.h new file mode 100644 index 00000000000..f8bb0f373fe --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/matching_elements_fields.h @@ -0,0 +1,51 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <set> +#include <map> + +namespace search { + +/** + * Keeps track of a set of field names to calculate MatchingElements for. + * + * Also has mapping of the full name of struct fields into the name of the enclosing field. + * Example: + * A map<string, string> field "my_map" could contain two struct fields: "my_map.key" and "my_map.value". + **/ +class MatchingElementsFields { +private: + std::set<vespalib::string> _fields; + std::map<vespalib::string, vespalib::string> _struct_fields; + +public: + MatchingElementsFields(); + ~MatchingElementsFields(); + bool empty() const { return _fields.empty(); } + void add_field(const vespalib::string &field_name) { + _fields.insert(field_name); + } + void add_mapping(const vespalib::string &field_name, + const vespalib::string &struct_field_name) { + _fields.insert(field_name); + _struct_fields[struct_field_name] = field_name; + } + bool has_field(const vespalib::string &field_name) const { + return (_fields.count(field_name) > 0); + } + bool has_struct_field(const vespalib::string &struct_field_name) const { + return (_struct_fields.find(struct_field_name) != _struct_fields.end()); + } + const vespalib::string &get_enclosing_field(const vespalib::string &struct_field_name) const { + static const vespalib::string empty; + auto res = _struct_fields.find(struct_field_name); + if (res == _struct_fields.end()) { + return empty; + } + return res->second; + } +}; + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/common/struct_field_mapper.cpp b/searchlib/src/vespa/searchlib/common/struct_field_mapper.cpp deleted file mode 100644 index 849cfd06ade..00000000000 --- a/searchlib/src/vespa/searchlib/common/struct_field_mapper.cpp +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "struct_field_mapper.h" - -namespace search { - -StructFieldMapper::StructFieldMapper() = default; -StructFieldMapper::~StructFieldMapper() = default; - -} // namespace search diff --git a/searchlib/src/vespa/searchlib/common/struct_field_mapper.h b/searchlib/src/vespa/searchlib/common/struct_field_mapper.h deleted file mode 100644 index 1d0604daec3..00000000000 --- a/searchlib/src/vespa/searchlib/common/struct_field_mapper.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <set> -#include <map> - -namespace search { - -/** - * Keeps track of a set of struct field names and enables mapping the - * full name of struct subfields into the name of the enclosing struct - * field. - **/ -class StructFieldMapper -{ -private: - std::set<vespalib::string> _struct_fields; - std::map<vespalib::string,vespalib::string> _struct_subfields; - -public: - StructFieldMapper(); - ~StructFieldMapper(); - bool empty() const { return _struct_fields.empty(); } - void add_mapping(const vespalib::string &struct_field_name, - const vespalib::string &struct_subfield_name) - { - _struct_fields.insert(struct_field_name); - _struct_subfields[struct_subfield_name] = struct_field_name; - } - bool is_struct_field(const vespalib::string &field_name) const { - return (_struct_fields.count(field_name) > 0); - } - bool is_struct_subfield(const vespalib::string &field_name) const { - return (_struct_subfields.find(field_name) != _struct_subfields.end()); - } - const vespalib::string &get_struct_field(const vespalib::string &struct_subfield_name) const { - static const vespalib::string empty; - auto res = _struct_subfields.find(struct_subfield_name); - if (res == _struct_subfields.end()) { - return empty; - } - return res->second; - } -}; - -} // namespace search diff --git a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp index 4febe165665..8231d0b4cd7 100644 --- a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> #include <vespa/searchlib/queryeval/equiv_blueprint.h> +#include <vespa/searchlib/queryeval/filter_wrapper.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> @@ -69,10 +70,12 @@ void DiskTermBlueprint::fetchPostings(const queryeval::ExecuteInfo &execInfo) { (void) execInfo; - _hasEquivParent = areAnyParentsEquiv(getParent()); - _bitVector = _diskIndex.readBitVector(*_lookupRes); - if (!_useBitVector || !_bitVector) { - _postingHandle = _diskIndex.readPostingList(*_lookupRes); + if (!_fetchPostingsDone) { + _hasEquivParent = areAnyParentsEquiv(getParent()); + _bitVector = _diskIndex.readBitVector(*_lookupRes); + if (!_useBitVector || !_bitVector) { + _postingHandle = _diskIndex.readPostingList(*_lookupRes); + } } _fetchPostingsDone = true; } @@ -96,4 +99,17 @@ DiskTermBlueprint::createLeafSearch(const TermFieldMatchDataArray & tfmda, bool return search; } +SearchIterator::UP +DiskTermBlueprint::createFilterSearch(bool strict, FilterConstraint) const +{ + auto wrapper = std::make_unique<queryeval::FilterWrapper>(getState().numFields()); + auto & tfmda = wrapper->tfmda(); + if (_bitVector) { + wrapper->wrap(BitVectorIterator::create(_bitVector.get(), *tfmda[0], strict)); + } else { + wrapper->wrap(_postingHandle->createIterator(_lookupRes->counts, tfmda, _useBitVector)); + } + return wrapper; } + +} // namespace diff --git a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.h b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.h index 39ac27bc448..4d6ef4589df 100644 --- a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.h +++ b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.h @@ -41,6 +41,8 @@ public: std::unique_ptr<queryeval::SearchIterator> createLeafSearch(const fef::TermFieldMatchDataArray & tfmda, bool strict) const override; void fetchPostings(const queryeval::ExecuteInfo &execInfo) override; + + std::unique_ptr<queryeval::SearchIterator> createFilterSearch(bool strict, FilterConstraint) const override; }; } diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp index 847cd3623c3..f60bf69d908 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp @@ -57,7 +57,7 @@ Chunk::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const Compre Chunk::Chunk(uint32_t id, const Config & config) : _id(id), _lastSerial(static_cast<uint64_t>(-1l)), - _format(new ChunkFormatV2(config.getMaxBytes())) + _format(std::make_unique<ChunkFormatV2>(config.getMaxBytes())) { _lids.reserve(4096/sizeof(Entry)); } diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index a5bed4c33ce..fbbdcff3c5d 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -81,33 +81,26 @@ ChunkFormat::deserialize(const void * buffer, size_t len, bool skipcrc) uint32_t crc32(0); raw >> crc32; raw.rp(currPos); - ChunkFormat::UP format; if (version == ChunkFormatV1::VERSION) { if (skipcrc) { - format.reset(new ChunkFormatV1(raw)); + return std::make_unique<ChunkFormatV1>(raw); } else { - format.reset(new ChunkFormatV1(raw, crc32)); + return std::make_unique<ChunkFormatV1>(raw, crc32); } } else if (version == ChunkFormatV2::VERSION) { if (skipcrc) { - format.reset(new ChunkFormatV2(raw)); + return std::make_unique<ChunkFormatV2>(raw); } else { - format.reset(new ChunkFormatV2(raw, crc32)); + return std::make_unique<ChunkFormatV2>(raw, crc32); } } else { throw ChunkException(make_string("Unknown version %d", version), VESPA_STRLOC); } - return format; } -ChunkFormat::ChunkFormat() : - _dataBuf() -{ -} +ChunkFormat::ChunkFormat() = default; -ChunkFormat::~ChunkFormat() -{ -} +ChunkFormat::~ChunkFormat() = default; ChunkFormat::ChunkFormat(size_t maxSize) : _dataBuf(maxSize) diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index 84d64954e40..b4ef45187ee 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -78,9 +78,10 @@ FileChunk::FileChunk(FileId fileId, NameId nameId, const vespalib::string & base _dataFileName(createDatFileName(_name)), _idxFileName(createIdxFileName(_name)), _chunkInfo(), + _lastPersistedSerialNum(0), _dataHeaderLen(0u), _idxHeaderLen(0u), - _lastPersistedSerialNum(0), + _numLids(0), _docIdLimit(std::numeric_limits<uint32_t>::max()), _modificationTime() { @@ -228,6 +229,7 @@ FileChunk::updateLidMap(const LockGuard &guard, ISetLid &ds, uint64_t serialNum, globalBucketMap.recordLid(bucketId); } ds.setLid(guard, lidMeta.getLid(), LidInfo(getFileId().getId(), _chunkInfo.size(), lidMeta.size())); + _numLids++; } else { remove(lidMeta.getLid(), lidMeta.size()); } diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.h b/searchlib/src/vespa/searchlib/docstore/filechunk.h index 0f139f507c6..b68db801d60 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.h @@ -154,6 +154,7 @@ public: FileId getFileId() const { return _fileId; } NameId getNameId() const { return _nameId; } + uint32_t getNumLids() const { return _numLids; } size_t getBloatCount() const { return _erasedCount; } size_t getAddedBytes() const { return _addedBytes; } size_t getErasedBytes() const { return _erasedBytes; } @@ -245,9 +246,10 @@ protected: vespalib::string _dataFileName; vespalib::string _idxFileName; ChunkInfoVector _chunkInfo; + uint64_t _lastPersistedSerialNum; uint32_t _dataHeaderLen; uint32_t _idxHeaderLen; - uint64_t _lastPersistedSerialNum; + uint32_t _numLids; uint32_t _docIdLimit; // Limit when the file was created. Stored in idx file header. vespalib::system_time _modificationTime; }; diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 91de6ba4276..ae0696d5824 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -16,6 +16,11 @@ LOG_SETUP(".searchlib.docstore.logdatastore"); namespace search { +namespace { + constexpr size_t DEFAULT_MAX_FILESIZE = 1000000000ul; + constexpr uint32_t DEFAULT_MAX_LIDS_PER_FILE = 32 * 1024 * 1024; +} + using vespalib::LockGuard; using vespalib::getLastErrorString; using vespalib::getErrorString; @@ -30,10 +35,11 @@ using docstore::BucketCompacter; using namespace std::literals; LogDataStore::Config::Config() - : _maxFileSize(1000000000ul), + : _maxFileSize(DEFAULT_MAX_FILESIZE), _maxDiskBloatFactor(0.2), _maxBucketSpread(2.5), _minFileSizeFactor(0.2), + _maxNumLids(DEFAULT_MAX_LIDS_PER_FILE), _skipCrcOnRead(false), _compactCompression(CompressionConfig::LZ4), _fileConfig() @@ -202,9 +208,9 @@ LogDataStore::requireSpace(LockGuard guard, WriteableFileChunk & active) { assert(active.getFileId() == getActiveFileId(guard)); size_t oldSz(active.getDiskFootprint()); - LOG(spam, "Checking file %s size %ld < %ld", - active.getName().c_str(), oldSz, _config.getMaxFileSize()); - if (oldSz > _config.getMaxFileSize()) { + LOG(spam, "Checking file %s size %ld < %ld AND #lids %u < %u", + active.getName().c_str(), oldSz, _config.getMaxFileSize(), active.getNumLids(), _config.getMaxNumLids()); + if ((oldSz > _config.getMaxFileSize()) || (active.getNumLids() >= _config.getMaxNumLids())) { FileId fileId = allocateFileId(guard); setNewFileChunk(guard, createWritableFile(fileId, active.getSerialNum())); setActive(guard, fileId); @@ -220,9 +226,9 @@ LogDataStore::requireSpace(LockGuard guard, WriteableFileChunk & active) active.flushPendingChunks(active.getSerialNum()); active.freeze(); // TODO: Delay create of new file - LOG(debug, "Closed file %s of size %ld due to maxsize of %ld reached. Bloat is %ld", - active.getName().c_str(), active.getDiskFootprint(), - _config.getMaxFileSize(), active.getDiskBloat()); + LOG(debug, "Closed file %s of size %ld and %u lids due to maxsize of %ld or maxlids %u reached. Bloat is %ld", + active.getName().c_str(), active.getDiskFootprint(), active.getNumLids(), + _config.getMaxFileSize(), _config.getMaxNumLids(), active.getDiskBloat()); } } diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index 39b2c2b7100..7bd55599611 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -40,6 +40,7 @@ public: Config(); Config & setMaxFileSize(size_t v) { _maxFileSize = v; return *this; } + Config & setMaxNumLids(size_t v) { _maxNumLids = v; return *this; } Config & setMaxDiskBloatFactor(double v) { _maxDiskBloatFactor = v; return *this; } Config & setMaxBucketSpread(double v) { _maxBucketSpread = v; return *this; } Config & setMinFileSizeFactor(double v) { _minFileSizeFactor = v; return *this; } @@ -51,6 +52,7 @@ public: double getMaxDiskBloatFactor() const { return _maxDiskBloatFactor; } double getMaxBucketSpread() const { return _maxBucketSpread; } double getMinFileSizeFactor() const { return _minFileSizeFactor; } + uint32_t getMaxNumLids() const { return _maxNumLids; } bool crcOnReadDisabled() const { return _skipCrcOnRead; } const CompressionConfig & compactCompression() const { return _compactCompression; } @@ -64,6 +66,7 @@ public: double _maxDiskBloatFactor; double _maxBucketSpread; double _minFileSizeFactor; + uint32_t _maxNumLids; bool _skipCrcOnRead; CompressionConfig _compactCompression; WriteableFileChunk::Config _fileConfig; diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp index 13de5687ee0..5e595d0bb14 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp @@ -92,7 +92,7 @@ StoreByBucket::drain(IWrite & drainer) chunks.resize(_chunks.size()); for (const auto & it : _chunks) { ConstBufferRef buf(it.second); - chunks[it.first].reset(new Chunk(it.first, buf.data(), buf.size())); + chunks[it.first] = std::make_unique<Chunk>(it.first, buf.data(), buf.size()); } _chunks.clear(); for (auto & it : _where) { diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index d12a3a54a93..3517595d00a 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -708,6 +708,7 @@ WriteableFileChunk::append(uint64_t serialNum, uint32_t lid, const void * buffer assert(serialNum >= _serialNum); _serialNum = serialNum; _addedBytes += adjustSize(len); + _numLids++; size_t oldSz(_active->size()); LidMeta lm = _active->append(lid, buffer, len); setDiskFootprint(FileChunk::getDiskFootprint() - oldSz + _active->size()); diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index 6d870cae019..65dd822ac1f 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -47,7 +47,7 @@ public: uint32_t docIdLimit, const Config & config, const TuneFileSummary &tune, const common::FileHeaderContext &fileHeaderContext, const IBucketizer * bucketizer, bool crcOnReadDisabled); - ~WriteableFileChunk(); + ~WriteableFileChunk() override; ssize_t read(uint32_t lid, SubChunkId chunk, vespalib::DataBuffer & buffer) const override; void read(LidInfoWithLidV::const_iterator begin, size_t count, IBufferVisitor & visitor) const override; @@ -128,8 +128,8 @@ private: bool _writeTaskIsRunning; vespalib::Monitor _writeMonitor; ProcessedChunkQ _writeQ; - vespalib::Executor & _executor; - ProcessedChunkMap _orderedChunks; + vespalib::Executor & _executor; + ProcessedChunkMap _orderedChunks; BucketDensityComputer _bucketMap; }; diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp index bae7b2a1157..bf52e5e0b5e 100644 --- a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp @@ -60,7 +60,6 @@ AttributeMatchExecutor<T>::Computer::Computer(const IQueryEnvironment & env, Att _valueCount(0), _md(nullptr) { - _buffer.allocate(_params.attribute->getMaxValueCount()); QueryTermHelper queryTerms(env); for (const QueryTerm & qt : queryTerms.terms()) { _totalTermWeight += qt.termData()->getWeight().percent(); diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index a8737a19eec..37fd98c9f20 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -256,7 +256,7 @@ namespace dotproduct::array { template <typename BaseType> DotProductExecutorBase<BaseType>::DotProductExecutorBase(const V & queryVector) : FeatureExecutor(), - _multiplier(IAccelrated::getAccelrator()), + _multiplier(IAccelrated::getAccelerator()), _queryVector(queryVector) { } diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt index 7786e3b45a1..a3273a4a39c 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_add_library(searchlib_features_fieldmatch OBJECT SOURCES computer.cpp + computer_shared_state.cpp metrics.cpp params.cpp segmentstart.cpp diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp index 6dba7b87c08..ab36f18bfa5 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp @@ -1,7 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "computer.h" +#include "computer_shared_state.h" #include <vespa/searchlib/features/utils.h> +#include <vespa/searchlib/fef/phrase_splitter_query_env.h> +#include <vespa/searchlib/fef/phrasesplitter.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/locale/c.h> @@ -14,56 +17,28 @@ using namespace search::fef; namespace search::features::fieldmatch { -Computer::Computer(const vespalib::string &propertyNamespace, const PhraseSplitter &splitter, - const FieldInfo &fieldInfo, const Params ¶ms) : - _splitter(splitter), - _fieldId(fieldInfo.id()), - _params(params), - _tracing(false), - _trace(), - _useCachedHits(true), - _queryTerms(), - _queryTermFieldMatch(), - _totalTermWeight(0), - _totalTermSignificance(0.0f), - _fieldLength(FieldPositionsIterator::UNKNOWN_LENGTH), - _currentMetrics(this), - _finalMetrics(this), - _simpleMetrics(params), - _segments(), - _alternativeSegmentationsTried(0), - _cachedHits() +Computer::Computer(const ComputerSharedState& shared_state, const PhraseSplitter& splitter) + : _shared_state(shared_state), + _splitter(splitter), + _fieldId(_shared_state.get_field_id()), + _params(_shared_state.get_params()), + _useCachedHits(_shared_state.get_use_cached_hits()), + _queryTerms(_shared_state.get_query_terms()), + _queryTermFieldMatch(_queryTerms.size()), + _totalTermWeight(_shared_state.get_total_term_weight()), + _totalTermSignificance(_shared_state.get_total_term_significance()), + _fieldLength(FieldPositionsIterator::UNKNOWN_LENGTH), + _currentMetrics(this), + _finalMetrics(this), + _simpleMetrics(_shared_state.get_simple_metrics()), + _segments(), + _alternativeSegmentationsTried(0), + _cachedHits(_queryTerms.size()) { - // Store term data for all terms searching in this field - _queryTermFieldMatch.reserve(splitter.getNumTerms()); - _cachedHits.reserve(splitter.getNumTerms()); - for (uint32_t i = 0; i < splitter.getNumTerms(); ++i) { - QueryTerm qt = QueryTermFactory::create(splitter, i, true); - _totalTermWeight += qt.termData()->getWeight().percent(); - _totalTermSignificance += qt.significance(); - _simpleMetrics.addQueryTerm(qt.termData()->getWeight().percent()); - const ITermFieldData *field = qt.termData()->lookupField(_fieldId); - if (field != nullptr) { - qt.fieldHandle(field->getHandle()); - _queryTerms.push_back(qt); - _simpleMetrics.addSearchedTerm(qt.termData()->getWeight().percent()); - _queryTermFieldMatch.emplace_back(nullptr); - _cachedHits.emplace_back(); - } - } - - _totalTermWeight = atoi(splitter.getProperties().lookup(propertyNamespace, "totalTermWeight"). - get(vespalib::make_string("%d", _totalTermWeight)).c_str()); - _totalTermSignificance = vespalib::locale::c::atof(splitter.getProperties().lookup(propertyNamespace, "totalTermSignificance"). - get(vespalib::make_string("%f", _totalTermSignificance)).c_str()); - if (splitter.getProperties().lookup(propertyNamespace, "totalTermWeight").found()) { - _simpleMetrics.setTotalWeightInQuery(_totalTermWeight); + for (const auto &qt : _queryTerms) { + // Record that we need normal term field match data + (void) qt.termData()->lookupField(_fieldId)->getHandle(MatchDataDetails::Normal); } - - // update current and final metrics after initialization - _currentMetrics = Metrics(this); - _finalMetrics = Metrics(this); - // num query terms searching in this field + 1 _segments.reserve(getNumQueryTerms() + 1); for (uint32_t i = 0; i < (getNumQueryTerms() + 1); ++i) { @@ -71,6 +46,8 @@ Computer::Computer(const vespalib::string &propertyNamespace, const PhraseSplitt } } +Computer::~Computer() = default; + void Computer::reset(uint32_t docId) { @@ -139,7 +116,7 @@ Computer::handleError(uint32_t fieldPos, uint32_t docId) const static int errcnt; if (errcnt < 1000) { errcnt++; - const FieldInfo * finfo = _splitter.getIndexEnvironment().getField(getFieldId()); + const FieldInfo * finfo = _splitter.get_query_env().getIndexEnvironment().getField(getFieldId()); LOG(debug, "Bad field position %u >= fieldLength %u for field '%s' document %u. " "Document was probably refed during query (Ticket 7104969)", fieldPos, _fieldLength, @@ -248,27 +225,6 @@ Computer::fieldIndexToSemanticDistance(int j, uint32_t zeroJ) const } } -Computer & -Computer::trace(const vespalib::string &str) -{ - if (_tracing) { - _trace.push_back(str); - //LOG(info, "%s", str.c_str()); - } - return *this; -} - -vespalib::string -Computer::getTrace() const -{ - vespalib::string ret = ""; - for (std::vector<vespalib::string>::const_iterator it = _trace.begin(); - it != _trace.end(); ++it) { - ret += *it; - } - return ret; -} - vespalib::string Computer::toString() const { @@ -280,41 +236,13 @@ Computer::toString() const void Computer::exploreSegments() { - if (isTracing()) { - trace(vespalib::make_string("Calculating matches for %d query terms, %d field terms.", - getNumQueryTerms(), _fieldLength)); - } - _segments[0].segment->reset(_currentMetrics); _segments[0].valid = true; SegmentStart *segment = _segments[0].segment.get(); while (segment != nullptr) { - if (isTracing()) { - trace(vespalib::make_string("Looking for segment from %s...", - segment->toString().c_str())); - } - _currentMetrics = segment->getMetrics(); // take a copy of the segment returned from the current segment. bool found = findAlternativeSegmentFrom(segment); - if (found) { - if (isTracing()) { - vespalib::string segments = "[ "; - const std::vector<uint32_t> &lst = _currentMetrics.getSegmentStarts(); - for (uint32_t i = 0; i < lst.size(); ++i) { - segments += vespalib::make_string("%d", lst[i]); - if (i < lst.size() - 1) { - segments += ", "; - } - } - segments += " ]"; - trace(vespalib::make_string("...found segments: %s, score %f.", - segments.c_str(), - _currentMetrics.getSegmentationScore())); - } - } else { - if (isTracing()) { - trace("...no complete and improved segment existed."); - } + if (!found) { segment->setOpen(false); } segment = findOpenSegment(segment->getI()); @@ -402,9 +330,6 @@ Computer::inSegment(int i, int j, int previousJ, int previousI) } else { _currentMetrics.onInSegmentGap(i, j, previousJ); - if (isTracing()) { - trace(vespalib::make_string(" in segment gap: %d -> %d", i, j)); - } } } @@ -415,18 +340,12 @@ Computer::segmentStart(int i, int j, int previousJ) if (previousJ >= 0) { _currentMetrics.onPair(i, j, previousJ); } - if (isTracing()) { - trace(vespalib::make_string(" new segment at: %d -> %d", i, j)); - } return true; } void Computer::segmentEnd(int i, int j) { - if (isTracing()) { - trace(vespalib::make_string(" segment ended at: %d -> %d", i, j)); - } SegmentStart *startOfNext = _segments[i + 1].segment.get(); if (!_segments[i + 1].valid) { startOfNext->reset(_currentMetrics, j, i + 1); diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h index 12535ac105c..e4dbde1248a 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.h @@ -5,15 +5,21 @@ #include "params.h" #include "segmentstart.h" #include "simplemetrics.h" -#include <vespa/searchlib/fef/iqueryenvironment.h> -#include <vespa/searchlib/fef/fieldinfo.h> -#include <vespa/searchlib/fef/matchdata.h> -#include <vespa/searchlib/fef/phrasesplitter.h> #include <vespa/searchlib/features/queryterm.h> #include <vespa/searchlib/common/allocatedbitvector.h> +#include <vespa/vespalib/util/arrayref.h> + +namespace search::fef { + +class PhraseSplitter; +class TermFieldMatchData; + +} namespace search::features::fieldmatch { +class ComputerSharedState; + /** * <p>Calculates a set of metrics capturing information about the degree of agreement between a query and a field * string. This algorithm attempts to capture the property of text that very close tokens are usuall part of the same @@ -57,13 +63,12 @@ public: /** * Constructs a new computer object. * - * @param propertyNamespace The namespace used in query properties. + * @param shared_state The shared state for this computer * @param splitter The environment that holds all query information. - * @param fieldInfo The info object of the matched field. - * @param params The parameter object for this computer. */ - Computer(const vespalib::string &propertyNamespace, const fef::PhraseSplitter &splitter, - const fef::FieldInfo &fieldInfo, const Params ¶ms); + Computer(const ComputerSharedState& shared_state, const fef::PhraseSplitter& splitter); + + ~Computer(); /** * Resets this object according to the given document id @@ -121,15 +126,6 @@ public: int fieldIndexToSemanticDistance(int j, uint32_t zeroJ) const; /** - * Returns the query environment of this. This contains information about the query. - * - * @return The query environment. - */ - const fef::IQueryEnvironment &getQueryEnvironment() const { - return _splitter; - } - - /** * Returns the id of the searched field. * * @return The field id. @@ -157,39 +153,6 @@ public: } /** - * Adds the given string to the trace of this, if tracing is enabled. - * - * @param str The string to trace. - * @return This, to allow chaining. - */ - Computer &trace(const vespalib::string &str); - - /** - * Returns a textual trace of the last execution of this algorithm, if tracing is on. - * - * @return The trace string. - */ - vespalib::string getTrace() const; - - /** - * Set to true to collect a textual trace from the computation, which can be retrieved using {@link #getTrace}. - * - * @param tracing Whether or not to trace. - * @return This, to allow chaining. - */ - Computer &setTracing(bool tracing) { - _tracing = tracing; - return *this; - } - - /** - * Returns whether tracing is on. - * - * @return True if tracing is on. - */ - bool isTracing() const { return _tracing; } - - /** * Returns the number of terms searching on this field. * * @return The number of terms. @@ -338,14 +301,13 @@ private: }; // per query + const ComputerSharedState& _shared_state; const search::fef::PhraseSplitter & _splitter; uint32_t _fieldId; - Params _params; - bool _tracing; - std::vector<vespalib::string> _trace; + const Params _params; bool _useCachedHits; - QueryTermVector _queryTerms; + const vespalib::ConstArrayRef<QueryTerm> _queryTerms; TermFieldMatchDataVector _queryTermFieldMatch; uint32_t _totalTermWeight; feature_t _totalTermSignificance; diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/computer_shared_state.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/computer_shared_state.cpp new file mode 100644 index 00000000000..033124e86bb --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/computer_shared_state.cpp @@ -0,0 +1,50 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "computer_shared_state.h" +#include <vespa/searchlib/features/utils.h> +#include <vespa/searchlib/fef/phrase_splitter_query_env.h> +#include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/locale/c.h> +#include <set> + +using namespace search::fef; + +namespace search::features::fieldmatch { + +ComputerSharedState::ComputerSharedState(const vespalib::string& propertyNamespace, const PhraseSplitterQueryEnv& splitter_query_env, + const FieldInfo& fieldInfo, const Params& params) + : _field_id(fieldInfo.id()), + _params(params), + _use_cached_hits(true), + _query_terms(), + _total_term_weight(0), + _total_term_significance(0.0f), + _simple_metrics(_params) +{ + // Store term data for all terms searching in this field + for (uint32_t i = 0; i < splitter_query_env.getNumTerms(); ++i) { + QueryTerm qt = QueryTermFactory::create(splitter_query_env, i, true); + _total_term_weight += qt.termData()->getWeight().percent(); + _total_term_significance += qt.significance(); + _simple_metrics.addQueryTerm(qt.termData()->getWeight().percent()); + const ITermFieldData *field = qt.termData()->lookupField(_field_id); + if (field != nullptr) { + qt.fieldHandle(field->getHandle()); + _query_terms.push_back(qt); + _simple_metrics.addSearchedTerm(qt.termData()->getWeight().percent()); + } + } + + _total_term_weight = atoi(splitter_query_env.getProperties().lookup(propertyNamespace, "totalTermWeight"). + get(vespalib::make_string("%d", _total_term_weight)).c_str()); + _total_term_significance = vespalib::locale::c::atof(splitter_query_env.getProperties().lookup(propertyNamespace, "totalTermSignificance"). + get(vespalib::make_string("%f", _total_term_significance)).c_str()); + if (splitter_query_env.getProperties().lookup(propertyNamespace, "totalTermWeight").found()) { + _simple_metrics.setTotalWeightInQuery(_total_term_weight); + } +} + +ComputerSharedState::~ComputerSharedState() = default; + +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/computer_shared_state.h b/searchlib/src/vespa/searchlib/features/fieldmatch/computer_shared_state.h new file mode 100644 index 00000000000..e6b46f0add9 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/computer_shared_state.h @@ -0,0 +1,52 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "params.h" +#include "simplemetrics.h" +#include <vespa/searchlib/features/queryterm.h> + +namespace search::fef { class PhraseSplitterQueryEnv; } + +namespace search::features::fieldmatch { + +/** + * Shared state for field match computer. + */ +class ComputerSharedState { +public: + /** + * Constructs a new computer shared state object. + * + * @param propertyNamespace The namespace used in query properties. + * @param splitter_query_env The environment that holds all query information. + * @param fieldInfo The info object of the matched field. + * @param params The parameter object for this computer. + */ + ComputerSharedState(const vespalib::string& propertyNamespace, const fef::PhraseSplitterQueryEnv& splitter_query_env, + const fef::FieldInfo& fieldInfo, const Params& params); + ~ComputerSharedState(); + + uint32_t get_field_id() const { return _field_id; } + const Params& get_params() const { return _params; } + bool get_use_cached_hits() const { return _use_cached_hits; } + const QueryTermVector& get_query_terms() const { return _query_terms; } + uint32_t get_total_term_weight() const { return _total_term_weight; } + feature_t get_total_term_significance() const { return _total_term_significance; } + const SimpleMetrics& get_simple_metrics() const { return _simple_metrics; } + +private: + + // per query + uint32_t _field_id; + Params _params; + bool _use_cached_hits; + + QueryTermVector _query_terms; + uint32_t _total_term_weight; + feature_t _total_term_significance; + + // portions per docid (not used here), portions per query + SimpleMetrics _simple_metrics; // The metrics used to compute simple features. +}; + +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp index 6a56cb606d2..dcdaf1681a4 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.cpp @@ -8,9 +8,7 @@ #include <cmath> #include <cstdlib> -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { Metrics::Metrics(const Computer *source) : _source(source), @@ -337,7 +335,4 @@ Metrics::toString() const return vespalib::make_string("Metrics(match %f)", getMatch()); } - -} // fieldmatch -} // features -} // search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h index eadda3209ad..a5f0934cd64 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/metrics.h @@ -5,9 +5,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vector> -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { class Computer; @@ -557,7 +555,4 @@ private: uint32_t _queryLength; // num terms searching this field }; -} // fieldmatch -} // features -} // search - +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp index facf1a5f64a..f17d3cffcd2 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/params.cpp @@ -4,9 +4,7 @@ #include <vespa/log/log.h> LOG_SETUP(".features.fieldmatch.params"); -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { Params::Params() : _proximityLimit(10), @@ -40,5 +38,3 @@ Params::valid() } } -} -} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/params.h b/searchlib/src/vespa/searchlib/features/fieldmatch/params.h index 21c35f19472..dcbd61676ad 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/params.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/params.h @@ -5,9 +5,7 @@ #include <vector> #include <cstdint> -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { /** * The parameters to a string match metric calculator. @@ -256,7 +254,4 @@ private: std::vector<feature_t> _proximityTable; }; -} // fieldmatch -} // features -} // search - +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp index 8f5f86d1628..6f99bbfab14 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.cpp @@ -5,9 +5,7 @@ #include "metrics.h" #include <vespa/vespalib/util/stringfmt.h> -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { SegmentStart::SegmentStart(Computer *owner, const Metrics & metrics, uint32_t previousJ, uint32_t i, uint32_t j) : _owner(owner), @@ -48,12 +46,6 @@ bool SegmentStart::offerHistory(int previousJ, const Metrics & metrics) { if (metrics.getSegmentationScore() <= _metrics.getSegmentationScore()) { - if (_owner->isTracing()) { - _owner->trace(vespalib::make_string(" Rejected offered history [score %f, ending at %d] at %s.\n", - metrics.getSegmentationScore(), - previousJ, - toString().c_str())); - } return false; // reject } @@ -66,12 +58,6 @@ SegmentStart::offerHistory(int previousJ, const Metrics & metrics) } #endif - if (_owner->isTracing()) { - _owner->trace(vespalib::make_string(" Accepted offered history [score %f, ending at %d] at %s.\n", - metrics.getSegmentationScore(), - previousJ, - toString().c_str())); - } _previousJ = previousJ; _metrics = metrics; // take a copy of the given metrics return true; // accept @@ -95,6 +81,4 @@ SegmentStart::toString() { } } -} // fieldmatch -} // features -} // search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.h b/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.h index dd4b9406036..9b3fbc7b987 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/segmentstart.h @@ -5,9 +5,7 @@ #include <limits> #include "metrics.h" -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { /** * <p>Information on segment start points stored temporarily during string match metric calculation.</p> @@ -179,7 +177,4 @@ private: bool _open; // There are possibly more j's to try at this starting point. }; -} // fieldmatch -} // features -} // search - +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp index 045bc277454..ca3b79f17bc 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.cpp @@ -3,9 +3,7 @@ #include "simplemetrics.h" #include <vespa/vespalib/stllike/asciistream.h> -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { SimpleMetrics::SimpleMetrics(const Params & params) : _params(params), @@ -30,7 +28,4 @@ vespalib::string SimpleMetrics::toString() const return ss.str(); } - -} // fieldmatch -} // features -} // search +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.h b/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.h index 6c5a67884d8..ee1c11215d1 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/simplemetrics.h @@ -5,9 +5,7 @@ #include <vespa/vespalib/stllike/string.h> #include "params.h" -namespace search { -namespace features { -namespace fieldmatch { +namespace search::features::fieldmatch { /** * The collection of simple metrics calculated when traversing the query terms of the query environment. @@ -180,7 +178,4 @@ public: vespalib::string toString() const; }; -} // fieldmatch -} // features -} // search - +} diff --git a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp index 94240422106..191df116012 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp @@ -3,8 +3,11 @@ #include "fieldmatchfeature.h" #include "utils.h" #include <vespa/searchlib/features/fieldmatch/computer.h> +#include <vespa/searchlib/features/fieldmatch/computer_shared_state.h> #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/indexproperties.h> +#include <vespa/searchlib/fef/phrase_splitter_query_env.h> +#include <vespa/searchlib/fef/phrasesplitter.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/locale/c.h> @@ -15,34 +18,50 @@ using CollectionType = FieldInfo::CollectionType; namespace search::features { +class FieldMatchExecutorSharedState : public Anything { +private: + PhraseSplitterQueryEnv _splitter_env; + fieldmatch::ComputerSharedState _cmp_shared_state; +public: + FieldMatchExecutorSharedState(const fef::IQueryEnvironment& query_env, + const fef::FieldInfo& field, + const fieldmatch::Params& params); + ~FieldMatchExecutorSharedState() override; + const PhraseSplitterQueryEnv& get_phrase_splitter_query_env() const { return _splitter_env; } + const fieldmatch::ComputerSharedState &get_computer_shared_state() const { return _cmp_shared_state; } +}; + +FieldMatchExecutorSharedState::FieldMatchExecutorSharedState(const IQueryEnvironment& query_env, + const FieldInfo& field, + const fieldmatch::Params& params) + : Anything(), + _splitter_env(query_env, field.id()), + _cmp_shared_state(vespalib::make_string("fieldMatch(%s)", field.name().c_str()), _splitter_env, field, params) +{ +} + +FieldMatchExecutorSharedState::~FieldMatchExecutorSharedState() = default; + /** * Implements the executor for THE field match feature. */ class FieldMatchExecutor : public fef::FeatureExecutor { private: fef::PhraseSplitter _splitter; - const fef::FieldInfo & _field; fieldmatch::Computer _cmp; void handle_bind_match_data(const fef::MatchData &md) override; public: - FieldMatchExecutor(const fef::IQueryEnvironment & queryEnv, - const fef::FieldInfo & field, - const fieldmatch::Params & params); + FieldMatchExecutor(const FieldMatchExecutorSharedState& shared_state); void execute(uint32_t docId) override; }; -FieldMatchExecutor::FieldMatchExecutor(const IQueryEnvironment & queryEnv, - const FieldInfo & field, - [[maybe_unused]] const fieldmatch::Params & params) : - FeatureExecutor(), - _splitter(queryEnv, field.id()), - _field(field), - _cmp(vespalib::make_string("fieldMatch(%s)", _field.name().c_str()), - _splitter, field, params) +FieldMatchExecutor::FieldMatchExecutor(const FieldMatchExecutorSharedState& shared_state) + : FeatureExecutor(), + _splitter(shared_state.get_phrase_splitter_query_env()), + _cmp(shared_state.get_computer_shared_state(), _splitter) { - // empty } void @@ -52,7 +71,6 @@ FieldMatchExecutor::execute(uint32_t docId) _splitter.update(); _cmp.reset(docId); - //_cmp.setTracing(true); const fieldmatch::SimpleMetrics & simple = _cmp.getSimpleMetrics(); @@ -113,6 +131,7 @@ FieldMatchExecutor::handle_bind_match_data(const fef::MatchData &md) FieldMatchBlueprint::FieldMatchBlueprint() : Blueprint("fieldMatch"), _field(nullptr), + _shared_state_key(), _params() { } @@ -184,6 +203,7 @@ FieldMatchBlueprint::setup(const IIndexEnvironment & env, const ParameterList & params) { _field = params[0].asField(); + _shared_state_key = "fef.fieldmatch." + _field->name(); const Properties & lst = env.getProperties(); Property obj; @@ -321,14 +341,17 @@ FieldMatchBlueprint::setup(const IIndexEnvironment & env, FeatureExecutor & FieldMatchBlueprint::createExecutor(const IQueryEnvironment & env, vespalib::Stash &stash) const { - return stash.create<FieldMatchExecutor>(env, *_field, _params); + auto *shared_state = dynamic_cast<const FieldMatchExecutorSharedState *>(env.getObjectStore().get(_shared_state_key)); + if (shared_state == nullptr) { + shared_state = &stash.create<FieldMatchExecutorSharedState>(env, *_field, _params); + } + return stash.create<FieldMatchExecutor>(*shared_state); } void FieldMatchBlueprint::prepareSharedState(const IQueryEnvironment &env, IObjectStore & store) const { - (void) env; - (void) store; - //TODO WE need too extract the const and costly parts from PhraseSpiltter and Computer - // and initialize it here for later reuse in the multiple search threads. + if (store.get(_shared_state_key) == nullptr) { + store.add(_shared_state_key, std::make_unique<FieldMatchExecutorSharedState>(env, *_field, _params)); + } } } diff --git a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.h b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.h index 0e5f873005d..f203574e588 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.h @@ -13,6 +13,7 @@ namespace search::features { class FieldMatchBlueprint : public fef::Blueprint { private: const fef::FieldInfo * _field; + vespalib::string _shared_state_key; fieldmatch::Params _params; public: diff --git a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp index 64ae94ddc90..376ed8cd3d3 100644 --- a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stash.h> using namespace search::fef; @@ -15,12 +16,44 @@ namespace search::features { const uint32_t NativeFieldMatchParam::NOT_DEF_FIELD_LENGTH(std::numeric_limits<uint32_t>::max()); +NativeFieldMatchExecutorSharedState::NativeFieldMatchExecutorSharedState(const IQueryEnvironment& env, + const NativeFieldMatchParams& params) + : fef::Anything(), + _params(params), + _query_terms(), + _divisor(0) +{ + QueryTermHelper queryTerms(env); + for (const QueryTerm & qtTmp : queryTerms.terms()) { + if (qtTmp.termData()->getWeight().percent() != 0) // only consider query terms with contribution + { + MyQueryTerm qt(qtTmp); + typedef search::fef::ITermFieldRangeAdapter FRA; + uint32_t totalFieldWeight = 0; + for (FRA iter(*qt.termData()); iter.valid(); iter.next()) { + const ITermFieldData& tfd = iter.get(); + uint32_t fieldId = tfd.getFieldId(); + if (_params.considerField(fieldId)) { // only consider fields with contribution + totalFieldWeight += _params.vector[fieldId].fieldWeight; + qt.handles().emplace_back(tfd.getHandle(), &tfd); + } + } + if (!qt.handles().empty()) { + _query_terms.push_back(qt); + _divisor += (qt.significance() * qt.termData()->getWeight().percent() * totalFieldWeight); + } + } + } +} + +NativeFieldMatchExecutorSharedState::~NativeFieldMatchExecutorSharedState() = default; + feature_t NativeFieldMatchExecutor::calculateScore(const MyQueryTerm &qt, uint32_t docId) { feature_t termScore = 0; for (size_t i = 0; i < qt.handles().size(); ++i) { - TermFieldHandle tfh = qt.handles()[i]; + TermFieldHandle tfh = qt.handles()[i].first; const TermFieldMatchData *tfmd = _md->resolveTermField(tfh); const NativeFieldMatchParam & param = _params.vector[tfmd->getFieldId()]; if (tfmd->getDocId() == docId) { // do we have a hit @@ -38,33 +71,17 @@ NativeFieldMatchExecutor::calculateScore(const MyQueryTerm &qt, uint32_t docId) return termScore; } -NativeFieldMatchExecutor::NativeFieldMatchExecutor(const IQueryEnvironment & env, - const NativeFieldMatchParams & params) : - FeatureExecutor(), - _params(params), - _queryTerms(), - _divisor(0), - _md(nullptr) +NativeFieldMatchExecutor::NativeFieldMatchExecutor(const NativeFieldMatchExecutorSharedState& shared_state) + : FeatureExecutor(), + _params(shared_state.get_params()), + _queryTerms(shared_state.get_query_terms()), + _divisor(shared_state.get_divisor()), + _md(nullptr) { - QueryTermHelper queryTerms(env); - for (const QueryTerm & qtTmp : queryTerms.terms()) { - if (qtTmp.termData()->getWeight().percent() != 0) // only consider query terms with contribution - { - MyQueryTerm qt(qtTmp); - typedef search::fef::ITermFieldRangeAdapter FRA; - uint32_t totalFieldWeight = 0; - for (FRA iter(*qt.termData()); iter.valid(); iter.next()) { - const ITermFieldData& tfd = iter.get(); - uint32_t fieldId = tfd.getFieldId(); - if (_params.considerField(fieldId)) { // only consider fields with contribution - totalFieldWeight += _params.vector[fieldId].fieldWeight; - qt.handles().push_back(tfd.getHandle()); - } - } - if (!qt.handles().empty()) { - _queryTerms.push_back(qt); - _divisor += (qt.significance() * qt.termData()->getWeight().percent() * totalFieldWeight); - } + for (const auto& qt : _queryTerms) { + for (const auto& handle : qt.handles()) { + // Record that we need normal term field match data + (void) handle.second->getHandle(MatchDataDetails::Normal); } } } @@ -92,7 +109,8 @@ NativeFieldMatchBlueprint::NativeFieldMatchBlueprint() : Blueprint("nativeFieldMatch"), _params(), _defaultFirstOcc("expdecay(8000,12.50)"), - _defaultNumOcc("loggrowth(1500,4000,19)") + _defaultNumOcc("loggrowth(1500,4000,19)"), + _shared_state_key() { } @@ -116,9 +134,12 @@ bool NativeFieldMatchBlueprint::setup(const IIndexEnvironment & env, const ParameterList & params) { + vespalib::asciistream shared_state_key_builder; _params.resize(env.getNumFields()); FieldWrapper fields(env, params, FieldType::INDEX); vespalib::string defaultFirstOccImportance = env.getProperties().lookup(getBaseName(), "firstOccurrenceImportance").get("0.5"); + shared_state_key_builder << "fef.nativeFieldMatch["; + bool first_field = true; for (uint32_t i = 0; i < fields.getNumFields(); ++i) { const FieldInfo * info = fields.getField(i); uint32_t fieldId = info->id(); @@ -160,8 +181,16 @@ NativeFieldMatchBlueprint::setup(const IIndexEnvironment & env, } if (param.field) { env.hintFieldAccess(fieldId); + if (first_field) { + first_field = false; + } else { + shared_state_key_builder << ","; + } + shared_state_key_builder << info->name(); } } + shared_state_key_builder << "]"; + _shared_state_key = shared_state_key_builder.str(); _params.minFieldLength = util::strToNum<uint32_t>(env.getProperties().lookup (getBaseName(), "minFieldLength").get("6")); @@ -172,17 +201,23 @@ NativeFieldMatchBlueprint::setup(const IIndexEnvironment & env, FeatureExecutor & NativeFieldMatchBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { - NativeFieldMatchExecutor &native = stash.create<NativeFieldMatchExecutor>(env, _params); - if (native.empty()) { + auto *shared_state = dynamic_cast<const NativeFieldMatchExecutorSharedState *>(env.getObjectStore().get(_shared_state_key)); + if (shared_state == nullptr) { + shared_state = &stash.create<NativeFieldMatchExecutorSharedState>(env, _params); + } + if (shared_state->empty()) { return stash.create<SingleZeroValueExecutor>(); } else { - return native; + return stash.create<NativeFieldMatchExecutor>(*shared_state); } } void NativeFieldMatchBlueprint::prepareSharedState(const IQueryEnvironment &queryEnv, IObjectStore &objectStore) const { QueryTermHelper::lookupAndStoreQueryTerms(queryEnv, objectStore); + if (objectStore.get(_shared_state_key) == nullptr) { + objectStore.add(_shared_state_key, std::make_unique<NativeFieldMatchExecutorSharedState>(queryEnv, _params)); + } } } diff --git a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h index 9b132561cd3..5e8d865e159 100644 --- a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h +++ b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h @@ -29,13 +29,12 @@ public: }; /** - * Implements the executor for calculating the native field match score. - **/ -class NativeFieldMatchExecutor : public fef::FeatureExecutor -{ -private: - typedef std::vector<fef::TermFieldHandle> HandleVector; - + * Class containing shared state for native field match executor. + */ +class NativeFieldMatchExecutorSharedState : public fef::Anything { +public: + using WrappedHandle = std::pair<fef::TermFieldHandle, const fef::ITermFieldData*>; + using HandleVector = std::vector<WrappedHandle>; class MyQueryTerm : public QueryTerm { private: @@ -45,8 +44,28 @@ private: HandleVector &handles() { return _handles; } const HandleVector &handles() const { return _handles; } }; +private: + const NativeFieldMatchParams& _params; + std::vector<MyQueryTerm> _query_terms; + feature_t _divisor; +public: + NativeFieldMatchExecutorSharedState(const fef::IQueryEnvironment& env, const NativeFieldMatchParams& params); + ~NativeFieldMatchExecutorSharedState(); + const NativeFieldMatchParams& get_params() const { return _params; } + const std::vector<MyQueryTerm>& get_query_terms() const { return _query_terms; } + feature_t get_divisor() const { return _divisor; } + bool empty() const { return _query_terms.empty(); } +}; + +/** + * Implements the executor for calculating the native field match score. + **/ +class NativeFieldMatchExecutor : public fef::FeatureExecutor +{ +private: + using MyQueryTerm = NativeFieldMatchExecutorSharedState::MyQueryTerm; const NativeFieldMatchParams & _params; - std::vector<MyQueryTerm> _queryTerms; + vespalib::ConstArrayRef<MyQueryTerm> _queryTerms; feature_t _divisor; const fef::MatchData *_md; @@ -74,8 +93,7 @@ private: virtual void handle_bind_match_data(const fef::MatchData &md) override; public: - NativeFieldMatchExecutor(const fef::IQueryEnvironment & env, - const NativeFieldMatchParams & params); + NativeFieldMatchExecutor(const NativeFieldMatchExecutorSharedState& shared_state); void execute(uint32_t docId) override; feature_t getFirstOccBoost(uint32_t field, uint32_t position, uint32_t fieldLength) const { @@ -85,7 +103,6 @@ public: feature_t getNumOccBoost(uint32_t field, uint32_t occs, uint32_t fieldLength) const { return getNumOccBoost(_params.vector[field], occs, fieldLength); } - bool empty() const { return _queryTerms.empty(); } }; @@ -97,6 +114,7 @@ private: NativeFieldMatchParams _params; vespalib::string _defaultFirstOcc; vespalib::string _defaultNumOcc; + vespalib::string _shared_state_key; public: NativeFieldMatchBlueprint(); diff --git a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp index 98fd41aad0e..d0b3a429e83 100644 --- a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp @@ -7,13 +7,74 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stash.h> -#include <map> using namespace search::fef; namespace search::features { +NativeProximityExecutorSharedState::NativeProximityExecutorSharedState(const IQueryEnvironment& env, + const NativeProximityParams& params) + : fef::Anything(), + _params(params), + _setups(), + _total_field_weight(0), + _fields() +{ + QueryTermHelper queryTerms(env); + for (const QueryTerm& qt : queryTerms.terms()) { + typedef search::fef::ITermFieldRangeAdapter FRA; + for (FRA iter(*qt.termData()); iter.valid(); iter.next()) { + uint32_t fieldId = iter.get().getFieldId(); + if (_params.considerField(fieldId)) { // only consider fields with contribution + QueryTerm myQt = qt; + myQt.fieldHandle(iter.get().getHandle()); + _fields[fieldId].push_back(myQt); + } + } + } + for (const auto& entry : _fields) { + if (entry.second.size() >= 2) { + FieldSetup setup(entry.first); + generateTermPairs(env, entry.second, _params.slidingWindow, setup); + if (!setup.pairs.empty()) { + _setups.push_back(std::move(setup)); + _total_field_weight += params.vector[entry.first].fieldWeight; + } + } + } +} + + +NativeProximityExecutorSharedState::~NativeProximityExecutorSharedState() = default; + +void +NativeProximityExecutorSharedState::generateTermPairs(const IQueryEnvironment& env, const QueryTermVector& terms, + uint32_t slidingWindow, FieldSetup& setup) +{ + TermPairVector& pairs = setup.pairs; + for (size_t i = 0; i < terms.size(); ++i) { + for (size_t j = i + 1; (j < i + slidingWindow) && (j < terms.size()); ++j) { + feature_t connectedness = 1; + for (size_t k = j; k > i; --k) { + connectedness = std::min(util::lookupConnectedness(env, terms[k].termData()->getUniqueId(), + terms[k-1].termData()->getUniqueId(), 0.1), + connectedness); + } + connectedness /= (j - i); + if (terms[i].termData()->getWeight().percent() != 0 || + terms[j].termData()->getWeight().percent() != 0) + { // only consider term pairs with contribution + pairs.push_back(TermPair(terms[i], terms[j], connectedness)); + setup.divisor += (terms[i].significance() * terms[i].termData()->getWeight().percent() + + terms[j].significance() * terms[j].termData()->getWeight().percent()) * connectedness; + } + } + } +} + + feature_t NativeProximityExecutor::calculateScoreForField(const FieldSetup & fs, uint32_t docId) { @@ -47,35 +108,18 @@ NativeProximityExecutor::calculateScoreForPair(const TermPair & pair, uint32_t f return score; } -NativeProximityExecutor::NativeProximityExecutor(const IQueryEnvironment & env, - const NativeProximityParams & params) : - FeatureExecutor(), - _params(params), - _setups(), - _totalFieldWeight(0), - _md(nullptr) +NativeProximityExecutor::NativeProximityExecutor(const NativeProximityExecutorSharedState& shared_state) + : FeatureExecutor(), + _params(shared_state.get_params()), + _setups(shared_state.get_setups()), + _totalFieldWeight(shared_state.get_total_field_weight()), + _md(nullptr) { - QueryTermHelper queryTerms(env); - std::map<uint32_t, QueryTermVector> fields; - for (const QueryTerm & qt : queryTerms.terms()) { - typedef search::fef::ITermFieldRangeAdapter FRA; - for (FRA iter(*qt.termData()); iter.valid(); iter.next()) { - uint32_t fieldId = iter.get().getFieldId(); - if (_params.considerField(fieldId)) { // only consider fields with contribution - QueryTerm myQt = qt; - myQt.fieldHandle(iter.get().getHandle()); - fields[fieldId].push_back(myQt); - } - } - } - for (const auto & entry : fields) { - if (entry.second.size() >= 2) { - FieldSetup setup(entry.first); - generateTermPairs(env, entry.second, _params.slidingWindow, setup); - if (!setup.pairs.empty()) { - _setups.push_back(std::move(setup)); - _totalFieldWeight += params.vector[entry.first].fieldWeight; - } + auto& fields = shared_state.get_fields(); + for (const auto& entry : fields) { + for (const auto& qt : entry.second) { + // Record that we need normal term field match data + (void) qt.termData()->lookupField(entry.first)->getHandle(MatchDataDetails::Normal); } } } @@ -99,37 +143,12 @@ NativeProximityExecutor::handle_bind_match_data(const fef::MatchData &md) _md = &md; } -void -NativeProximityExecutor::generateTermPairs(const IQueryEnvironment & env, const QueryTermVector & terms, - uint32_t slidingWindow, FieldSetup & setup) -{ - TermPairVector & pairs = setup.pairs; - for (size_t i = 0; i < terms.size(); ++i) { - for (size_t j = i + 1; (j < i + slidingWindow) && (j < terms.size()); ++j) { - feature_t connectedness = 1; - for (size_t k = j; k > i; --k) { - connectedness = std::min(util::lookupConnectedness(env, terms[k].termData()->getUniqueId(), - terms[k-1].termData()->getUniqueId(), 0.1), - connectedness); - } - connectedness /= (j - i); - if (terms[i].termData()->getWeight().percent() != 0 || - terms[j].termData()->getWeight().percent() != 0) - { // only consider term pairs with contribution - pairs.push_back(TermPair(terms[i], terms[j], connectedness)); - setup.divisor += (terms[i].significance() * terms[i].termData()->getWeight().percent() + - terms[j].significance() * terms[j].termData()->getWeight().percent()) * connectedness; - } - } - } -} - - NativeProximityBlueprint::NativeProximityBlueprint() : Blueprint("nativeProximity"), _params(), _defaultProximityBoost("expdecay(500,3)"), - _defaultRevProximityBoost("expdecay(400,3)") + _defaultRevProximityBoost("expdecay(400,3)"), + _shared_state_key() { } @@ -153,10 +172,13 @@ bool NativeProximityBlueprint::setup(const IIndexEnvironment & env, const ParameterList & params) { + vespalib::asciistream shared_state_key_builder; _params.resize(env.getNumFields()); _params.slidingWindow = util::strToNum<uint32_t>(env.getProperties().lookup(getBaseName(), "slidingWindowSize").get("4")); FieldWrapper fields(env, params, FieldType::INDEX); vespalib::string defaultProximityImportance = env.getProperties().lookup(getBaseName(), "proximityImportance").get("0.5"); + shared_state_key_builder << "fef.nativeProximity["; + bool first_field = true; for (uint32_t i = 0; i < fields.getNumFields(); ++i) { const FieldInfo * info = fields.getField(i); uint32_t fieldId = info->id(); @@ -193,8 +215,16 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env, } if (param.field) { env.hintFieldAccess(fieldId); + if (first_field) { + first_field = false; + } else { + shared_state_key_builder << ","; + } + shared_state_key_builder << info->name(); } } + shared_state_key_builder << "]"; + _shared_state_key = shared_state_key_builder.str(); describeOutput("score", "The native proximity score"); return true; @@ -203,11 +233,14 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env, FeatureExecutor & NativeProximityBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { - NativeProximityExecutor &native = stash.create<NativeProximityExecutor>(env, _params); - if (native.empty()) { + auto *shared_state = dynamic_cast<const NativeProximityExecutorSharedState *>(env.getObjectStore().get(_shared_state_key)); + if (shared_state == nullptr) { + shared_state = &stash.create<NativeProximityExecutorSharedState>(env, _params); + } + if (shared_state->empty()) { return stash.create<SingleZeroValueExecutor>(); } else { - return native; + return stash.create<NativeProximityExecutor>(*shared_state); } } @@ -215,6 +248,9 @@ NativeProximityBlueprint::createExecutor(const IQueryEnvironment &env, vespalib: void NativeProximityBlueprint::prepareSharedState(const IQueryEnvironment &queryEnv, IObjectStore &objectStore) const { QueryTermHelper::lookupAndStoreQueryTerms(queryEnv, objectStore); + if (objectStore.get(_shared_state_key) == nullptr) { + objectStore.add(_shared_state_key, std::make_unique<NativeProximityExecutorSharedState>(queryEnv, _params)); + } } } diff --git a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.h b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.h index 4241e81a95e..ab74152f486 100644 --- a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.h +++ b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.h @@ -4,6 +4,7 @@ #include "nativerankfeature.h" #include "termdistancecalculator.h" +#include <map> namespace search::features { @@ -26,9 +27,10 @@ public: }; /** - * Implements the executor for calculating the native proximity score. - **/ -class NativeProximityExecutor : public fef::FeatureExecutor { + * Class containing shared state for native proximity executor. + */ +class NativeProximityExecutorSharedState : public fef::Anything { +public: public: /** * Represents a term pair with connectedness and associated term distance calculator. @@ -50,10 +52,34 @@ public: feature_t divisor; FieldSetup(uint32_t fid) : fieldId(fid), pairs(), divisor(0) {} }; +private: + const NativeProximityParams& _params; + std::vector<FieldSetup> _setups; + uint32_t _total_field_weight; + std::map<uint32_t, QueryTermVector> _fields; + +public: + NativeProximityExecutorSharedState(const fef::IQueryEnvironment& env, const NativeProximityParams& params); + ~NativeProximityExecutorSharedState(); + static void generateTermPairs(const fef::IQueryEnvironment& env, const QueryTermVector& terms, + uint32_t slidingWindow, FieldSetup& setup); + const std::vector<FieldSetup>& get_setups() const { return _setups; } + const NativeProximityParams& get_params() const { return _params; } + uint32_t get_total_field_weight() const { return _total_field_weight; } + bool empty() const { return _setups.empty(); } + const std::map<uint32_t, QueryTermVector>& get_fields() const { return _fields; } +}; +/** + * Implements the executor for calculating the native proximity score. + **/ +class NativeProximityExecutor : public fef::FeatureExecutor { +public: + using TermPair = NativeProximityExecutorSharedState::TermPair; + using FieldSetup = NativeProximityExecutorSharedState::FieldSetup; private: const NativeProximityParams & _params; - std::vector<FieldSetup> _setups; + vespalib::ConstArrayRef<FieldSetup> _setups; uint32_t _totalFieldWeight; const fef::MatchData *_md; @@ -63,13 +89,8 @@ private: virtual void handle_bind_match_data(const fef::MatchData &md) override; public: - NativeProximityExecutor(const fef::IQueryEnvironment & env, const NativeProximityParams & params); + NativeProximityExecutor(const NativeProximityExecutorSharedState& shared_state); void execute(uint32_t docId) override; - - static void generateTermPairs(const fef::IQueryEnvironment & env, const QueryTermVector & terms, - uint32_t slidingWindow, FieldSetup & setup); - - bool empty() const { return _setups.empty(); } }; @@ -81,6 +102,7 @@ private: NativeProximityParams _params; vespalib::string _defaultProximityBoost; vespalib::string _defaultRevProximityBoost; + vespalib::string _shared_state_key; public: NativeProximityBlueprint(); diff --git a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt index 08e64701c05..396775b20c5 100644 --- a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt @@ -23,6 +23,7 @@ vespa_add_library(searchlib_fef OBJECT parameter.cpp parameterdescriptions.cpp parametervalidator.cpp + phrase_splitter_query_env.cpp phrasesplitter.cpp properties.cpp queryproperties.cpp diff --git a/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp new file mode 100644 index 00000000000..eb2915c4e90 --- /dev/null +++ b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp @@ -0,0 +1,76 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "phrase_splitter_query_env.h" + +namespace search::fef { + +void +PhraseSplitterQueryEnv::considerTerm(uint32_t termIdx, const ITermData &term, uint32_t fieldId) +{ + typedef search::fef::ITermFieldRangeAdapter FRA; + + for (FRA iter(term); iter.valid(); iter.next()) { + if (iter.get().getFieldId() == fieldId) { + TermFieldHandle h = iter.get().getHandle(); + _maxHandle = std::max(_maxHandle, h); + if (term.getPhraseLength() > 1) { + SimpleTermData prototype; + prototype.setWeight(term.getWeight()); + prototype.setPhraseLength(1); + prototype.setUniqueId(term.getUniqueId()); + prototype.addField(fieldId); + _phrase_terms.push_back(PhraseTerm(term, _terms.size(), h)); + for (uint32_t i = 0; i < term.getPhraseLength(); ++i) { + _terms.push_back(prototype); + _termIdxMap.push_back(TermIdx(_terms.size() - 1, true)); + } + return; + } + } + } + _termIdxMap.push_back(TermIdx(termIdx, false)); +} + +PhraseSplitterQueryEnv::PhraseSplitterQueryEnv(const IQueryEnvironment & queryEnv, uint32_t fieldId) + : _queryEnv(queryEnv), + _terms(), + _termIdxMap(), + _maxHandle(0), + _skipHandles(0), + _field_id(fieldId), + _phrase_terms() +{ + TermFieldHandle numHandles = 0; // how many handles existed in underlying data + for (uint32_t i = 0; i < queryEnv.getNumTerms(); ++i) { + const ITermData *td = queryEnv.getTerm(i); + assert(td != nullptr); + considerTerm(i, *td, fieldId); + numHandles += td->numFields(); + } + + _skipHandles = _maxHandle + 1 + numHandles; + TermFieldHandle term_handle = _skipHandles; + for (auto & term : _terms) { + // start at _skipHandles + 0 + term.field(0).setHandle(term_handle); + ++term_handle; + } + + for (uint32_t i = 0; i < _phrase_terms.size(); ++i) { + const PhraseTerm &pterm = _phrase_terms[i]; + + for (uint32_t j = 0; j < pterm.term.getPhraseLength(); ++j) { + const ITermData &splitp_td = _terms[pterm.idx + j]; + const ITermFieldData& splitp_tfd = splitp_td.field(0); + HowToCopy meta; + meta.orig_handle = pterm.orig_handle; + meta.split_handle = splitp_tfd.getHandle(); + meta.offsetInPhrase = j; + _copyInfo.push_back(meta); + } + } +} + +PhraseSplitterQueryEnv::~PhraseSplitterQueryEnv() = default; + +} diff --git a/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.h b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.h new file mode 100644 index 00000000000..b2a3d416f5a --- /dev/null +++ b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.h @@ -0,0 +1,90 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "iqueryenvironment.h" +#include "simpletermdata.h" + +namespace search::fef { + +/** + * This class is used together with PhraseSplitter to split all phrase + * terms in a query environment into separate terms. New TermData + * objects are created for each splitted phrase term and managed by + * this class. Unmodified single terms are served from the query + * environment and match data. + * + * Use this class and PhraseSplitter class if you want to handle a + * phrase term the same way as single terms. + **/ +class PhraseSplitterQueryEnv : public IQueryEnvironment +{ +private: + struct TermIdx { + uint32_t idx; // index into either query environment or vector of TermData objects + bool splitted; // whether this term has been splitted or not + TermIdx(uint32_t i, bool s) : idx(i), splitted(s) {} + }; +public: + struct PhraseTerm { + const ITermData & term; // for original phrase + uint32_t idx; // index into vector of our TermData objects + TermFieldHandle orig_handle; + PhraseTerm(const ITermData & t, uint32_t i, uint32_t h) : term(t), idx(i), orig_handle(h) {} + }; + struct HowToCopy { + TermFieldHandle orig_handle; + TermFieldHandle split_handle; + uint32_t offsetInPhrase; + }; +private: + const IQueryEnvironment &_queryEnv; + std::vector<SimpleTermData> _terms; // splitted terms + std::vector<HowToCopy> _copyInfo; + std::vector<TermIdx> _termIdxMap; // renumbering of terms + TermFieldHandle _maxHandle; // the largest among original term field handles + TermFieldHandle _skipHandles; // how many handles to skip + uint32_t _field_id; + std::vector<PhraseTerm> _phrase_terms; // data about original phrase terms + + void considerTerm(uint32_t termIdx, const ITermData &term, uint32_t fieldId); + +public: + /** + * Create a phrase splitter based on the given query environment. + * + * @param queryEnv the query environment to wrap. + * @param fieldId the field where we need to split phrases + **/ + PhraseSplitterQueryEnv(const IQueryEnvironment & queryEnv, uint32_t fieldId); + ~PhraseSplitterQueryEnv(); + + /** + * Update the underlying TermFieldMatchData objects based on the bound MatchData object. + **/ + uint32_t getNumTerms() const override { return _termIdxMap.size(); } + + const ITermData * getTerm(uint32_t idx) const override { + if (idx >= _termIdxMap.size()) { + return nullptr; + } + const TermIdx & ti = _termIdxMap[idx]; + return ti.splitted ? &_terms[ti.idx] : _queryEnv.getTerm(ti.idx); + } + + const Properties & getProperties() const override { return _queryEnv.getProperties(); } + const Location & getLocation() const override { return _queryEnv.getLocation(); } + const attribute::IAttributeContext & getAttributeContext() const override { return _queryEnv.getAttributeContext(); } + double get_average_field_length(const vespalib::string &field_name) const override { return _queryEnv.get_average_field_length(field_name); } + const IIndexEnvironment & getIndexEnvironment() const override { return _queryEnv.getIndexEnvironment(); } + + // Accessor methods used by PhraseSplitter + TermFieldHandle get_skip_handles() const { return _skipHandles; } + uint32_t get_num_phrase_split_terms() const { return _terms.size(); } + uint32_t get_field_id() const { return _field_id; } + const std::vector<HowToCopy>& get_copy_info() const { return _copyInfo; } + const std::vector<PhraseTerm>& get_phrase_terms() const { return _phrase_terms; } +}; + + +} diff --git a/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp b/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp index e84f61332e1..b07498e8608 100644 --- a/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp +++ b/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp @@ -1,76 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "phrasesplitter.h" +#include "phrase_splitter_query_env.h" namespace search::fef { -void -PhraseSplitter::considerTerm(uint32_t termIdx, const ITermData &term, std::vector<PhraseTerm> &phraseTerms, uint32_t fieldId) +PhraseSplitter::PhraseSplitter(const PhraseSplitterQueryEnv& phrase_splitter_query_env) + : _phrase_splitter_query_env(phrase_splitter_query_env), + _skipHandles(_phrase_splitter_query_env.get_skip_handles()), + _matchData(nullptr), + _termMatches(_phrase_splitter_query_env.get_num_phrase_split_terms()) { - typedef search::fef::ITermFieldRangeAdapter FRA; - - for (FRA iter(term); iter.valid(); iter.next()) { - if (iter.get().getFieldId() == fieldId) { - TermFieldHandle h = iter.get().getHandle(); - _maxHandle = std::max(_maxHandle, h); - if (term.getPhraseLength() > 1) { - SimpleTermData prototype; - prototype.setWeight(term.getWeight()); - prototype.setPhraseLength(1); - prototype.setUniqueId(term.getUniqueId()); - prototype.addField(fieldId); - phraseTerms.push_back(PhraseTerm(term, _terms.size(), h)); - for (uint32_t i = 0; i < term.getPhraseLength(); ++i) { - _terms.push_back(prototype); - _termIdxMap.push_back(TermIdx(_terms.size() - 1, true)); - } - return; - } - } + uint32_t field_id = _phrase_splitter_query_env.get_field_id(); + for (auto & term_match : _termMatches) { + term_match.setFieldId(field_id); } - _termIdxMap.push_back(TermIdx(termIdx, false)); -} - -PhraseSplitter::PhraseSplitter(const IQueryEnvironment & queryEnv, uint32_t fieldId) : - _queryEnv(queryEnv), - _matchData(nullptr), - _terms(), - _termMatches(), - _termIdxMap(), - _maxHandle(0), - _skipHandles(0) -{ - TermFieldHandle numHandles = 0; // how many handles existed in underlying data - std::vector<PhraseTerm> phraseTerms; // data about original phrase terms - - for (uint32_t i = 0; i < queryEnv.getNumTerms(); ++i) { - const ITermData *td = queryEnv.getTerm(i); - assert(td != nullptr); - considerTerm(i, *td, phraseTerms, fieldId); - numHandles += td->numFields(); - } - - _skipHandles = _maxHandle + 1 + numHandles; - _termMatches.reserve(_terms.size()); - for (auto & term : _terms) { - // start at _skipHandles + 0 - term.field(0).setHandle(_skipHandles + _termMatches.size()); - _termMatches.emplace_back(); - _termMatches.back().setFieldId(fieldId); - } - - for (uint32_t i = 0; i < phraseTerms.size(); ++i) { - const PhraseTerm &pterm = phraseTerms[i]; - - for (uint32_t j = 0; j < pterm.term.getPhraseLength(); ++j) { - const ITermData &splitp_td = _terms[pterm.idx + j]; - const ITermFieldData& splitp_tfd = splitp_td.field(0); - HowToCopy meta; - meta.orig_handle = pterm.orig_handle; - meta.split_handle = splitp_tfd.getHandle(); - meta.offsetInPhrase = j; - _copyInfo.push_back(meta); - } + auto &phrase_terms = _phrase_splitter_query_env.get_phrase_terms(); + for (const auto &phrase_term : phrase_terms) { + // Record that we need normal term field match data + (void) phrase_term.term.lookupField(field_id)->getHandle(MatchDataDetails::Normal); } } @@ -91,11 +39,11 @@ PhraseSplitter::copyTermFieldMatchData(TermFieldMatchData & dst, const TermField void PhraseSplitter::update() { - for (uint32_t i = 0; i < _copyInfo.size(); ++i) { - const TermFieldMatchData *src = _matchData->resolveTermField(_copyInfo[i].orig_handle); - TermFieldMatchData *dst = resolveSplittedTermField(_copyInfo[i].split_handle); + for (const auto ©_info : _phrase_splitter_query_env.get_copy_info()) { + const TermFieldMatchData *src = _matchData->resolveTermField(copy_info.orig_handle); + TermFieldMatchData *dst = resolveSplittedTermField(copy_info.split_handle); assert(src != nullptr && dst != nullptr); - copyTermFieldMatchData(*dst, *src, _copyInfo[i].offsetInPhrase); + copyTermFieldMatchData(*dst, *src, copy_info.offsetInPhrase); } } diff --git a/searchlib/src/vespa/searchlib/fef/phrasesplitter.h b/searchlib/src/vespa/searchlib/fef/phrasesplitter.h index 25944158445..49b0a8a5dbf 100644 --- a/searchlib/src/vespa/searchlib/fef/phrasesplitter.h +++ b/searchlib/src/vespa/searchlib/fef/phrasesplitter.h @@ -2,60 +2,34 @@ #pragma once -#include "iqueryenvironment.h" #include "matchdata.h" -#include "simpletermdata.h" #include "termfieldmatchdata.h" -#include "fieldinfo.h" namespace search::fef { +class PhraseSplitterQueryEnv; + /** - * This class is used to split all phrase terms in a query environment - * into separate terms. New TermData and TermFieldMatchData objects - * are created for each splitted phrase term and managed by this - * class. Unmodified single terms are served from the query - * environment and match data. + * This class is used together with PhraseSplitterQueryEnv to split + * all phrase terms in a query environment into separate terms. New + * TermFieldMatchData objects are created for each splitted phrase + * term and managed by this class. Unmodified single terms are served + * from the query environment and match data. * * The TermFieldMatchData objects managed by this class are updated * based on the TermFieldMatchData objects associated with the * original phrase terms. Positions are adjusted with +1 for each term * after the first one. * - * Use this class if you want to handle a phrase term the same way as - * single terms. + * Use this class and PhraseSplitterQueryEnv if you want to handle a + * phrase term the same way as single terms. **/ -class PhraseSplitter : public IQueryEnvironment +class PhraseSplitter { -private: - struct TermIdx { - uint32_t idx; // index into either query environment or vector of TermData objects - bool splitted; // whether this term has been splitted or not - TermIdx(uint32_t i, bool s) : idx(i), splitted(s) {} - }; - struct PhraseTerm { - const ITermData & term; // for original phrase - uint32_t idx; // index into vector of our TermData objects - TermFieldHandle orig_handle; - PhraseTerm(const ITermData & t, uint32_t i, uint32_t h) : term(t), idx(i), orig_handle(h) {} - }; - struct HowToCopy { - TermFieldHandle orig_handle; - TermFieldHandle split_handle; - uint32_t offsetInPhrase; - }; - - const IQueryEnvironment &_queryEnv; + const PhraseSplitterQueryEnv& _phrase_splitter_query_env; + TermFieldHandle _skipHandles; const MatchData *_matchData; - std::vector<SimpleTermData> _terms; // splitted terms std::vector<TermFieldMatchData> _termMatches; // match objects associated with splitted terms - std::vector<HowToCopy> _copyInfo; - std::vector<TermIdx> _termIdxMap; // renumbering of terms - TermFieldHandle _maxHandle; // the largest among original term field handles - TermFieldHandle _skipHandles; // how many handles to skip - - void considerTerm(uint32_t termIdx, const ITermData &term, std::vector<PhraseTerm> &phraseTerms, uint32_t fieldId); - void splitPhrase(const ITermData &phrase, std::vector<PhraseTerm> &phraseTerms, uint32_t fieldId); TermFieldMatchData *resolveSplittedTermField(TermFieldHandle handle) { return &_termMatches[handle - _skipHandles]; @@ -72,7 +46,7 @@ public: * @param queryEnv the query environment to wrap. * @param field the field where we need to split phrases **/ - PhraseSplitter(const IQueryEnvironment & queryEnv, uint32_t fieldId); + PhraseSplitter(const PhraseSplitterQueryEnv &phrase_splitter_query_env); ~PhraseSplitter(); /** @@ -89,15 +63,6 @@ public: * Update the underlying TermFieldMatchData objects based on the bound MatchData object. **/ void update(); - uint32_t getNumTerms() const override { return _termIdxMap.size(); } - - const ITermData * getTerm(uint32_t idx) const override { - if (idx >= _termIdxMap.size()) { - return nullptr; - } - const TermIdx & ti = _termIdxMap[idx]; - return ti.splitted ? &_terms[ti.idx] : _queryEnv.getTerm(ti.idx); - } /** * Inherit doc from MatchData. @@ -109,12 +74,8 @@ public: return handle < _skipHandles ? _matchData->resolveTermField(handle) : resolveSplittedTermField(handle); } - const Properties & getProperties() const override { return _queryEnv.getProperties(); } - const Location & getLocation() const override { return _queryEnv.getLocation(); } - const attribute::IAttributeContext & getAttributeContext() const override { return _queryEnv.getAttributeContext(); } - double get_average_field_length(const vespalib::string &field_name) const override { return _queryEnv.get_average_field_length(field_name); } - const IIndexEnvironment & getIndexEnvironment() const override { return _queryEnv.getIndexEnvironment(); } void bind_match_data(const fef::MatchData &md) { _matchData = &md; } + const PhraseSplitterQueryEnv& get_query_env() const { return _phrase_splitter_query_env; } }; } diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp index 781135bf246..7e3e0f5e4bc 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp @@ -104,7 +104,8 @@ TermFieldMatchData::swap(TermFieldMatchData &rhs) namespace { -constexpr size_t MAX_ELEMS = std::numeric_limits<uint16_t>::max(); +constexpr size_t MAX_ELEMS = std::numeric_limits<uint16_t>::max(); +constexpr size_t INITIAL_ELEMS = 1024/sizeof(TermFieldMatchDataPosition); } @@ -128,7 +129,7 @@ TermFieldMatchData::allocateVector() { assert(_sz < 2); assert(!allocated()); - size_t newSize = 2; + size_t newSize = INITIAL_ELEMS; TermFieldMatchDataPosition * n = new TermFieldMatchDataPosition[newSize]; if (_sz > 0) { n[0] = *getFixed(); diff --git a/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp b/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp index 97cb829e30c..973e11fc0d2 100644 --- a/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp +++ b/searchlib/src/vespa/searchlib/fef/termmatchdatamerger.cpp @@ -41,31 +41,52 @@ TermMatchDataMerger::merge(uint32_t docid, { _scratch.clear(); bool wasMatch = false; + bool needs_normal_features = out.needs_normal_features(); + bool needs_interleaved_features = out.needs_interleaved_features(); + uint32_t num_occs = 0u; + uint16_t field_length = 0u; for (size_t i = 0; i < in.size(); ++i) { const TermFieldMatchData *md = in[i].matchData; if (md->getDocId() == docid) { - for (const TermFieldMatchDataPosition &iter : *md) { - double exactness = in[i].exactness * iter.getMatchExactness(); - _scratch.push_back(iter); - _scratch.back().setMatchExactness(exactness); + if (needs_normal_features) { + for (const TermFieldMatchDataPosition &iter : *md) { + double exactness = in[i].exactness * iter.getMatchExactness(); + _scratch.push_back(iter); + _scratch.back().setMatchExactness(exactness); + } + } + if (needs_interleaved_features) { + num_occs += md->getNumOccs(); + field_length = std::max(field_length, md->getFieldLength()); } wasMatch = true; } } if (wasMatch) { out.reset(docid); - if (_scratch.size() > 0) { - std::sort(_scratch.begin(), _scratch.end(), - TermFieldMatchDataPosition::compareWithExactness); - TermFieldMatchDataPosition prev = _scratch[0]; - for (size_t i = 1; i < _scratch.size(); ++i) { - const TermFieldMatchDataPosition &curr = _scratch[i]; - if (prev.key() < curr.key()) { - out.appendPosition(prev); - prev = curr; + if (needs_normal_features) { + num_occs = 0; + if (_scratch.size() > 0) { + std::sort(_scratch.begin(), _scratch.end(), + TermFieldMatchDataPosition::compareWithExactness); + TermFieldMatchDataPosition prev = _scratch[0]; + for (size_t i = 1; i < _scratch.size(); ++i) { + const TermFieldMatchDataPosition &curr = _scratch[i]; + if (prev.key() < curr.key()) { + out.appendPosition(prev); + prev = curr; + ++num_occs; + } } + out.appendPosition(prev); + ++num_occs; } - out.appendPosition(prev); + } + if (needs_interleaved_features) { + constexpr uint32_t max_num_occs = std::numeric_limits<uint16_t>::max(); + uint16_t capped_num_occs = std::min(num_occs, max_num_occs); + out.setNumOccs(std::min(capped_num_occs, field_length)); + out.setFieldLength(field_length); } } } diff --git a/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp b/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp index a7ad475d6aa..4656a5e9edd 100644 --- a/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp +++ b/searchlib/src/vespa/searchlib/index/doctypebuilder.cpp @@ -10,38 +10,36 @@ using namespace document; namespace search::index { namespace { -TensorDataType tensorDataType(vespalib::eval::ValueType::from_spec("tensor(x{}, y{})")); - -const DataType *convert(Schema::DataType type) { +DataType::Type convert(Schema::DataType type) { switch (type) { case schema::DataType::BOOL: case schema::DataType::UINT2: case schema::DataType::UINT4: case schema::DataType::INT8: - return DataType::BYTE; + return DataType::T_BYTE; case schema::DataType::INT16: - return DataType::SHORT; + return DataType::T_SHORT; case schema::DataType::INT32: - return DataType::INT; + return DataType::T_INT; case schema::DataType::INT64: - return DataType::LONG; + return DataType::T_LONG; case schema::DataType::FLOAT: - return DataType::FLOAT; + return DataType::T_FLOAT; case schema::DataType::DOUBLE: - return DataType::DOUBLE; + return DataType::T_DOUBLE; case schema::DataType::STRING: - return DataType::STRING; + return DataType::T_STRING; case schema::DataType::RAW: - return DataType::RAW; + return DataType::T_RAW; case schema::DataType::BOOLEANTREE: - return DataType::PREDICATE; + return DataType::T_PREDICATE; case schema::DataType::TENSOR: - return &tensorDataType; + return DataType::T_TENSOR; default: break; } assert(!"Unknown datatype in schema"); - return 0; + return DataType::MAX; } void @@ -142,12 +140,12 @@ document::DocumenttypesConfig DocTypeBuilder::makeConfig() const { if (usf != usedFields.end()) { continue; // taken as index field } - const DataType *primitiveType = convert(field.getDataType()); - if (primitiveType->getId() == DataType::T_TENSOR) { - header_struct.addTensorField(field.getName(), dynamic_cast<const TensorDataType &>(*primitiveType).getTensorType().to_spec()); + auto type_id = convert(field.getDataType()); + if (type_id == DataType::T_TENSOR) { + header_struct.addTensorField(field.getName(), field.get_tensor_spec()); } else { header_struct.addField(field.getName(), type_cache.getType( - primitiveType->getId(), field.getCollectionType())); + type_id, field.getCollectionType())); } usedFields.insert(field.getName()); } @@ -158,12 +156,12 @@ document::DocumenttypesConfig DocTypeBuilder::makeConfig() const { if (usf != usedFields.end()) { continue; // taken as index field or attribute field } - const DataType *primitiveType(convert(field.getDataType())); - if (primitiveType->getId() == DataType::T_TENSOR) { - header_struct.addTensorField(field.getName(), dynamic_cast<const TensorDataType &>(*primitiveType).getTensorType().to_spec()); + auto type_id = convert(field.getDataType()); + if (type_id == DataType::T_TENSOR) { + header_struct.addTensorField(field.getName(), field.get_tensor_spec()); } else { header_struct.addField(field.getName(), type_cache.getType( - primitiveType->getId(), field.getCollectionType())); + type_id, field.getCollectionType())); } usedFields.insert(field.getName()); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index bc74725d620..fecb8116f90 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> #include <vespa/searchlib/queryeval/searchiterator.h> +#include <vespa/searchlib/queryeval/filter_wrapper.h> #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreenode.hpp> @@ -272,6 +273,13 @@ public: _field_id, _posting_itr.size()); return result; } + + SearchIterator::UP createFilterSearch(bool, FilterConstraint) const override { + auto wrapper = std::make_unique<queryeval::FilterWrapper>(getState().numFields()); + auto & tfmda = wrapper->tfmda(); + wrapper->wrap(make_search_iterator<interleaved_features>(_posting_itr, _feature_store, _field_id, tfmda)); + return wrapper; + } }; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp index d68fae69a52..f14cac54980 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp @@ -197,8 +197,8 @@ FieldInverter::sortWords() // Populate word numbers in word buffer and mapping from // word numbers to word reference. // TODO: shrink word buffer to only contain unique words - std::vector<uint32_t>::const_iterator w(_wordRefs.begin() + 1); - std::vector<uint32_t>::const_iterator we(_wordRefs.end()); + auto w(_wordRefs.begin() + 1); + auto we(_wordRefs.end()); uint32_t wordNum = 1; // First valid word number const char *lastWord = getWordFromRef(*w); updateWordNum(*w, wordNum); diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 1c074b6ee77..78a1cf6c171 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -9,9 +9,9 @@ #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/vespalib/stllike/allocator.h> #include <limits> #include <map> -#include <set> namespace search::index { class FieldLengthCalculator; } @@ -115,8 +115,8 @@ private: void set_field_length(uint32_t field_length) { _field_length = field_length; } }; - using ElemInfoVec = std::vector<ElemInfo>; - using PosInfoVec = std::vector<PosInfo>; + using ElemInfoVec = std::vector<ElemInfo, vespalib::allocator_large<ElemInfo>>; + using PosInfoVec = std::vector<PosInfo, vespalib::allocator_large<PosInfo>>; class CompareWordRef { const char *const _wordBuffer; @@ -161,6 +161,7 @@ private: uint32_t getLen() const { return _len; } }; + using UInt32Vector = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; // Current field state. uint32_t _fieldId; // current field id uint32_t _elem; // current element @@ -174,8 +175,8 @@ private: ElemInfoVec _elems; PosInfoVec _positions; index::DocIdAndPosOccFeatures _features; - std::vector<uint32_t> _elementWordRefs; - std::vector<uint32_t> _wordRefs; + UInt32Vector _elementWordRefs; + UInt32Vector _wordRefs; using SpanTerm = std::pair<document::Span, const document::FieldValue *>; using SpanTermVector = std::vector<SpanTerm>; @@ -184,18 +185,16 @@ private: // Info about aborted and pending documents. std::vector<PositionRange> _abortedDocs; std::map<uint32_t, PositionRange> _pendingDocs; - std::vector<uint32_t> _removeDocs; + UInt32Vector _removeDocs; FieldIndexRemover &_remover; IOrderedFieldIndexInserter &_inserter; index::FieldLengthCalculator &_calculator; - void - invertNormalDocTextField(const document::FieldValue &val); + void invertNormalDocTextField(const document::FieldValue &val); public: void startElement(int32_t weight); - void endElement(); private: @@ -253,14 +252,9 @@ public: processAnnotations(const document::StringFieldValue &value); private: - void - processNormalDocTextField(const document::StringFieldValue &field); - - void - processNormalDocArrayTextField(const document::ArrayFieldValue &field); - - void - processNormalDocWeightedSetTextField(const document::WeightedSetFieldValue &field); + void processNormalDocTextField(const document::StringFieldValue &field); + void processNormalDocArrayTextField(const document::ArrayFieldValue &field); + void processNormalDocWeightedSetTextField(const document::WeightedSetFieldValue &field); const index::Schema &getSchema() const { return _schema; } @@ -313,7 +307,7 @@ public: /** * Setup remove of word in old version of document. */ - virtual void remove(const vespalib::stringref word, uint32_t docId) override; + void remove(const vespalib::stringref word, uint32_t docId) override; void removeDocument(uint32_t docId) { abortPendingDoc(docId); diff --git a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt index 0dcb0393473..a080fbcd387 100644 --- a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt @@ -5,10 +5,12 @@ vespa_add_library(searchlib_queryeval OBJECT andsearch.cpp blueprint.cpp booleanmatchiteratorwrapper.cpp + children_iterators.cpp create_blueprint_visitor_helper.cpp document_weight_search_iterator.cpp dot_product_blueprint.cpp dot_product_search.cpp + elementiterator.cpp emptysearch.cpp equiv_blueprint.cpp equivsearch.cpp @@ -18,7 +20,10 @@ vespa_add_library(searchlib_queryeval OBJECT fake_search.cpp fake_searchable.cpp field_spec.cpp + filter_wrapper.cpp + full_search.cpp get_weight_from_node.cpp + global_filter.cpp hitcollector.cpp intermediate_blueprints.cpp isourceselector.cpp diff --git a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp index 5646af9e677..61d6d2d9259 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp @@ -51,7 +51,7 @@ public: * * @param children the search objects we are andnot'ing **/ - AndNotSearchStrict(const Children & children) : AndNotSearchStrictBase(children) + AndNotSearchStrict(Children children) : AndNotSearchStrictBase(std::move(children)) { } @@ -105,8 +105,8 @@ AndNotSearchStrict::internalSeek(uint32_t docid) } // namespace -OptimizedAndNotForBlackListing::OptimizedAndNotForBlackListing(const MultiSearch::Children & children) : - AndNotSearchStrictBase(children) +OptimizedAndNotForBlackListing::OptimizedAndNotForBlackListing(MultiSearch::Children children) : + AndNotSearchStrictBase(std::move(children)) { } @@ -131,23 +131,24 @@ void OptimizedAndNotForBlackListing::doUnpack(uint32_t docid) positive()->doUnpack(docid); } -SearchIterator * -AndNotSearch::create(const AndNotSearch::Children &children, bool strict) { +std::unique_ptr<SearchIterator> +AndNotSearch::create(ChildrenIterators children_in, bool strict) { + MultiSearch::Children children = std::move(children_in); if (strict) { - if ((children.size() == 2) && OptimizedAndNotForBlackListing::isBlackListIterator(children[1])) { - return new OptimizedAndNotForBlackListing(children); + if ((children.size() == 2) && OptimizedAndNotForBlackListing::isBlackListIterator(children[1].get())) { + return std::make_unique<OptimizedAndNotForBlackListing>(std::move(children)); } else { - return new AndNotSearchStrict(children); + return std::make_unique<AndNotSearchStrict>(std::move(children)); } } else { - return new AndNotSearch(children); + return SearchIterator::UP(new AndNotSearch(std::move(children))); } } BitVector::UP AndNotSearch::get_hits(uint32_t begin_id) { const Children &children = getChildren(); - BitVector::UP result = children.front()->get_hits(begin_id); + BitVector::UP result = children[0]->get_hits(begin_id); result->notSelf(); result = TermwiseHelper::orChildren(std::move(result), children.begin()+1, children.end(), begin_id); result->notSelf(); diff --git a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h index c29ef9407ad..53a9aad6cca 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h @@ -24,11 +24,10 @@ protected: * * @param children the search objects we are andnot'ing **/ - AndNotSearch(const Children & children) : MultiSearch(children) { } + AndNotSearch(MultiSearch::Children children) : MultiSearch(std::move(children)) { } public: - // Caller takes ownership of the returned SearchIterator. - static SearchIterator *create(const Children &children, bool strict); + static std::unique_ptr<SearchIterator> create(ChildrenIterators children, bool strict); std::unique_ptr<BitVector> get_hits(uint32_t begin_id) override; void or_hits_into(BitVector &result, uint32_t begin_id) override; @@ -43,7 +42,7 @@ private: class AndNotSearchStrictBase : public AndNotSearch { protected: - AndNotSearchStrictBase(const Children & children) : AndNotSearch(children) { } + AndNotSearchStrictBase(Children children) : AndNotSearch(std::move(children)) { } private: Trinary is_strict() const override { return Trinary::True; } UP andWith(UP filter, uint32_t estimate) override; @@ -61,7 +60,7 @@ private: //typedef FilterAttributeIteratorT<SingleValueSmallNumericAttribute::SingleSearchContext> BlackListIterator; typedef AttributeIteratorT<SingleValueSmallNumericAttribute::SingleSearchContext> BlackListIterator; public: - OptimizedAndNotForBlackListing(const MultiSearch::Children & children); + OptimizedAndNotForBlackListing(MultiSearch::Children children); static bool isBlackListIterator(const SearchIterator * iterator); uint32_t seekFast(uint32_t docid) { @@ -69,8 +68,8 @@ public: } void initRange(uint32_t beginid, uint32_t endid) override; private: - SearchIterator * positive() { return getChildren()[0]; } - BlackListIterator * blackList() { return static_cast<BlackListIterator *>(getChildren()[1]); } + SearchIterator * positive() { return getChildren()[0].get(); } + BlackListIterator * blackList() { return static_cast<BlackListIterator *>(getChildren()[1].get()); } template<bool doSeekOnly> uint32_t internalSeek(uint32_t docid) { uint32_t curr(docid); diff --git a/searchlib/src/vespa/searchlib/queryeval/andsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/andsearch.cpp index 3cbb30e5f89..229cd27afed 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/andsearch.cpp @@ -48,8 +48,8 @@ AndSearch::doUnpack(uint32_t docid) } } -AndSearch::AndSearch(const Children & children) : - MultiSearch(children), +AndSearch::AndSearch(Children children) : + MultiSearch(std::move(children)), _estimate(std::numeric_limits<uint32_t>::max()) { } @@ -99,31 +99,37 @@ private: } -AndSearch * -AndSearch::create(const MultiSearch::Children &children, bool strict) +std::unique_ptr<AndSearch> +AndSearch::create(ChildrenIterators children, bool strict) { UnpackInfo unpackInfo; unpackInfo.forceAll(); - return create(children, strict, unpackInfo); + return create(std::move(children), strict, unpackInfo); } -AndSearch * -AndSearch::create(const MultiSearch::Children &children, bool strict, const UnpackInfo & unpackInfo) { +std::unique_ptr<AndSearch> +AndSearch::create(ChildrenIterators children, bool strict, const UnpackInfo & unpackInfo) { if (strict) { if (unpackInfo.unpackAll()) { - return new AndSearchStrict<FullUnpack>(children, FullUnpack()); + using MyAnd = AndSearchStrict<FullUnpack>; + return std::make_unique<MyAnd>(std::move(children), FullUnpack()); } else if(unpackInfo.empty()) { - return new AndSearchStrict<NoUnpack>(children, NoUnpack()); + using MyAnd = AndSearchStrict<NoUnpack>; + return std::make_unique<MyAnd>(std::move(children), NoUnpack()); } else { - return new AndSearchStrict<SelectiveUnpack>(children, SelectiveUnpack(unpackInfo)); + using MyAnd = AndSearchStrict<SelectiveUnpack>; + return std::make_unique<MyAnd>(std::move(children), SelectiveUnpack(unpackInfo)); } } else { if (unpackInfo.unpackAll()) { - return new AndSearchNoStrict<FullUnpack>(children, FullUnpack()); + using MyAnd = AndSearchNoStrict<FullUnpack>; + return std::make_unique<MyAnd>(std::move(children), FullUnpack()); } else if (unpackInfo.empty()) { - return new AndSearchNoStrict<NoUnpack>(children, NoUnpack()); + using MyAnd = AndSearchNoStrict<NoUnpack>; + return std::make_unique<MyAnd>(std::move(children), NoUnpack()); } else { - return new AndSearchNoStrict<SelectiveUnpack>(children, SelectiveUnpack(unpackInfo)); + using MyAnd = AndSearchNoStrict<SelectiveUnpack>; + return std::make_unique<MyAnd>(std::move(children), SelectiveUnpack(unpackInfo)); } } } diff --git a/searchlib/src/vespa/searchlib/queryeval/andsearch.h b/searchlib/src/vespa/searchlib/queryeval/andsearch.h index 59c5769a1a2..b081951e826 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/andsearch.h @@ -13,9 +13,8 @@ namespace search::queryeval { class AndSearch : public MultiSearch { public: - // Caller takes ownership of the returned SearchIterator. - static AndSearch *create(const Children &children, bool strict, const UnpackInfo & unpackInfo); - static AndSearch *create(const Children &children, bool strict); + static std::unique_ptr<AndSearch> create(ChildrenIterators children, bool strict, const UnpackInfo & unpackInfo); + static std::unique_ptr<AndSearch> create(ChildrenIterators children, bool strict); std::unique_ptr<BitVector> get_hits(uint32_t begin_id) override; void or_hits_into(BitVector &result, uint32_t begin_id) override; @@ -24,7 +23,7 @@ public: AndSearch & estimate(uint32_t est) { _estimate = est; return *this; } uint32_t estimate() const { return _estimate; } protected: - AndSearch(const Children & children); + AndSearch(Children children); void doUnpack(uint32_t docid) override; UP andWith(UP filter, uint32_t estimate) override; UP offerFilterToChildren(UP filter, uint32_t estimate); diff --git a/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h b/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h index 1b1a5e77445..fac502f4c98 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h +++ b/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h @@ -22,8 +22,8 @@ public: * @param children the search objects we are and'ing * ownership of the children is taken by the MultiSearch base class. **/ - AndSearchNoStrict(const Children & children, const Unpack & unpacker) : - AndSearch(children), + AndSearchNoStrict(Children children, const Unpack & unpacker) : + AndSearch(std::move(children)), _unpacker(unpacker) { } diff --git a/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h b/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h index 87be4248a0a..7ec179404b6 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h +++ b/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h @@ -23,8 +23,8 @@ protected: Trinary is_strict() const override { return Trinary::True; } SearchIterator::UP andWith(SearchIterator::UP filter, uint32_t estimate) override; public: - AndSearchStrict(const MultiSearch::Children & children, const Unpack & unpacker) : - AndSearchNoStrict<Unpack>(children, unpacker) + AndSearchStrict(MultiSearch::Children children, const Unpack & unpacker) + : AndSearchNoStrict<Unpack>(std::move(children), unpacker) { } diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 1f0ed4ed76f..1a7a0c5eba2 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -3,6 +3,8 @@ #include "blueprint.h" #include "leaf_blueprints.h" #include "intermediate_blueprints.h" +#include "emptysearch.h" +#include "full_search.h" #include "field_spec.hpp" #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/objects/visit.hpp> @@ -105,7 +107,7 @@ Blueprint::get_replacement() } void -Blueprint::set_global_filter(std::shared_ptr<BitVector>) +Blueprint::set_global_filter(const GlobalFilter &) { } @@ -119,6 +121,17 @@ Blueprint::root() const return *bp; } +SearchIterator::UP +Blueprint::createFilterSearch(bool /*strict*/, FilterConstraint constraint) const +{ + if (constraint == FilterConstraint::UPPER_BOUND) { + return std::make_unique<FullSearch>(); + } else { + LOG_ASSERT(constraint == FilterConstraint::LOWER_BOUND); + return std::make_unique<EmptySearch>(); + } +} + vespalib::string Blueprint::asString() const { @@ -371,7 +384,7 @@ IntermediateBlueprint::optimize(Blueprint* &self) } void -IntermediateBlueprint::set_global_filter(std::shared_ptr<BitVector> global_filter) +IntermediateBlueprint::set_global_filter(const GlobalFilter &global_filter) { for (auto & child : _children) { if (child->getState().want_global_filter()) { @@ -387,10 +400,9 @@ IntermediateBlueprint::createSearch(fef::MatchData &md, bool strict) const subSearches.reserve(_children.size()); for (size_t i = 0; i < _children.size(); ++i) { bool strictChild = (strict && inheritStrict(i)); - SearchIterator::UP search = _children[i]->createSearch(md, strictChild); - subSearches.push_back(search.release()); + subSearches.push_back(_children[i]->createSearch(md, strictChild)); } - return createIntermediateSearch(subSearches, strict, md); + return createIntermediateSearch(std::move(subSearches), strict, md); } IntermediateBlueprint::IntermediateBlueprint() = default; diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index 091a5b924ae..ef15736073e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -5,6 +5,8 @@ #include "field_spec.h" #include "unpackinfo.h" #include "executeinfo.h" +#include "global_filter.h" +#include "multisearch.h" #include <vespa/searchlib/common/bitvector.h> namespace vespalib { class ObjectVisitor; } @@ -163,6 +165,12 @@ public: virtual bool check(const Blueprint & bp) const = 0; }; + // Signal if createFilterSearch should ensure the returned + // iterator is an upper bound (yielding a hit on at least + // all matching documents) or a lower bound (never yielding a + // hit that isn't certain to be a match). + enum class FilterConstraint { UPPER_BOUND, LOWER_BOUND }; + Blueprint(); Blueprint(const Blueprint &) = delete; Blueprint &operator=(const Blueprint &) = delete; @@ -186,7 +194,7 @@ public: virtual bool supports_termwise_children() const { return false; } virtual bool always_needs_unpack() const { return false; } - virtual void set_global_filter(std::shared_ptr<BitVector> global_filter); + virtual void set_global_filter(const GlobalFilter &global_filter); virtual const State &getState() const = 0; const Blueprint &root() const; @@ -198,6 +206,7 @@ public: bool frozen() const { return _frozen; } virtual SearchIteratorUP createSearch(fef::MatchData &md, bool strict) const = 0; + virtual SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const; // for debug dumping vespalib::string asString() const; @@ -273,7 +282,7 @@ public: void setDocIdLimit(uint32_t limit) override final; void optimize(Blueprint* &self) override final; - void set_global_filter(std::shared_ptr<BitVector> global_filter) override; + void set_global_filter(const GlobalFilter &global_filter) override; IndexList find(const IPredicate & check) const; size_t childCnt() const { return _children.size(); } @@ -289,7 +298,7 @@ public: virtual void sort(std::vector<Blueprint*> &children) const = 0; virtual bool inheritStrict(size_t i) const = 0; virtual SearchIteratorUP - createIntermediateSearch(const std::vector<SearchIterator *> &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const = 0; void visitMembers(vespalib::ObjectVisitor &visitor) const override; diff --git a/searchlib/src/vespa/searchlib/queryeval/children_iterators.cpp b/searchlib/src/vespa/searchlib/queryeval/children_iterators.cpp new file mode 100644 index 00000000000..3abbd6a1b81 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/children_iterators.cpp @@ -0,0 +1,3 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "children_iterators.h" diff --git a/searchlib/src/vespa/searchlib/queryeval/children_iterators.h b/searchlib/src/vespa/searchlib/queryeval/children_iterators.h new file mode 100644 index 00000000000..aa147e0299b --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/children_iterators.h @@ -0,0 +1,39 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "searchiterator.h" + +namespace search::queryeval { + +/** + * Convenience for constructing MultiSearch::Children + * holding them or passing ownership around. + **/ +class ChildrenIterators { + private: + std::vector<SearchIterator::UP> _data; + public: + ChildrenIterators(std::vector<SearchIterator::UP> data) + : _data(std::move(data)) {} + ChildrenIterators(ChildrenIterators && other) = default; + + // convenience constructors for unit tests: + template <typename... Args> + ChildrenIterators(SearchIterator::UP a, Args&&... args) { + _data.reserve(1 + sizeof...(Args)); + _data.push_back(std::move(a)); + (_data.emplace_back(std::forward<Args>(args)), ...); + } + template <typename... Args> + ChildrenIterators(SearchIterator *a, Args&&... args) { + _data.reserve(1 + sizeof...(Args)); + _data.emplace_back(a); + (_data.emplace_back(std::forward<Args>(args)), ...); + } + operator std::vector<SearchIterator::UP> () && { + return std::move(_data); + } +}; + +} // namespace diff --git a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp index 741aec98f4f..454db4e820a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp @@ -59,6 +59,7 @@ DotProductBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray const State &childState = _terms[i]->getState(); assert(childState.numFields() == 1); childMatch.push_back(childState.field(0).resolve(*md)); + // TODO: pass ownership with unique_ptr children[i] = _terms[i]->createSearch(*md, true).release(); } return DotProductSearch::create(children, *tfmda[0], childMatch, _weights, std::move(md)); diff --git a/searchlib/src/vespa/searchlib/queryeval/dot_product_search.h b/searchlib/src/vespa/searchlib/queryeval/dot_product_search.h index 0fa047bb215..8bdbdd18003 100644 --- a/searchlib/src/vespa/searchlib/queryeval/dot_product_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/dot_product_search.h @@ -25,6 +25,7 @@ protected: DotProductSearch() {} public: + // TODO: use MultiSearch::Children to pass ownership static SearchIterator::UP create(const std::vector<SearchIterator*> &children, search::fef::TermFieldMatchData &tmd, const std::vector<fef::TermFieldMatchData*> &childMatch, diff --git a/searchlib/src/vespa/searchlib/queryeval/elementiterator.cpp b/searchlib/src/vespa/searchlib/queryeval/elementiterator.cpp new file mode 100644 index 00000000000..da4a088adea --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/elementiterator.cpp @@ -0,0 +1,71 @@ +#include "elementiterator.h" +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/vespalib/objects/objectvisitor.h> + +namespace search::queryeval { + +void +ElementIterator::visitMembers(vespalib::ObjectVisitor &visitor) const { + visit(visitor, "iterator", _search.get()); +} + +ElementIteratorWrapper::ElementIteratorWrapper(SearchIterator::UP search, fef::TermFieldMatchData & tfmd) + : ElementIterator(std::move(search)), + _tfmd(tfmd) +{} + +ElementIteratorWrapper::~ElementIteratorWrapper() = default; + +void +ElementIteratorWrapper::getElementIds(uint32_t docId, std::vector<uint32_t> & elementIds) { + _search->unpack(docId); + int prevId(-1); + for (auto element : _tfmd) { + uint32_t id(element.getElementId()); + if (prevId != int(id)) { + elementIds.push_back(id); + prevId = id; + } + } +} + +void +ElementIteratorWrapper::mergeElementIds(uint32_t docId, std::vector<uint32_t> & elementIds) { + _search->unpack(docId); + size_t toKeep(0); + int32_t id(-1); + auto it = _tfmd.begin(); + for (int32_t candidate : elementIds) { + if (candidate > id) { + while ((it < _tfmd.end()) && (candidate > int(it->getElementId()))) { + it++; + } + if (it == _tfmd.end()) break; + id = it->getElementId(); + } + if (id == candidate) { + elementIds[toKeep++] = candidate; + } + } + elementIds.resize(toKeep); +} + +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, + const search::queryeval::ElementIterator *obj) +{ + if (obj != 0) { + self.openStruct(name, "ElementIterator"); + obj->visitMembers(self); + self.closeStruct(); + } else { + self.visitNull(name); + } +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, + const search::queryeval::ElementIterator &obj) +{ + visit(self, name, &obj); +} diff --git a/searchlib/src/vespa/searchlib/queryeval/elementiterator.h b/searchlib/src/vespa/searchlib/queryeval/elementiterator.h new file mode 100644 index 00000000000..a433c9be8f4 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/elementiterator.h @@ -0,0 +1,48 @@ +#pragma once + +#include <vespa/searchlib/queryeval/searchiterator.h> + +namespace search::fef { class TermFieldMatchData; } + +namespace search::queryeval { + +class ElementIterator { +public: + using UP = std::unique_ptr<ElementIterator>; + ElementIterator(SearchIterator::UP search) : _search(std::move(search)) { } + virtual ~ElementIterator() = default; + bool seek(uint32_t docId) { + return _search->seek(docId); + } + void initFullRange() { + _search->initFullRange(); + } + void initRange(uint32_t beginid, uint32_t endid) { + _search->initRange(beginid, endid); + } + uint32_t getDocId() const { + return _search->getDocId(); + } + virtual void getElementIds(uint32_t docId, std::vector<uint32_t> & elementIds) = 0; + virtual void mergeElementIds(uint32_t docId, std::vector<uint32_t> & elementIds) = 0; + virtual void visitMembers(vespalib::ObjectVisitor &visitor) const; +protected: + SearchIterator::UP _search; +}; + +class ElementIteratorWrapper : public ElementIterator { +public: + ElementIteratorWrapper(SearchIterator::UP search, fef::TermFieldMatchData & tfmd); + ~ElementIteratorWrapper() override; + void getElementIds(uint32_t docId, std::vector<uint32_t> & elementIds) override; + void mergeElementIds(uint32_t docId, std::vector<uint32_t> & elementIds) override; +private: + fef::TermFieldMatchData & _tfmd; +}; + +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, + const search::queryeval::ElementIterator &obj); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, + const search::queryeval::ElementIterator *obj); diff --git a/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp b/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp index bab047827ad..782c6fc1946 100644 --- a/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp @@ -2,8 +2,7 @@ #include "emptysearch.h" -namespace search { -namespace queryeval { +namespace search::queryeval { void EmptySearch::doSeek(uint32_t) @@ -15,6 +14,25 @@ EmptySearch::doUnpack(uint32_t) { } +void +EmptySearch::or_hits_into(BitVector &, uint32_t) +{ + // nop +} + +void +EmptySearch::and_hits_into(BitVector &result, uint32_t begin_id) +{ + result.clearInterval(begin_id, getEndId()); +} + +BitVector::UP +EmptySearch::get_hits(uint32_t begin_id) +{ + auto result = BitVector::create(begin_id, getEndId()); + return result; +} + EmptySearch::Trinary EmptySearch::is_strict() const { @@ -30,5 +48,4 @@ EmptySearch::~EmptySearch() { } -} // namespace queryeval -} // namespace search +} // namespace diff --git a/searchlib/src/vespa/searchlib/queryeval/emptysearch.h b/searchlib/src/vespa/searchlib/queryeval/emptysearch.h index 12d7430922c..3a9c6684db3 100644 --- a/searchlib/src/vespa/searchlib/queryeval/emptysearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/emptysearch.h @@ -3,15 +3,20 @@ #pragma once #include "searchiterator.h" +#include <vespa/searchlib/common/bitvector.h> namespace search { namespace queryeval { +/** Search iterator that never yields any hits. */ class EmptySearch : public SearchIterator { protected: void doSeek(uint32_t) override; void doUnpack(uint32_t) override; + void or_hits_into(BitVector &result, uint32_t begin_id) override; + void and_hits_into(BitVector &result, uint32_t begin_id) override; + BitVector::UP get_hits(uint32_t begin_id) override; void initRange(uint32_t begin, uint32_t end) override { SearchIterator::initRange(begin, end); setAtEnd(); @@ -25,4 +30,3 @@ public: } // namespace queryeval } // namespace search - diff --git a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp index 08a05b25772..6d044ca337d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp @@ -4,9 +4,42 @@ #include "equivsearch.h" #include "field_spec.hpp" #include <vespa/vespalib/objects/visit.hpp> +#include <vespa/vespalib/stllike/hash_map.hpp> namespace search::queryeval { +namespace { + +class UnpackNeed +{ + bool _needs_normal_features; + bool _needs_interleaved_features; +public: + UnpackNeed() + : _needs_normal_features(false), + _needs_interleaved_features(false) + { + } + + void observe(const fef::TermFieldMatchData &output) + { + if (output.needs_normal_features()) { + _needs_normal_features = true; + } + if (output.needs_interleaved_features()) { + _needs_interleaved_features = true; + } + } + + void notify(fef::TermFieldMatchData &input) const + { + input.setNeedNormalFeatures(_needs_normal_features); + input.setNeedInterleavedFeatures(_needs_interleaved_features); + } +}; + +}; + EquivBlueprint::EquivBlueprint(const FieldSpecBaseList &fields, fef::MatchDataLayout subtree_mdl) : ComplexLeafBlueprint(fields), @@ -24,16 +57,35 @@ SearchIterator::UP EquivBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &outputs, bool strict) const { fef::MatchData::UP md = _layout.createMatchData(); - MultiSearch::Children children(_terms.size()); + MultiSearch::Children children; + children.reserve(_terms.size()); fef::TermMatchDataMerger::Inputs childMatch; + vespalib::hash_map<uint16_t, UnpackNeed> unpack_needs(outputs.size()); + for (size_t i = 0; i < outputs.size(); ++i) { + unpack_needs[outputs[i]->getFieldId()].observe(*outputs[i]); + } for (size_t i = 0; i < _terms.size(); ++i) { const State &childState = _terms[i]->getState(); for (size_t j = 0; j < childState.numFields(); ++j) { - childMatch.emplace_back(childState.field(j).resolve(*md), _exactness[i]); + auto *child_term_field_match_data = childState.field(j).resolve(*md); + unpack_needs[child_term_field_match_data->getFieldId()].notify(*child_term_field_match_data); + childMatch.emplace_back(child_term_field_match_data, _exactness[i]); } - children[i] = _terms[i]->createSearch(*md, strict).release(); + children.push_back(_terms[i]->createSearch(*md, strict)); + } + return EquivSearch::create(std::move(children), std::move(md), childMatch, outputs, strict); +} + +SearchIterator::UP +EquivBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + MultiSearch::Children children; + children.reserve(_terms.size()); + for (size_t i = 0; i < _terms.size(); ++i) { + children.push_back(_terms[i]->createFilterSearch(strict, constraint)); } - return SearchIterator::UP(EquivSearch::create(children, std::move(md), childMatch, outputs, strict)); + UnpackInfo unpack_info; + return OrSearch::create(std::move(children), strict, unpack_info); } void diff --git a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.h index 59ed5ad5d3d..d1df1e0aada 100644 --- a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.h @@ -25,6 +25,7 @@ public: EquivBlueprint& addTerm(Blueprint::UP term, double exactness); SearchIteratorUP createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool strict) const override; + SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override; void visitMembers(vespalib::ObjectVisitor &visitor) const override; void fetchPostings(const ExecuteInfo &execInfo) override; diff --git a/searchlib/src/vespa/searchlib/queryeval/equivsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/equivsearch.cpp index 593701fd14f..95af4da01b0 100644 --- a/searchlib/src/vespa/searchlib/queryeval/equivsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/equivsearch.cpp @@ -21,19 +21,19 @@ public: * * @param children the search objects that should be equivalent **/ - EquivImpl(const MultiSearch::Children &children, + EquivImpl(MultiSearch::Children children, fef::MatchData::UP inputMatchData, const fef::TermMatchDataMerger::Inputs &inputs, const fef::TermFieldMatchDataArray &outputs); }; template<bool strict> -EquivImpl<strict>::EquivImpl(const MultiSearch::Children &children, +EquivImpl<strict>::EquivImpl(MultiSearch::Children children, fef::MatchData::UP inputMatchData, const fef::TermMatchDataMerger::Inputs &inputs, const fef::TermFieldMatchDataArray &outputs) - : OrLikeSearch<strict, NoUnpack>(children, NoUnpack()), + : OrLikeSearch<strict, NoUnpack>(std::move(children), NoUnpack()), _inputMatchData(std::move(inputMatchData)), _merger(inputs, outputs), _valid(outputs.valid()) @@ -50,17 +50,17 @@ EquivImpl<strict>::doUnpack(uint32_t docid) } } -SearchIterator * -EquivSearch::create(const Children &children, +SearchIterator::UP +EquivSearch::create(Children children, fef::MatchData::UP inputMatchData, const fef::TermMatchDataMerger::Inputs &inputs, const fef::TermFieldMatchDataArray &outputs, bool strict) { if (strict) { - return new EquivImpl<true>(children, std::move(inputMatchData), inputs, outputs); + return std::make_unique<EquivImpl<true>>(std::move(children), std::move(inputMatchData), inputs, outputs); } else { - return new EquivImpl<false>(children, std::move(inputMatchData), inputs, outputs); + return std::make_unique<EquivImpl<false>>(std::move(children), std::move(inputMatchData), inputs, outputs); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/equivsearch.h b/searchlib/src/vespa/searchlib/queryeval/equivsearch.h index 252e17e610a..7dc7f90ee23 100644 --- a/searchlib/src/vespa/searchlib/queryeval/equivsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/equivsearch.h @@ -18,12 +18,12 @@ class EquivSearch : public SearchIterator public: typedef MultiSearch::Children Children; - // Caller takes ownership of the returned SearchIterator. - static SearchIterator *create(const Children &children, - fef::MatchData::UP inputMD, - const fef::TermMatchDataMerger::Inputs &inputs, - const fef::TermFieldMatchDataArray &outputs, - bool strict); + static SearchIterator::UP + create(Children children, + fef::MatchData::UP inputMD, + const fef::TermMatchDataMerger::Inputs &inputs, + const fef::TermFieldMatchDataArray &outputs, + bool strict); }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp index 6f2d3cb6b2a..8e2ec3eabed 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp @@ -6,8 +6,7 @@ #include <vespa/vespalib/objects/visit.h> #include <vespa/searchcommon/attribute/i_search_context.h> -namespace search { -namespace queryeval { +namespace search::queryeval { void FakeSearch::doSeek(uint32_t docid) @@ -61,5 +60,4 @@ FakeSearch::visitMembers(vespalib::ObjectVisitor &visitor) const visit(visitor, "term", _term); } -} // namespace search::queryeval -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.h b/searchlib/src/vespa/searchlib/queryeval/fake_search.h index f5b95a94e99..4cbba5e5c24 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.h @@ -7,8 +7,7 @@ #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchcommon/attribute/i_search_context.h> -namespace search { -namespace queryeval { +namespace search::queryeval { class FakeSearch : public SearchIterator { @@ -41,9 +40,12 @@ public: bool is_attr() const { return (_ctx != nullptr); } void doSeek(uint32_t docid) override; void doUnpack(uint32_t docid) override; + void initRange(uint32_t begin, uint32_t end) override { + SearchIterator::initRange(begin, end); + _offset = 0; + } const PostingInfo *getPostingInfo() const override { return _result.postingInfo(); } void visitMembers(vespalib::ObjectVisitor &visitor) const override; }; -} // namespace queryeval -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/filter_wrapper.cpp b/searchlib/src/vespa/searchlib/queryeval/filter_wrapper.cpp new file mode 100644 index 00000000000..9128fccd20f --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/filter_wrapper.cpp @@ -0,0 +1,3 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "filter_wrapper.h" diff --git a/searchlib/src/vespa/searchlib/queryeval/filter_wrapper.h b/searchlib/src/vespa/searchlib/queryeval/filter_wrapper.h new file mode 100644 index 00000000000..27740df6ba6 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/filter_wrapper.h @@ -0,0 +1,63 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "searchiterator.h" +#include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/searchlib/fef/termfieldmatchdataarray.h> + +namespace search::queryeval { + +/** + * Wraps an iterator for use as a filter search. + * Owns TermFieldMatchData the wrapped iterator + * can wire to, and write to if necessary. + **/ +class FilterWrapper : public SearchIterator { +private: + std::vector<fef::TermFieldMatchData> _unused_md; + fef::TermFieldMatchDataArray _tfmda; + std::unique_ptr<SearchIterator> _wrapped_search; +public: + FilterWrapper(size_t num_fields) + : _unused_md(num_fields), + _tfmda(), + _wrapped_search() + { + for (size_t i = 0; i < num_fields; ++i) { + _tfmda.add(&_unused_md[i]); + } + } + const fef::TermFieldMatchDataArray& tfmda() const { return _tfmda; } + void wrap(std::unique_ptr<SearchIterator> wrapped) { + _wrapped_search = std::move(wrapped); + } + void wrap(SearchIterator *wrapped) { + _wrapped_search.reset(wrapped); + } + void doSeek(uint32_t docid) override { + _wrapped_search->seek(docid); // use outer seek for most robustness + setDocId(_wrapped_search->getDocId()); // propagate current iterator docid + } + void doUnpack(uint32_t) override {} + void initRange(uint32_t begin_id, uint32_t end_id) override { + SearchIterator::initRange(begin_id, end_id); + _wrapped_search->initRange(begin_id, end_id); + setDocId(_wrapped_search->getDocId()); + } + void or_hits_into(BitVector &result, uint32_t begin_id) override { + _wrapped_search->or_hits_into(result, begin_id); + } + void and_hits_into(BitVector &result, uint32_t begin_id) override { + _wrapped_search->and_hits_into(result, begin_id); + } + BitVector::UP get_hits(uint32_t begin_id) override { + return _wrapped_search->get_hits(begin_id); + } + Trinary is_strict() const override { + return _wrapped_search->is_strict(); + } +}; + +} // namespace diff --git a/searchlib/src/vespa/searchlib/queryeval/full_search.cpp b/searchlib/src/vespa/searchlib/queryeval/full_search.cpp new file mode 100644 index 00000000000..6f387b0ab41 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/full_search.cpp @@ -0,0 +1,44 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "full_search.h" + +namespace search::queryeval { + +void +FullSearch::doSeek(uint32_t docid) +{ + setDocId(docid); +} + +void +FullSearch::doUnpack(uint32_t) +{ +} + +void +FullSearch::or_hits_into(BitVector &result, uint32_t begin_id) +{ + result.setInterval(begin_id, getEndId()); +} + +void +FullSearch::and_hits_into(BitVector &, uint32_t) +{ + // nop +} + +BitVector::UP +FullSearch::get_hits(uint32_t begin_id) +{ + auto result = BitVector::create(begin_id, getEndId()); + result->setInterval(begin_id, getEndId()); + return result; +} + +FullSearch::FullSearch() : SearchIterator() +{ +} + +FullSearch::~FullSearch() = default; + +} // namespace diff --git a/searchlib/src/vespa/searchlib/queryeval/full_search.h b/searchlib/src/vespa/searchlib/queryeval/full_search.h new file mode 100644 index 00000000000..734c3a1443e --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/full_search.h @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "searchiterator.h" +#include <vespa/searchlib/common/bitvector.h> + +namespace search::queryeval { + +/** + * Search iterator that hits all documents. + * Note that it does not search any field, and + * does not unpack any ranking information. + **/ +class FullSearch : public SearchIterator +{ +private: + Trinary is_strict() const override { return Trinary::True; } + void doSeek(uint32_t) override; + void doUnpack(uint32_t) override; + void or_hits_into(BitVector &result, uint32_t begin_id) override; + void and_hits_into(BitVector &result, uint32_t begin_id) override; + BitVector::UP get_hits(uint32_t begin_id) override; + +public: + FullSearch(); + ~FullSearch(); +}; + +} diff --git a/searchlib/src/vespa/searchlib/queryeval/global_filter.cpp b/searchlib/src/vespa/searchlib/queryeval/global_filter.cpp new file mode 100644 index 00000000000..849700b250e --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/global_filter.cpp @@ -0,0 +1,3 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "global_filter.h" diff --git a/searchlib/src/vespa/searchlib/queryeval/global_filter.h b/searchlib/src/vespa/searchlib/queryeval/global_filter.h new file mode 100644 index 00000000000..b30162affa7 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/global_filter.h @@ -0,0 +1,46 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> +#include <vespa/searchlib/common/bitvector.h> + +namespace search::queryeval { + +/** + * Hold ownership of a global filter that can be taken + * into account by adaptive query operators. The owned + * bitvector should be a white-list (documents that may + * possibly become hits have their bit set, documents + * that are certain to be filtered away should have theirs + * cleared). + **/ +class GlobalFilter : public std::enable_shared_from_this<GlobalFilter> +{ +private: + struct ctor_tag {}; + std::unique_ptr<search::BitVector> bit_vector; + + GlobalFilter(const GlobalFilter &) = delete; + GlobalFilter(GlobalFilter &&) = delete; +public: + + GlobalFilter(ctor_tag, std::unique_ptr<search::BitVector> bit_vector_in) + : bit_vector(std::move(bit_vector_in)) + {} + + GlobalFilter(ctor_tag) : bit_vector() {} + + ~GlobalFilter() {} + + template<typename ... Params> + static std::shared_ptr<GlobalFilter> create(Params&& ... params) { + return std::make_shared<GlobalFilter>(ctor_tag(), std::forward<Params>(params)...); + } + + const search::BitVector *filter() const { return bit_vector.get(); } + + bool has_filter() const { return bool(bit_vector); } +}; + +} // namespace diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index 430bc3956e7..3d3a703cd7b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -82,6 +82,23 @@ need_normal_features_for_children(const IntermediateBlueprint &blueprint, fef::M } } +/** utility for operators that degrade to AND when creating filter */ +SearchIterator::UP createAndFilter(const IntermediateBlueprint &self, + bool strict, Blueprint::FilterConstraint constraint) +{ + MultiSearch::Children sub_searches; + sub_searches.reserve(self.childCnt()); + for (size_t i = 0; i < self.childCnt(); ++i) { + bool child_strict = strict && (i == 0); + auto search = self.getChild(i).createFilterSearch(child_strict, constraint); + sub_searches.push_back(std::move(search)); + } + UnpackInfo unpack_info; + auto search = AndSearch::create(std::move(sub_searches), strict, unpack_info); + search->estimate(self.getState().estimate().estHits); + return search; +} + } // namespace search::queryeval::<unnamed> //----------------------------------------------------------------------------- @@ -149,23 +166,51 @@ AndNotBlueprint::inheritStrict(size_t i) const } SearchIterator::UP -AndNotBlueprint::createIntermediateSearch(const MultiSearch::Children &subSearches, +AndNotBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &md) const { - UnpackInfo unpackInfo(calculateUnpackInfo(md)); - if (should_do_termwise_eval(unpackInfo, md.get_termwise_limit())) { - TermwiseBlueprintHelper helper(*this, subSearches, unpackInfo); + UnpackInfo unpack_info(calculateUnpackInfo(md)); + if (should_do_termwise_eval(unpack_info, md.get_termwise_limit())) { + TermwiseBlueprintHelper helper(*this, std::move(sub_searches), unpack_info); bool termwise_strict = (strict && inheritStrict(helper.first_termwise)); auto termwise_search = (helper.first_termwise == 0) - ? SearchIterator::UP(AndNotSearch::create(helper.termwise, termwise_strict)) - : SearchIterator::UP(OrSearch::create(helper.termwise, termwise_strict)); + ? AndNotSearch::create(helper.get_termwise_children(), termwise_strict) + : OrSearch::create(helper.get_termwise_children(), termwise_strict); helper.insert_termwise(std::move(termwise_search), termwise_strict); - if (helper.children.size() == 1) { - return SearchIterator::UP(helper.children.front()); + auto rearranged = helper.get_result(); + if (rearranged.size() == 1) { + return std::move(rearranged[0]); } - return SearchIterator::UP(AndNotSearch::create(helper.children, strict)); + return AndNotSearch::create(std::move(rearranged), strict); + } + return AndNotSearch::create(std::move(sub_searches), strict); +} + +namespace { +Blueprint::FilterConstraint invert(Blueprint::FilterConstraint constraint) { + if (constraint == Blueprint::FilterConstraint::UPPER_BOUND) { + return Blueprint::FilterConstraint::LOWER_BOUND; + } + if (constraint == Blueprint::FilterConstraint::LOWER_BOUND) { + return Blueprint::FilterConstraint::UPPER_BOUND; } - return SearchIterator::UP(AndNotSearch::create(subSearches, strict)); + abort(); +} +} // namespace <unnamed> + +SearchIterator::UP +AndNotBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + MultiSearch::Children sub_searches; + sub_searches.reserve(childCnt()); + for (size_t i = 0; i < childCnt(); ++i) { + bool child_strict = strict && inheritStrict(i); + auto search = (i == 0) + ? getChild(i).createFilterSearch(child_strict, constraint) + : getChild(i).createFilterSearch(child_strict, invert(constraint)); + sub_searches.push_back(std::move(search)); + } + return AndNotSearch::create(std::move(sub_searches), strict); } //----------------------------------------------------------------------------- @@ -221,26 +266,33 @@ AndBlueprint::inheritStrict(size_t i) const } SearchIterator::UP -AndBlueprint::createIntermediateSearch(const MultiSearch::Children &subSearches, +AndBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData & md) const { - UnpackInfo unpackInfo(calculateUnpackInfo(md)); - AndSearch * search = 0; - if (should_do_termwise_eval(unpackInfo, md.get_termwise_limit())) { - TermwiseBlueprintHelper helper(*this, subSearches, unpackInfo); + UnpackInfo unpack_info(calculateUnpackInfo(md)); + std::unique_ptr<AndSearch> search; + if (should_do_termwise_eval(unpack_info, md.get_termwise_limit())) { + TermwiseBlueprintHelper helper(*this, std::move(sub_searches), unpack_info); bool termwise_strict = (strict && inheritStrict(helper.first_termwise)); - auto termwise_search = SearchIterator::UP(AndSearch::create(helper.termwise, termwise_strict)); + auto termwise_search = AndSearch::create(helper.get_termwise_children(), termwise_strict); helper.insert_termwise(std::move(termwise_search), termwise_strict); - if (helper.children.size() == 1) { - return SearchIterator::UP(helper.children.front()); + auto rearranged = helper.get_result(); + if (rearranged.size() == 1) { + return std::move(rearranged[0]); } else { - search = AndSearch::create(helper.children, strict, helper.termwise_unpack); + search = AndSearch::create(std::move(rearranged), strict, helper.termwise_unpack); } } else { - search = AndSearch::create(subSearches, strict, unpackInfo); + search = AndSearch::create(std::move(sub_searches), strict, unpack_info); } search->estimate(getState().estimate().estHits); - return SearchIterator::UP(search); + return search; +} + +SearchIterator::UP +AndBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + return createAndFilter(*this, strict, constraint); } double @@ -303,21 +355,36 @@ OrBlueprint::inheritStrict(size_t) const } SearchIterator::UP -OrBlueprint::createIntermediateSearch(const MultiSearch::Children &subSearches, +OrBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData & md) const { - UnpackInfo unpackInfo(calculateUnpackInfo(md)); - if (should_do_termwise_eval(unpackInfo, md.get_termwise_limit())) { - TermwiseBlueprintHelper helper(*this, subSearches, unpackInfo); + UnpackInfo unpack_info(calculateUnpackInfo(md)); + if (should_do_termwise_eval(unpack_info, md.get_termwise_limit())) { + TermwiseBlueprintHelper helper(*this, std::move(sub_searches), unpack_info); bool termwise_strict = (strict && inheritStrict(helper.first_termwise)); - auto termwise_search = SearchIterator::UP(OrSearch::create(helper.termwise, termwise_strict)); + auto termwise_search = OrSearch::create(helper.get_termwise_children(), termwise_strict); helper.insert_termwise(std::move(termwise_search), termwise_strict); - if (helper.children.size() == 1) { - return SearchIterator::UP(helper.children.front()); + auto rearranged = helper.get_result(); + if (rearranged.size() == 1) { + return std::move(rearranged[0]); } - return SearchIterator::UP(OrSearch::create(helper.children, strict, helper.termwise_unpack)); + return OrSearch::create(std::move(rearranged), strict, helper.termwise_unpack); + } + return OrSearch::create(std::move(sub_searches), strict, unpack_info); +} + +SearchIterator::UP +OrBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + MultiSearch::Children sub_searches; + sub_searches.reserve(childCnt()); + for (size_t i = 0; i < childCnt(); ++i) { + bool child_strict = strict && inheritStrict(i); + auto search = getChild(i).createFilterSearch(child_strict, constraint); + sub_searches.push_back(std::move(search)); } - return SearchIterator::UP(OrSearch::create(subSearches, strict, unpackInfo)); + UnpackInfo unpack_info; + return OrSearch::create(std::move(sub_searches), strict, unpack_info); } //----------------------------------------------------------------------------- @@ -359,18 +426,19 @@ WeakAndBlueprint::always_needs_unpack() const } SearchIterator::UP -WeakAndBlueprint::createIntermediateSearch(const MultiSearch::Children &subSearches, +WeakAndBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &) const { WeakAndSearch::Terms terms; - assert(subSearches.size() == childCnt()); + assert(sub_searches.size() == childCnt()); assert(_weights.size() == childCnt()); - for (size_t i = 0; i < subSearches.size(); ++i) { - terms.push_back(wand::Term(subSearches[i], + for (size_t i = 0; i < sub_searches.size(); ++i) { + // TODO: pass ownership with unique_ptr + terms.push_back(wand::Term(sub_searches[i].release(), _weights[i], getChild(i).getState().estimate().estHits)); } - return SearchIterator::UP(WeakAndSearch::create(terms, _n, strict)); + return WeakAndSearch::create(terms, _n, strict); } //----------------------------------------------------------------------------- @@ -407,7 +475,7 @@ NearBlueprint::createSearch(fef::MatchData &md, bool strict) const } SearchIterator::UP -NearBlueprint::createIntermediateSearch(const MultiSearch::Children &subSearches, +NearBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &md) const { search::fef::TermFieldMatchDataArray tfmda; @@ -417,7 +485,17 @@ NearBlueprint::createIntermediateSearch(const MultiSearch::Children &subSearches tfmda.add(cs.field(j).resolve(md)); } } - return SearchIterator::UP(new NearSearch(subSearches, tfmda, _window, strict)); + return SearchIterator::UP(new NearSearch(std::move(sub_searches), tfmda, _window, strict)); +} + +SearchIterator::UP +NearBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + if (constraint == Blueprint::FilterConstraint::UPPER_BOUND) { + return createAndFilter(*this, strict, constraint); + } else { + return std::make_unique<EmptySearch>(); + } } //----------------------------------------------------------------------------- @@ -455,7 +533,7 @@ ONearBlueprint::createSearch(fef::MatchData &md, bool strict) const } SearchIterator::UP -ONearBlueprint::createIntermediateSearch(const MultiSearch::Children &subSearches, +ONearBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &md) const { search::fef::TermFieldMatchDataArray tfmda; @@ -465,9 +543,19 @@ ONearBlueprint::createIntermediateSearch(const MultiSearch::Children &subSearche tfmda.add(cs.field(j).resolve(md)); } } - // could sort subSearches here + // could sort sub_searches here // but then strictness inheritance would also need to be fixed - return SearchIterator::UP(new ONearSearch(subSearches, tfmda, _window, strict)); + return SearchIterator::UP(new ONearSearch(std::move(sub_searches), tfmda, _window, strict)); +} + +SearchIterator::UP +ONearBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + if (constraint == Blueprint::FilterConstraint::UPPER_BOUND) { + return createAndFilter(*this, strict, constraint); + } else { + return std::make_unique<EmptySearch>(); + } } //----------------------------------------------------------------------------- @@ -520,31 +608,38 @@ RankBlueprint::inheritStrict(size_t i) const } SearchIterator::UP -RankBlueprint::createIntermediateSearch(const MultiSearch::Children &subSearches, +RankBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData & md) const { - UnpackInfo unpackInfo(calculateUnpackInfo(md)); - if (unpackInfo.unpackAll()) { - return SearchIterator::UP(RankSearch::create(subSearches, strict)); + UnpackInfo unpack_info(calculateUnpackInfo(md)); + if (unpack_info.unpackAll()) { + return RankSearch::create(std::move(sub_searches), strict); } else { - MultiSearch::Children requireUnpack; - requireUnpack.reserve(subSearches.size()); - requireUnpack.push_back(subSearches[0]); - for (size_t i(1); i < subSearches.size(); i++) { - if (unpackInfo.needUnpack(i)) { - requireUnpack.push_back(subSearches[i]); + MultiSearch::Children require_unpack; + require_unpack.reserve(sub_searches.size()); + require_unpack.push_back(std::move(sub_searches[0])); + for (size_t i(1); i < sub_searches.size(); i++) { + if (unpack_info.needUnpack(i)) { + require_unpack.push_back(std::move(sub_searches[i])); } else { - delete subSearches[i]; + sub_searches[i].reset(); } } - if (requireUnpack.size() == 1) { - return SearchIterator::UP(requireUnpack[0]); + if (require_unpack.size() == 1) { + return SearchIterator::UP(std::move(require_unpack[0])); } else { - return SearchIterator::UP(RankSearch::create(requireUnpack, strict)); + return RankSearch::create(std::move(require_unpack), strict); } } } +SearchIterator::UP +RankBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + assert(childCnt() > 0); + return getChild(0).createFilterSearch(strict, constraint); +} + //----------------------------------------------------------------------------- SourceBlenderBlueprint::SourceBlenderBlueprint(const ISourceSelector &selector) @@ -597,18 +692,37 @@ SourceBlenderBlueprint::findSource(uint32_t sourceId) const } SearchIterator::UP -SourceBlenderBlueprint::createIntermediateSearch(const MultiSearch::Children &subSearches, +SourceBlenderBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &) const { SourceBlenderSearch::Children children; - assert(subSearches.size() == childCnt()); - for (size_t i = 0; i < subSearches.size(); ++i) { - children.push_back(SourceBlenderSearch::Child(subSearches[i], + assert(sub_searches.size() == childCnt()); + for (size_t i = 0; i < sub_searches.size(); ++i) { + // TODO: pass ownership with unique_ptr + children.push_back(SourceBlenderSearch::Child(sub_searches[i].release(), getChild(i).getSourceId())); assert(children.back().sourceId != 0xffffffff); } - return SearchIterator::UP(SourceBlenderSearch::create(_selector.createIterator(), - children, strict)); + return SourceBlenderSearch::create(_selector.createIterator(), + children, strict); +} + +SearchIterator::UP +SourceBlenderBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + if (constraint == FilterConstraint::UPPER_BOUND) { + MultiSearch::Children sub_searches; + sub_searches.reserve(childCnt()); + for (size_t i = 0; i < childCnt(); ++i) { + bool child_strict = strict && inheritStrict(i); + auto search = getChild(i).createFilterSearch(child_strict, constraint); + sub_searches.push_back(std::move(search)); + } + UnpackInfo unpack_info; + return OrSearch::create(std::move(sub_searches), strict, unpack_info); + } else { + return std::make_unique<EmptySearch>(); + } } bool diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h index 3a6b5e1a31a..6bbe4562641 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h @@ -22,14 +22,17 @@ public: void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; + SearchIterator::UP + createFilterSearch(bool strict, FilterConstraint constraint) const override; private: bool isPositive(size_t index) const override { return index == 0; } }; //----------------------------------------------------------------------------- +/** normal AND operator */ class AndBlueprint : public IntermediateBlueprint { public: @@ -46,12 +49,15 @@ public: void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; + SearchIterator::UP + createFilterSearch(bool strict, FilterConstraint constraint) const override; }; //----------------------------------------------------------------------------- +/** normal OR operator */ class OrBlueprint : public IntermediateBlueprint { public: @@ -63,8 +69,10 @@ public: void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; + SearchIterator::UP + createFilterSearch(bool strict, FilterConstraint constraint) const override; }; //----------------------------------------------------------------------------- @@ -82,7 +90,7 @@ public: bool inheritStrict(size_t i) const override; bool always_needs_unpack() const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; WeakAndBlueprint(uint32_t n) : _n(n) {} @@ -110,15 +118,16 @@ public: bool inheritStrict(size_t i) const override; SearchIteratorUP createSearch(fef::MatchData &md, bool strict) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; + SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; NearBlueprint(uint32_t window) : _window(window) {} }; //----------------------------------------------------------------------------- -class ONearBlueprint : public IntermediateBlueprint +class ONearBlueprint : public IntermediateBlueprint { private: uint32_t _window; @@ -131,8 +140,9 @@ public: bool inheritStrict(size_t i) const override; SearchIteratorUP createSearch(fef::MatchData &md, bool strict) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; + SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; ONearBlueprint(uint32_t window) : _window(window) {} }; @@ -149,8 +159,10 @@ public: void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; + SearchIterator::UP + createFilterSearch(bool strict, FilterConstraint constraint) const override; }; //----------------------------------------------------------------------------- @@ -173,8 +185,10 @@ public: */ ssize_t findSource(uint32_t sourceId) const; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; + SearchIterator::UP + createFilterSearch(bool strict, FilterConstraint constraint) const override; /** check if this blueprint has the same source selector as the other */ bool isCompatibleWith(const SourceBlenderBlueprint &other) const; diff --git a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h index a7b30594a5b..8f4f3a3744b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h +++ b/searchlib/src/vespa/searchlib/queryeval/iterator_pack.h @@ -23,10 +23,12 @@ public: SearchIteratorPack(SearchIteratorPack &&rhs) noexcept; SearchIteratorPack &operator=(SearchIteratorPack &&rhs) noexcept; + // TODO: use MultiSearch::Children to pass ownership SearchIteratorPack(const std::vector<SearchIterator*> &children, const std::vector<fef::TermFieldMatchData*> &childMatch, MatchDataUP md); + // TODO: use MultiSearch::Children to pass ownership SearchIteratorPack(const std::vector<SearchIterator*> &children, MatchDataUP md); uint32_t get_docid(uint32_t ref) const { diff --git a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp index 94cec9a63c5..fac6290d3f8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.cpp @@ -16,6 +16,12 @@ EmptyBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray &, return std::make_unique<EmptySearch>(); } +SearchIterator::UP +EmptyBlueprint::createFilterSearch(bool /*strict*/, FilterConstraint /* constraint */) const +{ + return std::make_unique<EmptySearch>(); +} + EmptyBlueprint::EmptyBlueprint(const FieldSpecBase &field) : SimpleLeafBlueprint(field) { @@ -42,6 +48,17 @@ SimpleBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray &, return search; } +SearchIterator::UP +SimpleBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + SimpleSearch *ss = new SimpleSearch(_result); + SearchIterator::UP search(ss); + ss->tag(_tag + + (strict ? "<strict," : "<nostrict,") + + (constraint == FilterConstraint::UPPER_BOUND ? "upper>" : "lower>")); + return search; +} + SimpleBlueprint::SimpleBlueprint(const SimpleResult &result) : SimpleLeafBlueprint(FieldSpecBaseList()), _tag(), diff --git a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h index 2dc2d938bb6..5bba87c7091 100644 --- a/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/leaf_blueprints.h @@ -14,13 +14,12 @@ namespace search::queryeval { class EmptyBlueprint : public SimpleLeafBlueprint { protected: - SearchIterator::UP - createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, bool strict) const override; - + SearchIterator::UP createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, bool strict) const override; public: EmptyBlueprint(const FieldSpecBaseList &fields); EmptyBlueprint(const FieldSpecBase &field); EmptyBlueprint(); + SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; }; //----------------------------------------------------------------------------- @@ -32,14 +31,14 @@ private: SimpleResult _result; protected: - SearchIterator::UP + SearchIterator::UP createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, bool strict) const override; - public: SimpleBlueprint(const SimpleResult &result); ~SimpleBlueprint(); SimpleBlueprint &tag(const vespalib::string &tag); const vespalib::string &tag() const { return _tag; } + SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; }; //----------------------------------------------------------------------------- diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp index bba331284f9..ca8513a3c91 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp @@ -1,19 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/queryeval/multibitvectoriterator.h> -#include <vespa/searchlib/queryeval/andsearch.h> -#include <vespa/searchlib/queryeval/andnotsearch.h> -#include <vespa/searchlib/queryeval/sourceblendersearch.h> -#include <vespa/searchlib/queryeval/orsearch.h> +#include "multibitvectoriterator.h" +#include "andsearch.h" +#include "andnotsearch.h" +#include "sourceblendersearch.h" #include <vespa/searchlib/common/bitvectoriterator.h> -#include <vespa/searchlib/attribute/attributeiterators.h> -#include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/util/optimized.h> +#include <vespa/vespalib/hwaccelrated/iaccelrated.h> namespace search::queryeval { using vespalib::Trinary; +using vespalib::hwaccelrated::IAccelrated; namespace { @@ -21,7 +20,16 @@ template<typename Update> class MultiBitVectorIterator : public MultiBitVectorIteratorBase { public: - MultiBitVectorIterator(const Children & children) : MultiBitVectorIteratorBase(children) { } + explicit MultiBitVectorIterator(Children children) + : MultiBitVectorIteratorBase(std::move(children)), + _update(), + _accel(IAccelrated::getAccelerator()), + _lastWords() + { + static_assert(sizeof(_lastWords) == 64, "Lastwords should have 64 byte size"); + static_assert(NumWordsInBatch == 8, "Batch size should be 8 words."); + memset(_lastWords, 0, sizeof(_lastWords)); + } protected: void updateLastValue(uint32_t docId); void strictSeek(uint32_t docId); @@ -29,33 +37,56 @@ private: void doSeek(uint32_t docId) override; Trinary is_strict() const override { return Trinary::False; } bool acceptExtraFilter() const override { return Update::isAnd(); } - Update _update; + Update _update; + const IAccelrated & _accel; + alignas(64) Word _lastWords[8]; + static constexpr size_t NumWordsInBatch = sizeof(_lastWords) / sizeof(Word); }; template<typename Update> class MultiBitVectorIteratorStrict : public MultiBitVectorIterator<Update> { public: - MultiBitVectorIteratorStrict(const MultiSearch::Children & children) : MultiBitVectorIterator<Update>(children) { } + explicit MultiBitVectorIteratorStrict(MultiSearch::Children children) + : MultiBitVectorIterator<Update>(std::move(children)) + { } private: void doSeek(uint32_t docId) override { this->strictSeek(docId); } Trinary is_strict() const override { return Trinary::True; } }; +struct And { + using Word = BitWord::Word; + void operator () (const IAccelrated & accel, size_t offset, const std::vector<std::pair<const void *, bool>> & src, void *dest) { + accel.and64(offset, src, dest); + } + static bool isAnd() { return true; } +}; + +struct Or { + using Word = BitWord::Word; + void operator () (const IAccelrated & accel, size_t offset, const std::vector<std::pair<const void *, bool>> & src, void *dest) { + accel.or64(offset, src, dest); + } + static bool isAnd() { return false; } +}; + template<typename Update> void MultiBitVectorIterator<Update>::updateLastValue(uint32_t docId) { if (docId >= _lastMaxDocIdLimit) { - if (__builtin_expect(docId < _numDocs, true)) { - const uint32_t index(wordNum(docId)); - _lastValue = _bvs[0][index]; - for(uint32_t i(1); i < _bvs.size(); i++) { - _lastValue = _update(_lastValue, _bvs[i][index]); - } - _lastMaxDocIdLimit = (index + 1) * WordLen; - } else { + if (__builtin_expect(docId >= _numDocs, false)) { setAtEnd(); + return; + } + const uint32_t index(wordNum(docId)); + if (docId >= _lastMaxDocIdLimitRequireFetch) { + uint32_t baseIndex = index & ~(NumWordsInBatch - 1); + _update(_accel, baseIndex*sizeof(Word), _bvs, _lastWords); + _lastMaxDocIdLimitRequireFetch = (baseIndex + NumWordsInBatch) * WordLen; } + _lastValue = _lastWords[index % NumWordsInBatch]; + _lastMaxDocIdLimit = (index + 1) * WordLen; } } @@ -75,7 +106,7 @@ template<typename Update> void MultiBitVectorIterator<Update>::strictSeek(uint32_t docId) { - for (updateLastValue(docId), _lastValue=_lastValue & checkTab(docId); + for (updateLastValue(docId), _lastValue = _lastValue & checkTab(docId); (_lastValue == 0) && __builtin_expect(! isAtEnd(), true); updateLastValue(_lastMaxDocIdLimit)); if (__builtin_expect(!isAtEnd(), true)) { @@ -88,21 +119,6 @@ MultiBitVectorIterator<Update>::strictSeek(uint32_t docId) } } -struct And { - typedef BitWord::Word Word; - Word operator () (const Word a, const Word b) { - return a & b; - } - static bool isAnd() { return true; } -}; - -struct Or { - typedef BitWord::Word Word; - Word operator () (const Word a, const Word b) { - return a | b; - } - static bool isAnd() { return false; } -}; typedef MultiBitVectorIterator<And> AndBVIterator; typedef MultiBitVectorIteratorStrict<And> AndBVIteratorStrict; @@ -112,7 +128,7 @@ typedef MultiBitVectorIteratorStrict<Or> OrBVIteratorStrict; bool hasAtLeast2Bitvectors(const MultiSearch::Children & children) { size_t count(0); - for (const SearchIterator * search : children) { + for (const auto & search : children) { if (search->isBitVector()) { count++; } @@ -133,17 +149,18 @@ bool canOptimize(const MultiSearch & s) { } -MultiBitVectorIteratorBase::MultiBitVectorIteratorBase(const Children & children) : - MultiSearch(children), +MultiBitVectorIteratorBase::MultiBitVectorIteratorBase(Children children) : + MultiSearch(std::move(children)), _numDocs(std::numeric_limits<unsigned int>::max()), - _lastValue(0), _lastMaxDocIdLimit(0), + _lastMaxDocIdLimitRequireFetch(0), + _lastValue(0), _bvs() { - _bvs.reserve(children.size()); - for (size_t i(0); i < children.size(); i++) { - const auto * bv = static_cast<const BitVectorIterator *>(children[i]); - _bvs.emplace_back(reinterpret_cast<const Word *>(bv->getBitValues()), bv->isInverted()); + _bvs.reserve(getChildren().size()); + for (const auto & child : getChildren()) { + const auto * bv = static_cast<const BitVectorIterator *>(child.get()); + _bvs.emplace_back(bv->getBitValues(), bv->isInverted()); _numDocs = std::min(_numDocs, bv->getDocIdLimit()); } } @@ -155,6 +172,7 @@ MultiBitVectorIteratorBase::initRange(uint32_t beginId, uint32_t endId) { MultiSearch::initRange(beginId, endId); _lastMaxDocIdLimit = 0; + _lastMaxDocIdLimitRequireFetch = 0; } SearchIterator::UP @@ -163,9 +181,10 @@ MultiBitVectorIteratorBase::andWith(UP filter, uint32_t estimate) (void) estimate; if (filter->isBitVector() && acceptExtraFilter()) { const auto & bv = static_cast<const BitVectorIterator &>(*filter); - _bvs.emplace_back(reinterpret_cast<const Word *>(bv.getBitValues()), bv.isInverted()); + _bvs.emplace_back(bv.getBitValues(), bv.isInverted()); insert(getChildren().size(), std::move(filter)); _lastMaxDocIdLimit = 0; // force reload + _lastMaxDocIdLimitRequireFetch = 0; } return filter; } @@ -177,7 +196,9 @@ MultiBitVectorIteratorBase::doUnpack(uint32_t docid) MultiSearch::doUnpack(docid); } else { auto &children = getChildren(); - _unpackInfo.each([&children,docid](size_t i){children[i]->doUnpack(docid);}, children.size()); + _unpackInfo.each([&children,docid](size_t i) { + static_cast<BitVectorIterator *>(children[i].get())->unpack(docid); + }, children.size()); } } @@ -216,7 +237,7 @@ MultiBitVectorIteratorBase::optimizeMultiSearch(SearchIterator::UP parentIt) if ( ! strict && (bit->is_strict() == Trinary::True)) { strict = true; } - stolen.push_back(bit.release()); + stolen.push_back(std::move(bit)); } else { it++; } @@ -224,21 +245,21 @@ MultiBitVectorIteratorBase::optimizeMultiSearch(SearchIterator::UP parentIt) SearchIterator::UP next; if (parent.isAnd()) { if (strict) { - next = std::make_unique<AndBVIteratorStrict>(stolen); + next = std::make_unique<AndBVIteratorStrict>(std::move(stolen)); } else { - next = std::make_unique<AndBVIterator>(stolen); + next = std::make_unique<AndBVIterator>(std::move(stolen)); } } else if (parent.isOr()) { if (strict) { - next = std::make_unique<OrBVIteratorStrict>(stolen); + next = std::make_unique<OrBVIteratorStrict>(std::move(stolen)); } else { - next = std::make_unique<OrBVIterator>(stolen); + next = std::make_unique<OrBVIterator>(std::move(stolen)); } } else if (parent.isAndNot()) { if (strict) { - next = std::make_unique<OrBVIteratorStrict>(stolen); + next = std::make_unique<OrBVIteratorStrict>(std::move(stolen)); } else { - next = std::make_unique<OrBVIterator>(stolen); + next = std::make_unique<OrBVIterator>(std::move(stolen)); } } auto & nextM(static_cast<MultiBitVectorIteratorBase &>(*next)); @@ -252,8 +273,8 @@ MultiBitVectorIteratorBase::optimizeMultiSearch(SearchIterator::UP parentIt) } } auto & toOptimize(const_cast<MultiSearch::Children &>(parent.getChildren())); - for (SearchIterator * & search : toOptimize) { - search = optimize(MultiSearch::UP(search)).release(); + for (auto & search : toOptimize) { + search = optimize(std::move(search)); } return parentIt; diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h index cc40f834114..29e92584ffe 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h @@ -11,7 +11,7 @@ namespace search::queryeval { class MultiBitVectorIteratorBase : public MultiSearch, protected BitWord { public: - ~MultiBitVectorIteratorBase(); + ~MultiBitVectorIteratorBase() override; void initRange(uint32_t beginId, uint32_t endId) override; void addUnpackIndex(size_t index) { _unpackInfo.add(index); } /** @@ -20,26 +20,21 @@ public: */ static SearchIterator::UP optimize(SearchIterator::UP parent); protected: - MultiBitVectorIteratorBase(const Children & children); - class MetaWord { - public: - MetaWord(const Word * words, bool inverted) : _words(words), _inverted(inverted) { } - Word operator [] (uint32_t index) const { return _inverted ? ~_words[index] : _words[index]; } - private: - const Word * _words; - bool _inverted; - }; + MultiBitVectorIteratorBase(Children hildren); + using MetaWord = std::pair<const void *, bool>; uint32_t _numDocs; - Word _lastValue; // Last value computed uint32_t _lastMaxDocIdLimit; // next documentid requiring recomputation. + uint32_t _lastMaxDocIdLimitRequireFetch; + Word _lastValue; // Last value computed std::vector<MetaWord> _bvs; private: virtual bool acceptExtraFilter() const = 0; UP andWith(UP filter, uint32_t estimate) override; void doUnpack(uint32_t docid) override; - UnpackInfo _unpackInfo; static SearchIterator::UP optimizeMultiSearch(SearchIterator::UP parent); + + UnpackInfo _unpackInfo; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp index b63a54785a4..51098f50b37 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp @@ -10,7 +10,7 @@ void MultiSearch::insert(size_t index, SearchIterator::UP search) { assert(index <= _children.size()); - _children.insert(_children.begin()+index, search.release()); + _children.insert(_children.begin()+index, std::move(search)); onInsert(index); } @@ -18,7 +18,7 @@ SearchIterator::UP MultiSearch::remove(size_t index) { assert(index < _children.size()); - SearchIterator::UP search(_children[index]); + SearchIterator::UP search = std::move(_children[index]); _children.erase(_children.begin() + index); onRemove(index); return search; @@ -27,7 +27,7 @@ MultiSearch::remove(size_t index) void MultiSearch::doUnpack(uint32_t docid) { - for (SearchIterator *child: _children) { + for (auto &child: _children) { if (__builtin_expect(child->getDocId() < docid, false)) { child->doSeek(docid); } @@ -37,23 +37,20 @@ MultiSearch::doUnpack(uint32_t docid) } } -MultiSearch::MultiSearch(const Children & children) - : _children(children) +MultiSearch::MultiSearch(Children children) + : _children(std::move(children)) { } MultiSearch::~MultiSearch() { - for (SearchIterator * child : _children) { - delete child; - } } void MultiSearch::initRange(uint32_t beginid, uint32_t endid) { SearchIterator::initRange(beginid, endid); - for (SearchIterator * child : _children) { + for (auto & child : _children) { child->initRange(beginid, endid); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/multisearch.h b/searchlib/src/vespa/searchlib/queryeval/multisearch.h index af96734b26a..bd916f7953b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multisearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/multisearch.h @@ -3,6 +3,7 @@ #pragma once #include "searchiterator.h" +#include "children_iterators.h" struct MultiSearchRemoveTest; @@ -12,25 +13,26 @@ class MultiBitVectorIteratorBase; /** * A virtual intermediate class that serves as the basis for combining searches - * like and, or any or others that take a list of children. + * like AND, OR, RANK or others that take a list of children. **/ class MultiSearch : public SearchIterator { friend struct ::MultiSearchRemoveTest; friend class ::search::queryeval::MultiBitVectorIteratorBase; + friend class MySearch; public: /** - * Defines how to represent the children iterators. vespalib::Array usage - * generates faster and more compact code then using std::vector. + * Defines how to represent the children iterators. */ - typedef std::vector<SearchIterator *> Children; + using Children = std::vector<SearchIterator::UP>; + /** * Create a new Multi Search with the given children. * * @param children the search objects we are and'ing * this object takes ownership of the children. **/ - MultiSearch(const Children & children); + MultiSearch(Children children); virtual ~MultiSearch() override; const Children & getChildren() const { return _children; } virtual bool isAnd() const { return false; } @@ -40,6 +42,7 @@ public: virtual bool needUnpack(size_t index) const { (void) index; return true; } void initRange(uint32_t beginId, uint32_t endId) override; protected: + MultiSearch() {} void doUnpack(uint32_t docid) override; void visitMembers(vespalib::ObjectVisitor &visitor) const override; private: diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index 68be4d35972..d8b63909142 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -9,6 +9,9 @@ #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/distance_function_factory.h> +#include <vespa/log/log.h> + +LOG_SETUP(".searchlib.queryeval.nearest_neighbor_blueprint"); using vespalib::tensor::DenseTensorView; using vespalib::tensor::DenseTensor; @@ -42,7 +45,7 @@ convert_cells<double,double>(std::unique_ptr<DenseTensorView> &, vespalib::eval: struct ConvertCellsSelector { template <typename LCT, typename RCT> - static auto get_fun() { return convert_cells<LCT, RCT>; } + static auto invoke() { return convert_cells<LCT, RCT>; } }; } // namespace <unnamed> @@ -59,23 +62,21 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _explore_additional_hits(explore_additional_hits), _fallback_dist_fun(), _distance_heap(target_num_hits), - _found_hits() + _found_hits(), + _global_filter(GlobalFilter::create()) { auto lct = _query_tensor->cellsRef().type; auto rct = _attr_tensor.getTensorType().cell_type(); - auto fixup_fun = vespalib::tensor::select_2<ConvertCellsSelector>(lct, rct); + using MyTypify = vespalib::eval::TypifyCellType; + auto fixup_fun = vespalib::typify_invoke<2,MyTypify,ConvertCellsSelector>(lct, rct); fixup_fun(_query_tensor, _attr_tensor.getTensorType()); - auto def_dm = search::attribute::DistanceMetric::Euclidean; - _fallback_dist_fun = search::tensor::make_distance_function(def_dm, rct); + _fallback_dist_fun = search::tensor::make_distance_function(_attr_tensor.getConfig().distance_metric(), rct); _dist_fun = _fallback_dist_fun.get(); auto nns_index = _attr_tensor.nearest_neighbor_index(); if (nns_index) { _dist_fun = nns_index->distance_function(); } uint32_t est_hits = _attr_tensor.getNumDocs(); - if (_approximate && nns_index) { - est_hits = std::min(target_num_hits, est_hits); - } setEstimate(HitEstimate(est_hits, false)); set_want_global_filter(true); } @@ -83,10 +84,29 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f NearestNeighborBlueprint::~NearestNeighborBlueprint() = default; void -NearestNeighborBlueprint::set_global_filter(std::shared_ptr<BitVector> global_filter) +NearestNeighborBlueprint::set_global_filter(const GlobalFilter &global_filter) { - // XXX do something with global_filter here - (void) global_filter; + _global_filter = global_filter.shared_from_this(); + auto nns_index = _attr_tensor.nearest_neighbor_index(); + LOG(debug, "set_global_filter with: %s / %s / %s", + (_approximate ? "approximate" : "exact"), + (nns_index ? "nns_index" : "no_index"), + (_global_filter->has_filter() ? "has_filter" : "no_filter")); + if (_approximate && nns_index) { + uint32_t est_hits = _attr_tensor.getNumDocs(); + if (_global_filter->has_filter()) { + uint32_t max_hits = _global_filter->filter()->countTrueBits(); + LOG(debug, "set_global_filter getNumDocs: %u / max_hits %u", est_hits, max_hits); + if (max_hits * 10 < est_hits) { + LOG(debug, "too many hits filtered out, consider using brute force implementation"); + } + est_hits = std::min(est_hits, max_hits); + } + est_hits = std::min(est_hits, _target_num_hits); + setEstimate(HitEstimate(est_hits, false)); + perform_top_k(); + LOG(debug, "perform_top_k found %zu hits", _found_hits.size()); + } } void @@ -96,22 +116,20 @@ NearestNeighborBlueprint::perform_top_k() if (_approximate && nns_index) { auto lhs_type = _query_tensor->fast_type(); auto rhs_type = _attr_tensor.getTensorType(); - // XXX deal with different cell types later + // different cell types should be converted already if (lhs_type == rhs_type) { auto lhs = _query_tensor->cellsRef(); uint32_t k = _target_num_hits; - _found_hits = nns_index->find_top_k(k, lhs, k + _explore_additional_hits); + if (_global_filter->has_filter()) { + auto filter = _global_filter->filter(); + _found_hits = nns_index->find_top_k_with_filter(k, lhs, *filter, k + _explore_additional_hits); + } else { + _found_hits = nns_index->find_top_k(k, lhs, k + _explore_additional_hits); + } } } } -void -NearestNeighborBlueprint::fetchPostings(const ExecuteInfo &execInfo) { - if (execInfo.isStrict()) { - perform_top_k(); - } -} - std::unique_ptr<SearchIterator> NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray& tfmda, bool strict) const { @@ -121,7 +139,8 @@ NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchData return NnsIndexIterator::create(tfmd, _found_hits, _dist_fun); } const vespalib::tensor::DenseTensorView &qT = *_query_tensor; - return NearestNeighborIterator::create(strict, tfmd, qT, _attr_tensor, _distance_heap, _dist_fun); + return NearestNeighborIterator::create(strict, tfmd, qT, _attr_tensor, + _distance_heap, _global_filter->filter(), _dist_fun); } void diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h index cdb2b23e318..3e402b46a43 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h @@ -28,6 +28,7 @@ private: const search::tensor::DistanceFunction *_dist_fun; mutable NearestNeighborDistanceHeap _distance_heap; std::vector<search::tensor::NearestNeighborIndex::Neighbor> _found_hits; + std::shared_ptr<const GlobalFilter> _global_filter; void perform_top_k(); public: @@ -41,13 +42,13 @@ public: const tensor::DenseTensorAttribute& get_attribute_tensor() const { return _attr_tensor; } const vespalib::tensor::DenseTensorView& get_query_tensor() const { return *_query_tensor; } uint32_t get_target_num_hits() const { return _target_num_hits; } - void set_global_filter(std::shared_ptr<BitVector> global_filter) override; + void set_global_filter(const GlobalFilter &global_filter) override; + bool may_approximate() const { return _approximate; } std::unique_ptr<SearchIterator> createLeafSearch(const search::fef::TermFieldMatchDataArray& tfmda, bool strict) const override; void visitMembers(vespalib::ObjectVisitor& visitor) const override; bool always_needs_unpack() const override; - void fetchPostings(const ExecuteInfo &execInfo) override; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp index 68c6a1603d0..07e10271c55 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "nearest_neighbor_iterator.h" +#include <vespa/searchlib/common/bitvector.h> using search::tensor::DenseTensorAttribute; using vespalib::ConstArrayRef; @@ -29,7 +30,7 @@ is_compatible(const vespalib::eval::ValueType& lhs, * Keeps a heap of the K best hit distances. * Currently always does brute-force scanning, which is very expensive. **/ -template <bool strict> +template <bool strict, bool has_filter> class NearestNeighborImpl : public NearestNeighborIterator { public: @@ -48,11 +49,13 @@ public: void doSeek(uint32_t docId) override { double distanceLimit = params().distanceHeap.distanceLimit(); while (__builtin_expect((docId < getEndId()), true)) { - double d = computeDistance(docId, distanceLimit); - if (d <= distanceLimit) { - _lastScore = d; - setDocId(docId); - return; + if ((!has_filter) || params().filter->testBit(docId)) { + double d = computeDistance(docId, distanceLimit); + if (d <= distanceLimit) { + _lastScore = d; + setDocId(docId); + return; + } } if (strict) { ++docId; @@ -83,22 +86,23 @@ private: double _lastScore; }; -template <bool strict> -NearestNeighborImpl<strict>::~NearestNeighborImpl() = default; +template <bool strict, bool has_filter> +NearestNeighborImpl<strict, has_filter>::~NearestNeighborImpl() = default; namespace { +template <bool has_filter> std::unique_ptr<NearestNeighborIterator> -resolve_strict_LCT_RCT(bool strict, const NearestNeighborIterator::Params ¶ms) +resolve_strict(bool strict, const NearestNeighborIterator::Params ¶ms) { CellType lct = params.queryTensor.fast_type().cell_type(); CellType rct = params.tensorAttribute.getTensorType().cell_type(); if (lct != rct) abort(); if (strict) { - using NNI = NearestNeighborImpl<true>; + using NNI = NearestNeighborImpl<true, has_filter>; return std::make_unique<NNI>(params); } else { - using NNI = NearestNeighborImpl<false>; + using NNI = NearestNeighborImpl<false, has_filter>; return std::make_unique<NNI>(params); } } @@ -112,11 +116,16 @@ NearestNeighborIterator::create( const vespalib::tensor::DenseTensorView &queryTensor, const search::tensor::DenseTensorAttribute &tensorAttribute, NearestNeighborDistanceHeap &distanceHeap, + const search::BitVector *filter, const search::tensor::DistanceFunction *dist_fun) { - Params params(tfmd, queryTensor, tensorAttribute, distanceHeap, dist_fun); - return resolve_strict_LCT_RCT(strict, params); + Params params(tfmd, queryTensor, tensorAttribute, distanceHeap, filter, dist_fun); + if (filter) { + return resolve_strict<true>(strict, params); + } else { + return resolve_strict<false>(strict, params); + } } } // namespace diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h index 2a800f96710..9cbb1d39a91 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h @@ -25,17 +25,20 @@ public: const DenseTensorView &queryTensor; const DenseTensorAttribute &tensorAttribute; NearestNeighborDistanceHeap &distanceHeap; + const search::BitVector *filter; const search::tensor::DistanceFunction *distanceFunction; Params(fef::TermFieldMatchData &tfmd_in, const DenseTensorView &queryTensor_in, const DenseTensorAttribute &tensorAttribute_in, NearestNeighborDistanceHeap &distanceHeap_in, + const search::BitVector *filter_in, const search::tensor::DistanceFunction *distanceFunction_in) : tfmd(tfmd_in), queryTensor(queryTensor_in), tensorAttribute(tensorAttribute_in), distanceHeap(distanceHeap_in), + filter(filter_in), distanceFunction(distanceFunction_in) {} }; @@ -50,6 +53,7 @@ public: const vespalib::tensor::DenseTensorView &queryTensor, const search::tensor::DenseTensorAttribute &tensorAttribute, NearestNeighborDistanceHeap &distanceHeap, + const search::BitVector *filter, const search::tensor::DistanceFunction *dist_fun); const Params& params() const { return _params; } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp index 4ecd84fb7c3..f7b83de5f4b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp @@ -31,11 +31,11 @@ void setup_fields(uint32_t window, std::vector<T> &matchers, const TermFieldMatc } // namespace search::queryeval::<unnamed> -NearSearchBase::NearSearchBase(const Children & terms, +NearSearchBase::NearSearchBase(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict) - : AndSearch(terms), + : AndSearch(std::move(terms)), _data_size(data.size()), _window(window), _strict(strict) @@ -107,8 +107,7 @@ NearSearchBase::doSeek(uint32_t docId) const Children & terms(getChildren()); bool foundHit = true; for (uint32_t i = 0, len = terms.size(); i < len; ++i) { - SearchIterator *term = terms[i]; - if (!term->seek(docId)) { + if (! terms[i]->seek(docId)) { LOG(debug, "Term %d does not occur in document %d.", i, docId); foundHit = false; break; @@ -123,11 +122,11 @@ NearSearchBase::doSeek(uint32_t docId) } } -NearSearch::NearSearch(const Children & terms, +NearSearch::NearSearch(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict) - : NearSearchBase(terms, data, window, strict), + : NearSearchBase(std::move(terms), data, window, strict), _matchers() { setup_fields(window, _matchers, data); @@ -224,11 +223,11 @@ NearSearch::match(uint32_t docId) return false; } -ONearSearch::ONearSearch(const Children & terms, +ONearSearch::ONearSearch(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict) - : NearSearchBase(terms, data, window, strict), + : NearSearchBase(std::move(terms), data, window, strict), _matchers() { setup_fields(window, _matchers, data); diff --git a/searchlib/src/vespa/searchlib/queryeval/nearsearch.h b/searchlib/src/vespa/searchlib/queryeval/nearsearch.h index 27c862735b1..13bd1c53383 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearsearch.h @@ -72,7 +72,7 @@ public: * @param window The size of the window in which all terms must occur. * @param strict Whether or not to skip to next matching document if seek fails. */ - NearSearchBase(const Children & terms, + NearSearchBase(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict); @@ -106,7 +106,7 @@ public: * @param window The size of the window in which all terms must occur. * @param strict Whether or not to skip to next matching document if seek fails. */ - NearSearch(const Children & terms, + NearSearch(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict = true); @@ -138,7 +138,7 @@ public: * @param window The size of the window in which all terms must occur. * @param strict Whether or not to skip to next matching document if seek fails. */ - ONearSearch(const Children & terms, + ONearSearch(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict = true); diff --git a/searchlib/src/vespa/searchlib/queryeval/orlikesearch.h b/searchlib/src/vespa/searchlib/queryeval/orlikesearch.h index 1dec50ffc97..57a2ff11997 100644 --- a/searchlib/src/vespa/searchlib/queryeval/orlikesearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/orlikesearch.h @@ -46,8 +46,8 @@ public: * * @param children the search objects we are or'ing **/ - OrLikeSearch(const Children &children, const Unpack & unpacker) : - OrSearch(children), + OrLikeSearch(Children children, const Unpack & unpacker) + : OrSearch(std::move(children)), _unpacker(unpacker) { } private: diff --git a/searchlib/src/vespa/searchlib/queryeval/orsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/orsearch.cpp index 977080f8266..9d1dd252796 100644 --- a/searchlib/src/vespa/searchlib/queryeval/orsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/orsearch.cpp @@ -5,8 +5,7 @@ #include "termwise_helper.h" #include <vespa/searchlib/common/bitvector.h> -namespace search { -namespace queryeval { +namespace search::queryeval { namespace { @@ -79,34 +78,38 @@ OrSearch::or_hits_into(BitVector &result, uint32_t begin_id) TermwiseHelper::orChildren(result, getChildren().begin(), getChildren().end(), begin_id); } -SearchIterator * -OrSearch::create(const MultiSearch::Children &children, bool strict) { +SearchIterator::UP +OrSearch::create(ChildrenIterators children, bool strict) { UnpackInfo unpackInfo; unpackInfo.forceAll(); - return create(children, strict, unpackInfo); + return create(std::move(children), strict, unpackInfo); } -SearchIterator * -OrSearch::create(const MultiSearch::Children &children, bool strict, const UnpackInfo & unpackInfo) { - (void) unpackInfo; +SearchIterator::UP +OrSearch::create(ChildrenIterators children, bool strict, const UnpackInfo & unpackInfo) { if (strict) { if (unpackInfo.unpackAll()) { - return new OrLikeSearch<true, FullUnpack>(children, FullUnpack()); + using MyOr = OrLikeSearch<true, FullUnpack>; + return std::make_unique<MyOr>(std::move(children), FullUnpack()); } else if(unpackInfo.empty()) { - return new OrLikeSearch<true, NoUnpack>(children, NoUnpack()); + using MyOr = OrLikeSearch<true, NoUnpack>; + return std::make_unique<MyOr>(std::move(children), NoUnpack()); } else { - return new OrLikeSearch<true, SelectiveUnpack>(children, SelectiveUnpack(unpackInfo)); + using MyOr = OrLikeSearch<true, SelectiveUnpack>; + return std::make_unique<MyOr>(std::move(children), SelectiveUnpack(unpackInfo)); } } else { if (unpackInfo.unpackAll()) { - return new OrLikeSearch<false, FullUnpack>(children, FullUnpack()); + using MyOr = OrLikeSearch<false, FullUnpack>; + return std::make_unique<MyOr>(std::move(children), FullUnpack()); } else if(unpackInfo.empty()) { - return new OrLikeSearch<false, NoUnpack>(children, NoUnpack()); + using MyOr = OrLikeSearch<false, NoUnpack>; + return std::make_unique<MyOr>(std::move(children), NoUnpack()); } else { - return new OrLikeSearch<false, SelectiveUnpack>(children, SelectiveUnpack(unpackInfo)); + using MyOr = OrLikeSearch<false, SelectiveUnpack>; + return std::make_unique<MyOr>(std::move(children), SelectiveUnpack(unpackInfo)); } } } -} // namespace queryeval -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/orsearch.h b/searchlib/src/vespa/searchlib/queryeval/orsearch.h index e3d74573db8..79da1b85b6f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/orsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/orsearch.h @@ -15,16 +15,15 @@ class OrSearch : public MultiSearch public: typedef MultiSearch::Children Children; - // Caller takes ownership of the returned SearchIterator. - static SearchIterator *create(const Children &children, bool strict); - static SearchIterator *create(const Children &children, bool strict, const UnpackInfo & unpackInfo); + static SearchIterator::UP create(ChildrenIterators children, bool strict); + static SearchIterator::UP create(ChildrenIterators children, bool strict, const UnpackInfo & unpackInfo); std::unique_ptr<BitVector> get_hits(uint32_t begin_id) override; void or_hits_into(BitVector &result, uint32_t begin_id) override; void and_hits_into(BitVector &result, uint32_t begin_id) override; protected: - OrSearch(const Children & children) : MultiSearch(children) { } + OrSearch(Children children) : MultiSearch(std::move(children)) { } private: bool isOr() const override { return true; } diff --git a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp index 034f57bbb36..97e7044ec0f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp @@ -163,7 +163,8 @@ PredicateBlueprint::PredicateBlueprint(const FieldSpecBase &field, _bounds_btree_iterators(), _bounds_vector_iterators(), _zstar_btree_iterator(), - _zstar_vector_iterator() + _zstar_vector_iterator(), + _fetch_postings_done(false) { const auto &interval_index = _index.getIntervalIndex(); const auto zero_constraints_docs = _index.getZeroConstraintDocs(); @@ -234,36 +235,39 @@ namespace { } void PredicateBlueprint::fetchPostings(const ExecuteInfo &) { - const auto &interval_index = _index.getIntervalIndex(); - const auto &bounds_index = _index.getBoundsIndex(); - lookupPostingLists(_interval_dict_entries, _interval_vector_iterators, - _interval_btree_iterators, interval_index); - lookupPostingLists(_bounds_dict_entries, _bounds_vector_iterators, - _bounds_btree_iterators, bounds_index); - - // Lookup zstar interval iterator - if (_zstar_dict_entry.valid()) { - auto vector_iterator = interval_index.getVectorPostingList(Constants::z_star_compressed_hash); - if (vector_iterator) { - _zstar_vector_iterator.emplace(std::move(*vector_iterator)); - } else { - _zstar_btree_iterator.emplace(interval_index.getBTreePostingList(_zstar_dict_entry)); + if (!_fetch_postings_done) { + const auto &interval_index = _index.getIntervalIndex(); + const auto &bounds_index = _index.getBoundsIndex(); + lookupPostingLists(_interval_dict_entries, _interval_vector_iterators, + _interval_btree_iterators, interval_index); + lookupPostingLists(_bounds_dict_entries, _bounds_vector_iterators, + _bounds_btree_iterators, bounds_index); + + // Lookup zstar interval iterator + if (_zstar_dict_entry.valid()) { + auto vector_iterator = interval_index.getVectorPostingList(Constants::z_star_compressed_hash); + if (vector_iterator) { + _zstar_vector_iterator.emplace(std::move(*vector_iterator)); + } else { + _zstar_btree_iterator.emplace(interval_index.getBTreePostingList(_zstar_dict_entry)); + } } - } - PredicateAttribute::MinFeatureHandle mfh = predicate_attribute().getMinFeatureVector(); - Alloc kv(Alloc::alloc(mfh.second, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE*4)); - _kVBacking.swap(kv); - _kV = BitVectorCache::CountVector(static_cast<uint8_t *>(_kVBacking.get()), mfh.second); - _index.computeCountVector(_cachedFeatures, _kV); - for (const auto & entry : _bounds_dict_entries) { - addBoundsPostingToK(entry.feature); - } - for (const auto & entry : _interval_dict_entries) { - addPostingToK(entry.feature); + PredicateAttribute::MinFeatureHandle mfh = predicate_attribute().getMinFeatureVector(); + Alloc kv(Alloc::alloc(mfh.second, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE*4)); + _kVBacking.swap(kv); + _kV = BitVectorCache::CountVector(static_cast<uint8_t *>(_kVBacking.get()), mfh.second); + _index.computeCountVector(_cachedFeatures, _kV); + for (const auto & entry : _bounds_dict_entries) { + addBoundsPostingToK(entry.feature); + } + for (const auto & entry : _interval_dict_entries) { + addPostingToK(entry.feature); + } + addPostingToK(Constants::z_star_compressed_hash); + addZeroConstraintToK(); + _fetch_postings_done = true; } - addPostingToK(Constants::z_star_compressed_hash); - addZeroConstraintToK(); } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h index c9a19a0f5bb..9609cd4f6c9 100644 --- a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h @@ -87,6 +87,7 @@ private: // The zstar iterator is either a vector or a btree iterator. optional<BTreeIterator> _zstar_btree_iterator; optional<VectorIterator> _zstar_vector_iterator; + bool _fetch_postings_done; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp b/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp index 2bcf2267b1d..a915442bf0c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp @@ -32,7 +32,7 @@ public: * * @param children the search objects we are rank'ing **/ - RankSearchStrict(const Children & children) : RankSearch(children) { } + RankSearchStrict(Children children) : RankSearch(std::move(children)) { } }; SearchIterator::UP @@ -49,12 +49,12 @@ RankSearchStrict::doSeek(uint32_t docid) } } // namespace -SearchIterator * -RankSearch::create(const RankSearch::Children &children, bool strict) { +SearchIterator::UP +RankSearch::create(ChildrenIterators children, bool strict) { if (strict) { - return new RankSearchStrict(children); + return UP(new RankSearchStrict(std::move(children))); } else { - return new RankSearch(children); + return UP(new RankSearch(std::move(children))); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/ranksearch.h b/searchlib/src/vespa/searchlib/queryeval/ranksearch.h index 60efed3c694..443202f3e59 100644 --- a/searchlib/src/vespa/searchlib/queryeval/ranksearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/ranksearch.h @@ -20,11 +20,10 @@ protected: * * @param children the search objects we are rank'ing **/ - RankSearch(const Children & children) : MultiSearch(children) { } + RankSearch(Children children) : MultiSearch(std::move(children)) { } public: - // Caller takes ownership of the returned SearchIterator. - static SearchIterator *create(const Children &children, bool strict); + static SearchIterator::UP create(ChildrenIterators children, bool strict); }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp index 22a392ca208..ac4f164b09f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp @@ -4,19 +4,19 @@ #include "same_element_search.h" #include "field_spec.hpp" #include <vespa/searchlib/fef/termfieldmatchdata.h> -#include <vespa/searchlib/attribute/elementiterator.h> +#include <vespa/searchlib/attribute/searchcontextelementiterator.h> #include <vespa/vespalib/objects/visit.hpp> #include <algorithm> #include <map> namespace search::queryeval { -SameElementBlueprint::SameElementBlueprint(const vespalib::string &struct_field_name_in, bool expensive) +SameElementBlueprint::SameElementBlueprint(const vespalib::string &field_name_in, bool expensive) : ComplexLeafBlueprint(FieldSpecBaseList()), _estimate(), _layout(), _terms(), - _struct_field_name(struct_field_name_in) + _field_name(field_name_in) { if (expensive) { set_cost_tier(State::COST_TIER_EXPENSIVE); @@ -48,8 +48,7 @@ void SameElementBlueprint::optimize_self() { std::sort(_terms.begin(), _terms.end(), - [](const auto &a, const auto &b) - { + [](const auto &a, const auto &b) { return (a->getState().estimate() < b->getState().estimate()); }); } @@ -66,34 +65,23 @@ std::unique_ptr<SameElementSearch> SameElementBlueprint::create_same_element_search(bool strict) const { fef::MatchDataLayout my_layout = _layout; - std::vector<fef::TermFieldHandle> extra_handles; - for (size_t i = 0; i < _terms.size(); ++i) { - const State &childState = _terms[i]->getState(); - assert(childState.numFields() == 1); - extra_handles.push_back(my_layout.allocTermField(childState.field(0).getFieldId())); - } fef::MatchData::UP md = my_layout.createMatchData(); - search::fef::TermFieldMatchDataArray childMatch; - std::vector<SearchIterator::UP> children(_terms.size()); + std::vector<ElementIterator::UP> children(_terms.size()); for (size_t i = 0; i < _terms.size(); ++i) { const State &childState = _terms[i]->getState(); SearchIterator::UP child = _terms[i]->createSearch(*md, (strict && (i == 0))); const attribute::ISearchContext *context = _terms[i]->get_attribute_search_context(); if (context == nullptr) { - children[i] = std::move(child); - childMatch.add(childState.field(0).resolve(*md)); + children[i] = std::make_unique<ElementIteratorWrapper>(std::move(child), *childState.field(0).resolve(*md)); } else { - fef::TermFieldMatchData *child_tfmd = md->resolveTermField(extra_handles[i]); - children[i] = std::make_unique<attribute::ElementIterator>(std::move(child), *context, *child_tfmd); - childMatch.add(child_tfmd); + children[i] = std::make_unique<attribute::SearchContextElementIterator>(std::move(child), *context); } } - return std::make_unique<SameElementSearch>(std::move(md), std::move(children), childMatch, strict); + return std::make_unique<SameElementSearch>(std::move(md), std::move(children), strict); } SearchIterator::UP -SameElementBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, - bool strict) const +SameElementBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, bool strict) const { (void) tfmda; assert(!tfmda.valid()); diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h index fc19abe4c5e..8d647ac3a32 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h @@ -17,10 +17,10 @@ private: HitEstimate _estimate; fef::MatchDataLayout _layout; std::vector<Blueprint::UP> _terms; - vespalib::string _struct_field_name; + vespalib::string _field_name; public: - SameElementBlueprint(const vespalib::string &struct_field_name_in, bool expensive); + SameElementBlueprint(const vespalib::string &field_name_in, bool expensive); SameElementBlueprint(const SameElementBlueprint &) = delete; SameElementBlueprint &operator=(const SameElementBlueprint &) = delete; ~SameElementBlueprint(); @@ -42,7 +42,7 @@ public: bool strict) const override; void visitMembers(vespalib::ObjectVisitor &visitor) const override; const std::vector<Blueprint::UP> &terms() const { return _terms; } - const vespalib::string &struct_field_name() const { return _struct_field_name; } + const vespalib::string &field_name() const { return _field_name; } }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp index 37f8eccadd9..18f42597121 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp @@ -11,26 +11,6 @@ using TFMD = search::fef::TermFieldMatchData; namespace search::queryeval { -namespace { - -template <typename It> -int32_t try_match(const fef::TermFieldMatchDataArray &match, std::vector<It> &iterators, uint32_t cand) { - for (size_t i = 0; i < iterators.size(); ++i) { - while ((iterators[i] != match[i]->end()) && (iterators[i]->getElementId() < cand)) { - ++iterators[i]; - } - if (iterators[i] == match[i]->end()) { - return -1; - } - if (iterators[i]->getElementId() != cand) { - return iterators[i]->getElementId(); - } - } - return cand; -} - -} - bool SameElementSearch::check_docid_match(uint32_t docid) { @@ -43,41 +23,30 @@ SameElementSearch::check_docid_match(uint32_t docid) } void -SameElementSearch::unpack_children(uint32_t docid) +SameElementSearch::fetch_matching_elements(uint32_t docid, std::vector<uint32_t> & elems) { - for (const auto &child: _children) { - child->doUnpack(docid); - } - for (size_t i = 0; i < _childMatch.size(); ++i) { - _iterators[i] = _childMatch[i]->begin(); + _children.front()->getElementIds(docid, elems); + for (auto it(_children.begin() + 1); it != _children.end(); it++) { + (*it)->mergeElementIds(docid, elems); } } bool SameElementSearch::check_element_match(uint32_t docid) { - unpack_children(docid); - int32_t cand = 0; - int32_t next = try_match(_childMatch, _iterators, cand); - while (next > cand) { - cand = next; - next = try_match(_childMatch, _iterators, cand); - } - return (cand == next); + _matchingElements.clear(); + fetch_matching_elements(docid, _matchingElements); + return !_matchingElements.empty(); } SameElementSearch::SameElementSearch(fef::MatchData::UP md, - std::vector<SearchIterator::UP> children, - const fef::TermFieldMatchDataArray &childMatch, + std::vector<ElementIterator::UP> children, bool strict) : _md(std::move(md)), _children(std::move(children)), - _childMatch(childMatch), - _iterators(childMatch.size()), _strict(strict) { assert(!_children.empty()); - assert(_childMatch.valid()); } void @@ -118,17 +87,7 @@ void SameElementSearch::find_matching_elements(uint32_t docid, std::vector<uint32_t> &dst) { if (check_docid_match(docid)) { - unpack_children(docid); - int32_t cand = 0; - while (cand >= 0) { - int32_t next = try_match(_childMatch, _iterators, cand); - if (next == cand) { - dst.push_back(cand); - ++cand; - } else { - cand = next; - } - } + fetch_matching_elements(docid, dst); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_search.h b/searchlib/src/vespa/searchlib/queryeval/same_element_search.h index ae7ecab781d..e5f185e06e6 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_search.h @@ -2,7 +2,7 @@ #pragma once -#include "searchiterator.h" +#include "elementiterator.h" #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> @@ -21,26 +21,24 @@ class SameElementSearch : public SearchIterator private: using It = fef::TermFieldMatchData::PositionsIterator; - fef::MatchData::UP _md; - std::vector<SearchIterator::UP> _children; - fef::TermFieldMatchDataArray _childMatch; - std::vector<It> _iterators; - bool _strict; + fef::MatchData::UP _md; + std::vector<ElementIterator::UP> _children; + std::vector<uint32_t> _matchingElements; + bool _strict; - void unpack_children(uint32_t docid); + void fetch_matching_elements(uint32_t docid, std::vector<uint32_t> &dst); bool check_docid_match(uint32_t docid); bool check_element_match(uint32_t docid); public: SameElementSearch(fef::MatchData::UP md, - std::vector<SearchIterator::UP> children, - const fef::TermFieldMatchDataArray &childMatch, + std::vector<ElementIterator::UP> children, bool strict); void initRange(uint32_t begin_id, uint32_t end_id) override; void doSeek(uint32_t docid) override; void doUnpack(uint32_t) override {} void visitMembers(vespalib::ObjectVisitor &visitor) const override; - const std::vector<SearchIterator::UP> &children() const { return _children; } + const std::vector<ElementIterator::UP> &children() const { return _children; } // used during docsum fetching to identify matching elements // initRange must be called before use. diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h index afc1bc8ce15..3ce808de7a1 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h @@ -160,14 +160,14 @@ public: /** * Find all hits in the currently searched range (specified by - * initRange) and OR them into the given temporary result. This + * initRange) and AND them into the given temporary result. This * function will perform term-at-a-time evaluation and should only * be used for terms not needed for ranking. Calling this function * will exhaust this iterator and no more results will be * available in the currently searched range after this function * returns. * - * @param result result to be augmented by adding hits from this + * @param result result to be augmented by clearing non-hits from this * iterator. * @param begin_id the lowest document id that may be a hit * (we might not remember beginId from initRange) diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp index c52cf6ddae1..83d5a5e1739 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp @@ -2,6 +2,7 @@ #include "simple_phrase_blueprint.h" #include "simple_phrase_search.h" +#include "emptysearch.h" #include "field_spec.hpp" #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.hpp> @@ -60,7 +61,8 @@ SimplePhraseBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &tfmd assert(tfmda.size() == 1); fef::MatchData::UP md = _layout.createMatchData(); fef::TermFieldMatchDataArray childMatch; - SimplePhraseSearch::Children children(_terms.size()); + SimplePhraseSearch::Children children; + children.reserve(_terms.size()); std::multimap<uint32_t, uint32_t> order_map; for (size_t i = 0; i < _terms.size(); ++i) { const State &childState = _terms[i]->getState(); @@ -69,7 +71,7 @@ SimplePhraseBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &tfmd child_term_field_match_data->setNeedInterleavedFeatures(tfmda[0]->needs_interleaved_features()); child_term_field_match_data->setNeedNormalFeatures(true); childMatch.add(child_term_field_match_data); - children[i] = _terms[i]->createSearch(*md, strict).release(); + children.push_back(_terms[i]->createSearch(*md, strict)); order_map.insert(std::make_pair(childState.estimate().estHits, i)); } std::vector<uint32_t> eval_order; @@ -77,12 +79,29 @@ SimplePhraseBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &tfmd eval_order.push_back(child.second); } - auto phrase = std::make_unique<SimplePhraseSearch>(children, std::move(md), childMatch, + auto phrase = std::make_unique<SimplePhraseSearch>(std::move(children), + std::move(md), childMatch, eval_order, *tfmda[0], strict); phrase->setDoom(& _doom); return phrase; } +SearchIterator::UP +SimplePhraseBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +{ + if (constraint == FilterConstraint::UPPER_BOUND) { + MultiSearch::Children children; + children.reserve(_terms.size()); + for (size_t i = 0; i < _terms.size(); ++i) { + bool child_strict = strict && (i == 0); + children.push_back(_terms[i]->createFilterSearch(child_strict, constraint)); + } + UnpackInfo unpack_info; + return AndSearch::create(std::move(children), strict, unpack_info); + } else { + return std::make_unique<EmptySearch>(); + } +} void SimplePhraseBlueprint::fetchPostings(const ExecuteInfo &execInfo) diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.h index 0fb1599d68f..b20866fdd1f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.h @@ -33,8 +33,8 @@ public: // used by create visitor void addTerm(Blueprint::UP term); - SearchIteratorUP - createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool strict) const override; + SearchIteratorUP createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool strict) const override; + SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override; void visitMembers(vespalib::ObjectVisitor &visitor) const override; void fetchPostings(const ExecuteInfo &execInfo) override; }; diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp index df0dff06582..43a9ee4ab91 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp @@ -157,23 +157,23 @@ SimplePhraseSearch::phraseSeek(uint32_t doc_id) { } -SimplePhraseSearch::SimplePhraseSearch(const Children &children, +SimplePhraseSearch::SimplePhraseSearch(Children children, fef::MatchData::UP md, const fef::TermFieldMatchDataArray &childMatch, vector<uint32_t> eval_order, TermFieldMatchData &tmd, bool strict) - : AndSearch(children), + : AndSearch(std::move(children)), _md(std::move(md)), _childMatch(childMatch), _eval_order(std::move(eval_order)), _tmd(tmd), _doom(nullptr), _strict(strict), - _iterators(children.size()) + _iterators(getChildren().size()) { - assert(!children.empty()); - assert(children.size() == _childMatch.size()); - assert(children.size() == _eval_order.size()); + assert(getChildren().size() > 0); + assert(getChildren().size() == _childMatch.size()); + assert(getChildren().size() == _eval_order.size()); } void diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h index d45e67ed4cb..5cec039e733 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h @@ -43,7 +43,7 @@ public: * terms. The term with fewest hits should be * evaluated first. **/ - SimplePhraseSearch(const Children &children, + SimplePhraseSearch(Children children, fef::MatchData::UP md, const fef::TermFieldMatchDataArray &childMatch, std::vector<uint32_t> eval_order, diff --git a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp index 3280e0ac2cc..b50e71d5d53 100644 --- a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp @@ -10,6 +10,14 @@ namespace search::queryeval { EmptySearch SourceBlenderSearch::_emptySearch; +class SourceBlenderSearchNonStrict : public SourceBlenderSearch +{ +public: + SourceBlenderSearchNonStrict(std::unique_ptr<Iterator> sourceSelector, const Children &children) + : SourceBlenderSearch(std::move(sourceSelector), children) + {} +}; + class SourceBlenderSearchStrict : public SourceBlenderSearch { public: @@ -158,14 +166,14 @@ SourceBlenderSearch::setChild(size_t index, SearchIterator::UP child) { _sources[_children[index]] = child.release(); } -SourceBlenderSearch * +SearchIterator::UP SourceBlenderSearch::create(std::unique_ptr<sourceselector::Iterator> sourceSelector, const Children &children, bool strict) { if (strict) { - return new SourceBlenderSearchStrict(std::move(sourceSelector), children); + return std::make_unique<SourceBlenderSearchStrict>(std::move(sourceSelector), children); } else { - return new SourceBlenderSearch(std::move(sourceSelector), children); + return std::make_unique<SourceBlenderSearchNonStrict>(std::move(sourceSelector), children); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h index 9b4d8f58034..f209e0f7fd8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h @@ -68,8 +68,9 @@ public: * @param strict whether this search is strict * (a strict search will locate its next hit when seeking fails) **/ - static SourceBlenderSearch * create(std::unique_ptr<Iterator> sourceSelector, - const Children &children, bool strict); + static SearchIterator::UP create(std::unique_ptr<Iterator> sourceSelector, + const Children &children, + bool strict); ~SourceBlenderSearch() override; size_t getNumChildren() const { return _children.size(); } SearchIterator::UP steal(size_t index) { diff --git a/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.cpp b/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.cpp index ae21fd93ba3..e26a1652441 100644 --- a/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.cpp @@ -6,27 +6,27 @@ namespace search::queryeval { TermwiseBlueprintHelper::TermwiseBlueprintHelper(const IntermediateBlueprint &self, - const MultiSearch::Children &subSearches, + MultiSearch::Children subSearches, UnpackInfo &unpackInfo) - : children(), - termwise(), + : termwise_ch(), + other_ch(), first_termwise(subSearches.size()), termwise_unpack() { - children.reserve(subSearches.size()); - termwise.reserve(subSearches.size()); + other_ch.reserve(subSearches.size()); + termwise_ch.reserve(subSearches.size()); for (size_t i = 0; i < subSearches.size(); ++i) { bool need_unpack = unpackInfo.needUnpack(i); bool allow_termwise = self.getChild(i).getState().allow_termwise_eval(); if (need_unpack || !allow_termwise) { if (need_unpack) { - size_t index = (i < first_termwise) ? children.size() : (children.size() + 1); + size_t index = (i < first_termwise) ? other_ch.size() : (other_ch.size() + 1); termwise_unpack.add(index); } - children.push_back(subSearches[i]); + other_ch.push_back(std::move(subSearches[i])); } else { first_termwise = std::min(i, first_termwise); - termwise.push_back(subSearches[i]); + termwise_ch.push_back(std::move(subSearches[i])); } } } @@ -37,7 +37,7 @@ void TermwiseBlueprintHelper::insert_termwise(SearchIterator::UP search, bool strict) { auto termwise_search = make_termwise(std::move(search), strict); - children.insert(children.begin() + first_termwise, termwise_search.release()); + other_ch.insert(other_ch.begin() + first_termwise, std::move(termwise_search)); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.h b/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.h index e6b46cfb7d2..2917ce4d8c9 100644 --- a/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.h +++ b/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.h @@ -15,13 +15,18 @@ namespace search::queryeval { * termwise and non-termwise parts with each other. **/ struct TermwiseBlueprintHelper { - MultiSearch::Children children; - MultiSearch::Children termwise; +private: + MultiSearch::Children termwise_ch; + MultiSearch::Children other_ch; +public: size_t first_termwise; UnpackInfo termwise_unpack; + MultiSearch::Children get_termwise_children() { return std::move(termwise_ch); } + MultiSearch::Children get_result() { return std::move(other_ch); } + TermwiseBlueprintHelper(const IntermediateBlueprint &self, - const MultiSearch::Children &subSearches, UnpackInfo &unpackInfo); + MultiSearch::Children subSearches, UnpackInfo &unpackInfo); ~TermwiseBlueprintHelper(); void insert_termwise(SearchIterator::UP search, bool strict); diff --git a/searchlib/src/vespa/searchlib/queryeval/test/leafspec.h b/searchlib/src/vespa/searchlib/queryeval/test/leafspec.h index 47b5ed26b60..fa14941844e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/test/leafspec.h +++ b/searchlib/src/vespa/searchlib/queryeval/test/leafspec.h @@ -18,7 +18,7 @@ struct LeafSpec int32_t weight; int32_t maxWeight; FakeResult result; - SearchIterator *search; + SearchIterator::UP search; LeafSpec(const std::string &n, int32_t w = 100) : name(n), weight(w), @@ -26,31 +26,36 @@ struct LeafSpec result(), search() {} + LeafSpec(LeafSpec && other) = default; ~LeafSpec() {} - LeafSpec &doc(uint32_t docid) { + LeafSpec && doc(uint32_t docid) && { result.doc(docid); - return *this; + return std::move(*this); } - LeafSpec &doc(uint32_t docid, int32_t w) { + LeafSpec && doc(uint32_t docid, int32_t w) && { result.doc(docid); result.weight(w); result.pos(0); maxWeight = std::max(maxWeight, w); - return *this; + return std::move(*this); } - LeafSpec &itr(SearchIterator *si) { - search = si; - return *this; + LeafSpec && itr(SearchIterator::UP si) && { + search = std::move(si); + return std::move(*this); } - SearchIterator *create(SearchHistory &hist, fef::TermFieldMatchData *tfmd) const { - if (search != nullptr) { - return new TrackedSearch(name, hist, search); + LeafSpec && itr(SearchIterator *si) && { + search.reset(si); + return std::move(*this); + } + SearchIterator::UP create(SearchHistory &hist, fef::TermFieldMatchData *tfmd) { + if (search) { + return SearchIterator::UP(new TrackedSearch(name, hist, std::move(search))); } else if (tfmd != nullptr) { - return new TrackedSearch(name, hist, result, *tfmd, - MinMaxPostingInfo(0, maxWeight)); + return SearchIterator::UP(new TrackedSearch(name, hist, result, *tfmd, + MinMaxPostingInfo(0, maxWeight))); } - return new TrackedSearch(name, hist, result, - MinMaxPostingInfo(0, maxWeight)); + return SearchIterator::UP(new TrackedSearch(name, hist, result, + MinMaxPostingInfo(0, maxWeight))); } }; diff --git a/searchlib/src/vespa/searchlib/queryeval/test/trackedsearch.h b/searchlib/src/vespa/searchlib/queryeval/test/trackedsearch.h index 6cb4c1a9dda..ae04d35e658 100644 --- a/searchlib/src/vespa/searchlib/queryeval/test/trackedsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/test/trackedsearch.h @@ -56,6 +56,12 @@ public: _search(new FakeSearch("<tag>", "<field>", "<term>", result, makeArray(tfmd))), _minMaxPostingInfo(new MinMaxPostingInfo(minMaxPostingInfo)) { setDocId(_search->getDocId()); } + + // wraps a generic search (typically wand) + TrackedSearch(const std::string &name, SearchHistory &hist, SearchIterator::UP search) + : _name(name), _history(hist), _matchData(), _search(std::move(search)), _minMaxPostingInfo() + { setDocId(_search->getDocId()); } + // wraps a generic search (typically wand) TrackedSearch(const std::string &name, SearchHistory &hist, SearchIterator *search) : _name(name), _history(hist), _matchData(), _search(search), _minMaxPostingInfo() diff --git a/searchlib/src/vespa/searchlib/queryeval/test/wandspec.h b/searchlib/src/vespa/searchlib/queryeval/test/wandspec.h index bf456c287d6..2fb2b3bc9e2 100644 --- a/searchlib/src/vespa/searchlib/queryeval/test/wandspec.h +++ b/searchlib/src/vespa/searchlib/queryeval/test/wandspec.h @@ -26,8 +26,8 @@ private: public: WandSpec() : _leafs(), _layout(), _handles(), _history() {} ~WandSpec() {} - WandSpec &leaf(const LeafSpec &l) { - _leafs.push_back(l); + WandSpec &leaf(LeafSpec && l) { + _leafs.emplace_back(std::move(l)); _handles.push_back(_layout.allocTermField(0)); return *this; } @@ -35,7 +35,7 @@ public: wand::Terms terms; for (size_t i = 0; i < _leafs.size(); ++i) { fef::TermFieldMatchData *tfmd = (matchData != NULL ? matchData->resolveTermField(_handles[i]) : NULL); - terms.push_back(wand::Term(_leafs[i].create(_history, tfmd), + terms.push_back(wand::Term(_leafs[i].create(_history, tfmd).release(), _leafs[i].weight, _leafs[i].result.inspect().size(), tfmd)); diff --git a/searchlib/src/vespa/searchlib/queryeval/truesearch.h b/searchlib/src/vespa/searchlib/queryeval/truesearch.h index 6cf1e4dec02..35a609cb3a7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/truesearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/truesearch.h @@ -7,6 +7,10 @@ namespace search::queryeval { +/** + * Search iterator for testing, yielding a hit on all documents. + * Unpacks (sets docid) to the given TermFieldMatchData. + **/ class TrueSearch : public SearchIterator { private: diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp index 7ad4a36f871..d4f16e2f91c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp @@ -86,6 +86,7 @@ ParallelWeakAndBlueprint::createLeafSearch(const search::fef::TermFieldMatchData for (size_t i = 0; i < _terms.size(); ++i) { const State &childState = _terms[i]->getState(); assert(childState.numFields() == 1); + // TODO: pass ownership with unique_ptr terms.push_back(wand::Term(_terms[i]->createSearch(*childrenMatchData, true).release(), _weights[i], childState.estimate().estHits, diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp index 789bfc40782..3a7d4d8273a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp @@ -128,7 +128,7 @@ insertMonitoringSearchIterator(const wand::Terms &terms) } template <typename FutureHeap, typename PastHeap, bool IS_STRICT> -SearchIterator * +SearchIterator::UP createWand(const wand::Terms &terms, const ParallelWeakAndSearch::MatchParams &matchParams, ParallelWeakAndSearch::RankParams &&rankParams) @@ -149,9 +149,9 @@ createWand(const wand::Terms &terms, std::move(rankParams.childrenMatchData)), matchParams)), false)); - return new MonitoringDumpIterator(std::move(monitoringIterator)); + return std::make_unique<MonitoringDumpIterator>(std::move(monitoringIterator)); } - return new WandType(rankParams.rootMatchData, + return std::make_unique<WandType>(rankParams.rootMatchData, VectorizedIteratorTerms(terms, DotProductScorer(), matchParams.docIdLimit, @@ -163,7 +163,7 @@ createWand(const wand::Terms &terms, } // namespace search::queryeval::wand -SearchIterator * +SearchIterator::UP ParallelWeakAndSearch::createArrayWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, @@ -176,7 +176,7 @@ ParallelWeakAndSearch::createArrayWand(const Terms &terms, } } -SearchIterator * +SearchIterator::UP ParallelWeakAndSearch::createHeapWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, @@ -189,7 +189,7 @@ ParallelWeakAndSearch::createHeapWand(const Terms &terms, } } -SearchIterator * +SearchIterator::UP ParallelWeakAndSearch::create(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, @@ -251,7 +251,7 @@ ParallelWeakAndSearch::create(search::fef::TermFieldMatchData &tfmd, childrenMatchData->resolveTermField(handles[i]))); } assert(terms.size() == dict_entries.size()); - return SearchIterator::UP(create(terms, matchParams, RankParams(tfmd, std::move(childrenMatchData)), strict)); + return create(terms, matchParams, RankParams(tfmd, std::move(childrenMatchData)), strict); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.h b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.h index ba0812aad48..3e1ab1400e8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.h @@ -64,9 +64,9 @@ struct ParallelWeakAndSearch : public SearchIterator virtual score_t get_max_score(size_t idx) const = 0; virtual const MatchParams &getMatchParams() const = 0; - static SearchIterator *createArrayWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict); - static SearchIterator *createHeapWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict); - static SearchIterator *create(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict); + static SearchIterator::UP createArrayWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict); + static SearchIterator::UP createHeapWand(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict); + static SearchIterator::UP create(const Terms &terms, const MatchParams &matchParams, RankParams &&rankParams, bool strict); static SearchIterator::UP create(fef::TermFieldMatchData &tmd, const MatchParams &matchParams, diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h index bd60473e05d..071d6d99470 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h @@ -35,6 +35,7 @@ typedef std::vector<AttrDictEntry> AttrDictEntries; * Wrapper used to specify underlying terms during setup **/ struct Term { + // TODO: use unique_ptr for ownership SearchIterator *search; int32_t weight; uint32_t estHits; @@ -44,6 +45,7 @@ struct Term { : search(s), weight(w), estHits(e), matchData(tfmd) {} Term() : Term(nullptr, 0, 0, nullptr){} Term(SearchIterator *s, int32_t w, uint32_t e) : Term(s, w, e, nullptr) {} + Term(SearchIterator::UP s, int32_t w, uint32_t e) : Term(s.release(), w, e, nullptr) {} }; //----------------------------------------------------------------------------- diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp index f290d3e19ea..d94dc6d8ae8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp @@ -106,27 +106,27 @@ WeakAndSearch::visitMembers(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- -SearchIterator * +SearchIterator::UP WeakAndSearch::createArrayWand(const Terms &terms, uint32_t n, bool strict) { if (strict) { - return new wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, true>(terms, n); + return SearchIterator::UP(new wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, true>(terms, n)); } else { - return new wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, false>(terms, n); + return SearchIterator::UP(new wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, false>(terms, n)); } } -SearchIterator * +SearchIterator::UP WeakAndSearch::createHeapWand(const Terms &terms, uint32_t n, bool strict) { if (strict) { - return new wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, true>(terms, n); + return SearchIterator::UP(new wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, true>(terms, n)); } else { - return new wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, false>(terms, n); + return SearchIterator::UP(new wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, false>(terms, n)); } } -SearchIterator * +SearchIterator::UP WeakAndSearch::create(const Terms &terms, uint32_t n, bool strict) { if (terms.size() < 128) { diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h index 5b09d087873..e51b5ca102d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h @@ -16,9 +16,9 @@ struct WeakAndSearch : SearchIterator { virtual const Terms &getTerms() const = 0; virtual uint32_t getN() const = 0; void visitMembers(vespalib::ObjectVisitor &visitor) const override; - static SearchIterator *createArrayWand(const Terms &terms, uint32_t n, bool strict); - static SearchIterator *createHeapWand(const Terms &terms, uint32_t n, bool strict); - static SearchIterator *create(const Terms &terms, uint32_t n, bool strict); + static SearchIterator::UP createArrayWand(const Terms &terms, uint32_t n, bool strict); + static SearchIterator::UP createHeapWand(const Terms &terms, uint32_t n, bool strict); + static SearchIterator::UP create(const Terms &terms, uint32_t n, bool strict); }; } // namespace queryeval diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp index 36378439c01..cea35d976f0 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp @@ -50,6 +50,7 @@ WeightedSetTermBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &t fef::MatchData::UP md = _layout.createMatchData(); std::vector<SearchIterator*> children(_terms.size()); for (size_t i = 0; i < _terms.size(); ++i) { + // TODO: pass ownership with unique_ptr children[i] = _terms[i]->createSearch(*md, true).release(); } return SearchIterator::UP(WeightedSetTermSearch::create(children, *tfmda[0], _weights, std::move(md))); diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp index 2801f1c5e0c..71270c84c63 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp @@ -131,8 +131,8 @@ public: //----------------------------------------------------------------------------- -SearchIterator * -WeightedSetTermSearch::create(const std::vector<SearchIterator*> &children, +SearchIterator::UP +WeightedSetTermSearch::create(const std::vector<SearchIterator *> &children, TermFieldMatchData &tmd, const std::vector<int32_t> &weights, fef::MatchData::UP match_data) @@ -141,9 +141,9 @@ WeightedSetTermSearch::create(const std::vector<SearchIterator*> &children, typedef WeightedSetTermSearchImpl<vespalib::LeftHeap, SearchIteratorPack> HeapImpl; if (children.size() < 128) { - return new ArrayHeapImpl(tmd, weights, SearchIteratorPack(children, std::move(match_data))); + return SearchIterator::UP(new ArrayHeapImpl(tmd, weights, SearchIteratorPack(children, std::move(match_data)))); } - return new HeapImpl(tmd, weights, SearchIteratorPack(children, std::move(match_data))); + return SearchIterator::UP(new HeapImpl(tmd, weights, SearchIteratorPack(children, std::move(match_data)))); } //----------------------------------------------------------------------------- diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h index 397ac0caf2e..ecc620a3adb 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h @@ -27,10 +27,11 @@ protected: WeightedSetTermSearch() {} public: - static SearchIterator* create(const std::vector<SearchIterator*> &children, - search::fef::TermFieldMatchData &tmd, - const std::vector<int32_t> &weights, - fef::MatchData::UP match_data); + // TODO: pass ownership with unique_ptr + static SearchIterator::UP create(const std::vector<SearchIterator *> &children, + search::fef::TermFieldMatchData &tmd, + const std::vector<int32_t> &weights, + fef::MatchData::UP match_data); static SearchIterator::UP create(search::fef::TermFieldMatchData &tmd, const std::vector<int32_t> &weights, diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h index 79f987c740c..d37495e85da 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h @@ -17,7 +17,7 @@ template <typename FloatType> class SquaredEuclideanDistance : public DistanceFunction { public: SquaredEuclideanDistance() - : _computer(vespalib::hwaccelrated::IAccelrated::getAccelrator()) + : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()) {} double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override { auto lhs_vector = lhs.typify<FloatType>(); @@ -60,7 +60,7 @@ template <typename FloatType> class AngularDistance : public DistanceFunction { public: AngularDistance() - : _computer(vespalib::hwaccelrated::IAccelrated::getAccelrator()) + : _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()) {} double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override { auto lhs_vector = lhs.typify<FloatType>(); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp index 0f58a72e794..564676a2d44 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp @@ -11,13 +11,16 @@ HnswGraph::HnswGraph() : node_refs(), nodes(HnswIndex::make_default_node_store_config()), links(HnswIndex::make_default_link_store_config()), - entry_docid(0), // Note that docid 0 is reserved and never used - entry_level(-1) -{} + entry_docid_and_level() +{ + node_refs.ensure_size(1, AtomicEntryRef()); + EntryNode entry; + set_entry_node(entry); +} HnswGraph::~HnswGraph() {} -void +HnswGraph::NodeRef HnswGraph::make_node_for_document(uint32_t docid, uint32_t num_levels) { node_refs.ensure_size(docid + 1, AtomicEntryRef()); @@ -27,15 +30,23 @@ HnswGraph::make_node_for_document(uint32_t docid, uint32_t num_levels) vespalib::Array<AtomicEntryRef> levels(num_levels, AtomicEntryRef()); auto node_ref = nodes.add(levels); node_refs[docid].store_release(node_ref); + return node_ref; } void HnswGraph::remove_node_for_document(uint32_t docid) { auto node_ref = node_refs[docid].load_acquire(); - nodes.remove(node_ref); + assert(node_ref.valid()); + auto levels = nodes.get(node_ref); vespalib::datastore::EntryRef invalid; node_refs[docid].store_release(invalid); + // Ensure data referenced through the old ref can be recycled: + nodes.remove(node_ref); + for (size_t i = 0; i < levels.size(); ++i) { + auto old_links_ref = levels[i].load_acquire(); + links.remove(old_links_ref); + } } void @@ -45,6 +56,7 @@ HnswGraph::set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& ne auto node_ref = node_refs[docid].load_acquire(); assert(node_ref.valid()); auto levels = nodes.get_writable(node_ref); + assert(level < levels.size()); auto old_links_ref = levels[level].load_acquire(); levels[level].store_release(new_links_ref); links.remove(old_links_ref); diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h index d1d308def99..8b40eb87bae 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h @@ -23,6 +23,7 @@ struct HnswGraph { // Provides mapping from document id -> node reference. // The reference is used to lookup the node data in NodeStore. using NodeRefVector = vespalib::RcuVector<AtomicEntryRef>; + using NodeRef = vespalib::datastore::EntryRef; // This stores the level arrays for all nodes. // Each node consists of an array of levels (from level 0 to n) where each entry is a reference to the link array at that level. @@ -38,34 +39,116 @@ struct HnswGraph { NodeRefVector node_refs; NodeStore nodes; LinkStore links; - uint32_t entry_docid; - int32_t entry_level; + + std::atomic<uint64_t> entry_docid_and_level; HnswGraph(); ~HnswGraph(); - void make_node_for_document(uint32_t docid, uint32_t num_levels); + NodeRef make_node_for_document(uint32_t docid, uint32_t num_levels); void remove_node_for_document(uint32_t docid); + NodeRef get_node_ref(uint32_t docid) const { + return node_refs[docid].load_acquire(); + } + + bool still_valid(uint32_t docid, NodeRef node_ref) const { + return node_ref.valid() && (get_node_ref(docid) == node_ref); + } + + LevelArrayRef get_level_array(NodeRef node_ref) const { + if (node_ref.valid()) { + return nodes.get(node_ref); + } + return LevelArrayRef(); + } + LevelArrayRef get_level_array(uint32_t docid) const { - auto node_ref = node_refs[docid].load_acquire(); - assert(node_ref.valid()); - return nodes.get(node_ref); + auto node_ref = get_node_ref(docid); + return get_level_array(node_ref); + } + + LinkArrayRef get_link_array(LevelArrayRef levels, uint32_t level) const { + if (level < levels.size()) { + auto links_ref = levels[level].load_acquire(); + if (links_ref.valid()) { + return links.get(links_ref); + } + } + return LinkArrayRef(); } LinkArrayRef get_link_array(uint32_t docid, uint32_t level) const { auto levels = get_level_array(docid); - assert(level < levels.size()); - return links.get(levels[level].load_acquire()); + return get_link_array(levels, level); } - + + LinkArrayRef get_link_array(NodeRef node_ref, uint32_t level) const { + auto levels = get_level_array(node_ref); + return get_link_array(levels, level); + } + void set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& new_links); - void set_entry_node(uint32_t docid, int32_t level) { - entry_docid = docid; - entry_level = level; + struct EntryNode { + uint32_t docid; + NodeRef node_ref; + int32_t level; + EntryNode() + : docid(0), // Note that docid 0 is reserved and never used + node_ref(), + level(-1) + {} + EntryNode(uint32_t docid_in, NodeRef node_ref_in, int32_t level_in) + : docid(docid_in), + node_ref(node_ref_in), + level(level_in) + {} + }; + + void set_entry_node(EntryNode node) { + uint64_t value = node.level; + value <<= 32; + value |= node.docid; + if (node.node_ref.valid()) { + assert(node.level >= 0); + assert(node.docid > 0); + } else { + assert(node.level == -1); + assert(node.docid == 0); + } + entry_docid_and_level.store(value, std::memory_order_release); + } + + uint64_t get_entry_atomic() const { + return entry_docid_and_level.load(std::memory_order_acquire); + } + + EntryNode get_entry_node() const { + EntryNode entry; + while (true) { + uint64_t value = get_entry_atomic(); + entry.docid = (uint32_t)value; + entry.node_ref = get_node_ref(entry.docid); + entry.level = (int32_t)(value >> 32); + if ((entry.docid == 0) + && (entry.level == -1) + && ! entry.node_ref.valid()) + { + // invalid in every way + return entry; + } + if ((entry.docid > 0) + && (entry.level > -1) + && entry.node_ref.valid() + && (get_entry_atomic() == value)) + { + // valid in every way + return entry; + } + } } size_t size() const { return node_refs.size(); } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 612f30cc64f..36d970dfd01 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -11,6 +11,9 @@ #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/datastore/array_store.hpp> #include <vespa/vespalib/util/rcuvector.hpp> +#include <vespa/log/log.h> + +LOG_SETUP(".searchlib.tensor.hnsw_index"); namespace search::tensor { @@ -69,10 +72,10 @@ HnswIndex::max_links_for_level(uint32_t level) const } bool -HnswIndex::have_closer_distance(HnswCandidate candidate, const LinkArrayRef& result) const +HnswIndex::have_closer_distance(HnswCandidate candidate, const HnswCandidateVector& result) const { - for (uint32_t result_docid : result) { - double dist = calc_distance(candidate.docid, result_docid); + for (const auto & neighbor : result) { + double dist = calc_distance(candidate.docid, neighbor.docid); if (dist < candidate.distance) { return true; } @@ -88,7 +91,7 @@ HnswIndex::select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_ SelectResult result; for (const auto & candidate : sorted) { if (result.used.size() < max_links) { - result.used.push_back(candidate.docid); + result.used.push_back(candidate); } else { result.unused.push_back(candidate.docid); } @@ -111,7 +114,7 @@ HnswIndex::select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint result.unused.push_back(candidate.docid); continue; } - result.used.push_back(candidate.docid); + result.used.push_back(candidate); if (result.used.size() == max_links) { while (!nearest.empty()) { candidate = nearest.top(); @@ -145,7 +148,12 @@ HnswIndex::shrink_if_needed(uint32_t docid, uint32_t level) neighbors.emplace_back(neighbor_docid, dist); } auto split = select_neighbors(neighbors, max_links); - _graph.set_link_array(docid, level, split.used); + LinkArray new_links; + new_links.reserve(split.used.size()); + for (const auto & neighbor : split.used) { + new_links.push_back(neighbor.docid); + } + _graph.set_link_array(docid, level, new_links); for (uint32_t removed_docid : split.unused) { remove_link_to(removed_docid, docid, level); } @@ -198,10 +206,13 @@ HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& e bool keep_searching = true; while (keep_searching) { keep_searching = false; - for (uint32_t neighbor_docid : _graph.get_link_array(nearest.docid, level)) { + for (uint32_t neighbor_docid : _graph.get_link_array(nearest.node_ref, level)) { + auto neighbor_ref = _graph.get_node_ref(neighbor_docid); double dist = calc_distance(input, neighbor_docid); - if (dist < nearest.distance) { - nearest = HnswCandidate(neighbor_docid, dist); + if (_graph.still_valid(neighbor_docid, neighbor_ref) + && dist < nearest.distance) + { + nearest = HnswCandidate(neighbor_docid, neighbor_ref, dist); keep_searching = true; } } @@ -210,15 +221,23 @@ HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& e } void -HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level) const +HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, + FurthestPriQ& best_neighbors, uint32_t level, const search::BitVector *filter) const { NearestPriQ candidates; uint32_t doc_id_limit = _graph.node_refs.size(); + if (filter) { + assert(filter->size() >= doc_id_limit); + } auto visited = _visited_set_pool.get(doc_id_limit); for (const auto &entry : best_neighbors.peek()) { assert(entry.docid < doc_id_limit); candidates.push(entry); visited.mark(entry.docid); + if (filter && !filter->testBit(entry.docid)) { + assert(best_neighbors.size() == 1); + best_neighbors.pop(); + } } double limit_dist = std::numeric_limits<double>::max(); @@ -228,18 +247,24 @@ HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, Fur break; } candidates.pop(); - for (uint32_t neighbor_docid : _graph.get_link_array(cand.docid, level)) { - if ((neighbor_docid >= doc_id_limit) || visited.is_marked(neighbor_docid)) { + for (uint32_t neighbor_docid : _graph.get_link_array(cand.node_ref, level)) { + auto neighbor_ref = _graph.get_node_ref(neighbor_docid); + if ((! neighbor_ref.valid()) + || (neighbor_docid >= doc_id_limit) + || visited.is_marked(neighbor_docid)) + { continue; } visited.mark(neighbor_docid); double dist_to_input = calc_distance(input, neighbor_docid); if (dist_to_input < limit_dist) { - candidates.emplace(neighbor_docid, dist_to_input); - best_neighbors.emplace(neighbor_docid, dist_to_input); - if (best_neighbors.size() > neighbors_to_find) { - best_neighbors.pop(); - limit_dist = best_neighbors.top().distance; + candidates.emplace(neighbor_docid, neighbor_ref, dist_to_input); + if ((!filter) || filter->testBit(neighbor_docid)) { + best_neighbors.emplace(neighbor_docid, neighbor_ref, dist_to_input); + if (best_neighbors.size() > neighbors_to_find) { + best_neighbors.pop(); + limit_dist = best_neighbors.top().distance; + } } } } @@ -262,38 +287,105 @@ HnswIndex::~HnswIndex() = default; void HnswIndex::add_document(uint32_t docid) { - auto input = get_vector(docid); + PreparedAddDoc op = internal_prepare_add(docid, get_vector(docid)); + internal_complete_add(docid, op); +} + +HnswIndex::PreparedAddDoc +HnswIndex::internal_prepare_add(uint32_t docid, TypedCells input_vector) const +{ // TODO: Add capping on num_levels int level = _level_generator->max_level(); - _graph.make_node_for_document(docid, level + 1); - uint32_t entry_docid = get_entry_docid(); - if (entry_docid == 0) { - _graph.set_entry_node(docid, level); - return; - } - - int search_level = get_entry_level(); - double entry_dist = calc_distance(input, entry_docid); - HnswCandidate entry_point(entry_docid, entry_dist); - while (search_level > level) { - entry_point = find_nearest_in_layer(input, entry_point, search_level); + PreparedAddDoc op(docid, level); + auto entry = _graph.get_entry_node(); + if (entry.docid == 0) { + // graph has no entry point + return op; + } + int search_level = entry.level; + double entry_dist = calc_distance(input_vector, entry.docid); + // TODO: check if entry docid/node_ref is still valid here + HnswCandidate entry_point(entry.docid, entry.node_ref, entry_dist); + while (search_level > op.max_level) { + entry_point = find_nearest_in_layer(input_vector, entry_point, search_level); --search_level; } FurthestPriQ best_neighbors; best_neighbors.push(entry_point); - search_level = std::min(level, search_level); + search_level = std::min(op.max_level, search_level); - // Insert the added document in each level it should exist in. + // Find neighbors of the added document in each level it should exist in. while (search_level >= 0) { - // TODO: Rename to search_level? - search_layer(input, _cfg.neighbors_to_explore_at_construction(), best_neighbors, search_level); + search_layer(input_vector, _cfg.neighbors_to_explore_at_construction(), best_neighbors, search_level); auto neighbors = select_neighbors(best_neighbors.peek(), _cfg.max_links_on_inserts()); - connect_new_node(docid, neighbors.used, search_level); + op.connections[search_level].reserve(neighbors.used.size()); + for (const auto & neighbor : neighbors.used) { + auto neighbor_levels = _graph.get_level_array(neighbor.node_ref); + if (size_t(search_level) < neighbor_levels.size()) { + op.connections[search_level].emplace_back(neighbor.docid, neighbor.node_ref); + } else { + LOG(warning, "in prepare_add(%u), selected neighbor %u is missing level %d (has %zu levels)", + docid, neighbor.docid, search_level, neighbor_levels.size()); + } + } --search_level; } - if (level > get_entry_level()) { - _graph.set_entry_node(docid, level); + return op; +} + +HnswIndex::LinkArray +HnswIndex::filter_valid_docids(uint32_t level, const PreparedAddDoc::Links &neighbors, uint32_t self_docid) +{ + LinkArray valid; + valid.reserve(neighbors.size()); + for (const auto & neighbor : neighbors) { + uint32_t docid = neighbor.first; + HnswGraph::NodeRef node_ref = neighbor.second; + if (_graph.still_valid(docid, node_ref)) { + assert(docid != self_docid); + auto levels = _graph.get_level_array(node_ref); + if (level < levels.size()) { + valid.push_back(docid); + } + } + } + return valid; +} + +void +HnswIndex::internal_complete_add(uint32_t docid, PreparedAddDoc &op) +{ + auto node_ref = _graph.make_node_for_document(docid, op.max_level + 1); + for (int level = 0; level <= op.max_level; ++level) { + auto neighbors = filter_valid_docids(level, op.connections[level], docid); + connect_new_node(docid, neighbors, level); + } + if (op.max_level > get_entry_level()) { + _graph.set_entry_node({docid, node_ref, op.max_level}); + } +} + +std::unique_ptr<PrepareResult> +HnswIndex::prepare_add_document(uint32_t docid, + TypedCells vector, + vespalib::GenerationHandler::Guard read_guard) const +{ + PreparedAddDoc op = internal_prepare_add(docid, vector); + (void) read_guard; // must keep guard until this point + return std::make_unique<PreparedAddDoc>(std::move(op)); +} + +void +HnswIndex::complete_add_document(uint32_t docid, std::unique_ptr<PrepareResult> prepare_result) +{ + auto prepared = dynamic_cast<PreparedAddDoc *>(prepare_result.get()); + if (prepared && (prepared->docid == docid)) { + internal_complete_add(docid, *prepared); + } else { + LOG(warning, "complete_add_document called with invalid prepare_result"); + // fallback to normal add + add_document(docid); } } @@ -327,22 +419,22 @@ void HnswIndex::remove_document(uint32_t docid) { bool need_new_entrypoint = (docid == get_entry_docid()); - LinkArray empty; LevelArrayRef node_levels = _graph.get_level_array(docid); for (int level = node_levels.size(); level-- > 0; ) { LinkArrayRef my_links = _graph.get_link_array(docid, level); for (uint32_t neighbor_id : my_links) { if (need_new_entrypoint) { - _graph.set_entry_node(neighbor_id, level); + auto entry_node_ref = _graph.get_node_ref(neighbor_id); + _graph.set_entry_node({neighbor_id, entry_node_ref, level}); need_new_entrypoint = false; } remove_link_to(neighbor_id, docid, level); } mutual_reconnect(my_links, level); - _graph.set_link_array(docid, level, empty); } if (need_new_entrypoint) { - _graph.set_entry_node(0, -1); + HnswGraph::EntryNode entry; + _graph.set_entry_node(entry); } _graph.remove_node_for_document(docid); } @@ -397,8 +489,9 @@ HnswIndex::get_state(const vespalib::slime::Inserter& inserter) const uint32_t reachable = count_reachable_nodes(); uint32_t unreachable = valid_nodes - reachable; object.setLong("unreachable_nodes", unreachable); - object.setLong("entry_docid", _graph.entry_docid); - object.setLong("entry_level", _graph.entry_level); + auto entry_node = _graph.get_entry_node(); + object.setLong("entry_docid", entry_node.docid); + object.setLong("entry_level", entry_node.level); auto& cfgObj = object.setObject("cfg"); cfgObj.setLong("max_links_at_level_0", _cfg.max_links_at_level_0()); cfgObj.setLong("max_links_on_inserts", _cfg.max_links_on_inserts()); @@ -429,10 +522,11 @@ struct NeighborsByDocId { }; std::vector<NearestNeighborIndex::Neighbor> -HnswIndex::find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const +HnswIndex::top_k_by_docid(uint32_t k, TypedCells vector, + const BitVector *filter, uint32_t explore_k) const { std::vector<Neighbor> result; - FurthestPriQ candidates = top_k_candidates(vector, std::max(k, explore_k)); + FurthestPriQ candidates = top_k_candidates(vector, std::max(k, explore_k), filter); while (candidates.size() > k) { candidates.pop(); } @@ -444,23 +538,38 @@ HnswIndex::find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const return result; } +std::vector<NearestNeighborIndex::Neighbor> +HnswIndex::find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const +{ + return top_k_by_docid(k, vector, nullptr, explore_k); +} + +std::vector<NearestNeighborIndex::Neighbor> +HnswIndex::find_top_k_with_filter(uint32_t k, TypedCells vector, + const BitVector &filter, uint32_t explore_k) const +{ + return top_k_by_docid(k, vector, &filter, explore_k); +} + FurthestPriQ -HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k) const +HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k, const BitVector *filter) const { FurthestPriQ best_neighbors; - if (get_entry_level() < 0) { + auto entry = _graph.get_entry_node(); + if (entry.docid == 0) { + // graph has no entry point return best_neighbors; } - uint32_t entry_docid = get_entry_docid(); - int search_level = get_entry_level(); - double entry_dist = calc_distance(vector, entry_docid); - HnswCandidate entry_point(entry_docid, entry_dist); + int search_level = entry.level; + double entry_dist = calc_distance(vector, entry.docid); + // TODO: check if entry docid/node_ref is still valid here + HnswCandidate entry_point(entry.docid, entry.node_ref, entry_dist); while (search_level > 0) { entry_point = find_nearest_in_layer(vector, entry_point, search_level); --search_level; } best_neighbors.push(entry_point); - search_layer(vector, k, best_neighbors, 0); + search_layer(vector, k, best_neighbors, 0, filter); return best_neighbors; } @@ -487,13 +596,13 @@ HnswIndex::set_node(uint32_t docid, const HnswNode &node) { size_t num_levels = node.size(); assert(num_levels > 0); - _graph.make_node_for_document(docid, num_levels); + auto node_ref = _graph.make_node_for_document(docid, num_levels); for (size_t level = 0; level < num_levels; ++level) { connect_new_node(docid, node.level(level), level); } int max_level = num_levels - 1; if (get_entry_level() < max_level) { - _graph.set_entry_node(docid, max_level); + _graph.set_entry_node({docid, node_ref, max_level}); } } @@ -512,6 +621,8 @@ HnswIndex::check_link_symmetry() const auto neighbor_links = _graph.get_link_array(neighbor_docid, level); if (! has_link_to(neighbor_links, docid)) { all_sym = false; + LOG(warning, "check_link_symmetry: docid %zu links to %u on level %u, but no backlink", + docid, neighbor_docid, level); } } ++level; @@ -524,15 +635,15 @@ HnswIndex::check_link_symmetry() const uint32_t HnswIndex::count_reachable_nodes() const { - int search_level = get_entry_level(); + auto entry = _graph.get_entry_node(); + int search_level = entry.level; if (search_level < 0) { return 0; } auto visited = _visited_set_pool.get(_graph.size()); - uint32_t entry_id = get_entry_docid(); LinkArray found_links; - found_links.push_back(entry_id); - visited.mark(entry_id); + found_links.push_back(entry.docid); + visited.mark(entry.docid); while (search_level >= 0) { for (uint32_t idx = 0; idx < found_links.size(); ++idx) { uint32_t docid = found_links[idx]; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 6a7496e8696..ab3eced8fdc 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -91,10 +91,11 @@ protected: * where the candidate is located. * Used by select_neighbors_heuristic(). */ - bool have_closer_distance(HnswCandidate candidate, const LinkArrayRef& curr_result) const; + bool have_closer_distance(HnswCandidate candidate, const HnswCandidateVector& curr_result) const; struct SelectResult { - LinkArray used; + HnswCandidateVector used; LinkArray unused; + ~SelectResult() {} }; SelectResult select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const; SelectResult select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const; @@ -115,8 +116,25 @@ protected: * Performs a greedy search in the given layer to find the candidate that is nearest the input vector. */ HnswCandidate find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) const; - void search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, uint32_t level) const; - + void search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, + uint32_t level, const search::BitVector *filter = nullptr) const; + std::vector<Neighbor> top_k_by_docid(uint32_t k, TypedCells vector, + const BitVector *filter, uint32_t explore_k) const; + + struct PreparedAddDoc : public PrepareResult { + uint32_t docid; + int32_t max_level; + using Links = std::vector<std::pair<uint32_t, HnswGraph::NodeRef>>; + std::vector<Links> connections; + PreparedAddDoc(uint32_t docid_in, int32_t max_level_in) + : docid(docid_in), max_level(max_level_in), connections(max_level+1) + {} + ~PreparedAddDoc() = default; + PreparedAddDoc(PreparedAddDoc&& other) = default; + }; + PreparedAddDoc internal_prepare_add(uint32_t docid, TypedCells input_vector) const; + LinkArray filter_valid_docids(uint32_t level, const PreparedAddDoc::Links &neighbors, uint32_t me); + void internal_complete_add(uint32_t docid, PreparedAddDoc &op); public: HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func, RandomLevelGenerator::UP level_generator, const Config& cfg); @@ -126,6 +144,10 @@ public: // Implements NearestNeighborIndex void add_document(uint32_t docid) override; + std::unique_ptr<PrepareResult> prepare_add_document(uint32_t docid, + TypedCells vector, + vespalib::GenerationHandler::Guard read_guard) const override; + void complete_add_document(uint32_t docid, std::unique_ptr<PrepareResult> prepare_result) override; void remove_document(uint32_t docid) override; void transfer_hold_lists(generation_t current_gen) override; void trim_hold_lists(generation_t first_used_gen) override; @@ -136,12 +158,14 @@ public: bool load(const fileutil::LoadedBuffer& buf) override; std::vector<Neighbor> find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const override; + std::vector<Neighbor> find_top_k_with_filter(uint32_t k, TypedCells vector, + const BitVector &filter, uint32_t explore_k) const override; const DistanceFunction *distance_function() const override { return _distance_func.get(); } - FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k) const; + FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k, const BitVector *filter) const; - uint32_t get_entry_docid() const { return _graph.entry_docid; } - int32_t get_entry_level() const { return _graph.entry_level; } + uint32_t get_entry_docid() const { return _graph.get_entry_node().docid; } + int32_t get_entry_level() const { return _graph.get_entry_node().level; } // Should only be used by unit tests. HnswNode get_node(uint32_t docid) const; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp index f02ead86a8d..ac98b28d105 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp @@ -39,7 +39,8 @@ HnswIndexLoader::load(const fileutil::LoadedBuffer& buf) } if (_failed) return false; _graph.node_refs.ensure_size(num_nodes); - _graph.set_entry_node(entry_docid, entry_level); + auto entry_node_ref = _graph.get_node_ref(entry_docid); + _graph.set_entry_node({entry_docid, entry_node_ref, entry_level}); return true; } diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp index 46a988d575e..6593a60d6b5 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp @@ -11,8 +11,9 @@ HnswIndexSaver::~HnswIndexSaver() {} HnswIndexSaver::HnswIndexSaver(const HnswGraph &graph) : _graph_links(graph.links), _meta_data() { - _meta_data.entry_docid = graph.entry_docid; - _meta_data.entry_level = graph.entry_level; + auto entry = graph.get_entry_node(); + _meta_data.entry_docid = entry.docid; + _meta_data.entry_level = entry.level; size_t num_nodes = graph.node_refs.size(); _meta_data.nodes.reserve(num_nodes); for (size_t i = 0; i < num_nodes; ++i) { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h index b11d3f36a7a..99266505780 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h @@ -5,6 +5,7 @@ #include <cstdint> #include <queue> #include <vector> +#include "hnsw_graph.h" namespace search::tensor { @@ -13,8 +14,12 @@ namespace search::tensor { */ struct HnswCandidate { uint32_t docid; + HnswGraph::NodeRef node_ref; double distance; - HnswCandidate(uint32_t docid_in, double distance_in) : docid(docid_in), distance(distance_in) {} + HnswCandidate(uint32_t docid_in, double distance_in) + : docid(docid_in), node_ref(), distance(distance_in) {} + HnswCandidate(uint32_t docid_in, HnswGraph::NodeRef node_ref_in, double distance_in) + : docid(docid_in), node_ref(node_ref_in), distance(distance_in) {} }; struct GreaterDistance { diff --git a/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h index 2f7f9f4445e..e70830a78f8 100644 --- a/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h +++ b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h @@ -2,6 +2,7 @@ #include "random_level_generator.h" #include <random> +#include <mutex> namespace search::tensor { @@ -16,17 +17,24 @@ namespace search::tensor { class InvLogLevelGenerator : public RandomLevelGenerator { std::mt19937_64 _rng; + std::mutex _mutex; std::uniform_real_distribution<double> _uniform; - double _levelMultiplier; + const double _levelMultiplier; + + double get_uniform() { + std::lock_guard<std::mutex> guard(_mutex); + return _uniform(_rng); + } public: InvLogLevelGenerator(uint32_t m) : _rng(0x1234deadbeef5678uLL), + _mutex(), _uniform(0.0, 1.0), _levelMultiplier(1.0 / log(1.0 * m)) {} uint32_t max_level() override { - double unif = _uniform(_rng); + double unif = get_uniform(); double r = -log(1.0-unif) * _levelMultiplier; return (uint32_t) r; } diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h index aca2ce2af66..b46c19ac88a 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h @@ -3,6 +3,7 @@ #pragma once #include "distance_function.h" +#include "prepare_result.h" #include <vespa/eval/tensor/dense/typed_cells.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/util/memoryusage.h> @@ -14,6 +15,8 @@ namespace vespalib::slime { struct Inserter; } namespace search::fileutil { class LoadedBuffer; } +namespace search { class BitVector; } + namespace search::tensor { class NearestNeighborIndexSaver; @@ -34,6 +37,26 @@ public: }; virtual ~NearestNeighborIndex() {} virtual void add_document(uint32_t docid) = 0; + + /** + * Performs the prepare step in a two-phase operation to add a document to the index. + * + * This function can be called by any thread. + * The document to add is represented by the given vector as it is _not_ stored in the enclosing tensor attribute at this point in time. + * It should return the result of the costly and non-modifying part of this operation. + * The given read guard must be kept in the result. + */ + virtual std::unique_ptr<PrepareResult> prepare_add_document(uint32_t docid, + vespalib::tensor::TypedCells vector, + vespalib::GenerationHandler::Guard read_guard) const = 0; + /** + * Performs the complete step in a two-phase operation to add a document to the index. + * + * This function is only called by the attribute writer thread. + * It uses the result from the prepare step to do the modifying changes. + */ + virtual void complete_add_document(uint32_t docid, std::unique_ptr<PrepareResult> prepare_result) = 0; + virtual void remove_document(uint32_t docid) = 0; virtual void transfer_hold_lists(generation_t current_gen) = 0; virtual void trim_hold_lists(generation_t first_used_gen) = 0; @@ -53,6 +76,12 @@ public: vespalib::tensor::TypedCells vector, uint32_t explore_k) const = 0; + // only return neighbors where the corresponding filter bit is set + virtual std::vector<Neighbor> find_top_k_with_filter(uint32_t k, + vespalib::tensor::TypedCells vector, + const BitVector &filter, + uint32_t explore_k) const = 0; + virtual const DistanceFunction *distance_function() const = 0; }; diff --git a/searchlib/src/vespa/searchlib/tensor/prepare_result.h b/searchlib/src/vespa/searchlib/tensor/prepare_result.h new file mode 100644 index 00000000000..05300684497 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/prepare_result.h @@ -0,0 +1,15 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::tensor { + +/** + * Interface for a class used to keep the result of the prepare step of a two-phase operation. + */ +class PrepareResult { +public: + virtual ~PrepareResult() {} +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 95af9f0471b..6cf4f6d2689 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -20,7 +20,6 @@ using vespalib::tensor::SparseTensor; using vespalib::tensor::Tensor; using vespalib::tensor::TypedDenseTensorBuilder; using vespalib::tensor::WrappedSimpleTensor; -using vespalib::tensor::dispatch_0; using search::StateExplorerUtils; namespace search::tensor { @@ -34,7 +33,7 @@ constexpr size_t DEAD_SLACK = 0x10000u; struct CallMakeEmptyTensor { template <typename CT> - static Tensor::UP call(const ValueType &type) { + static Tensor::UP invoke(const ValueType &type) { TypedDenseTensorBuilder<CT> builder(type); return builder.build(); } @@ -46,7 +45,8 @@ createEmptyTensor(const ValueType &type) if (type.is_sparse()) { return std::make_unique<SparseTensor>(type, SparseTensor::Cells()); } else if (type.is_dense()) { - return dispatch_0<CallMakeEmptyTensor>(type.cell_type(), type); + using MyTypify = vespalib::eval::TypifyCellType; + return vespalib::typify_invoke<1,MyTypify,CallMakeEmptyTensor>(type.cell_type(), type); } else { return std::make_unique<WrappedSimpleTensor>(std::make_unique<SimpleTensor>(type, SimpleTensor::Cells())); } @@ -253,6 +253,23 @@ TensorAttribute::getRefCopy() const return RefCopyVector(&_refVector[0], &_refVector[0] + size); } +std::unique_ptr<PrepareResult> +TensorAttribute::prepare_set_tensor(DocId docid, const Tensor& tensor) const +{ + (void) docid; + (void) tensor; + return std::unique_ptr<PrepareResult>(); +} + +void +TensorAttribute::complete_set_tensor(DocId docid, const Tensor& tensor, + std::unique_ptr<PrepareResult> prepare_result) +{ + (void) docid; + (void) tensor; + (void) prepare_result; +} + IMPLEMENT_IDENTIFIABLE_ABSTRACT(TensorAttribute, AttributeVector); } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index e8efd2170c9..8380e485172 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -3,8 +3,9 @@ #pragma once #include "i_tensor_attribute.h" -#include <vespa/searchlib/attribute/not_implemented_attribute.h> +#include "prepare_result.h" #include "tensor_store.h" +#include <vespa/searchlib/attribute/not_implemented_attribute.h> #include <vespa/vespalib/util/rcuvector.h> namespace search::tensor { @@ -51,6 +52,23 @@ public: uint32_t getVersion() const override; RefCopyVector getRefCopy() const; virtual void setTensor(DocId docId, const Tensor &tensor) = 0; + + /** + * Performs the prepare step in a two-phase operation to set a tensor for a document. + * + * This function can be called by any thread. + * It should return the result of the costly and non-modifying part of such operation. + */ + virtual std::unique_ptr<PrepareResult> prepare_set_tensor(DocId docid, const Tensor& tensor) const; + + /** + * Performs the complete step in a two-phase operation to set a tensor for a document. + * + * This function is only called by the attribute writer thread. + * It uses the result from the prepare step to do the modifying changes. + */ + virtual void complete_set_tensor(DocId docid, const Tensor& tensor, std::unique_ptr<PrepareResult> prepare_result); + virtual void compactWorst() = 0; }; diff --git a/searchlib/src/vespa/searchlib/test/directory_handler.h b/searchlib/src/vespa/searchlib/test/directory_handler.h index 72f9cf83d85..66e5a710870 100644 --- a/searchlib/src/vespa/searchlib/test/directory_handler.h +++ b/searchlib/src/vespa/searchlib/test/directory_handler.h @@ -5,8 +5,7 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/string.h> -namespace search { -namespace test { +namespace search::test { class DirectoryHandler { @@ -17,11 +16,8 @@ private: public: DirectoryHandler(const vespalib::string &mkdir) - : _mkdir(mkdir), - _rmdir(mkdir), - _cleanup(true) + : DirectoryHandler(mkdir, mkdir) { - vespalib::mkdir(_mkdir); } DirectoryHandler(const vespalib::string &mkdir, const vespalib::string &rmdir) @@ -37,8 +33,7 @@ public: } } void cleanup(bool v) { _cleanup = v; } + const vespalib::string & getDir() const { return _mkdir; } }; } -} - diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h index 3fcc2427880..ac315c805d5 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h @@ -13,11 +13,7 @@ using search::fef::TermFieldMatchDataPosition; #include <vector> #include <string> -namespace search -{ - -namespace fakedata -{ +namespace search::fakedata { /* * Base class for faked posting list formats. @@ -41,56 +37,36 @@ public: /* * Size of posting list, in bits. */ - virtual size_t - bitSize() const = 0; - - virtual size_t - skipBitSize() const; - - virtual size_t - l1SkipBitSize() const; - - virtual size_t - l2SkipBitSize() const; - - virtual size_t - l3SkipBitSize() const; - - virtual size_t - l4SkipBitSize() const; - - virtual bool - hasWordPositions() const = 0; - + virtual size_t bitSize() const = 0; + virtual size_t skipBitSize() const; + virtual size_t l1SkipBitSize() const; + virtual size_t l2SkipBitSize() const; + virtual size_t l3SkipBitSize() const; + virtual size_t l4SkipBitSize() const; + virtual bool hasWordPositions() const = 0; virtual bool has_interleaved_features() const; - virtual bool enable_unpack_normal_features() const; - virtual bool enable_unpack_interleaved_features() const; /* * Single posting list performance, without feature unpack. */ - virtual int - lowLevelSinglePostingScan() const = 0; + virtual int lowLevelSinglePostingScan() const = 0; /* * Single posting list performance, with feature unpack. */ - virtual int - lowLevelSinglePostingScanUnpack() const = 0; + virtual int lowLevelSinglePostingScanUnpack() const = 0; /* * Two posting lists performance (same format) without feature unpack. */ - virtual int - lowLevelAndPairPostingScan(const FakePosting &rhs) const = 0; + virtual int lowLevelAndPairPostingScan(const FakePosting &rhs) const = 0; /* * Two posting lists performance (same format) with feature unpack. */ - virtual int - lowLevelAndPairPostingScanUnpack(const FakePosting &rhs) const = 0; + virtual int lowLevelAndPairPostingScanUnpack(const FakePosting &rhs) const = 0; /* @@ -105,7 +81,4 @@ public: } }; -} // namespace fakedata - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h index ffe9e2e54d0..97b0d3a44d3 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h @@ -12,9 +12,7 @@ #include <vespa/searchlib/diskindex/fieldreader.h> #include <vespa/searchlib/diskindex/fieldwriter.h> -namespace search { - -namespace fakedata { +namespace search::fakedata { /** @@ -275,7 +273,4 @@ public: void addDocIdBias(uint32_t docIdBias); }; -} // namespace fakedata - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/test/initrange.h b/searchlib/src/vespa/searchlib/test/initrange.h index 9431740ac08..a143dfdb119 100644 --- a/searchlib/src/vespa/searchlib/test/initrange.h +++ b/searchlib/src/vespa/searchlib/test/initrange.h @@ -25,6 +25,7 @@ public: void verify(SearchIterator & iterator) const; /// Convenience that takes ownership of the pointer. void verify(SearchIterator * iterator) const; + void verify(SearchIterator::UP iterator) const { verify(*iterator); } private: void verify(SearchIterator & iterator, bool strict) const; void verify(SearchIterator & iterator, const Ranges & ranges, bool strict) const; diff --git a/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp b/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp index fce768ab0d2..ec53d6d9d00 100644 --- a/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp +++ b/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp @@ -170,9 +170,9 @@ void SearchIteratorVerifier::verifyAnd(bool strict) const { fef::TermFieldMatchData tfmd; MultiSearch::Children children; - children.emplace_back(create(strict).release()); - children.emplace_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, false).release()); - SearchIterator::UP search(AndSearch::create(children, strict, UnpackInfo())); + children.push_back(create(strict)); + children.push_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, false)); + auto search = AndSearch::create(std::move(children), strict, UnpackInfo()); TEST_DO(verify(*search, strict, _expectedAnd)); TEST_DO(verifyTermwise(std::move(search), strict, _expectedAnd)); } @@ -183,18 +183,18 @@ SearchIteratorVerifier::verifyAndNot(bool strict) const { { for (bool notStrictness : {false, true}) { MultiSearch::Children children; - children.emplace_back(create(strict).release()); - children.emplace_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, notStrictness).release()); - SearchIterator::UP search(AndNotSearch::create(children, strict)); + children.push_back(create(strict)); + children.push_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, notStrictness)); + auto search = AndNotSearch::create(std::move(children), strict); TEST_DO(verify(*search, strict, _expectedAndNotPositive)); TEST_DO(verifyTermwise(std::move(search), strict, _expectedAndNotPositive)); } } { MultiSearch::Children children; - children.emplace_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, true).release()); - children.emplace_back(create(strict).release()); - SearchIterator::UP search(AndNotSearch::create(children, strict)); + children.push_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, true)); + children.push_back(create(strict)); + auto search = AndNotSearch::create(std::move(children), strict); TEST_DO(verify(*search, strict, _expectedAndNotNegative)); TEST_DO(verifyTermwise(std::move(search), strict, _expectedAndNotNegative)); } @@ -205,9 +205,9 @@ void SearchIteratorVerifier::verifyOr(bool strict) const { fef::TermFieldMatchData tfmd; MultiSearch::Children children; - children.emplace_back(create(strict).release()); - children.emplace_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, strict).release()); - SearchIterator::UP search(OrSearch::create(children, strict, UnpackInfo())); + children.push_back(create(strict)); + children.push_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, strict)); + SearchIterator::UP search(OrSearch::create(std::move(children), strict, UnpackInfo())); TEST_DO(verify(*search, strict, _expectedOr)); TEST_DO(verifyTermwise(std::move(search), strict, _expectedOr)); } diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index 2a23dd4c495..3792f2b6218 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -20,11 +20,13 @@ vespa_define_module( src/vespa/searchsummary src/vespa/searchsummary/config src/vespa/searchsummary/docsummary + src/vespa/searchsummary/test TESTS src/tests/docsumformat src/tests/docsummary src/tests/docsummary/attribute_combiner + src/tests/docsummary/attributedfw src/tests/docsummary/matched_elements_filter src/tests/docsummary/slime_summary src/tests/extractkeywords diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt index cffdef25e5b..3ac95211aec 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_executable(searchsummary_attribute_combiner_test_app TEST attribute_combiner_test.cpp DEPENDS searchsummary + searchsummary_test GTest::GTest ) vespa_add_test(NAME searchsummary_attribute_combiner_test_app COMMAND searchsummary_attribute_combiner_test_app) diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index 16e6d402764..eaeaa27f053 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -1,202 +1,43 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcommon/common/undefinedvalues.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/attribute/attributemanager.h> #include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/attributevector.hpp> -#include <vespa/searchlib/attribute/floatbase.h> -#include <vespa/searchlib/attribute/integerbase.h> -#include <vespa/searchlib/attribute/stringbase.h> #include <vespa/searchlib/common/matching_elements.h> -#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> +#include <vespa/searchsummary/docsummary/docsumfieldwriter.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/docsum_field_writer_state.h> #include <vespa/searchsummary/docsummary/attribute_combiner_dfw.h> +#include <vespa/searchsummary/test/mock_attribute_manager.h> +#include <vespa/searchsummary/test/mock_state_callback.h> +#include <vespa/searchsummary/test/slime_value.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("attribute_combiner_test"); -using search::AttributeFactory; -using search::AttributeManager; -using search::AttributeVector; -using search::IntegerAttribute; -using search::FloatingPointAttribute; -using search::MatchingElements; -using search::StringAttribute; using search::attribute::BasicType; -using search::attribute::CollectionType; -using search::attribute::Config; -using search::attribute::IAttributeVector; using search::attribute::getUndefined; using search::docsummary::AttributeCombinerDFW; using search::docsummary::GetDocsumsState; using search::docsummary::GetDocsumsStateCallback; using search::docsummary::IDocsumEnvironment; using search::docsummary::IDocsumFieldWriter; +using search::docsummary::test::MockAttributeManager; +using search::docsummary::test::MockStateCallback; +using search::docsummary::test::SlimeValue; namespace { -vespalib::string -toCompactJsonString(const vespalib::Slime &slime) -{ - vespalib::SimpleBuffer buf; - vespalib::slime::JsonFormat::encode(slime, buf, true); - return buf.get().make_string(); -} - -struct FieldBlock { - vespalib::string input; - vespalib::Slime slime; - search::RawBuf binary; - vespalib::string json; - - explicit FieldBlock(const vespalib::string &jsonInput); - ~FieldBlock(); - const char *data() const { return binary.GetDrainPos(); } - size_t dataLen() const { return binary.GetUsedLen(); } -}; - -FieldBlock::FieldBlock(const vespalib::string &jsonInput) - : input(jsonInput), slime(), binary(1024), json() -{ - size_t used = vespalib::slime::JsonFormat::decode(jsonInput, slime); - EXPECT_TRUE(used > 0); - json = toCompactJsonString(slime); - search::SlimeOutputRawBufAdapter adapter(binary); - vespalib::slime::BinaryFormat::encode(slime, adapter); -} - -FieldBlock::~FieldBlock() = default; - -struct AttributeManagerFixture -{ - AttributeManager mgr; - - AttributeManagerFixture(); - - ~AttributeManagerFixture(); - - template <typename AttributeType, typename ValueType> - void - buildAttribute(const vespalib::string &name, - BasicType type, - std::vector<std::vector<ValueType>> values); - - void - buildStringAttribute(const vespalib::string &name, - std::vector<std::vector<vespalib::string>> values); - void - buildFloatAttribute(const vespalib::string &name, - std::vector<std::vector<double>> values); - - void - buildIntegerAttribute(const vespalib::string &name, - BasicType type, - std::vector<std::vector<IAttributeVector::largeint_t>> values); -}; - -AttributeManagerFixture::AttributeManagerFixture() - : mgr() -{ - buildStringAttribute("array.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); - buildIntegerAttribute("array.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}}); - buildFloatAttribute("array.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}}); - buildStringAttribute("smap.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1", "k3.2"}, {"", "k4.2"}, {}}); - buildStringAttribute("smap.value.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); - buildIntegerAttribute("smap.value.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}}); - buildFloatAttribute("smap.value.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}}); - buildStringAttribute("map.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1"}, {"", "k4.2"}, {}}); - buildStringAttribute("map.value", {{"n1.1", "n1.2"}, {}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); -} - -AttributeManagerFixture::~AttributeManagerFixture() = default; - -template <typename AttributeType, typename ValueType> -void -AttributeManagerFixture::buildAttribute(const vespalib::string &name, - BasicType type, - std::vector<std::vector<ValueType>> values) -{ - Config cfg(type, CollectionType::Type::ARRAY); - auto attrBase = AttributeFactory::createAttribute(name, cfg); - EXPECT_TRUE(attrBase); - auto attr = std::dynamic_pointer_cast<AttributeType>(attrBase); - EXPECT_TRUE(attr); - attr->addReservedDoc(); - for (const auto &docValues : values) { - uint32_t docId = 0; - EXPECT_TRUE(attr->addDoc(docId)); - EXPECT_NE(0u, docId); - for (const auto &value : docValues) { - attr->append(docId, value, 1); - } - attr->commit(); - } - EXPECT_TRUE(mgr.add(attr)); -} - -void -AttributeManagerFixture::buildStringAttribute(const vespalib::string &name, - std::vector<std::vector<vespalib::string>> values) -{ - buildAttribute<StringAttribute, vespalib::string>(name, BasicType::Type::STRING, std::move(values)); -} - -void -AttributeManagerFixture::buildFloatAttribute(const vespalib::string &name, - std::vector<std::vector<double>> values) -{ - buildAttribute<FloatingPointAttribute, double>(name, BasicType::Type::DOUBLE, std::move(values)); -} - -void -AttributeManagerFixture::buildIntegerAttribute(const vespalib::string &name, - BasicType type, - std::vector<std::vector<IAttributeVector::largeint_t>> values) -{ - buildAttribute<IntegerAttribute, IAttributeVector::largeint_t>(name, type, std::move(values)); -} - - -class DummyStateCallback : public GetDocsumsStateCallback -{ -public: - MatchingElements _matching_elements; - - DummyStateCallback(); - void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } - void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } - void ParseLocation(GetDocsumsState *) override { } - std::unique_ptr<MatchingElements> fill_matching_elements(const search::StructFieldMapper &) override { return std::make_unique<MatchingElements>(_matching_elements); } - ~DummyStateCallback() override { } -}; - -DummyStateCallback::DummyStateCallback() - : GetDocsumsStateCallback(), - _matching_elements() -{ - _matching_elements.add_matching_elements(1, "array", {1}); - _matching_elements.add_matching_elements(3, "array", {0}); - _matching_elements.add_matching_elements(4, "array", {1}); - _matching_elements.add_matching_elements(1, "smap", {1}); - _matching_elements.add_matching_elements(3, "smap", {0}); - _matching_elements.add_matching_elements(4, "smap", {1}); - _matching_elements.add_matching_elements(1, "map", {1}); - _matching_elements.add_matching_elements(3, "map", {0}); - _matching_elements.add_matching_elements(4, "map", {1}); -} - struct AttributeCombinerTest : public ::testing::Test { - AttributeManagerFixture attrs; + MockAttributeManager attrs; std::unique_ptr<IDocsumFieldWriter> writer; - DummyStateCallback stateCallback; + MockStateCallback callback; GetDocsumsState state; - std::shared_ptr<search::StructFieldMapper> _struct_field_mapper; + std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields; AttributeCombinerTest(); ~AttributeCombinerTest(); @@ -207,11 +48,31 @@ struct AttributeCombinerTest : public ::testing::Test AttributeCombinerTest::AttributeCombinerTest() : attrs(), writer(), - stateCallback(), - state(stateCallback), - _struct_field_mapper() -{ - state._attrCtx = attrs.mgr.createContext(); + callback(), + state(callback), + _matching_elems_fields() +{ + attrs.build_string_attribute("array.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); + attrs.build_int_attribute("array.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}}); + attrs.build_float_attribute("array.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}}); + attrs.build_string_attribute("smap.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1", "k3.2"}, {"", "k4.2"}, {}}); + attrs.build_string_attribute("smap.value.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); + attrs.build_int_attribute("smap.value.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}}); + attrs.build_float_attribute("smap.value.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}}); + attrs.build_string_attribute("map.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1"}, {"", "k4.2"}, {}}); + attrs.build_string_attribute("map.value", {{"n1.1", "n1.2"}, {}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); + + callback.add_matching_elements(1, "array", {1}); + callback.add_matching_elements(3, "array", {0}); + callback.add_matching_elements(4, "array", {1}); + callback.add_matching_elements(1, "smap", {1}); + callback.add_matching_elements(3, "smap", {0}); + callback.add_matching_elements(4, "smap", {1}); + callback.add_matching_elements(1, "map", {1}); + callback.add_matching_elements(3, "map", {0}); + callback.add_matching_elements(4, "map", {1}); + + state._attrCtx = attrs.mgr().createContext(); } AttributeCombinerTest::~AttributeCombinerTest() = default; @@ -220,131 +81,121 @@ void AttributeCombinerTest::set_field(const vespalib::string &field_name, bool filter_elements) { if (filter_elements) { - _struct_field_mapper = std::make_shared<search::StructFieldMapper>(); + _matching_elems_fields = std::make_shared<search::MatchingElementsFields>(); } - writer = AttributeCombinerDFW::create(field_name, *state._attrCtx, filter_elements, _struct_field_mapper); + writer = AttributeCombinerDFW::create(field_name, *state._attrCtx, filter_elements, _matching_elems_fields); EXPECT_TRUE(writer->setFieldWriterStateIndex(0)); state._fieldWriterStates.resize(1); } void -AttributeCombinerTest::assertWritten(const vespalib::string &expectedJson, uint32_t docId) +AttributeCombinerTest::assertWritten(const vespalib::string &exp_slime_as_json, uint32_t docId) { - vespalib::Slime target; - vespalib::slime::SlimeInserter inserter(target); + vespalib::Slime act; + vespalib::slime::SlimeInserter inserter(act); writer->insertField(docId, nullptr, &state, search::docsummary::RES_JSONSTRING, inserter); - search::RawBuf binary(1024); - vespalib::string json = toCompactJsonString(target); - search::SlimeOutputRawBufAdapter adapter(binary); - vespalib::slime::BinaryFormat::encode(target, adapter); - FieldBlock block(expectedJson); - EXPECT_EQ(block.dataLen(), binary.GetUsedLen()); - EXPECT_EQ(0, memcmp(block.data(), binary.GetDrainPos(), block.dataLen())); - if (block.dataLen() != binary.GetUsedLen() || - memcmp(block.data(), binary.GetDrainPos(), block.dataLen()) != 0) { - LOG(error, "Expected '%s'", expectedJson.c_str()); - LOG(error, "Expected normalized '%s'", block.json.c_str()); - LOG(error, "Got '%s'", json.c_str()); - } + + SlimeValue exp(exp_slime_as_json); + EXPECT_EQ(exp.slime, act); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_array_of_struct) { set_field("array", false); - assertWritten("[ { fval: 110.0, name: \"n1.1\", val: 10}, { name: \"n1.2\", val: 11}]", 1); - assertWritten("[ { fval: 120.0, name: \"n2\", val: 20}, { fval: 121.0, val: 21 }]", 2); - assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30}, { fval: 131.0, name: \"n3.2\"} ]", 3); - assertWritten("[ { }, { fval: 141.0, name: \"n4.2\", val: 41} ]", 4); + assertWritten("[ { fval: 110.0, name: 'n1.1', val: 10}, { name: 'n1.2', val: 11}]", 1); + assertWritten("[ { fval: 120.0, name: 'n2', val: 20}, { fval: 121.0, val: 21 }]", 2); + assertWritten("[ { fval: 130.0, name: 'n3.1', val: 30}, { fval: 131.0, name: 'n3.2'} ]", 3); + assertWritten("[ { }, { fval: 141.0, name: 'n4.2', val: 41} ]", 4); assertWritten("null", 5); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_map_of_struct) { set_field("smap", false); - assertWritten("[ { key: \"k1.1\", value: { fval: 110.0, name: \"n1.1\", val: 10} }, { key: \"k1.2\", value: { name: \"n1.2\", val: 11} }]", 1); - assertWritten("[ { key: \"k2\", value: { fval: 120.0, name: \"n2\", val: 20} }, { value: { fval: 121.0, val: 21 } }]", 2); - assertWritten("[ { key: \"k3.1\", value: { fval: 130.0, name: \"n3.1\", val: 30} }, { key: \"k3.2\", value: { fval: 131.0, name: \"n3.2\"} } ]", 3); - assertWritten("[ { value: { } }, { key: \"k4.2\", value: { fval: 141.0, name: \"n4.2\", val: 41} } ]", 4); + assertWritten("[ { key: 'k1.1', value: { fval: 110.0, name: 'n1.1', val: 10} }, { key: 'k1.2', value: { name: 'n1.2', val: 11} }]", 1); + assertWritten("[ { key: 'k2', value: { fval: 120.0, name: 'n2', val: 20} }, { key: '', value: { fval: 121.0, val: 21 } }]", 2); + assertWritten("[ { key: 'k3.1', value: { fval: 130.0, name: 'n3.1', val: 30} }, { key: 'k3.2', value: { fval: 131.0, name: 'n3.2'} } ]", 3); + assertWritten("[ { key: '', value: { } }, { key: 'k4.2', value: { fval: 141.0, name: 'n4.2', val: 41} } ]", 4); assertWritten("null", 5); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_map_of_string) { set_field("map", false); - assertWritten("[ { key: \"k1.1\", value: \"n1.1\" }, { key: \"k1.2\", value: \"n1.2\"}]", 1); - assertWritten("[ { key: \"k2\"}]", 2); - assertWritten("[ { key: \"k3.1\", value: \"n3.1\" }, { value: \"n3.2\"} ]", 3); - assertWritten("[ { }, { key: \"k4.2\", value: \"n4.2\" } ]", 4); + assertWritten("[ { key: 'k1.1', value: 'n1.1' }, { key: 'k1.2', value: 'n1.2'}]", 1); + assertWritten("[ { key: 'k2', value: '' }]", 2); + assertWritten("[ { key: 'k3.1', value: 'n3.1' }, { key: '', value: 'n3.2'} ]", 3); + assertWritten("[ { key: '', value: '' }, { key: 'k4.2', value: 'n4.2' } ]", 4); assertWritten("null", 5); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_array_of_struct) { set_field("array", true); - assertWritten("[ { name: \"n1.2\", val: 11}]", 1); + assertWritten("[ { name: 'n1.2', val: 11}]", 1); assertWritten("[ ]", 2); - assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30} ]", 3); - assertWritten("[ { fval: 141.0, name: \"n4.2\", val: 41} ]", 4); + assertWritten("[ { fval: 130.0, name: 'n3.1', val: 30} ]", 3); + assertWritten("[ { fval: 141.0, name: 'n4.2', val: 41} ]", 4); assertWritten("null", 5); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_map_of_struct) { set_field("smap", true); - assertWritten("[ { key: \"k1.2\", value: { name: \"n1.2\", val: 11} }]", 1); + assertWritten("[ { key: 'k1.2', value: { name: 'n1.2', val: 11} }]", 1); assertWritten("[ ]", 2); - assertWritten("[ { key: \"k3.1\", value: { fval: 130.0, name: \"n3.1\", val: 30} } ]", 3); - assertWritten("[ { key: \"k4.2\", value: { fval: 141.0, name: \"n4.2\", val: 41} } ]", 4); + assertWritten("[ { key: 'k3.1', value: { fval: 130.0, name: 'n3.1', val: 30} } ]", 3); + assertWritten("[ { key: 'k4.2', value: { fval: 141.0, name: 'n4.2', val: 41} } ]", 4); assertWritten("null", 5); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_map_of_string) { set_field("map", true); - assertWritten("[ { key: \"k1.2\", value: \"n1.2\"}]", 1); + assertWritten("[ { key: 'k1.2', value: 'n1.2'}]", 1); assertWritten("[ ]", 2); - assertWritten("[ { key: \"k3.1\", value: \"n3.1\" } ]", 3); - assertWritten("[ { key: \"k4.2\", value: \"n4.2\" } ]", 4); + assertWritten("[ { key: 'k3.1', value: 'n3.1' } ]", 3); + assertWritten("[ { key: 'k4.2', value: 'n4.2' } ]", 4); assertWritten("null", 5); } -TEST_F(AttributeCombinerTest, require_that_struct_field_mapper_is_setup_for_filtered_array_of_struct) +TEST_F(AttributeCombinerTest, require_that_matching_elems_fields_is_setup_for_filtered_array_of_struct) { set_field("array", true); - EXPECT_TRUE(_struct_field_mapper); - EXPECT_TRUE(_struct_field_mapper->is_struct_field("array")); - EXPECT_FALSE(_struct_field_mapper->is_struct_field("map")); - EXPECT_FALSE(_struct_field_mapper->is_struct_field("smap")); - EXPECT_EQ("", _struct_field_mapper->get_struct_field("array.foo")); - EXPECT_EQ("array", _struct_field_mapper->get_struct_field("array.name")); - EXPECT_EQ("array", _struct_field_mapper->get_struct_field("array.val")); - EXPECT_EQ("array", _struct_field_mapper->get_struct_field("array.fval")); + EXPECT_TRUE(_matching_elems_fields); + EXPECT_TRUE(_matching_elems_fields->has_field("array")); + EXPECT_FALSE(_matching_elems_fields->has_field("map")); + EXPECT_FALSE(_matching_elems_fields->has_field("smap")); + EXPECT_EQ("", _matching_elems_fields->get_enclosing_field("array.foo")); + EXPECT_EQ("array", _matching_elems_fields->get_enclosing_field("array.name")); + EXPECT_EQ("array", _matching_elems_fields->get_enclosing_field("array.val")); + EXPECT_EQ("array", _matching_elems_fields->get_enclosing_field("array.fval")); } -TEST_F(AttributeCombinerTest, require_that_struct_field_mapper_is_setup_for_filtered_map_of_struct) +TEST_F(AttributeCombinerTest, require_that_matching_elems_fields_is_setup_for_filtered_map_of_struct) { set_field("smap", true); - EXPECT_TRUE(_struct_field_mapper); - EXPECT_FALSE(_struct_field_mapper->is_struct_field("array")); - EXPECT_FALSE(_struct_field_mapper->is_struct_field("map")); - EXPECT_TRUE(_struct_field_mapper->is_struct_field("smap")); - EXPECT_EQ("", _struct_field_mapper->get_struct_field("smap.foo")); - EXPECT_EQ("smap", _struct_field_mapper->get_struct_field("smap.key")); - EXPECT_EQ("smap", _struct_field_mapper->get_struct_field("smap.value.name")); - EXPECT_EQ("smap", _struct_field_mapper->get_struct_field("smap.value.val")); - EXPECT_EQ("smap", _struct_field_mapper->get_struct_field("smap.value.fval")); + EXPECT_TRUE(_matching_elems_fields); + EXPECT_FALSE(_matching_elems_fields->has_field("array")); + EXPECT_FALSE(_matching_elems_fields->has_field("map")); + EXPECT_TRUE(_matching_elems_fields->has_field("smap")); + EXPECT_EQ("", _matching_elems_fields->get_enclosing_field("smap.foo")); + EXPECT_EQ("smap", _matching_elems_fields->get_enclosing_field("smap.key")); + EXPECT_EQ("smap", _matching_elems_fields->get_enclosing_field("smap.value.name")); + EXPECT_EQ("smap", _matching_elems_fields->get_enclosing_field("smap.value.val")); + EXPECT_EQ("smap", _matching_elems_fields->get_enclosing_field("smap.value.fval")); } -TEST_F(AttributeCombinerTest, require_that_struct_field_mapper_is_setup_for_filtered_map_of_string) +TEST_F(AttributeCombinerTest, require_that_matching_elems_fields_is_setup_for_filtered_map_of_string) { set_field("map", true); - EXPECT_TRUE(_struct_field_mapper); - EXPECT_FALSE(_struct_field_mapper->is_struct_field("array")); - EXPECT_TRUE(_struct_field_mapper->is_struct_field("map")); - EXPECT_FALSE(_struct_field_mapper->is_struct_field("smap")); - EXPECT_EQ("", _struct_field_mapper->get_struct_field("map.foo")); - EXPECT_EQ("map", _struct_field_mapper->get_struct_field("map.key")); - EXPECT_EQ("map", _struct_field_mapper->get_struct_field("map.value")); + EXPECT_TRUE(_matching_elems_fields); + EXPECT_FALSE(_matching_elems_fields->has_field("array")); + EXPECT_TRUE(_matching_elems_fields->has_field("map")); + EXPECT_FALSE(_matching_elems_fields->has_field("smap")); + EXPECT_EQ("", _matching_elems_fields->get_enclosing_field("map.foo")); + EXPECT_EQ("map", _matching_elems_fields->get_enclosing_field("map.key")); + EXPECT_EQ("map", _matching_elems_fields->get_enclosing_field("map.value")); } } diff --git a/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt b/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt new file mode 100644 index 00000000000..8bdf30273f2 --- /dev/null +++ b/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) +vespa_add_executable(searchsummary_attributedfw_test_app TEST + SOURCES + attributedfw_test.cpp + DEPENDS + searchsummary + searchsummary_test + GTest::GTest +) +vespa_add_test(NAME searchsummary_attributedfw_test_app COMMAND searchsummary_attributedfw_test_app) diff --git a/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp new file mode 100644 index 00000000000..7bea92ec8f3 --- /dev/null +++ b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp @@ -0,0 +1,150 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/common/matching_elements_fields.h> +#include <vespa/searchsummary/docsummary/attributedfw.h> +#include <vespa/searchsummary/test/mock_attribute_manager.h> +#include <vespa/searchsummary/test/mock_state_callback.h> +#include <vespa/searchsummary/test/slime_value.h> +#include <vespa/vespalib/gtest/gtest.h> + +#include <vespa/log/log.h> +LOG_SETUP("attributedfw_test"); + +using search::MatchingElements; +using search::MatchingElementsFields; +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::docsummary::AttributeDFWFactory; +using search::docsummary::GetDocsumsState; +using search::docsummary::IDocsumFieldWriter; +using search::docsummary::test::MockAttributeManager; +using search::docsummary::test::MockStateCallback; +using search::docsummary::test::SlimeValue; + +using ElementVector = std::vector<uint32_t>; + +class AttributeDFWTest : public ::testing::Test { +protected: + MockAttributeManager _attrs; + std::unique_ptr<IDocsumFieldWriter> _writer; + MockStateCallback _callback; + GetDocsumsState _state; + std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields; + vespalib::string _field_name; + +public: + AttributeDFWTest() + : _attrs(), + _writer(), + _callback(), + _state(_callback), + _matching_elems_fields(), + _field_name() + { + _attrs.build_string_attribute("array_str", { {"a", "b", "c"}, {} }); + _attrs.build_int_attribute("array_int", BasicType::INT32, { {10, 20, 30}, {} }); + _attrs.build_float_attribute("array_float", { {10.5, 20.5, 30.5}, {} }); + + _attrs.build_string_attribute("wset_str", { {"a", "b", "c"}, {} }, CollectionType::WSET); + _attrs.build_int_attribute("wset_int", BasicType::INT32, { {10, 20, 30}, {} }, CollectionType::WSET); + _attrs.build_float_attribute("wset_float", { {10.5, 20.5, 30.5}, {} }, CollectionType::WSET); + + _state._attrCtx = _attrs.mgr().createContext(); + } + ~AttributeDFWTest() {} + + void setup(const vespalib::string& field_name, bool filter_elements) { + if (filter_elements) { + _matching_elems_fields = std::make_shared<MatchingElementsFields>(); + } + _writer = AttributeDFWFactory::create(_attrs.mgr(), field_name, filter_elements, _matching_elems_fields); + _writer->setIndex(0); + _field_name = field_name; + _state._attributes.resize(1); + _state._attributes[0] = _state._attrCtx->getAttribute(field_name); + } + + void expect_field(const vespalib::string& exp_slime_as_json, uint32_t docid) { + vespalib::Slime act; + vespalib::slime::SlimeInserter inserter(act); + _writer->insertField(docid, nullptr, &_state, search::docsummary::RES_JSONSTRING, inserter); + + SlimeValue exp(exp_slime_as_json); + EXPECT_EQ(exp.slime, act); + } + + void expect_filtered(const ElementVector& matching_elems, const std::string& exp_slime_as_json, uint32_t docid = 1) { + _callback.clear(); + _callback.add_matching_elements(docid, _field_name, matching_elems); + _state._matching_elements = std::unique_ptr<MatchingElements>(); + expect_field(exp_slime_as_json, docid); + } +}; + +TEST_F(AttributeDFWTest, outputs_slime_for_array_of_string) +{ + setup("array_str", false); + expect_field("[ 'a', 'b', 'c' ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, outputs_slime_for_array_of_int) +{ + setup("array_int", false); + expect_field("[ 10, 20, 30 ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, outputs_slime_for_array_of_float) +{ + setup("array_float", false); + expect_field("[ 10.5, 20.5, 30.5 ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, outputs_slime_for_wset_of_string) +{ + setup("wset_str", false); + expect_field("[ {'item':'a', 'weight':1}, {'item':'b', 'weight':1}, {'item':'c', 'weight':1} ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, outputs_slime_for_wset_of_int) +{ + setup("wset_int", false); + expect_field("[ {'item':10, 'weight':1}, {'item':20, 'weight':1}, {'item':30, 'weight':1} ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, outputs_slime_for_wset_of_float) +{ + setup("wset_float", false); + expect_field("[ {'item':10.5, 'weight':1}, {'item':20.5, 'weight':1}, {'item':30.5, 'weight':1} ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, matched_elements_fields_is_populated) +{ + setup("array_str", true); + EXPECT_TRUE(_matching_elems_fields->has_field("array_str")); +} + +TEST_F(AttributeDFWTest, filteres_matched_elements_in_array_attribute) +{ + setup("array_str", true); + expect_filtered({}, "[]"); + expect_filtered({0}, "[ 'a' ]"); + expect_filtered({1, 2}, "[ 'b', 'c' ]"); + expect_filtered({3}, "[]"); +} + +TEST_F(AttributeDFWTest, filteres_matched_elements_in_wset_attribute) +{ + setup("wset_str", true); + expect_filtered({}, "[]"); + expect_filtered({0}, "[ {'item':'a', 'weight':1} ]"); + expect_filtered({1, 2}, "[ {'item':'b', 'weight':1}, {'item':'c', 'weight':1} ]"); + expect_filtered({3}, "[]"); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index dff3acc5b89..0ac2f09e1b0 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/common/matching_elements.h> -#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> @@ -15,7 +15,7 @@ #include <vespa/searchsummary/docsummary/resultconfig.h> #include <vespa/searchsummary/docsummary/resultpacker.h> #include <vespa/searchsummary/docsummary/summaryfieldconverter.h> -#include <vespa/vespalib/data/slime/json_format.h> +#include <vespa/searchsummary/test/slime_value.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> #include <iostream> @@ -26,12 +26,13 @@ LOG_SETUP("matched_elements_filter_test"); using search::AttributeFactory; using search::AttributeVector; using search::MatchingElements; -using search::StructFieldMapper; +using search::MatchingElementsFields; using search::attribute::BasicType; using search::attribute::CollectionType; using search::attribute::Config; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; +using search::docsummary::test::SlimeValue; using vespalib::Slime; using namespace document; @@ -40,17 +41,6 @@ using namespace vespalib::slime; using ElementVector = std::vector<uint32_t>; -struct SlimeValue { - Slime slime; - - SlimeValue(const std::string& json_input) - : slime() - { - size_t used = JsonFormat::decode(json_input, slime); - EXPECT_GT(used, 0); - } -}; - StructDataType::UP make_struct_elem_type() { @@ -178,7 +168,7 @@ public: void FillSummaryFeatures(GetDocsumsState*, IDocsumEnvironment*) override {} void FillRankFeatures(GetDocsumsState*, IDocsumEnvironment*) override {} void ParseLocation(GetDocsumsState*) override {} - std::unique_ptr<MatchingElements> fill_matching_elements(const StructFieldMapper&) override { + std::unique_ptr<MatchingElements> fill_matching_elements(const MatchingElementsFields&) override { auto result = std::make_unique<MatchingElements>(); result->add_matching_elements(doc_id, _field_name, _matching_elements); return result; @@ -189,7 +179,7 @@ class MatchedElementsFilterTest : public ::testing::Test { private: DocsumStore _doc_store; AttributeContext _attr_ctx; - std::shared_ptr<StructFieldMapper> _mapper; + std::shared_ptr<MatchingElementsFields> _fields; Slime run_filter_field_writer(const std::string& input_field_name, const ElementVector& matching_elements) { auto writer = make_field_writer(input_field_name); @@ -209,21 +199,21 @@ public: MatchedElementsFilterTest() : _doc_store(), _attr_ctx(), - _mapper(std::make_shared<StructFieldMapper>()) + _fields(std::make_shared<MatchingElementsFields>()) { } ~MatchedElementsFilterTest() {} std::unique_ptr<IDocsumFieldWriter> make_field_writer(const std::string& input_field_name) { int input_field_enum = _doc_store.get_config().GetFieldNameEnum().Lookup(input_field_name.c_str()); return MatchedElementsFilterDFW::create(input_field_name, input_field_enum, - _attr_ctx, _mapper); + _attr_ctx, _fields); } void expect_filtered(const std::string& input_field_name, const ElementVector& matching_elements, const std::string& exp_slime_as_json) { Slime act = run_filter_field_writer(input_field_name, matching_elements); SlimeValue exp(exp_slime_as_json); EXPECT_EQ(exp.slime, act); } - const StructFieldMapper& mapper() const { return *_mapper; } + const MatchingElementsFields& fields() const { return *_fields; } }; TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value) @@ -249,12 +239,12 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value_when_inp expect_filtered("array_in_doc", {0, 1, 100}, "[]"); } -TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_array_field_value) +TEST_F(MatchedElementsFilterTest, matching_elements_fields_is_setup_for_array_field_value) { auto writer = make_field_writer("array"); - EXPECT_TRUE(mapper().is_struct_field("array")); - EXPECT_EQ("", mapper().get_struct_field("array.name")); - EXPECT_EQ("array", mapper().get_struct_field("array.weight")); + EXPECT_TRUE(fields().has_field("array")); + EXPECT_EQ("", fields().get_enclosing_field("array.name")); + EXPECT_EQ("array", fields().get_enclosing_field("array.weight")); } TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value) @@ -280,21 +270,21 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value_when_input expect_filtered("map_in_doc", {0, 1, 100}, "[]"); } -TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_map_field_value) +TEST_F(MatchedElementsFilterTest, matching_elements_fields_is_setup_for_map_field_value) { { auto writer = make_field_writer("map"); - EXPECT_TRUE(mapper().is_struct_field("map")); - EXPECT_EQ("", mapper().get_struct_field("map.key")); - EXPECT_EQ("map", mapper().get_struct_field("map.value.name")); - EXPECT_EQ("", mapper().get_struct_field("map.value.weight")); + EXPECT_TRUE(fields().has_field("map")); + EXPECT_EQ("", fields().get_enclosing_field("map.key")); + EXPECT_EQ("map", fields().get_enclosing_field("map.value.name")); + EXPECT_EQ("", fields().get_enclosing_field("map.value.weight")); } { auto writer = make_field_writer("map2"); - EXPECT_TRUE(mapper().is_struct_field("map2")); - EXPECT_EQ("map2", mapper().get_struct_field("map2.key")); - EXPECT_EQ("", mapper().get_struct_field("map2.value.name")); - EXPECT_EQ("", mapper().get_struct_field("map2.value.weight")); + EXPECT_TRUE(fields().has_field("map2")); + EXPECT_EQ("map2", fields().get_enclosing_field("map2.key")); + EXPECT_EQ("", fields().get_enclosing_field("map2.value.name")); + EXPECT_EQ("", fields().get_enclosing_field("map2.value.weight")); } } diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index 4cad98a8e01..f54b4c19dc3 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -110,7 +110,7 @@ struct MyGetDocsumsStateCallback : GetDocsumsStateCallback { virtual void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override {} virtual void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override {} virtual void ParseLocation(GetDocsumsState *) override {} - std::unique_ptr<MatchingElements> fill_matching_elements(const StructFieldMapper &) override { abort(); } + std::unique_ptr<MatchingElements> fill_matching_elements(const MatchingElementsFields &) override { abort(); } }; template <typename AttrType> diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index 5b74666cbec..6fceef37f09 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -78,7 +78,7 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } void ParseLocation(GetDocsumsState *) override { } - std::unique_ptr<MatchingElements> fill_matching_elements(const search::StructFieldMapper &) override { abort(); } + std::unique_ptr<MatchingElements> fill_matching_elements(const search::MatchingElementsFields &) override { abort(); } }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp index a1f3ce4d392..7a656eb0422 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp @@ -7,7 +7,7 @@ #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchlib/common/matching_elements.h> -#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/data/slime/cursor.h> #include <cassert> @@ -30,7 +30,8 @@ public: const std::vector<vespalib::string> &attributeNames, IAttributeContext &context, const vespalib::string &field_name, - const MatchingElements* matching_elements); + const MatchingElements* matching_elements, + bool is_map_of_scalar); ~ArrayAttributeFieldWriterState() override; void insert_element(uint32_t element_index, Cursor &array); void insertField(uint32_t docId, vespalib::slime::Inserter &target) override; @@ -40,10 +41,11 @@ ArrayAttributeFieldWriterState::ArrayAttributeFieldWriterState(const std::vector const std::vector<vespalib::string> &attributeNames, IAttributeContext &context, const vespalib::string &field_name, - const MatchingElements *matching_elements) + const MatchingElements *matching_elements, + bool is_map_of_scalar) : DocsumFieldWriterState(), _writers(), - _field_name(field_name), + _field_name(field_name), _matching_elements(matching_elements) { size_t fields = fieldNames.size(); @@ -51,7 +53,7 @@ ArrayAttributeFieldWriterState::ArrayAttributeFieldWriterState(const std::vector for (uint32_t field = 0; field < fields; ++field) { const IAttributeVector *attr = context.getAttribute(attributeNames[field]); if (attr != nullptr) { - _writers.emplace_back(AttributeFieldWriter::create(fieldNames[field], *attr)); + _writers.emplace_back(AttributeFieldWriter::create(fieldNames[field], *attr, is_map_of_scalar)); } } } @@ -103,13 +105,14 @@ ArrayAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Ins ArrayAttributeCombinerDFW::ArrayAttributeCombinerDFW(const vespalib::string &fieldName, const StructFieldsResolver& fields_resolver, bool filter_elements, - std::shared_ptr<StructFieldMapper> struct_field_mapper) - : AttributeCombinerDFW(fieldName, filter_elements, std::move(struct_field_mapper)), + std::shared_ptr<MatchingElementsFields> matching_elems_fields) + : AttributeCombinerDFW(fieldName, filter_elements, std::move(matching_elems_fields)), _fields(fields_resolver.get_array_fields()), - _attributeNames(fields_resolver.get_array_attributes()) + _attributeNames(fields_resolver.get_array_attributes()), + _is_map_of_scalar(fields_resolver.is_map_of_scalar()) { - if (filter_elements && _struct_field_mapper && !_struct_field_mapper->is_struct_field(fieldName)) { - fields_resolver.apply_to(*_struct_field_mapper); + if (filter_elements && _matching_elems_fields && !_matching_elems_fields->has_field(fieldName)) { + fields_resolver.apply_to(*_matching_elems_fields); } } @@ -118,7 +121,8 @@ ArrayAttributeCombinerDFW::~ArrayAttributeCombinerDFW() = default; std::unique_ptr<DocsumFieldWriterState> ArrayAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context, const MatchingElements* matching_elements) { - return std::make_unique<ArrayAttributeFieldWriterState>(_fields, _attributeNames, context, _fieldName, matching_elements); + return std::make_unique<ArrayAttributeFieldWriterState>(_fields, _attributeNames, context, + _fieldName, matching_elements, _is_map_of_scalar); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h index c3e686965cf..742128a229d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h @@ -20,13 +20,14 @@ class ArrayAttributeCombinerDFW : public AttributeCombinerDFW { std::vector<vespalib::string> _fields; std::vector<vespalib::string> _attributeNames; + bool _is_map_of_scalar; std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context, const MatchingElements* matching_elements) override; public: ArrayAttributeCombinerDFW(const vespalib::string &fieldName, const StructFieldsResolver& fields_resolver, bool filter_elements, - std::shared_ptr<StructFieldMapper> struct_field_mapper); + std::shared_ptr<MatchingElementsFields> matching_elems_fields); ~ArrayAttributeCombinerDFW() override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp index 8eb77c0ed9c..4ad42133f14 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp @@ -6,7 +6,7 @@ #include "docsumstate.h" #include "struct_fields_resolver.h" #include "struct_map_attribute_combiner_dfw.h" -#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <algorithm> #include <vespa/log/log.h> @@ -16,12 +16,13 @@ using search::attribute::IAttributeContext; namespace search::docsummary { -AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper) +AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName, bool filter_elements, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) : ISimpleDFW(), _stateIndex(0), _filter_elements(filter_elements), _fieldName(fieldName), - _struct_field_mapper(std::move(struct_field_mapper)) + _matching_elems_fields(std::move(matching_elems_fields)) { } @@ -41,15 +42,16 @@ AttributeCombinerDFW::setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) } std::unique_ptr<IDocsumFieldWriter> -AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeContext &attrCtx, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper) +AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeContext &attrCtx, bool filter_elements, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) { StructFieldsResolver structFields(fieldName, attrCtx, true); if (structFields.has_error()) { return std::unique_ptr<IDocsumFieldWriter>(); } else if (structFields.is_map_of_struct()) { - return std::make_unique<StructMapAttributeCombinerDFW>(fieldName, structFields, filter_elements, std::move(struct_field_mapper)); + return std::make_unique<StructMapAttributeCombinerDFW>(fieldName, structFields, filter_elements, std::move(matching_elems_fields)); } - return std::make_unique<ArrayAttributeCombinerDFW>(fieldName, structFields, filter_elements, std::move(struct_field_mapper)); + return std::make_unique<ArrayAttributeCombinerDFW>(fieldName, structFields, filter_elements, std::move(matching_elems_fields)); } void @@ -59,7 +61,7 @@ AttributeCombinerDFW::insertField(uint32_t docid, GetDocsumsState *state, ResTyp if (!fieldWriterState) { const MatchingElements *matching_elements = nullptr; if (_filter_elements) { - matching_elements = &state->get_matching_elements(*_struct_field_mapper); + matching_elements = &state->get_matching_elements(*_matching_elems_fields); } fieldWriterState = allocFieldWriterState(*state->_attrCtx, matching_elements); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h index a8ab5f2f8f5..bb635d4d280 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h @@ -6,7 +6,7 @@ namespace search { class MatchingElements; -class StructFieldMapper; +class MatchingElementsFields; } namespace search::attribute { class IAttributeContext; } @@ -25,15 +25,17 @@ protected: uint32_t _stateIndex; const bool _filter_elements; vespalib::string _fieldName; - std::shared_ptr<StructFieldMapper> _struct_field_mapper; - AttributeCombinerDFW(const vespalib::string &fieldName, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper); + std::shared_ptr<MatchingElementsFields> _matching_elems_fields; + AttributeCombinerDFW(const vespalib::string &fieldName, bool filter_elements, + std::shared_ptr<MatchingElementsFields> matching_elems_fields); protected: virtual std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context, const MatchingElements* matching_elements) = 0; public: ~AttributeCombinerDFW() override; bool IsGenerated() const override; bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override; - static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, search::attribute::IAttributeContext &attrCtx, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper); + static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, search::attribute::IAttributeContext &attrCtx, + bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields); void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp index 99c5d8b7f30..91fec74fce9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.cpp @@ -46,6 +46,15 @@ public: void print(uint32_t idx, Cursor &cursor) override; }; +class WriteStringFieldNeverSkip : public WriteField<search::attribute::ConstCharContent> +{ +public: + WriteStringFieldNeverSkip(vespalib::Memory fieldName, + const IAttributeVector &attr) + : WriteField(fieldName, attr) {} + ~WriteStringFieldNeverSkip() override {} + void print(uint32_t idx, Cursor &cursor) override; +}; class WriteFloatField : public WriteField<search::attribute::FloatContent> { @@ -104,6 +113,17 @@ WriteStringField::print(uint32_t idx, Cursor &cursor) } } +void +WriteStringFieldNeverSkip::print(uint32_t idx, Cursor &cursor) +{ + if (idx < _size) { + const char *s = _content[idx]; + cursor.setString(_fieldName, vespalib::Memory(s)); + } else { + cursor.setString(_fieldName, vespalib::Memory("")); + } +} + WriteFloatField::WriteFloatField(vespalib::Memory fieldName, const IAttributeVector &attr) : WriteField(fieldName, attr) @@ -147,7 +167,7 @@ WriteIntField::print(uint32_t idx, Cursor &cursor) } std::unique_ptr<AttributeFieldWriter> -AttributeFieldWriter::create(vespalib::Memory fieldName, const IAttributeVector &attr) +AttributeFieldWriter::create(vespalib::Memory fieldName, const IAttributeVector &attr, bool keep_empty_strings) { switch (attr.getBasicType()) { case BasicType::INT8: @@ -162,7 +182,11 @@ AttributeFieldWriter::create(vespalib::Memory fieldName, const IAttributeVector case BasicType::DOUBLE: return std::make_unique<WriteFloatField>(fieldName, attr); case BasicType::STRING: - return std::make_unique<WriteStringField>(fieldName, attr); + if (keep_empty_strings) { + return std::make_unique<WriteStringFieldNeverSkip>(fieldName, attr); + } else { + return std::make_unique<WriteStringField>(fieldName, attr); + } default: assert(false); abort(); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h index 3fed6edb1b5..b6c6bc82325 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_field_writer.h @@ -27,7 +27,7 @@ public: virtual ~AttributeFieldWriter(); virtual void fetch(uint32_t docId) = 0; virtual void print(uint32_t idx, vespalib::slime::Cursor &cursor) = 0; - static std::unique_ptr<AttributeFieldWriter> create(vespalib::Memory fieldName, const search::attribute::IAttributeVector &attr); + static std::unique_ptr<AttributeFieldWriter> create(vespalib::Memory fieldName, const search::attribute::IAttributeVector &attr, bool keep_empty_strings = false); uint32_t size() const { return _size; } }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index f1b12d8a227..05ba12ddff9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -1,25 +1,29 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "docsumwriter.h" #include "attributedfw.h" #include "docsumstate.h" -#include <vespa/searchlib/attribute/stringbase.h> -#include <vespa/searchlib/attribute/integerbase.h> +#include "docsumwriter.h" +#include <vespa/eval/tensor/serialization/typed_binary_format.h> +#include <vespa/eval/tensor/tensor.h> +#include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/attribute/iattributemanager.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/tensor/i_tensor_attribute.h> -#include <vespa/searchcommon/attribute/iattributecontext.h> -#include <vespa/eval/tensor/tensor.h> -#include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.attributedfw"); using namespace search; +using search::attribute::BasicType; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; -using search::attribute::BasicType; +using vespalib::Memory; +using vespalib::slime::Cursor; using vespalib::slime::Inserter; namespace search::docsummary { @@ -30,7 +34,7 @@ AttrDFW::AttrDFW(const vespalib::string & attrName) : } const attribute::IAttributeVector & -AttrDFW::vec(const GetDocsumsState & s) const { +AttrDFW::get_attribute(const GetDocsumsState& s) const { return *s.getAttribute(getIndex()); } @@ -48,13 +52,13 @@ public: bool SingleAttrDFW::isDefaultValue(uint32_t docid, const GetDocsumsState * state) const { - return vec(*state).isUndefined(docid); + return get_attribute(*state).isUndefined(docid); } void SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType type, Inserter &target) { - const IAttributeVector & v = vec(*state); + const auto& v = get_attribute(*state); switch (type) { case RES_INT: { uint32_t val = v.getInt(docid); @@ -133,46 +137,115 @@ SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType type //----------------------------------------------------------------------------- -class MultiAttrDFW : public AttrDFW -{ -public: - explicit MultiAttrDFW(const vespalib::string & attrName) : AttrDFW(attrName) {} - void insertField(uint32_t docid, GetDocsumsState *state, ResType type, Inserter &target) override; +template <typename DataType> +class MultiAttrDFW : public AttrDFW { +private: + bool _is_weighted_set; + bool _filter_elements; + std::shared_ptr<MatchingElementsFields> _matching_elems_fields; +public: + explicit MultiAttrDFW(const vespalib::string& attr_name, bool is_weighted_set, + bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields) + : AttrDFW(attr_name), + _is_weighted_set(is_weighted_set), + _filter_elements(filter_elements), + _matching_elems_fields(std::move(matching_elems_fields)) + { + if (filter_elements && _matching_elems_fields) { + _matching_elems_fields->add_field(attr_name); + } + } + void insertField(uint32_t docid, GetDocsumsState* state, ResType type, Inserter& target) override; }; void -MultiAttrDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, Inserter &target) +insert_element(const std::vector<IAttributeVector::WeightedString>& elements, + size_t idx, bool is_weighted_set, Cursor& arr) +{ + const vespalib::string& sv = elements[idx].getValue(); + Memory value(sv.c_str(), sv.size()); + if (is_weighted_set) { + Cursor& elem = arr.addObject(); + elem.setString("item", value); + elem.setLong("weight", elements[idx].getWeight()); + } else { + arr.addString(value); + } +} + +void +insert_element(const std::vector<IAttributeVector::WeightedInt>& elements, + size_t idx, bool is_weighted_set, Cursor& arr) +{ + if (is_weighted_set) { + Cursor& elem = arr.addObject(); + elem.setLong("item", elements[idx].getValue()); + elem.setLong("weight", elements[idx].getWeight()); + } else { + arr.addLong(elements[idx].getValue()); + } +} + +void +insert_element(const std::vector<IAttributeVector::WeightedFloat>& elements, + size_t idx, bool is_weighted_set, Cursor& arr) { - using vespalib::slime::Cursor; - using vespalib::Memory; - const IAttributeVector & v = vec(*state); - bool isWeightedSet = v.hasWeightedSetType(); + if (is_weighted_set) { + Cursor& elem = arr.addObject(); + elem.setDouble("item", elements[idx].getValue()); + elem.setLong("weight", elements[idx].getWeight()); + } else { + arr.addDouble(elements[idx].getValue()); + } +} - uint32_t entries = v.getValueCount(docid); +template <typename DataType> +void +MultiAttrDFW<DataType>::insertField(uint32_t docid, GetDocsumsState* state, ResType, Inserter& target) +{ + const auto& attr = get_attribute(*state); + uint32_t entries = attr.getValueCount(docid); if (entries == 0) { return; // Don't insert empty fields } Cursor &arr = target.insertArray(); - BasicType::Type t = v.getBasicType(); - switch (t) { - case BasicType::NONE: - case BasicType::STRING: { - std::vector<IAttributeVector::WeightedString> elements(entries); - entries = std::min(entries, v.get(docid, &elements[0], entries)); - for (uint32_t i = 0; i < entries; ++i) { - const vespalib::string &sv = elements[i].getValue(); - Memory value(sv.c_str(), sv.size()); - if (isWeightedSet) { - Cursor &elem = arr.addObject(); - elem.setString("item", value); - elem.setLong("weight", elements[i].getWeight()); - } else { - arr.addString(value); + std::vector<DataType> elements(entries); + entries = std::min(entries, attr.get(docid, elements.data(), entries)); + + if (_filter_elements) { + const auto& matching_elems = state->get_matching_elements(*_matching_elems_fields) + .get_matching_elements(docid, getAttributeName()); + if (!matching_elems.empty() && matching_elems.back() < entries) { + for (uint32_t id_to_keep : matching_elems) { + insert_element(elements, id_to_keep, _is_weighted_set, arr); } } - return; } + } else { + for (uint32_t i = 0; i < entries; ++i) { + insert_element(elements, i, _is_weighted_set, arr); + } + } +} + +//----------------------------------------------------------------------------- + +namespace { + +std::unique_ptr<IDocsumFieldWriter> +create_multi_writer(const IAttributeVector& attr, + bool filter_elements, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) +{ + auto type = attr.getBasicType(); + bool is_weighted_set = attr.hasWeightedSetType(); + switch (type) { + case BasicType::NONE: + case BasicType::STRING: { + return std::make_unique<MultiAttrDFW<IAttributeVector::WeightedString>>(attr.getName(), is_weighted_set, + filter_elements, std::move(matching_elems_fields)); + } case BasicType::BOOL: case BasicType::UINT2: case BasicType::UINT4: @@ -180,54 +253,39 @@ MultiAttrDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, Inser case BasicType::INT16: case BasicType::INT32: case BasicType::INT64: { - std::vector<IAttributeVector::WeightedInt> elements(entries); - entries = std::min(entries, v.get(docid, &elements[0], entries)); - for (uint32_t i = 0; i < entries; ++i) { - if (isWeightedSet) { - Cursor &elem = arr.addObject(); - elem.setLong("item", elements[i].getValue()); - elem.setLong("weight", elements[i].getWeight()); - } else { - arr.addLong(elements[i].getValue()); - } - } - return; } + return std::make_unique<MultiAttrDFW<IAttributeVector::WeightedInt>>(attr.getName(), is_weighted_set, + filter_elements, std::move(matching_elems_fields)); + } case BasicType::FLOAT: case BasicType::DOUBLE: { - std::vector<IAttributeVector::WeightedFloat> elements(entries); - entries = std::min(entries, v.get(docid, &elements[0], entries)); - for (uint32_t i = 0; i < entries; ++i) { - if (isWeightedSet) { - Cursor &elem = arr.addObject(); - elem.setDouble("item", elements[i].getValue()); - elem.setLong("weight", elements[i].getWeight()); - } else { - arr.addDouble(elements[i].getValue()); - } - } - return; } + return std::make_unique<MultiAttrDFW<IAttributeVector::WeightedFloat>>(attr.getName(), is_weighted_set, + filter_elements, std::move(matching_elems_fields)); + } default: // should not happen - LOG(error, "bad value for type: %u\n", t); + LOG(error, "Bad value for attribute type: %u", type); LOG_ASSERT(false); } } -//----------------------------------------------------------------------------- +} -IDocsumFieldWriter * -AttributeDFWFactory::create(IAttributeManager & vecMan, const char *vecName) +std::unique_ptr<IDocsumFieldWriter> +AttributeDFWFactory::create(IAttributeManager& attr_mgr, + const vespalib::string& attr_name, + bool filter_elements, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) { - IAttributeContext::UP ctx = vecMan.createContext(); - const IAttributeVector * vec = ctx->getAttribute(vecName); - if (vec == nullptr) { - LOG(warning, "No valid attribute vector found: %s", vecName); - return nullptr; - } - if (vec->hasMultiValue()) { - return new MultiAttrDFW(vec->getName()); + auto ctx = attr_mgr.createContext(); + const auto* attr = ctx->getAttribute(attr_name); + if (attr == nullptr) { + LOG(warning, "No valid attribute vector found: '%s'", attr_name.c_str()); + return std::unique_ptr<IDocsumFieldWriter>(); + } + if (attr->hasMultiValue()) { + return create_multi_writer(*attr, filter_elements, std::move(matching_elems_fields)); } else { - return new SingleAttrDFW(vec->getName()); + return std::make_unique<SingleAttrDFW>(attr->getName()); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h index 211becea16c..55a30f0bb7b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h @@ -4,16 +4,30 @@ #include "docsumfieldwriter.h" + +namespace search { class MatchingElementsFields; } namespace search::attribute { class IAttributeVector; } namespace search::docsummary { +/** + * Factory to create an IDocsumFieldWriter to write an attribute vector to slime. + */ +class AttributeDFWFactory { +public: + static std::unique_ptr<IDocsumFieldWriter> create(IAttributeManager& attr_mgr, + const vespalib::string& attr_name, + bool filter_elements = false, + std::shared_ptr<MatchingElementsFields> matching_elems_fields + = std::shared_ptr<MatchingElementsFields>()); +}; + class AttrDFW : public ISimpleDFW { private: vespalib::string _attrName; protected: - const attribute::IAttributeVector & vec(const GetDocsumsState & s) const; + const attribute::IAttributeVector& get_attribute(const GetDocsumsState& s) const; const vespalib::string & getAttributeName() const override { return _attrName; } public: AttrDFW(const vespalib::string & attrName); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index 9235ce2b181..f41ada8b2e8 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -10,7 +10,7 @@ #include "positionsdfw.h" #include "rankfeaturesdfw.h" #include "textextractordfw.h" -#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> @@ -25,7 +25,7 @@ DynamicDocsumConfig::getResultConfig() const { } IDocsumFieldWriter::UP -DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<StructFieldMapper> struct_field_mapper) +DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<MatchingElementsFields> matching_elems_fields) { const ResultConfig & resultConfig = getResultConfig(); rc = false; @@ -88,23 +88,21 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & rc = fieldWriter.get(); } } else if (overrideName == "attribute") { - const char *vectorName = argument.c_str(); if (getEnvironment() && getEnvironment()->getAttributeManager()) { - IDocsumFieldWriter *fw = AttributeDFWFactory::create(*getEnvironment()->getAttributeManager(), vectorName); - fieldWriter.reset(fw); - rc = fw != NULL; + fieldWriter = AttributeDFWFactory::create(*getEnvironment()->getAttributeManager(), argument); + rc = static_cast<bool>(fieldWriter); } } else if (overrideName == "attributecombiner") { if (getEnvironment() && getEnvironment()->getAttributeManager()) { auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); - fieldWriter = AttributeCombinerDFW::create(fieldName, *attr_ctx, false, std::shared_ptr<StructFieldMapper>()); + fieldWriter = AttributeCombinerDFW::create(fieldName, *attr_ctx, false, std::shared_ptr<MatchingElementsFields>()); rc = static_cast<bool>(fieldWriter); } } else if (overrideName == "matchedattributeelementsfilter") { string source_field = argument.empty() ? fieldName : argument; if (getEnvironment() && getEnvironment()->getAttributeManager()) { auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); - fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, struct_field_mapper); + fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, matching_elems_fields); rc = static_cast<bool>(fieldWriter); } } else if (overrideName == "matchedelementsfilter") { @@ -112,7 +110,7 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & if (getEnvironment() && getEnvironment()->getAttributeManager()) { auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); fieldWriter = MatchedElementsFilterDFW::create(source_field, resultConfig.GetFieldNameEnum().Lookup(source_field.c_str()), - *attr_ctx, struct_field_mapper); + *attr_ctx, matching_elems_fields); rc = static_cast<bool>(fieldWriter); } } else { @@ -125,14 +123,14 @@ void DynamicDocsumConfig::configure(const vespa::config::search::SummarymapConfig &cfg) { std::vector<string> strCfg; - auto struct_field_mapper = std::make_shared<StructFieldMapper>(); + auto matching_elems_fields = std::make_shared<MatchingElementsFields>(); if ((cfg.defaultoutputclass != -1) && !_writer->SetDefaultOutputClass(cfg.defaultoutputclass)) { throw IllegalArgumentException(make_string("could not set default output class to %d", cfg.defaultoutputclass)); } for (size_t i = 0; i < cfg.override.size(); ++i) { const vespa::config::search::SummarymapConfig::Override & o = cfg.override[i]; bool rc(false); - IDocsumFieldWriter::UP fieldWriter = createFieldWriter(o.field, o.command, o.arguments, rc, struct_field_mapper); + IDocsumFieldWriter::UP fieldWriter = createFieldWriter(o.field, o.command, o.arguments, rc, matching_elems_fields); if (rc && fieldWriter.get() != NULL) { rc = _writer->Override(o.field.c_str(), fieldWriter.release()); // OBJECT HAND-OVER } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h index 34ca49824c5..f6f1939cc62 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h @@ -4,7 +4,7 @@ #include <vespa/config-summarymap.h> -namespace search { class StructFieldMapper; } +namespace search { class MatchingElementsFields; } namespace search::docsummary { class IDocsumEnvironment; @@ -29,7 +29,7 @@ protected: virtual std::unique_ptr<IDocsumFieldWriter> createFieldWriter(const string & fieldName, const string & overrideName, - const string & argument, bool & rc, std::shared_ptr<StructFieldMapper> struct_field_mapper); + const string & argument, bool & rc, std::shared_ptr<MatchingElementsFields> matching_elems_fields); private: IDocsumEnvironment * _env; DynamicDocsumWriter * _writer; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h index 43375fb47f3..a40f105a1cb 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h @@ -84,14 +84,5 @@ public: vespalib::slime::Inserter &target) override; }; -//-------------------------------------------------------------------------- - -class AttributeDFWFactory -{ -private: - AttributeDFWFactory(); -public: - static IDocsumFieldWriter *create(IAttributeManager & vecMan, const char *vecName); -}; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index 3270e4b4c98..ebbf97e9f55 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -50,10 +50,10 @@ GetDocsumsState::~GetDocsumsState() } const MatchingElements & -GetDocsumsState::get_matching_elements(const StructFieldMapper &struct_field_mapper) +GetDocsumsState::get_matching_elements(const MatchingElementsFields &matching_elems_fields) { if (!_matching_elements) { - _matching_elements = _callback.fill_matching_elements(struct_field_mapper); + _matching_elements = _callback.fill_matching_elements(matching_elems_fields); } return *_matching_elements; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index 350e2410c9d..57cae341682 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -15,7 +15,7 @@ namespace juniper { namespace search { class MatchingElements; -class StructFieldMapper; +class MatchingElementsFields; } namespace search::common { class Location; } namespace search::attribute { @@ -35,7 +35,7 @@ public: virtual void FillSummaryFeatures(GetDocsumsState * state, IDocsumEnvironment * env) = 0; virtual void FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) = 0; virtual void ParseLocation(GetDocsumsState * state) = 0; - virtual std::unique_ptr<MatchingElements> fill_matching_elements(const StructFieldMapper &struct_field_mapper) = 0; + virtual std::unique_ptr<MatchingElements> fill_matching_elements(const MatchingElementsFields &matching_elems_fields) = 0; virtual ~GetDocsumsStateCallback(void) { } GetDocsumsStateCallback(const GetDocsumsStateCallback &) = delete; GetDocsumsStateCallback & operator = (const GetDocsumsStateCallback &) = delete; @@ -96,7 +96,7 @@ public: GetDocsumsState& operator=(const GetDocsumsState &) = delete; GetDocsumsState(GetDocsumsStateCallback &callback); ~GetDocsumsState(); - const MatchingElements &get_matching_elements(const StructFieldMapper &struct_field_mapper); + const MatchingElements &get_matching_elements(const MatchingElementsFields &matching_elems_fields); }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index ae3d6acde43..df510d5bcbc 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -47,7 +47,7 @@ GeoPositionDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType, using vespalib::slime::Symbol; using vespalib::slime::ArrayInserter; - const IAttributeVector & attribute = vec(*dsState); + const auto& attribute = get_attribute(*dsState); if (attribute.hasMultiValue()) { uint32_t entries = attribute.getValueCount(docid); Cursor &arr = target.insertArray(); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp index 443634b3e3f..6991d3acb29 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp @@ -8,7 +8,7 @@ #include <vespa/document/fieldvalue/literalfieldvalue.h> #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/common/matching_elements.h> -#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/data/slime/binary_format.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/smart_buffer.h> @@ -29,35 +29,35 @@ namespace search::docsummary { const std::vector<uint32_t>& MatchedElementsFilterDFW::get_matching_elements(uint32_t docid, GetDocsumsState& state) const { - return state.get_matching_elements(*_struct_field_mapper).get_matching_elements(docid, _input_field_name); + return state.get_matching_elements(*_matching_elems_fields).get_matching_elements(docid, _input_field_name); } MatchedElementsFilterDFW::MatchedElementsFilterDFW(const std::string& input_field_name, uint32_t input_field_enum, - std::shared_ptr<StructFieldMapper> struct_field_mapper) + std::shared_ptr<MatchingElementsFields> matching_elems_fields) : _input_field_name(input_field_name), _input_field_enum(input_field_enum), - _struct_field_mapper(std::move(struct_field_mapper)) + _matching_elems_fields(std::move(matching_elems_fields)) { } std::unique_ptr<IDocsumFieldWriter> MatchedElementsFilterDFW::create(const std::string& input_field_name, uint32_t input_field_enum, - std::shared_ptr<StructFieldMapper> struct_field_mapper) + std::shared_ptr<MatchingElementsFields> matching_elems_fields) { - return std::make_unique<MatchedElementsFilterDFW>(input_field_name, input_field_enum, std::move(struct_field_mapper)); + return std::make_unique<MatchedElementsFilterDFW>(input_field_name, input_field_enum, std::move(matching_elems_fields)); } std::unique_ptr<IDocsumFieldWriter> MatchedElementsFilterDFW::create(const std::string& input_field_name, uint32_t input_field_enum, search::attribute::IAttributeContext& attr_ctx, - std::shared_ptr<StructFieldMapper> struct_field_mapper) + std::shared_ptr<MatchingElementsFields> matching_elems_fields) { StructFieldsResolver resolver(input_field_name, attr_ctx, false); if (resolver.has_error()) { return std::unique_ptr<IDocsumFieldWriter>(); } - resolver.apply_to(*struct_field_mapper); - return std::make_unique<MatchedElementsFilterDFW>(input_field_name, input_field_enum, std::move(struct_field_mapper)); + resolver.apply_to(*matching_elems_fields); + return std::make_unique<MatchedElementsFilterDFW>(input_field_name, input_field_enum, std::move(matching_elems_fields)); } MatchedElementsFilterDFW::~MatchedElementsFilterDFW() = default; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h index 966b5b95fa7..087ddfd8d40 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h @@ -16,18 +16,18 @@ class MatchedElementsFilterDFW : public IDocsumFieldWriter { private: std::string _input_field_name; uint32_t _input_field_enum; - std::shared_ptr<StructFieldMapper> _struct_field_mapper; + std::shared_ptr<MatchingElementsFields> _matching_elems_fields; const std::vector<uint32_t>& get_matching_elements(uint32_t docid, GetDocsumsState& state) const; public: MatchedElementsFilterDFW(const std::string& input_field_name, uint32_t input_field_enum, - std::shared_ptr<StructFieldMapper> struct_field_mapper); + std::shared_ptr<MatchingElementsFields> matching_elems_fields); static std::unique_ptr<IDocsumFieldWriter> create(const std::string& input_field_name, uint32_t input_field_enum, - std::shared_ptr<StructFieldMapper> struct_field_mapper); + std::shared_ptr<MatchingElementsFields> matching_elems_fields); static std::unique_ptr<IDocsumFieldWriter> create(const std::string& input_field_name, uint32_t input_field_enum, search::attribute::IAttributeContext& attr_ctx, - std::shared_ptr<StructFieldMapper> struct_field_mapper); + std::shared_ptr<MatchingElementsFields> matching_elems_fields); ~MatchedElementsFilterDFW(); bool IsGenerated() const override { return false; } void insertField(uint32_t docid, GeneralResult* result, GetDocsumsState *state, diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 6b003553f49..ecdde13b919 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -30,7 +30,7 @@ uint64_t AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState *state) { search::common::Location &location = *state->_parsedLocation; - const IAttributeVector & attribute(vec(*state)); + const auto& attribute = get_attribute(*state); uint64_t absdist = std::numeric_limits<int64_t>::max(); int32_t docx = 0; @@ -221,10 +221,10 @@ void PositionsDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType type, vespalib::slime::Inserter &target) { if (type == RES_XMLSTRING) { - insertFromAttr(vec(*dsState), docid, target); + insertFromAttr(get_attribute(*dsState), docid, target); return; } - vespalib::asciistream val(formatField(vec(*dsState), docid, type)); + vespalib::asciistream val(formatField(get_attribute(*dsState), docid, type)); target.insertString(vespalib::Memory(val.c_str(), val.size())); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.cpp b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.cpp index c29f0324ce2..9caad947335 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.cpp @@ -2,7 +2,7 @@ #include "struct_fields_resolver.h" #include <vespa/searchcommon/attribute/iattributecontext.h> -#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <algorithm> #include <vespa/log/log.h> @@ -22,12 +22,14 @@ StructFieldsResolver::StructFieldsResolver(const vespalib::string& field_name, c _array_fields(), _array_attributes(), _has_map_key(false), + _has_map_value(false), _error(false) { std::vector<const search::attribute::IAttributeVector *> attrs; attr_ctx.getAttributeList(attrs); vespalib::string prefix = field_name + "."; _map_key_attribute = prefix + "key"; + vespalib::string map_value_attribute_name = prefix + "value"; vespalib::string value_prefix = prefix + "value."; for (const auto attr : attrs) { vespalib::string name = attr->getName(); @@ -45,6 +47,8 @@ StructFieldsResolver::StructFieldsResolver(const vespalib::string& field_name, c _array_fields.emplace_back(name.substr(prefix.size())); if (name == _map_key_attribute) { _has_map_key = true; + } else if (name == map_value_attribute_name) { + _has_map_value = true; } } } @@ -74,18 +78,18 @@ StructFieldsResolver::StructFieldsResolver(const vespalib::string& field_name, c StructFieldsResolver::~StructFieldsResolver() = default; void -StructFieldsResolver::apply_to(StructFieldMapper& mapper) const +StructFieldsResolver::apply_to(MatchingElementsFields& fields) const { if (is_map_of_struct()) { if (_has_map_key) { - mapper.add_mapping(_field_name, _map_key_attribute); + fields.add_mapping(_field_name, _map_key_attribute); } for (const auto& sub_field : _map_value_attributes) { - mapper.add_mapping(_field_name, sub_field); + fields.add_mapping(_field_name, sub_field); } } else { for (const auto& sub_field : _array_attributes) { - mapper.add_mapping(_field_name, sub_field); + fields.add_mapping(_field_name, sub_field); } } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.h b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.h index 66d9fd69db4..e1db36d8a43 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.h @@ -7,7 +7,7 @@ namespace search { namespace attribute { class IAttributeContext; } -class StructFieldMapper; +class MatchingElementsFields; } namespace search::docsummary { @@ -26,12 +26,18 @@ private: StringVector _array_fields; StringVector _array_attributes; bool _has_map_key; + bool _has_map_value; bool _error; public: StructFieldsResolver(const vespalib::string& field_name, const search::attribute::IAttributeContext& attr_ctx, bool require_all_struct_fields_as_attributes); ~StructFieldsResolver(); + bool is_map_of_scalar() const { return (_has_map_key && + _has_map_value && + (_array_fields.size() == 2u) && + _map_value_fields.empty()); + } bool is_map_of_struct() const { return !_map_value_fields.empty(); } const vespalib::string& get_map_key_attribute() const { return _map_key_attribute; } const StringVector& get_map_value_fields() const { return _map_value_fields; } @@ -39,7 +45,7 @@ public: const StringVector& get_array_fields() const { return _array_fields; } const StringVector& get_array_attributes() const { return _array_attributes; } bool has_error() const { return _error; } - void apply_to(StructFieldMapper& mapper) const; + void apply_to(MatchingElementsFields& fields) const; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp index d5922ddf46b..5344162b402 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp @@ -7,7 +7,7 @@ #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchlib/common/matching_elements.h> -#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/data/slime/cursor.h> #include <cassert> @@ -55,7 +55,7 @@ StructMapAttributeFieldWriterState::StructMapAttributeFieldWriterState(const ves { const IAttributeVector *attr = context.getAttribute(keyAttributeName); if (attr != nullptr) { - _keyWriter = AttributeFieldWriter::create(keyName, *attr); + _keyWriter = AttributeFieldWriter::create(keyName, *attr, true); } size_t fields = valueFieldNames.size(); _valueWriters.reserve(fields); @@ -124,14 +124,14 @@ StructMapAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime: StructMapAttributeCombinerDFW::StructMapAttributeCombinerDFW(const vespalib::string &fieldName, const StructFieldsResolver& fields_resolver, bool filter_elements, - std::shared_ptr<StructFieldMapper> struct_field_mapper) - : AttributeCombinerDFW(fieldName, filter_elements, std::move(struct_field_mapper)), + std::shared_ptr<MatchingElementsFields> matching_elems_fields) + : AttributeCombinerDFW(fieldName, filter_elements, std::move(matching_elems_fields)), _keyAttributeName(fields_resolver.get_map_key_attribute()), _valueFields(fields_resolver.get_map_value_fields()), _valueAttributeNames(fields_resolver.get_map_value_attributes()) { - if (filter_elements && _struct_field_mapper && !_struct_field_mapper->is_struct_field(fieldName)) { - fields_resolver.apply_to(*_struct_field_mapper); + if (filter_elements && _matching_elems_fields && !_matching_elems_fields->has_field(fieldName)) { + fields_resolver.apply_to(*_matching_elems_fields); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h index a28e487fb1c..c2fdbebc0b6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h @@ -26,7 +26,7 @@ public: StructMapAttributeCombinerDFW(const vespalib::string &fieldName, const StructFieldsResolver& fields_resolver, bool filter_elements, - std::shared_ptr<StructFieldMapper> struct_field_mapper); + std::shared_ptr<MatchingElementsFields> matching_elems_fields); ~StructMapAttributeCombinerDFW() override; }; diff --git a/searchsummary/src/vespa/searchsummary/test/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/test/CMakeLists.txt new file mode 100644 index 00000000000..e79a75de5a2 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/test/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(searchsummary_test + SOURCES + mock_attribute_manager.cpp + AFTER + searchsummary_config +) diff --git a/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.cpp b/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.cpp new file mode 100644 index 00000000000..bd7307d1624 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.cpp @@ -0,0 +1,72 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "mock_attribute_manager.h" +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/searchlib/attribute/floatbase.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/stringbase.h> +#include <cassert> + +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::attribute::Config; + +namespace search::docsummary::test { + +template <typename AttributeType, typename ValueType> +void +MockAttributeManager::build_attribute(const vespalib::string& name, BasicType type, + search::attribute::CollectionType col_type, + const std::vector<std::vector<ValueType>>& values) +{ + Config cfg(type, col_type); + auto attr_base = AttributeFactory::createAttribute(name, cfg); + assert(attr_base); + auto attr = std::dynamic_pointer_cast<AttributeType>(attr_base); + assert(attr); + attr->addReservedDoc(); + for (const auto& docValues : values) { + uint32_t docId = 0; + attr->addDoc(docId); + for (const auto& value : docValues) { + attr->append(docId, value, 1); + } + attr->commit(); + } + _mgr.add(attr); +} + +MockAttributeManager::MockAttributeManager() + : _mgr() +{ +} + +MockAttributeManager::~MockAttributeManager() = default; + +void +MockAttributeManager::build_string_attribute(const vespalib::string& name, + const std::vector<std::vector<vespalib::string>>& values, + CollectionType col_type) +{ + build_attribute<StringAttribute, vespalib::string>(name, BasicType::Type::STRING, col_type, values); +} + +void +MockAttributeManager::build_float_attribute(const vespalib::string& name, + const std::vector<std::vector<double>>& values, + CollectionType col_type) +{ + build_attribute<FloatingPointAttribute, double>(name, BasicType::Type::DOUBLE, col_type, values); +} + +void +MockAttributeManager::build_int_attribute(const vespalib::string& name, BasicType type, + const std::vector<std::vector<int64_t>>& values, + CollectionType col_type) +{ + build_attribute<IntegerAttribute, int64_t>(name, type, col_type, values); +} + +} diff --git a/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.h b/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.h new file mode 100644 index 00000000000..a7e425e50b6 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.h @@ -0,0 +1,37 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcommon/attribute/basictype.h> +#include <vespa/searchlib/attribute/attributemanager.h> + +namespace search::docsummary::test { + +/** + * Class used to build attributes and populate a manager for testing. + */ +class MockAttributeManager { +private: + AttributeManager _mgr; + + template <typename AttributeType, typename ValueType> + void build_attribute(const vespalib::string& name, search::attribute::BasicType type, + search::attribute::CollectionType col_type, + const std::vector<std::vector<ValueType>>& values); + +public: + MockAttributeManager(); + ~MockAttributeManager(); + AttributeManager& mgr() { return _mgr; } + + void build_string_attribute(const vespalib::string& name, + const std::vector<std::vector<vespalib::string>>& values, + search::attribute::CollectionType col_type = search::attribute::CollectionType::ARRAY); + void build_float_attribute(const vespalib::string& name, + const std::vector<std::vector<double>>& values, + search::attribute::CollectionType col_type = search::attribute::CollectionType::ARRAY); + void build_int_attribute(const vespalib::string& name, search::attribute::BasicType type, + const std::vector<std::vector<int64_t>>& values, + search::attribute::CollectionType col_type = search::attribute::CollectionType::ARRAY); + +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h b/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h new file mode 100644 index 00000000000..b3ee405c856 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchsummary/docsummary/docsumstate.h> + +namespace search::docsummary::test { + +class MockStateCallback : public GetDocsumsStateCallback { +private: + MatchingElements _matching_elems; + +public: + MockStateCallback() + : GetDocsumsStateCallback(), + _matching_elems() + { + } + ~MockStateCallback() override { } + void FillSummaryFeatures(GetDocsumsState*, IDocsumEnvironment*) override { } + void FillRankFeatures(GetDocsumsState*, IDocsumEnvironment*) override { } + void ParseLocation(GetDocsumsState*) override { } + std::unique_ptr<MatchingElements> fill_matching_elements(const search::MatchingElementsFields&) override { + return std::make_unique<MatchingElements>(_matching_elems); + } + + void add_matching_elements(uint32_t docid, const vespalib::string& field_name, + const std::vector<uint32_t>& elements) { + _matching_elems.add_matching_elements(docid, field_name, elements); + } + void clear() { + _matching_elems = MatchingElements(); + } +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/test/slime_value.h b/searchsummary/src/vespa/searchsummary/test/slime_value.h new file mode 100644 index 00000000000..3cc461d04ca --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/test/slime_value.h @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/data/slime/slime.h> +#include <cassert> + +namespace search::docsummary::test { + +/** + * Utility class that wraps a slime object generated from json. + */ +struct SlimeValue { + vespalib::Slime slime; + + SlimeValue(const vespalib::string& json_input) + : slime() + { + size_t used = vespalib::slime::JsonFormat::decode(json_input, slime); + assert(used > 0); + } +}; + +} diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateWithKey.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateWithKey.java new file mode 100644 index 00000000000..4772de5c1fb --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateWithKey.java @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security; + +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.Collections; +import java.util.List; + +/** + * Wraps a {@link java.security.cert.X509Certificate} with its {@link java.security.PrivateKey}. + * Primary motivation is APIs where the callee must correctly observe an atomic update of both certificate and key. + * + * @author bjorncs + */ +public class X509CertificateWithKey { + + private final List<X509Certificate> certificate; + private final PrivateKey privateKey; + + public X509CertificateWithKey(X509Certificate certificate, PrivateKey privateKey) { + this(Collections.singletonList(certificate), privateKey); + } + + public X509CertificateWithKey(List<X509Certificate> certificate, PrivateKey privateKey) { + if (certificate.isEmpty()) throw new IllegalArgumentException(); + this.certificate = certificate; + this.privateKey = privateKey; + } + + public X509Certificate certificate() { return certificate.get(0); } + public List<X509Certificate> certificateWithIntermediates() { return certificate; } + public PrivateKey privateKey() { return privateKey; } +} diff --git a/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java b/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java index 18764f51dc5..d4e74e22e40 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java @@ -5,19 +5,20 @@ import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.X509CertificateWithKey; import javax.net.ssl.SSLEngine; import javax.net.ssl.X509ExtendedKeyManager; import java.io.IOException; import java.io.UncheckedIOException; import java.net.Socket; -import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyStore; import java.security.Principal; import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.time.Duration; +import java.util.Arrays; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -59,6 +60,13 @@ public class AutoReloadingX509KeyManager extends X509ExtendedKeyManager implemen return new AutoReloadingX509KeyManager(privateKeyFile, certificatesFile); } + public X509CertificateWithKey getCurrentCertificateWithKey() { + X509ExtendedKeyManager manager = mutableX509KeyManager.currentManager(); + X509Certificate[] certificateChain = manager.getCertificateChain(CERTIFICATE_ALIAS); + PrivateKey privateKey = manager.getPrivateKey(CERTIFICATE_ALIAS); + return new X509CertificateWithKey(Arrays.asList(certificateChain), privateKey); + } + private static KeyStore createKeystore(Path privateKey, Path certificateChain) { try { return KeyStoreBuilder.withType(KeyStoreType.PKCS12) diff --git a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java index 2b001ca2ca0..83742950dbc 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java @@ -23,18 +23,22 @@ class TransportSecurityOptionsEntity { @JsonProperty("accepted-ciphers") @JsonInclude(NON_EMPTY) List<String> acceptedCiphers; @JsonProperty("disable-hostname-validation") @JsonInclude(NON_NULL) Boolean isHostnameValidationDisabled; + @JsonIgnoreProperties(ignoreUnknown = true) static class Files { @JsonProperty("private-key") String privateKeyFile; @JsonProperty("certificates") String certificatesFile; @JsonProperty("ca-certificates") String caCertificatesFile; } + @JsonIgnoreProperties(ignoreUnknown = true) static class AuthorizedPeer { @JsonProperty("required-credentials") List<RequiredCredential> requiredCredentials; @JsonProperty("name") String name; + @JsonProperty("description") @JsonInclude(NON_NULL) String description; @JsonProperty("roles") @JsonInclude(NON_EMPTY) List<String> roles; } + @JsonIgnoreProperties(ignoreUnknown = true) static class RequiredCredential { @JsonProperty("field") CredentialField field; @JsonProperty("must-match") String matchExpression; diff --git a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java index 3cba434912c..4f6d9264f51 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java @@ -96,7 +96,7 @@ public class TransportSecurityOptionsJsonSerializer { if (authorizedPeer.requiredCredentials == null) { throw missingFieldException("required-credentials"); } - return new PeerPolicy(authorizedPeer.name, toRoles(authorizedPeer.roles), toRequestPeerCredentials(authorizedPeer.requiredCredentials)); + return new PeerPolicy(authorizedPeer.name, authorizedPeer.description, toRoles(authorizedPeer.roles), toRequestPeerCredentials(authorizedPeer.requiredCredentials)); } private static Set<Role> toRoles(List<String> roles) { @@ -144,6 +144,7 @@ public class TransportSecurityOptionsJsonSerializer { AuthorizedPeer authorizedPeer = new AuthorizedPeer(); authorizedPeer.name = peerPolicy.policyName(); authorizedPeer.requiredCredentials = new ArrayList<>(); + authorizedPeer.description = peerPolicy.description().orElse(null); for (RequiredPeerCredential requiredPeerCredential : peerPolicy.requiredCredentials()) { RequiredCredential requiredCredential = new RequiredCredential(); requiredCredential.field = toField(requiredPeerCredential.field()); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/policy/PeerPolicy.java b/security-utils/src/main/java/com/yahoo/security/tls/policy/PeerPolicy.java index 294f8543f43..4e0a4815f79 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/policy/PeerPolicy.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/policy/PeerPolicy.java @@ -4,6 +4,7 @@ package com.yahoo.security.tls.policy; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.Set; /** @@ -12,11 +13,18 @@ import java.util.Set; public class PeerPolicy { private final String policyName; + private final String description; private final Set<Role> assumedRoles; private final List<RequiredPeerCredential> requiredCredentials; public PeerPolicy(String policyName, Set<Role> assumedRoles, List<RequiredPeerCredential> requiredCredentials) { + this(policyName, null, assumedRoles, requiredCredentials); + } + + public PeerPolicy( + String policyName, String description, Set<Role> assumedRoles, List<RequiredPeerCredential> requiredCredentials) { this.policyName = policyName; + this.description = description; this.assumedRoles = assumedRoles; this.requiredCredentials = Collections.unmodifiableList(requiredCredentials); } @@ -25,6 +33,8 @@ public class PeerPolicy { return policyName; } + public Optional<String> description() { return Optional.ofNullable(description); } + public Set<Role> assumedRoles() { return assumedRoles; } @@ -37,6 +47,7 @@ public class PeerPolicy { public String toString() { return "PeerPolicy{" + "policyName='" + policyName + '\'' + + ", description='" + description + '\'' + ", assumedRoles=" + assumedRoles + ", requiredCredentials=" + requiredCredentials + '}'; @@ -48,12 +59,13 @@ public class PeerPolicy { if (o == null || getClass() != o.getClass()) return false; PeerPolicy that = (PeerPolicy) o; return Objects.equals(policyName, that.policyName) && + Objects.equals(description, that.description) && Objects.equals(assumedRoles, that.assumedRoles) && Objects.equals(requiredCredentials, that.requiredCredentials); } @Override public int hashCode() { - return Objects.hash(policyName, assumedRoles, requiredCredentials); + return Objects.hash(policyName, description, assumedRoles, requiredCredentials); } } diff --git a/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java b/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java index 0dec75fa711..d996b21442a 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java @@ -47,7 +47,7 @@ public class TransportSecurityOptionsJsonSerializerTest { .withAuthorizedPeers( new AuthorizedPeers( new HashSet<>(Arrays.asList( - new PeerPolicy("cfgserver", singleton(new Role("myrole")), Arrays.asList( + new PeerPolicy("cfgserver", "cfgserver policy description", singleton(new Role("myrole")), Arrays.asList( new RequiredPeerCredential(CN, new HostGlobPattern("mycfgserver")), new RequiredPeerCredential(SAN_DNS, new HostGlobPattern("*.suffix.com")))), new PeerPolicy("node", singleton(new Role("anotherrole")), Collections.singletonList(new RequiredPeerCredential(CN, new HostGlobPattern("hostname")))))))) diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp index 86cb8bdf757..c3e5c4bdd15 100644 --- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp @@ -200,7 +200,7 @@ ProcessMemoryStats::create(uint64_t sizeEpsilon) i, (samples.rbegin()+1)->toString().c_str(), samples.back().toString().c_str()); } std::sort(samples.begin(), samples.end()); - LOG(warning, "We failed to find 2 consecutive samples that where similar with epsilon of %" PRIu64 ".\nSmallest is '%s',\n median is '%s',\n largest is '%s'", + LOG(info, "We failed to find 2 consecutive samples that where similar with epsilon of %" PRIu64 ".\nSmallest is '%s',\n median is '%s',\n largest is '%s'", sizeEpsilon, samples.front().toString().c_str(), samples[samples.size()/2].toString().c_str(), samples.back().toString().c_str()); return samples[samples.size()/2]; } diff --git a/standalone-container/vespa-standalone-container.spec b/standalone-container/vespa-standalone-container.spec index 532706eb36f..5624d9c02dc 100644 --- a/standalone-container/vespa-standalone-container.spec +++ b/standalone-container/vespa-standalone-container.spec @@ -72,9 +72,9 @@ cp vespajlib/target/vespajlib.jar "$jars_dir" # Copy from submodules, so must be done separately cp zookeeper-server/zookeeper-server-common/target/zookeeper-server-common-jar-with-dependencies.jar "$jars_dir" -cp zookeeper-server/zookeeper-server-3.5/target/zookeeper-server-3.5-jar-with-dependencies.jar "$jars_dir" +cp zookeeper-server/zookeeper-server-3.5.6/target/zookeeper-server-3.5.6-jar-with-dependencies.jar "$jars_dir" # Symlink to default version -ln -s zookeeper-server-3.5-jar-with-dependencies.jar "$jars_dir"/zookeeper-server-jar-with-dependencies.jar +ln -s zookeeper-server-3.5.6-jar-with-dependencies.jar "$jars_dir"/zookeeper-server-jar-with-dependencies.jar declare -a libexec_files=( standalone-container/src/main/sh/standalone-container.sh diff --git a/storage/src/tests/bucketdb/lockablemaptest.cpp b/storage/src/tests/bucketdb/lockablemaptest.cpp index a55e258129c..101b9d014fa 100644 --- a/storage/src/tests/bucketdb/lockablemaptest.cpp +++ b/storage/src/tests/bucketdb/lockablemaptest.cpp @@ -241,12 +241,12 @@ TEST(LockableMapTest, chunked_iteration_is_transparent_across_chunk_sizes) { map.insert(14, A(42, 0, 0), "foo", preExisted); NonConstProcessor ncproc; // Increments 2nd value in all entries. // chunkedAll with chunk size of 1 - map.chunkedAll(ncproc, "foo", 1); + map.chunkedAll(ncproc, "foo", 1us, 1); EXPECT_EQ(A(4, 7, 0), *map.get(11, "foo")); EXPECT_EQ(A(42, 1, 0), *map.get(14, "foo")); EXPECT_EQ(A(1, 3, 3), *map.get(16, "foo")); // chunkedAll with chunk size larger than db size - map.chunkedAll(ncproc, "foo", 100); + map.chunkedAll(ncproc, "foo", 1us, 100); EXPECT_EQ(A(4, 8, 0), *map.get(11, "foo")); EXPECT_EQ(A(42, 2, 0), *map.get(14, "foo")); EXPECT_EQ(A(1, 4, 3), *map.get(16, "foo")); @@ -263,7 +263,7 @@ TEST(LockableMapTest, can_abort_during_chunked_iteration) { decisions.push_back(Map::CONTINUE); decisions.push_back(Map::ABORT); EntryProcessor proc(decisions); - map.chunkedAll(proc, "foo", 100); + map.chunkedAll(proc, "foo", 1us, 100); std::string expected("11 - A(4, 6, 0)\n" "14 - A(42, 0, 0)\n"); EXPECT_EQ(expected, proc.toString()); @@ -641,155 +641,6 @@ TEST(LockableMapTest, find_all_inconsistent_below_16_bits) { EXPECT_EQ(A(3,4,5), *results[id3.stripUnused()]); // sub bucket } -TEST(LockableMapTest, create) { - Map map; - { - document::BucketId id1(58, 0x43d6c878000004d2ull); - - auto entries = map.getContained(id1, "foo"); - - EXPECT_EQ(0, entries.size()); - - Map::WrappedEntry entry = map.createAppropriateBucket(36, "", id1); - EXPECT_EQ(document::BucketId(36,0x8000004d2ull), entry.getBucketId()); - } - { - document::BucketId id1(58, 0x423bf1e0000004d2ull); - - auto entries = map.getContained(id1, "foo"); - EXPECT_EQ(0, entries.size()); - - Map::WrappedEntry entry = map.createAppropriateBucket(36, "", id1); - EXPECT_EQ(document::BucketId(36,0x0000004d2ull), entry.getBucketId()); - } - - EXPECT_EQ(2, map.size()); -} - -TEST(LockableMapTest, create_2) { - Map map; - { - document::BucketId id1(58, 0xeaf77782000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(58, 0x00000000000004d2); - auto entries = map.getContained(id1, "foo"); - - EXPECT_EQ(0, entries.size()); - - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - - EXPECT_EQ(document::BucketId(34, 0x0000004d2ull), entry.getBucketId()); - } - - EXPECT_EQ(2, map.size()); -} - -TEST(LockableMapTest, create_3) { - Map map; - { - document::BucketId id1(58, 0xeaf77780000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(58, 0xeaf77782000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(58, 0x00000000000004d2); - auto entries = map.getContained(id1, "foo"); - - EXPECT_EQ(0, entries.size()); - - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - EXPECT_EQ(document::BucketId(40, 0x0000004d2ull), entry.getBucketId()); - } -} - -TEST(LockableMapTest, create_4) { - Map map; - { - document::BucketId id1(16, 0x00000000000004d1); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(40, 0x00000000000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(58, 0x00000000010004d2); - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - - EXPECT_EQ(document::BucketId(25, 0x0010004d2ull), entry.getBucketId()); - } -} - -TEST(LockableMapTest, create_5) { - Map map; - { - document::BucketId id1(0x8c000000000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - - { - document::BucketId id1(0xeb54b3ac000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - - { - document::BucketId id1(0x88000002000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(0x84000001000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(0xe9944a44000004d2); - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - EXPECT_EQ(document::BucketId(0x90000004000004d2), entry.getBucketId()); - } -} - -TEST(LockableMapTest, create_6) { - Map map; - { - document::BucketId id1(58, 0xeaf77780000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(40, 0x00000000000004d1); - - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(58, 0x00000000010004d2); - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - EXPECT_EQ(document::BucketId(25, 0x0010004d2ull), entry.getBucketId()); - } -} - -TEST(LockableMapTest, create_empty) { - Map map; - { - document::BucketId id1(58, 0x00000000010004d2); - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - EXPECT_EQ(document::BucketId(16, 0x0000004d2ull), entry.getBucketId()); - } -} - TEST(LockableMapTest, is_consistent) { Map map; document::BucketId id1(16, 0x00001); // contains id2-id3 diff --git a/storage/src/tests/distributor/getoperationtest.cpp b/storage/src/tests/distributor/getoperationtest.cpp index f14b78094d1..d4d14314790 100644 --- a/storage/src/tests/distributor/getoperationtest.cpp +++ b/storage/src/tests/distributor/getoperationtest.cpp @@ -66,7 +66,9 @@ struct GetOperationTest : Test, DistributorTestUtil { void sendReply(uint32_t idx, api::ReturnCode::Result result, - std::string authorVal, uint32_t timestamp) + std::string authorVal, + uint32_t timestamp, + bool is_tombstone = false) { if (idx == LastCommand) { idx = _sender.commands().size() - 1; @@ -75,7 +77,8 @@ struct GetOperationTest : Test, DistributorTestUtil { std::shared_ptr<api::StorageCommand> msg2 = _sender.command(idx); ASSERT_EQ(api::MessageType::GET, msg2->getType()); - auto* tmp = static_cast<api::GetCommand*>(msg2.get()); + auto* tmp = dynamic_cast<api::GetCommand*>(msg2.get()); + assert(tmp != nullptr); document::Document::SP doc; if (!authorVal.empty()) { @@ -86,12 +89,16 @@ struct GetOperationTest : Test, DistributorTestUtil { document::StringFieldValue(authorVal)); } - auto reply = std::make_shared<api::GetReply>(*tmp, doc, timestamp); + auto reply = std::make_shared<api::GetReply>(*tmp, doc, timestamp, false, is_tombstone); reply->setResult(result); op->receive(_sender, reply); } + void reply_with_tombstone(uint32_t idx, uint32_t tombstone_ts) { + sendReply(idx, api::ReturnCode::OK, "", tombstone_ts, true); + } + void replyWithFailure() { sendReply(LastCommand, api::ReturnCode::IO_FAILURE, "", 0); } @@ -126,6 +133,13 @@ struct GetOperationTest : Test, DistributorTestUtil { return dynamic_cast<api::GetReply&>(msg).had_consistent_replicas(); } + bool last_reply_has_document() { + assert(!_sender.replies().empty()); + auto& msg = *_sender.replies().back(); + assert(msg.getType() == api::MessageType::GET_REPLY); + return (dynamic_cast<api::GetReply&>(msg).getDocument().get() != nullptr); + } + void setClusterState(const std::string& clusterState) { enableDistributorClusterState(clusterState); } @@ -138,8 +152,8 @@ GetOperationTest::~GetOperationTest() = default; namespace { -NewestReplica replica_of(api::Timestamp ts, const document::BucketId& bucket_id, uint16_t node) { - return NewestReplica::of(ts, bucket_id, node); +NewestReplica replica_of(api::Timestamp ts, const document::BucketId& bucket_id, uint16_t node, bool is_tombstone) { + return NewestReplica::of(ts, bucket_id, node, is_tombstone); } } @@ -161,7 +175,7 @@ TEST_F(GetOperationTest, simple) { EXPECT_FALSE(op->any_replicas_failed()); EXPECT_TRUE(last_reply_had_consistent_replicas()); ASSERT_TRUE(op->newest_replica().has_value()); - EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 0), *op->newest_replica()); + EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 0, false), *op->newest_replica()); } TEST_F(GetOperationTest, ask_all_checksum_groups_if_inconsistent_even_if_trusted_replica_available) { @@ -182,7 +196,7 @@ TEST_F(GetOperationTest, ask_all_checksum_groups_if_inconsistent_even_if_trusted EXPECT_FALSE(op->any_replicas_failed()); EXPECT_FALSE(last_reply_had_consistent_replicas()); ASSERT_TRUE(op->newest_replica().has_value()); - EXPECT_EQ(replica_of(api::Timestamp(2), bucketId, 0), *op->newest_replica()); + EXPECT_EQ(replica_of(api::Timestamp(2), bucketId, 0, false), *op->newest_replica()); } TEST_F(GetOperationTest, ask_all_nodes_if_bucket_is_inconsistent) { @@ -205,7 +219,7 @@ TEST_F(GetOperationTest, ask_all_nodes_if_bucket_is_inconsistent) { EXPECT_FALSE(op->any_replicas_failed()); EXPECT_FALSE(last_reply_had_consistent_replicas()); ASSERT_TRUE(op->newest_replica().has_value()); - EXPECT_EQ(replica_of(api::Timestamp(2), bucketId, 1), *op->newest_replica()); + EXPECT_EQ(replica_of(api::Timestamp(2), bucketId, 1, false), *op->newest_replica()); } TEST_F(GetOperationTest, send_to_all_invalid_copies) { @@ -273,7 +287,7 @@ TEST_F(GetOperationTest, inconsistent_split) { EXPECT_FALSE(last_reply_had_consistent_replicas()); // Bucket with highest timestamp should be returned. In this case it's the one on node 0. ASSERT_TRUE(op->newest_replica().has_value()); - EXPECT_EQ(replica_of(api::Timestamp(2), BucketId(16, 0x0593), 0), *op->newest_replica()); + EXPECT_EQ(replica_of(api::Timestamp(2), BucketId(16, 0x0593), 0, false), *op->newest_replica()); } TEST_F(GetOperationTest, multi_inconsistent_bucket_not_found) { @@ -317,7 +331,7 @@ TEST_F(GetOperationTest, multi_inconsistent_bucket_not_found_deleted) { EXPECT_FALSE(op->any_replicas_failed()); EXPECT_FALSE(last_reply_had_consistent_replicas()); ASSERT_TRUE(op->newest_replica().has_value()); - EXPECT_EQ(replica_of(api::Timestamp(3), bucketId, 1), *op->newest_replica()); + EXPECT_EQ(replica_of(api::Timestamp(3), bucketId, 1, false), *op->newest_replica()); } TEST_F(GetOperationTest, multi_inconsistent_bucket) { @@ -367,7 +381,7 @@ TEST_F(GetOperationTest, multi_inconsistent_bucket_fail) { EXPECT_FALSE(last_reply_had_consistent_replicas()); ASSERT_TRUE(op->newest_replica().has_value()); // First send to node 2 fails, second is to node 3 which returned the highest timestamp - EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 3), *op->newest_replica()); + EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 3, false), *op->newest_replica()); } TEST_F(GetOperationTest, return_not_found_when_bucket_not_in_db) { @@ -408,7 +422,7 @@ TEST_F(GetOperationTest, not_found) { // the caller may want to perform special logic if all replicas are in sync // but are missing the document. // FIXME make sure all callers are aware of this! - EXPECT_EQ(replica_of(api::Timestamp(0), bucketId, 0), *op->newest_replica()); + EXPECT_EQ(replica_of(api::Timestamp(0), bucketId, 0, false), *op->newest_replica()); } TEST_F(GetOperationTest, not_found_on_subset_of_replicas_marks_get_as_inconsistent) { @@ -453,7 +467,7 @@ TEST_F(GetOperationTest, resend_on_storage_failure) { // Replica had read failure, but they're still in sync. An immutable Get won't change that fact. EXPECT_TRUE(last_reply_had_consistent_replicas()); ASSERT_TRUE(op->newest_replica().has_value()); - EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 2), *op->newest_replica()); + EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 2, false), *op->newest_replica()); } TEST_F(GetOperationTest, storage_failure_of_out_of_sync_replica_is_tracked_as_inconsistent) { @@ -470,7 +484,7 @@ TEST_F(GetOperationTest, storage_failure_of_out_of_sync_replica_is_tracked_as_in EXPECT_TRUE(op->any_replicas_failed()); EXPECT_FALSE(last_reply_had_consistent_replicas()); ASSERT_TRUE(op->newest_replica().has_value()); - EXPECT_EQ(replica_of(api::Timestamp(3), bucketId, 2), *op->newest_replica()); + EXPECT_EQ(replica_of(api::Timestamp(3), bucketId, 2, false), *op->newest_replica()); } TEST_F(GetOperationTest, resend_on_storage_failure_all_fail) { @@ -519,7 +533,7 @@ TEST_F(GetOperationTest, send_to_ideal_copy_if_bucket_in_sync) { _sender.getLastReply()); EXPECT_TRUE(last_reply_had_consistent_replicas()); ASSERT_TRUE(op->newest_replica().has_value()); - EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 1), *op->newest_replica()); + EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 1, false), *op->newest_replica()); } TEST_F(GetOperationTest, multiple_copies_with_failure_on_local_node) { @@ -550,7 +564,7 @@ TEST_F(GetOperationTest, multiple_copies_with_failure_on_local_node) { EXPECT_TRUE(op->any_replicas_failed()); EXPECT_TRUE(last_reply_had_consistent_replicas()); ASSERT_TRUE(op->newest_replica().has_value()); - EXPECT_EQ(replica_of(api::Timestamp(3), BucketId(16, 0x0593), 2), *op->newest_replica()); + EXPECT_EQ(replica_of(api::Timestamp(3), BucketId(16, 0x0593), 2, false), *op->newest_replica()); } TEST_F(GetOperationTest, can_get_documents_when_all_replica_nodes_retired) { @@ -580,4 +594,58 @@ TEST_F(GetOperationTest, can_send_gets_with_weak_internal_read_consistency) { do_test_read_consistency_is_propagated(api::InternalReadConsistency::Weak); } +TEST_F(GetOperationTest, replicas_considered_consistent_if_all_equal_tombstone_timestamps) { + setClusterState("distributor:1 storage:4"); + addNodesToBucketDB(bucketId, "0=100,2=100,1=200,3=200"); + sendGet(); + ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true)); + + ASSERT_NO_FATAL_FAILURE(reply_with_tombstone(0, 100)); + ASSERT_NO_FATAL_FAILURE(reply_with_tombstone(1, 100)); + + ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, timestamp 0) ReturnCode(NONE)", + _sender.getLastReply()); + + EXPECT_FALSE(op->any_replicas_failed()); + EXPECT_TRUE(last_reply_had_consistent_replicas()); + EXPECT_FALSE(last_reply_has_document()); + EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 0, true), *op->newest_replica()); +} + +TEST_F(GetOperationTest, newer_tombstone_hides_older_document) { + setClusterState("distributor:1 storage:4"); + addNodesToBucketDB(bucketId, "0=100,2=100,1=200,3=200"); + sendGet(); + ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true)); + + ASSERT_NO_FATAL_FAILURE(reply_with_tombstone(1, 200)); + ASSERT_NO_FATAL_FAILURE(sendReply(0, api::ReturnCode::OK, "newauthor", 100)); + + ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, timestamp 0) ReturnCode(NONE)", + _sender.getLastReply()); + + EXPECT_FALSE(op->any_replicas_failed()); + EXPECT_FALSE(last_reply_had_consistent_replicas()); + EXPECT_FALSE(last_reply_has_document()); + EXPECT_EQ(replica_of(api::Timestamp(200), bucketId, 1, true), *op->newest_replica()); +} + +TEST_F(GetOperationTest, older_tombstone_does_not_hide_newer_document) { + setClusterState("distributor:1 storage:4"); + addNodesToBucketDB(bucketId, "0=100,2=100,1=200,3=200"); + sendGet(); + ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true)); + + ASSERT_NO_FATAL_FAILURE(reply_with_tombstone(1, 100)); + ASSERT_NO_FATAL_FAILURE(sendReply(0, api::ReturnCode::OK, "newauthor", 200)); + + ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, timestamp 200) ReturnCode(NONE)", + _sender.getLastReply()); + + EXPECT_FALSE(op->any_replicas_failed()); + EXPECT_FALSE(last_reply_had_consistent_replicas()); + EXPECT_TRUE(last_reply_has_document()); + EXPECT_EQ(replica_of(api::Timestamp(200), bucketId, 0, false), *op->newest_replica()); +} + } diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp index 962ce085cb0..e42e7684f81 100644 --- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -90,6 +90,12 @@ struct TwoPhaseUpdateOperationTest : Test, DistributorTestUtil { api::ReturnCode::Result result = api::ReturnCode::OK, const std::string& trace_msg = ""); + void reply_to_get_with_tombstone( + Operation& callback, + DistributorMessageSenderStub& sender, + uint32_t index, + uint64_t old_timestamp); + struct UpdateOptions { bool _makeInconsistentSplit; bool _createIfNonExistent; @@ -253,6 +259,18 @@ TwoPhaseUpdateOperationTest::reply_to_metadata_get( callback.receive(sender, reply); } +void +TwoPhaseUpdateOperationTest::reply_to_get_with_tombstone( + Operation& callback, + DistributorMessageSenderStub& sender, + uint32_t index, + uint64_t old_timestamp) +{ + auto& get = dynamic_cast<const api::GetCommand&>(*sender.command(index)); + auto reply = std::make_shared<api::GetReply>(get, std::shared_ptr<Document>(), old_timestamp, false, true); + callback.receive(sender, reply); +} + namespace { struct DummyTransportContext : api::TransportContext { @@ -1317,6 +1335,45 @@ TEST_F(ThreePhaseUpdateTest, single_full_get_reply_received_after_close_is_no_op ASSERT_EQ("", _sender.getCommands(true, false, 3)); // Nothing new sent. } +TEST_F(ThreePhaseUpdateTest, single_full_get_tombstone_is_no_op_without_auto_create) { + setupDistributor(2, 2, "storage:2 distributor:1"); + getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); + getConfig().set_update_fast_path_restart_enabled(true); + auto cb = sendUpdate("0=1/2/3,1=2/3/4"); + cb->start(_sender, framework::MilliSecTime(0)); + + ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true)); + reply_to_metadata_get(*cb, _sender, 0, 1000U); + reply_to_metadata_get(*cb, _sender, 1, 2000U); + ASSERT_EQ("Get => 1", _sender.getCommands(true, false, 2)); + reply_to_get_with_tombstone(*cb, _sender, 2, 2000U); + // No puts should be sent, as Get returned a tombstone and no auto-create is set. + ASSERT_EQ("", _sender.getCommands(true, false, 3)); + // Nothing was updated. + EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 0) " + "ReturnCode(NONE)", + _sender.getLastReply(true)); +} + +TEST_F(ThreePhaseUpdateTest, single_full_get_tombstone_sends_puts_with_auto_create) { + setupDistributor(2, 2, "storage:2 distributor:1"); + getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true); + getConfig().set_update_fast_path_restart_enabled(true); + auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().createIfNonExistent(true)); + cb->start(_sender, framework::MilliSecTime(0)); + + ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true)); + reply_to_metadata_get(*cb, _sender, 0, 1000U); + reply_to_metadata_get(*cb, _sender, 1, 2000U); + ASSERT_EQ("Get => 1", _sender.getCommands(true, false, 2)); + reply_to_get_with_tombstone(*cb, _sender, 2, 2000U); + // Tombstone is treated as Not Found in this case, which auto-creates a new + // document version locally and pushes it out with Puts as expected. + ASSERT_EQ("Put => 1,Put => 0", _sender.getCommands(true, false, 3)); +} + // XXX currently differs in behavior from content nodes in that updates for // document IDs without explicit doctypes will _not_ be auto-failed on the // distributor. diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.h b/storage/src/vespa/storage/bucketdb/lockablemap.h index 2b2c2cbe7a8..8b4e403b899 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.h +++ b/storage/src/vespa/storage/bucketdb/lockablemap.h @@ -19,6 +19,7 @@ #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/document/bucket/bucketid.h> +#include <vespa/vespalib/util/time.h> #include <mutex> #include <condition_variable> #include <cassert> @@ -162,7 +163,7 @@ public: const key_type& first = key_type(), const key_type& last = key_type() - 1 ); - static constexpr uint32_t DEFAULT_CHUNK_SIZE = 10000; + static constexpr uint32_t DEFAULT_CHUNK_SIZE = 1000; /** * Iterate over the entire database contents, holding the global database @@ -173,6 +174,7 @@ public: template <typename Functor> void chunkedAll(Functor& functor, const char* clientId, + vespalib::duration yieldTime = 10us, uint32_t chunkSize = DEFAULT_CHUNK_SIZE); void print(std::ostream& out, bool verbose, const std::string& indent) const override; @@ -185,11 +187,6 @@ public: std::map<BucketId, WrappedEntry> getContained(const BucketId& bucketId, const char* clientId); - WrappedEntry - createAppropriateBucket(uint16_t newBucketBits, - const char* clientId, - const BucketId& bucket); - typedef std::map<BucketId, WrappedEntry> EntryMap; /** diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.hpp b/storage/src/vespa/storage/bucketdb/lockablemap.hpp index 3cef17c9025..2ca2183ae26 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.hpp +++ b/storage/src/vespa/storage/bucketdb/lockablemap.hpp @@ -405,6 +405,7 @@ template <typename Functor> void LockableMap<Map>::chunkedAll(Functor& functor, const char* clientId, + vespalib::duration yieldTime, uint32_t chunkSize) { key_type key{}; @@ -416,7 +417,7 @@ LockableMap<Map>::chunkedAll(Functor& functor, // This is a pragmatic stop-gap solution; a more robust change requires // the redesign of bucket DB locking and signalling semantics in the // face of blocked point lookups. - std::this_thread::sleep_for(std::chrono::microseconds(100)); + std::this_thread::sleep_for(yieldTime); } } @@ -590,40 +591,6 @@ LockableMap<Map>::addAndLockResults( uint8_t getMinDiffBits(uint16_t minBits, const document::BucketId& a, const document::BucketId& b); template<typename Map> -typename LockableMap<Map>::WrappedEntry -LockableMap<Map>::createAppropriateBucket( - uint16_t newBucketBits, - const char* clientId, - const BucketId& bucket) -{ - std::unique_lock<std::mutex> guard(_lock); - typename Map::const_iterator iter = _map.lower_bound(bucket.toKey()); - - // Find the two buckets around the possible new bucket. The new - // bucket's used bits should be the highest used bits it can be while - // still being different from both of these. - if (iter != _map.end()) { - newBucketBits = getMinDiffBits(newBucketBits, BucketId(BucketId::keyToBucketId(iter->first)), bucket); - } - - if (iter != _map.begin()) { - --iter; - newBucketBits = getMinDiffBits(newBucketBits, BucketId(BucketId::keyToBucketId(iter->first)), bucket); - } - - BucketId newBucket(newBucketBits, bucket.getRawId()); - newBucket.setUsedBits(newBucketBits); - BucketId::Type key = newBucket.stripUnused().toKey(); - - LockId lid(key, clientId); - acquireKey(lid, guard); - bool preExisted; - typename Map::iterator it = _map.find(key, true, preExisted); - _lockedKeys.insert(LockId(key, clientId)); - return WrappedEntry(*this, key, it->second, clientId, preExisted); -} - -template<typename Map> std::map<document::BucketId, typename LockableMap<Map>::WrappedEntry> LockableMap<Map>::getContained(const BucketId& bucket, const char* clientId) diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp index b59405a73cd..bbe477be9ef 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp @@ -168,8 +168,9 @@ GetOperation::onReceive(DistributorMessageSender& sender, const std::shared_ptr< if (!_newest_replica.has_value() || getreply->getLastModifiedTimestamp() > _newest_replica->timestamp) { _returnCode = getreply->getResult(); assert(response.second[i].to_node != UINT16_MAX); - _newest_replica = NewestReplica::of(getreply->getLastModifiedTimestamp(), bucket_id, send_state.to_node); - _doc = getreply->getDocument(); + _newest_replica = NewestReplica::of(getreply->getLastModifiedTimestamp(), bucket_id, + send_state.to_node, getreply->is_tombstone()); + _doc = getreply->getDocument(); // May be empty (tombstones). } } else { _any_replicas_failed = true; @@ -228,7 +229,12 @@ void GetOperation::sendReply(DistributorMessageSender& sender) { if (_msg.get()) { - const auto timestamp = _newest_replica.value_or(NewestReplica::make_empty()).timestamp; + const auto newest = _newest_replica.value_or(NewestReplica::make_empty()); + // If the newest entry is a tombstone (remove entry), the externally visible + // behavior is as if the document was not found. In this case _doc will also + // be empty. This means we also currently don't propagate tombstone status outside + // of this operation (except via the newest_replica() functionality). + const auto timestamp = (newest.is_tombstone ? api::Timestamp(0) : newest.timestamp); auto repl = std::make_shared<api::GetReply>(*_msg, _doc, timestamp, !_has_replica_inconsistency); repl->setResult(_returnCode); update_internal_metrics(); diff --git a/storage/src/vespa/storage/distributor/operations/external/newest_replica.cpp b/storage/src/vespa/storage/distributor/operations/external/newest_replica.cpp index 8ca3b9bf411..2520db3e57a 100644 --- a/storage/src/vespa/storage/distributor/operations/external/newest_replica.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/newest_replica.cpp @@ -7,7 +7,9 @@ namespace storage::distributor { std::ostream& operator<<(std::ostream& os, const NewestReplica& nr) { os << "NewestReplica(timestamp " << nr.timestamp << ", bucket_id " << nr.bucket_id - << ", node " << nr.node << ')'; + << ", node " << nr.node + << ", is_tombstone " << nr.is_tombstone + << ')'; return os; } diff --git a/storage/src/vespa/storage/distributor/operations/external/newest_replica.h b/storage/src/vespa/storage/distributor/operations/external/newest_replica.h index 9eb9c1b8bd0..ec2b73e52ab 100644 --- a/storage/src/vespa/storage/distributor/operations/external/newest_replica.h +++ b/storage/src/vespa/storage/distributor/operations/external/newest_replica.h @@ -17,21 +17,24 @@ struct NewestReplica { api::Timestamp timestamp {0}; document::BucketId bucket_id; uint16_t node {UINT16_MAX}; + bool is_tombstone {false}; static NewestReplica of(api::Timestamp timestamp, const document::BucketId& bucket_id, - uint16_t node) noexcept { - return {timestamp, bucket_id, node}; + uint16_t node, + bool is_tombstone) noexcept { + return {timestamp, bucket_id, node, is_tombstone}; } static NewestReplica make_empty() { - return {api::Timestamp(0), document::BucketId(), 0}; + return {api::Timestamp(0), document::BucketId(), 0, false}; } bool operator==(const NewestReplica& rhs) const noexcept { return ((timestamp == rhs.timestamp) && (bucket_id == rhs.bucket_id) && - (node == rhs.node)); + (node == rhs.node) && + (is_tombstone == rhs.is_tombstone)); } bool operator!=(const NewestReplica& rhs) const noexcept { return !(*this == rhs); diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 2e6fe4020c2..43cf43b02b6 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -463,6 +463,9 @@ void TwoPhaseUpdateOperation::handle_safe_path_received_metadata_get( // Timestamps were not in sync, so we have to fetch the document from the highest // timestamped replica, apply the update to it and then explicitly Put the result // to all replicas. + // Note that this timestamp may be for a tombstone (remove) entry, in which case + // conditional create-if-missing behavior kicks in as usual. + // TODO avoid sending the Get at all if the newest replica is marked as a tombstone. _single_get_latency_timer.emplace(_manager.getClock()); document::Bucket bucket(_updateCmd->getBucket().getBucketSpace(), newest_replica->bucket_id); LOG(debug, "Update(%s): sending single payload Get to %s on node %u (had timestamp %" PRIu64 ")", diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 5627ade2a11..fbdbac27b7c 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -14,6 +14,10 @@ #include <vespa/storageapi/message/stat.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> +#ifndef XXH_INLINE_ALL +# define XXH_INLINE_ALL // Let XXH64 be inlined for fixed hash size (bucket ID) +#endif +#include <xxhash.h> #include <vespa/log/log.h> LOG_SETUP(".persistence.filestor.handler.impl"); @@ -894,6 +898,11 @@ FileStorHandlerImpl::Disk::broadcast() } } +uint64_t FileStorHandlerImpl::Disk::dispersed_bucket_bits(const document::Bucket& bucket) noexcept { + const uint64_t raw_id = bucket.getBucketId().getId(); + return XXH64(&raw_id, sizeof(uint64_t), 0); +} + bool FileStorHandlerImpl::Disk::schedule(const std::shared_ptr<api::StorageMessage>& msg) { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index 7a4f9000e82..00714c291b7 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -197,12 +197,7 @@ public: std::string dumpQueue() const; void dumpActiveHtml(std::ostream & os) const; void dumpQueueHtml(std::ostream & os) const; - static uint64_t dispersed_bucket_bits(const document::Bucket& bucket) noexcept { - // Disperse bucket bits by multiplying with the 64-bit FNV-1 prime. - // This avoids an inherent affinity between the LSB of a bucket's bits - // and the stripe an operation ends up on. - return bucket.getBucketId().getId() * 1099511628211ULL; - } + static uint64_t dispersed_bucket_bits(const document::Bucket& bucket) noexcept; // We make a fairly reasonable assumption that there will be less than 64k stripes. uint16_t stripe_index(const document::Bucket& bucket) const noexcept { return static_cast<uint16_t>(dispersed_bucket_bits(bucket) % _stripes.size()); diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index c53367c16f6..76c241d0627 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -295,7 +295,8 @@ PersistenceThread::handleGet(api::GetCommand& cmd, MessageTracker::UP tracker) if (!result.hasDocument()) { _env._metrics.get[cmd.getLoadType()].notFound.inc(); } - tracker->setReply(std::make_shared<api::GetReply>(cmd, result.getDocumentPtr(), result.getTimestamp())); + tracker->setReply(std::make_shared<api::GetReply>(cmd, result.getDocumentPtr(), result.getTimestamp(), + false, result.is_tombstone())); } return tracker; @@ -906,7 +907,9 @@ PersistenceThread::processLockedMessage(FileStorHandler::LockedMessage lock) { LOG(debug, "Partition %d, nodeIndex %d, ptr=%p", _env._partition, _env._nodeIndex, lock.second.get()); api::StorageMessage & msg(*lock.second); - auto tracker = std::make_unique<MessageTracker>(_env, _env._fileStorHandler, std::move(lock.first), std::move(lock.second)); + // Important: we _copy_ the message shared_ptr instead of moving to ensure that `msg` remains + // valid even if the tracker is destroyed by an exception in processMessage(). + auto tracker = std::make_unique<MessageTracker>(_env, _env._fileStorHandler, std::move(lock.first), lock.second); tracker = processMessage(msg, std::move(tracker)); if (tracker) { tracker->sendReply(); diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index 069ba1b8b92..da221e312e2 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -480,10 +480,9 @@ StateManager::onGetNodeState(const api::GetNodeStateCommand::SP& cmd) : cmd->getExpectedState()->toString().c_str(), _nodeState->toString().c_str()); reply = std::make_shared<api::GetNodeStateReply>(*cmd, *_nodeState); + mark_controller_as_having_observed_explicit_node_state(lock, cmd->getSourceIndex()); lock.unlock(); - std::string nodeInfo(getNodeInfo()); - reply->setNodeInfo(nodeInfo); - mark_controller_as_having_observed_explicit_node_state(cmd->getSourceIndex()); + reply->setNodeInfo(getNodeInfo()); } } if (reply) { @@ -492,7 +491,7 @@ StateManager::onGetNodeState(const api::GetNodeStateCommand::SP& cmd) return true; } -void StateManager::mark_controller_as_having_observed_explicit_node_state(uint16_t controller_index) { +void StateManager::mark_controller_as_having_observed_explicit_node_state(const vespalib::LockGuard &, uint16_t controller_index) { _controllers_observed_explicit_node_state.emplace(controller_index); } @@ -557,7 +556,7 @@ StateManager::sendGetNodeStateReplies(framework::MilliSecTime olderThanTime, { std::vector<std::shared_ptr<api::GetNodeStateReply>> replies; { - vespalib::MonitorGuard guard(_stateLock); + vespalib::LockGuard guard(_stateLock); for (auto it = _queuedStateRequests.begin(); it != _queuedStateRequests.end();) { if (node != 0xffff && node != it->second->getSourceIndex()) { ++it; @@ -567,7 +566,7 @@ StateManager::sendGetNodeStateReplies(framework::MilliSecTime olderThanTime, replies.emplace_back(std::make_shared<api::GetNodeStateReply>(*it->second, *_nodeState)); auto eraseIt = it++; - mark_controller_as_having_observed_explicit_node_state(eraseIt->second->getSourceIndex()); + mark_controller_as_having_observed_explicit_node_state(guard, eraseIt->second->getSourceIndex()); _queuedStateRequests.erase(eraseIt); } else { ++it; diff --git a/storage/src/vespa/storage/storageserver/statemanager.h b/storage/src/vespa/storage/storageserver/statemanager.h index 57f0e02a136..3d231c070ef 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.h +++ b/storage/src/vespa/storage/storageserver/statemanager.h @@ -106,7 +106,7 @@ private: bool sendGetNodeStateReplies( framework::MilliSecTime olderThanTime = framework::MilliSecTime(0), uint16_t index = 0xffff); - void mark_controller_as_having_observed_explicit_node_state(uint16_t controller_index); + void mark_controller_as_having_observed_explicit_node_state(const vespalib::LockGuard &, uint16_t controller_index); lib::Node thisNode() const; diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index bdd066e8a4a..c9cda047784 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -14,7 +14,7 @@ #include <sstream> #include <vespa/log/log.h> -LOG_SETUP(".visitor.instance"); +LOG_SETUP(".visitor.instance.visitor"); using document::BucketSpace; @@ -367,8 +367,7 @@ Visitor::sendReplyOnce() { assert(_initiatingCmd.get()); if (!_hasSentReply) { - std::shared_ptr<api::StorageReply> reply( - _initiatingCmd->makeReply().release()); + std::shared_ptr<api::StorageReply> reply(_initiatingCmd->makeReply()); _hitCounter->updateVisitorStatistics(_visitorStatistics); static_cast<api::CreateVisitorReply*>(reply.get()) @@ -563,8 +562,7 @@ Visitor::attach(std::shared_ptr<api::StorageCommand> initiatingCmd, _priority = initiatingCmd->getPriority(); _timeToDie = _component.getClock().getTimeInMicros() + timeout.getMicros(); if (_initiatingCmd.get()) { - std::shared_ptr<api::StorageReply> reply( - _initiatingCmd->makeReply().release()); + std::shared_ptr<api::StorageReply> reply(_initiatingCmd->makeReply()); reply->setResult(api::ReturnCode::ABORTED); _messageHandler->send(reply); } @@ -594,7 +592,7 @@ Visitor::attach(std::shared_ptr<api::StorageCommand> initiatingCmd, // In case there was no messages to resend we need to call // continueVisitor to provoke it to resume. - for (uint32_t i=0; i<_visitorOptions._maxParallelOneBucket; ++i) { + for (uint32_t i = 0; i < _visitorOptions._maxParallelOneBucket; ++i) { if (!continueVisitor()) return; } } @@ -669,8 +667,7 @@ Visitor::handleDocumentApiReply(mbus::Reply::UP reply, return; } assert(!meta.message); - meta.message.reset( - static_cast<documentapi::DocumentMessage*>(message.release())); + meta.message.reset(static_cast<documentapi::DocumentMessage*>(message.release())); meta.retryCount++; const size_t retryCount = meta.retryCount; @@ -702,7 +699,7 @@ Visitor::onCreateIteratorReply( const std::shared_ptr<CreateIteratorReply>& reply, VisitorThreadMetrics& /*metrics*/) { - std::list<BucketIterationState*>::reverse_iterator it = _bucketStates.rbegin(); + auto it = _bucketStates.rbegin(); document::Bucket bucket(reply->getBucket()); document::BucketId bucketId(bucket.getBucketId()); @@ -752,7 +749,7 @@ Visitor::onGetIterReply(const std::shared_ptr<GetIterReply>& reply, _id.c_str(), reply->getBucketId().toString().c_str(), reply->getResult().toString().c_str()); - std::list<BucketIterationState*>::reverse_iterator it = _bucketStates.rbegin(); + auto it = _bucketStates.rbegin(); // New requests will be pushed on end of list.. So searching // in reverse order should quickly get correct result. @@ -803,7 +800,7 @@ Visitor::onGetIterReply(const std::shared_ptr<GetIterReply>& reply, LOG(debug, "Visitor %s handling block of %zu documents.", _id.c_str(), reply->getEntries().size()); - try{ + try { framework::MilliSecTimer processingTimer(_component.getClock()); handleDocuments(reply->getBucketId(), reply->getEntries(), @@ -814,18 +811,16 @@ Visitor::onGetIterReply(const std::shared_ptr<GetIterReply>& reply, MBUS_TRACE(reply->getTrace(), 5, "Done processing data block in visitor plugin"); uint64_t size = 0; - for (size_t i = 0; i < reply->getEntries().size(); ++i) { - size += reply->getEntries()[i]->getPersistedDocumentSize(); + for (const auto& entry : reply->getEntries()) { + size += entry->getPersistedDocumentSize(); } _visitorStatistics.setDocumentsVisited( _visitorStatistics.getDocumentsVisited() + reply->getEntries().size()); - _visitorStatistics.setBytesVisited( - _visitorStatistics.getBytesVisited() + size); + _visitorStatistics.setBytesVisited(_visitorStatistics.getBytesVisited() + size); } catch (std::exception& e) { - LOG(warning, "handleDocuments threw exception %s", - e.what()); + LOG(warning, "handleDocuments threw exception %s", e.what()); reportProblem(e.what()); } } @@ -849,8 +844,7 @@ Visitor::sendDueQueuedMessages(framework::MicroSecTime timeNow) while (!_visitorTarget._queuedMessages.empty() && (_visitorTarget._pendingMessages.size() < _visitorOptions._maxPending)) { - VisitorTarget::MessageQueue::iterator it( - _visitorTarget._queuedMessages.begin()); + auto it = _visitorTarget._queuedMessages.begin(); if (it->first < timeNow) { auto& msgMeta = _visitorTarget.metaForMessageId(it->second); _visitorTarget._queuedMessages.erase(it); @@ -1204,13 +1198,10 @@ Visitor::getIterators() if (sentCount == 0) { if (LOG_WOULD_LOG(debug)) { LOG(debug, "Enough iterators being processed. Doing nothing for " - "visitor '%s' bucketStates = %d.", - _id.c_str(), (int)_bucketStates.size()); - for (std::list<BucketIterationState*>::iterator it( - _bucketStates.begin()); - it != _bucketStates.end(); ++it) - { - LOG(debug, "Existing: %s", (*it)->toString().c_str()); + "visitor '%s' bucketStates = %zu.", + _id.c_str(), _bucketStates.size()); + for (const auto& state : _bucketStates) { + LOG(debug, "Existing: %s", state->toString().c_str()); } } } diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index 006af5edf7d..c6e75735690 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -140,8 +140,7 @@ VisitorThread::shutdown() .getType() != PropagateVisitorConfig::ID)) { std::shared_ptr<api::StorageReply> reply( - static_cast<api::StorageCommand&>(*it->_message) - .makeReply().release()); + static_cast<api::StorageCommand&>(*it->_message).makeReply()); reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, "Shutting down storage node.")); _messageSender.send(reply); @@ -225,9 +224,9 @@ VisitorThread::run(framework::ThreadHandle& thread) if (entry._message.get()) { // If visitor doesn't exist, log failure only if it wasn't // recently deleted - if (_currentlyRunningVisitor == _visitors.end() && - entry._message->getType() != api::MessageType::VISITOR_CREATE && - entry._message->getType() != api::MessageType::INTERNAL) + if ((_currentlyRunningVisitor == _visitors.end()) && + (entry._message->getType() != api::MessageType::VISITOR_CREATE) && + (entry._message->getType() != api::MessageType::INTERNAL)) { handleNonExistingVisitorCall(entry, result); } else { @@ -264,10 +263,8 @@ VisitorThread::run(framework::ThreadHandle& thread) if (!handled && entry._message.get() && !entry._message->getType().isReply()) { - api::StorageCommand& cmd( - dynamic_cast<api::StorageCommand&>(*entry._message)); - std::shared_ptr<api::StorageReply> reply( - cmd.makeReply().release()); + auto& cmd = dynamic_cast<api::StorageCommand&>(*entry._message); + std::shared_ptr<api::StorageReply> reply(cmd.makeReply()); reply->setResult(result); _messageSender.send(reply); } @@ -278,10 +275,8 @@ void VisitorThread::tick() { // Give all visitors an event - for (VisitorMap::iterator it = _visitors.begin(); it != _visitors.end();) - { - LOG(spam, "Giving tick to visitor %s.", - it->second->getVisitorName().c_str()); + for (auto it = _visitors.begin(); it != _visitors.end();) { + LOG(spam, "Giving tick to visitor %s.", it->second->getVisitorName().c_str()); it->second->continueVisitor(); if (it->second->isCompleted()) { LOG(debug, "Closing visitor %s. Visitor marked as completed", @@ -312,11 +307,9 @@ VisitorThread::close() } else { _metrics.completedVisitors[loadType].inc(1); } - framework::SecondTime currentTime( - _component.getClock().getTimeInSeconds()); + framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); trimRecentlyCompletedList(currentTime); - _recentlyCompleted.push_back(std::make_pair( - _currentlyRunningVisitor->first, currentTime)); + _recentlyCompleted.emplace_back(_currentlyRunningVisitor->first, currentTime); _visitors.erase(_currentlyRunningVisitor); _currentlyRunningVisitor = _visitors.end(); } @@ -324,8 +317,7 @@ VisitorThread::close() void VisitorThread::trimRecentlyCompletedList(framework::SecondTime currentTime) { - framework::SecondTime recentLimit( - currentTime - framework::SecondTime(30)); + framework::SecondTime recentLimit(currentTime - framework::SecondTime(30)); // Dump all elements that aren't recent anymore while (!_recentlyCompleted.empty() && _recentlyCompleted.front().second < recentLimit) @@ -339,16 +331,12 @@ VisitorThread::handleNonExistingVisitorCall(const Event& entry, ReturnCode& code) { // Get current time. Set the time that is the oldest still recent. - framework::SecondTime currentTime( - _component.getClock().getTimeInSeconds());; + framework::SecondTime currentTime(_component.getClock().getTimeInSeconds());; trimRecentlyCompletedList(currentTime); // Go through all recent visitors. Ignore request if recent - for (std::deque<std::pair<api::VisitorId, framework::SecondTime> > - ::iterator it = _recentlyCompleted.begin(); - it != _recentlyCompleted.end(); ++it) - { - if (it->first == entry._visitorId) { + for (const auto& e : _recentlyCompleted) { + if (e.first == entry._visitorId) { code = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, "Visitor recently completed/failed/aborted."); return; @@ -371,13 +359,13 @@ VisitorThread::createVisitor(vespalib::stringref libName, vespalib::string str = libName; std::transform(str.begin(), str.end(), str.begin(), tolower); - VisitorFactory::Map::iterator it(_visitorFactories.find(str)); + auto it = _visitorFactories.find(str); if (it == _visitorFactories.end()) { error << "Visitor library " << str << " not found."; return std::shared_ptr<Visitor>(); } - LibMap::iterator libIter = _libs.find(str); + auto libIter = _libs.find(str); if (libIter == _libs.end()) { _libs[str] = std::shared_ptr<VisitorEnvironment>( it->second->makeVisitorEnvironment(_component).release()); @@ -402,17 +390,15 @@ namespace { std::unique_ptr<api::StorageMessageAddress> getDataAddress(const api::CreateVisitorCommand& cmd) { - return std::unique_ptr<api::StorageMessageAddress>( - new api::StorageMessageAddress( - mbus::Route::parse(cmd.getDataDestination()))); + return std::make_unique<api::StorageMessageAddress>( + mbus::Route::parse(cmd.getDataDestination())); } std::unique_ptr<api::StorageMessageAddress> getControlAddress(const api::CreateVisitorCommand& cmd) { - return std::unique_ptr<api::StorageMessageAddress>( - new api::StorageMessageAddress( - mbus::Route::parse(cmd.getControlDestination()))); + return std::make_unique<api::StorageMessageAddress>( + mbus::Route::parse(cmd.getControlDestination())); } void @@ -447,28 +433,27 @@ VisitorThread::onCreateVisitor( std::unique_ptr<api::StorageMessageAddress> dataAddress; std::shared_ptr<Visitor> visitor; do { - // If no buckets are specified, fail command - if (cmd->getBuckets().size() == 0) { + // If no buckets are specified, fail command + if (cmd->getBuckets().empty()) { result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, "No buckets specified"); LOG(warning, "CreateVisitor(%s): No buckets specified. Aborting.", cmd->getInstanceId().c_str()); break; } - // Get the source address + // Get the source address controlAddress = getControlAddress(*cmd); dataAddress = getDataAddress(*cmd); - // Attempt to load library containing visitor + // Attempt to load library containing visitor vespalib::asciistream errors; - visitor = createVisitor(cmd->getLibraryName(), cmd->getParameters(), - errors); - if (visitor.get() == 0) { + visitor = createVisitor(cmd->getLibraryName(), cmd->getParameters(), errors); + if (!visitor) { result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, errors.str()); LOG(warning, "CreateVisitor(%s): Failed to create visitor: %s", cmd->getInstanceId().c_str(), errors.str().data()); break; } - // Set visitor parameters + // Set visitor parameters if (cmd->getMaximumPendingReplyCount() != 0) { visitor->setMaxPending(cmd->getMaximumPendingReplyCount()); } else { @@ -494,11 +479,9 @@ VisitorThread::onCreateVisitor( // Parse document selection try{ - if (cmd->getDocumentSelection() != "") { - std::shared_ptr<const document::DocumentTypeRepo> repo( - _component.getTypeRepo()); - const document::BucketIdFactory& idFactory( - _component.getBucketIdFactory()); + if (!cmd->getDocumentSelection().empty()) { + std::shared_ptr<const document::DocumentTypeRepo> repo(_component.getTypeRepo()); + const document::BucketIdFactory& idFactory(_component.getBucketIdFactory()); document::select::Parser parser(*repo, idFactory); docSelection = parser.parse(cmd->getDocumentSelection()); validateDocumentSelection(*repo, *docSelection); @@ -522,11 +505,11 @@ VisitorThread::onCreateVisitor( } LOG(debug, "CreateVisitor(%s): Successfully created visitor", cmd->getInstanceId().c_str()); - // Insert visitor prior to creating successful reply. + // Insert visitor prior to creating successful reply. } while (false); - // Start the visitor last, as to ensure client will receive - // visitor create reply first, and that all errors we could detect - // resulted in proper error code in reply.. + // Start the visitor last, as to ensure client will receive + // visitor create reply first, and that all errors we could detect + // resulted in proper error code in reply.. if (result.success()) { _visitors[cmd->getVisitorId()] = visitor; try{ @@ -548,18 +531,17 @@ VisitorThread::onCreateVisitor( visitor->attach(cmd, *controlAddress, *dataAddress, framework::MilliSecTime(vespalib::count_ms(cmd->getTimeout()))); } catch (std::exception& e) { - // We don't handle exceptions from this code, as we've - // added visitor to internal structs we'll end up calling - // close() twice. + // We don't handle exceptions from this code, as we've + // added visitor to internal structs we'll end up calling + // close() twice. LOG(error, "Got exception we can't handle: %s", e.what()); assert(false); } _metrics.createdVisitors[visitor->getLoadType()].inc(1); visitorTimer.stop(_metrics.averageVisitorCreationTime[visitor->getLoadType()]); } else { - // Send reply - std::shared_ptr<api::CreateVisitorReply> reply( - new api::CreateVisitorReply(*cmd)); + // Send reply + auto reply = std::make_shared<api::CreateVisitorReply>(*cmd); reply->setResult(result); _messageSender.closed(cmd->getVisitorId()); _messageSender.send(reply); @@ -572,7 +554,7 @@ VisitorThread::handleMessageBusReply(mbus::Reply::UP reply, Visitor& visitor) { vespalib::MonitorGuard sync(_queueMonitor); - _queue.push_back(Event(visitor.getVisitorId(), std::move(reply))); + _queue.emplace_back(visitor.getVisitorId(), std::move(reply)); sync.broadcast(); } @@ -582,8 +564,7 @@ VisitorThread::onInternal(const std::shared_ptr<api::InternalCommand>& cmd) switch (cmd->getType()) { case PropagateVisitorConfig::ID: { - PropagateVisitorConfig& pcmd( - dynamic_cast<PropagateVisitorConfig&>(*cmd)); + auto& pcmd = dynamic_cast<PropagateVisitorConfig&>(*cmd); const vespa::config::content::core::StorVisitorConfig& config(pcmd.getConfig()); if (_defaultDocBlockSize != 0) { // Live update LOG(config, "Updating visitor thread configuration in visitor " @@ -655,12 +636,10 @@ VisitorThread::onInternal(const std::shared_ptr<api::InternalCommand>& cmd) case RequestStatusPage::ID: { LOG(spam, "Got RequestStatusPage request"); - RequestStatusPage& rsp(dynamic_cast<RequestStatusPage&>(*cmd)); + auto& rsp = dynamic_cast<RequestStatusPage&>(*cmd); vespalib::asciistream ost; getStatus(ost, rsp.getPath()); - std::shared_ptr<RequestStatusPageReply> reply( - new RequestStatusPageReply(rsp, ost.str())); - _messageSender.send(reply); + _messageSender.send(std::make_shared<RequestStatusPageReply>(rsp, ost.str())); break; } default: @@ -679,11 +658,9 @@ VisitorThread::onInternalReply(const std::shared_ptr<api::InternalReply>& r) switch (r->getType()) { case GetIterReply::ID: { - std::shared_ptr<GetIterReply> reply( - std::dynamic_pointer_cast<GetIterReply>(r)); + auto reply = std::dynamic_pointer_cast<GetIterReply>(r); assert(reply.get()); - _currentlyRunningVisitor->second->onGetIterReply( - reply, _metrics); + _currentlyRunningVisitor->second->onGetIterReply(reply, _metrics); if (_currentlyRunningVisitor->second->isCompleted()) { LOG(debug, "onGetIterReply(%s): Visitor completed.", _currentlyRunningVisitor->second->getVisitorName().c_str()); @@ -693,11 +670,9 @@ VisitorThread::onInternalReply(const std::shared_ptr<api::InternalReply>& r) } case CreateIteratorReply::ID: { - std::shared_ptr<CreateIteratorReply> reply( - std::dynamic_pointer_cast<CreateIteratorReply>(r)); + auto reply = std::dynamic_pointer_cast<CreateIteratorReply>(r); assert(reply.get()); - _currentlyRunningVisitor->second->onCreateIteratorReply( - reply, _metrics); + _currentlyRunningVisitor->second->onCreateIteratorReply(reply, _metrics); break; } default: @@ -721,25 +696,21 @@ VisitorThread::getStatus(vespalib::asciistream& out, if (status && verbose) { out << "<h3>Visitor libraries loaded</h3>\n<ul>\n"; - if (_libs.size() == 0) { + if (_libs.empty()) { out << "None\n"; } - for (LibMap::const_iterator it = _libs.begin(); it != _libs.end(); ++it) - { - out << "<li>" << it->first << "\n"; + for (const auto& lib : _libs) { + out << "<li>" << lib.first << "\n"; } out << "</ul>\n"; out << "<h3>Recently completed/failed/aborted visitors</h3>\n<ul>\n"; - if (_recentlyCompleted.size() == 0) { + if (_recentlyCompleted.empty()) { out << "None\n"; } - for (std::deque<std::pair<api::VisitorId, framework::SecondTime> > - ::const_iterator it = _recentlyCompleted.begin(); - it != _recentlyCompleted.end(); ++it) - { - out << "<li> Visitor " << it->first << " done at " - << it->second.getTime() << "\n"; + for (const auto& cv : _recentlyCompleted) { + out << "<li> Visitor " << cv.first << " done at " + << cv.second.getTime() << "\n"; } out << "</ul>\n"; out << "<h3>Current queue size: " << _queue.size() << "</h3>\n"; @@ -764,17 +735,15 @@ VisitorThread::getStatus(vespalib::asciistream& out, << "</table>\n"; } if (showAll) { - for (VisitorMap::const_iterator it = _visitors.begin(); - it != _visitors.end(); ++it) - { - out << "<h3>Visitor " << it->first << "</h3>\n"; + for (const auto& v : _visitors) { + out << "<h3>Visitor " << v.first << "</h3>\n"; std::ostringstream tmp; - it->second->getStatus(tmp, verbose); + v.second->getStatus(tmp, verbose); out << tmp.str(); } } else if (path.hasAttribute("visitor")) { out << "<h3>Visitor " << visitor << "</h3>\n"; - VisitorMap::const_iterator it = _visitors.find(visitor); + auto it = _visitors.find(visitor); if (it == _visitors.end()) { out << "Not found\n"; } else { @@ -784,7 +753,7 @@ VisitorThread::getStatus(vespalib::asciistream& out, } } else { // List visitors out << "<h3>Active visitors</h3>\n"; - if (_visitors.size() == 0) { + if (_visitors.empty()) { out << "None\n"; } for (VisitorMap::const_iterator it = _visitors.begin(); diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index 0f628f59aac..37159ab0011 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -291,6 +291,7 @@ TEST_P(StorageProtocolTest, get) { EXPECT_EQ(_testDoc->getId(), reply2->getDocumentId()); EXPECT_EQ(Timestamp(123), reply2->getBeforeTimestamp()); EXPECT_EQ(Timestamp(100), reply2->getLastModifiedTimestamp()); + EXPECT_FALSE(reply2->is_tombstone()); EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); } @@ -316,6 +317,40 @@ TEST_P(StorageProtocolTest, can_set_internal_read_consistency_on_get_commands) { EXPECT_EQ(cmd2->internal_read_consistency(), InternalReadConsistency::Strong); } +TEST_P(StorageProtocolTest, tombstones_propagated_for_gets) { + // Only supported on protocol version 7+. + if (GetParam().getMajor() < 7) { + return; + } + auto cmd = std::make_shared<GetCommand>(_bucket, _testDocId, "foo,bar", 123); + auto reply = std::make_shared<GetReply>(*cmd, std::shared_ptr<Document>(), 100, false, true); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + + EXPECT_TRUE(reply2->getDocument().get() == nullptr); + EXPECT_EQ(_testDoc->getId(), reply2->getDocumentId()); + EXPECT_EQ(Timestamp(123), reply2->getBeforeTimestamp()); + EXPECT_EQ(Timestamp(100), reply2->getLastModifiedTimestamp()); // In this case, the tombstone timestamp. + EXPECT_TRUE(reply2->is_tombstone()); +} + +// TODO remove this once pre-protobuf serialization is removed +TEST_P(StorageProtocolTest, old_serialization_format_treats_tombstone_get_replies_as_not_found) { + if (GetParam().getMajor() >= 7) { + return; + } + auto cmd = std::make_shared<GetCommand>(_bucket, _testDocId, "foo,bar", 123); + auto reply = std::make_shared<GetReply>(*cmd, std::shared_ptr<Document>(), 100, false, true); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + + EXPECT_TRUE(reply2->getDocument().get() == nullptr); + EXPECT_EQ(_testDoc->getId(), reply2->getDocumentId()); + EXPECT_EQ(Timestamp(123), reply2->getBeforeTimestamp()); + EXPECT_EQ(Timestamp(0), reply2->getLastModifiedTimestamp()); + EXPECT_FALSE(reply2->is_tombstone()); // Protocol version doesn't understand explicit tombstones. +} + TEST_P(StorageProtocolTest, remove) { auto cmd = std::make_shared<RemoveCommand>(_bucket, _testDocId, 159); auto cmd2 = copyCommand(cmd); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto index 12dbaf59146..f38c5bfb56a 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto @@ -73,6 +73,9 @@ message GetResponse { uint64 last_modified_timestamp = 2; BucketInfo bucket_info = 3; BucketId remapped_bucket_id = 4; + // Note: last_modified_timestamp and tombstone_timestamp are mutually exclusive. + // Tracked separately (rather than being a flag bool) to avoid issues during rolling upgrades. + uint64 tombstone_timestamp = 5; } message RevertRequest { diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp index aeb6d382997..8a0204c8fc9 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp @@ -168,7 +168,8 @@ ProtocolSerialization5_0::onDecodeUpdateReply(const SCmd& cmd, BBuf& buf) const void ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::GetReply& msg) const { SH::putDocument(msg.getDocument().get(), buf); - buf.putLong(msg.getLastModifiedTimestamp()); + // Old protocol version doesn't understand tombstones. Make it appear as Not Found. + buf.putLong(msg.is_tombstone() ? api::Timestamp(0) : msg.getLastModifiedTimestamp()); onEncodeBucketInfoReply(buf, msg); } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 90c8d1c7d2a..8ea946eede4 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -567,7 +567,17 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetReply& msg) cons if (msg.getDocument()) { set_document(*res.mutable_document(), *msg.getDocument()); } - res.set_last_modified_timestamp(msg.getLastModifiedTimestamp()); + if (!msg.is_tombstone()) { + res.set_last_modified_timestamp(msg.getLastModifiedTimestamp()); + } else { + // This field will be ignored by older versions, making the behavior as if + // a timestamp of zero was returned for tombstones, as it the legacy behavior. + res.set_tombstone_timestamp(msg.getLastModifiedTimestamp()); + // Will not be encoded onto the wire, but we include it here to hammer down the + // point that it's intentional to have the last modified time appear as a not + // found document for older versions. + res.set_last_modified_timestamp(0); + } }); } @@ -585,8 +595,12 @@ api::StorageReply::UP ProtocolSerialization7::onDecodeGetReply(const SCmd& cmd, return decode_bucket_info_response<protobuf::GetResponse>(buf, [&](auto& res) { try { auto document = get_document(res.document(), type_repo()); + const bool is_tombstone = (res.tombstone_timestamp() != 0); + const auto effective_timestamp = (is_tombstone ? res.tombstone_timestamp() + : res.last_modified_timestamp()); return std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), - std::move(document), res.last_modified_timestamp()); + std::move(document), effective_timestamp, + false, is_tombstone); } catch (std::exception& e) { auto reply = std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), std::shared_ptr<document::Document>(), 0u); diff --git a/storageapi/src/vespa/storageapi/message/persistence.cpp b/storageapi/src/vespa/storageapi/message/persistence.cpp index 7fd789a8c81..af0b7ae0288 100644 --- a/storageapi/src/vespa/storageapi/message/persistence.cpp +++ b/storageapi/src/vespa/storageapi/message/persistence.cpp @@ -211,14 +211,16 @@ GetCommand::print(std::ostream& out, bool verbose, const std::string& indent) co GetReply::GetReply(const GetCommand& cmd, const DocumentSP& doc, Timestamp lastModified, - bool had_consistent_replicas) + bool had_consistent_replicas, + bool is_tombstone) : BucketInfoReply(cmd), _docId(cmd.getDocumentId()), _fieldSet(cmd.getFieldSet()), _doc(doc), _beforeTimestamp(cmd.getBeforeTimestamp()), _lastModifiedTime(lastModified), - _had_consistent_replicas(had_consistent_replicas) + _had_consistent_replicas(had_consistent_replicas), + _is_tombstone(is_tombstone) { } diff --git a/storageapi/src/vespa/storageapi/message/persistence.h b/storageapi/src/vespa/storageapi/message/persistence.h index e6fe2b6dae5..24601803266 100644 --- a/storageapi/src/vespa/storageapi/message/persistence.h +++ b/storageapi/src/vespa/storageapi/message/persistence.h @@ -224,24 +224,27 @@ class GetReply : public BucketInfoReply { Timestamp _beforeTimestamp; Timestamp _lastModifiedTime; bool _had_consistent_replicas; - + bool _is_tombstone; public: - GetReply(const GetCommand& cmd, - const DocumentSP& doc = DocumentSP(), - Timestamp lastModified = 0, - bool had_consistent_replicas = false); + explicit GetReply(const GetCommand& cmd, + const DocumentSP& doc = DocumentSP(), + Timestamp lastModified = 0, + bool had_consistent_replicas = false, + bool is_tombstone = false); + ~GetReply() override; const DocumentSP& getDocument() const { return _doc; } const document::DocumentId& getDocumentId() const { return _docId; } const vespalib::string& getFieldSet() const { return _fieldSet; } - Timestamp getLastModifiedTimestamp() const { return _lastModifiedTime; } - Timestamp getBeforeTimestamp() const { return _beforeTimestamp; } + Timestamp getLastModifiedTimestamp() const noexcept { return _lastModifiedTime; } + Timestamp getBeforeTimestamp() const noexcept { return _beforeTimestamp; } - bool had_consistent_replicas() const noexcept { return _had_consistent_replicas; } + [[nodiscard]] bool had_consistent_replicas() const noexcept { return _had_consistent_replicas; } + [[nodiscard]] bool is_tombstone() const noexcept { return _is_tombstone; } - bool wasFound() const { return (_doc.get() != 0); } + bool wasFound() const { return (_doc.get() != nullptr); } void print(std::ostream& out, bool verbose, const std::string& indent) const override; DECLARE_STORAGEREPLY(GetReply, onGetReply) }; diff --git a/streamingvisitors/src/tests/matching_elements_filler/matching_elements_filler_test.cpp b/streamingvisitors/src/tests/matching_elements_filler/matching_elements_filler_test.cpp index 8135f378dd3..b40e045f8b0 100644 --- a/streamingvisitors/src/tests/matching_elements_filler/matching_elements_filler_test.cpp +++ b/streamingvisitors/src/tests/matching_elements_filler/matching_elements_filler_test.cpp @@ -1,27 +1,27 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/datatype/datatypes.h> -#include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/mapfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/structfieldvalue.h> -#include <vespa/searchlib/fef/matchdata.h> -#include <vespa/searchlib/common/struct_field_mapper.h> #include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchlib/common/matching_elements_fields.h> +#include <vespa/searchlib/fef/matchdata.h> +#include <vespa/searchlib/query/streaming/query.h> +#include <vespa/searchlib/query/streaming/queryterm.h> #include <vespa/searchlib/query/tree/querybuilder.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/query/tree/stackdumpcreator.h> -#include <vespa/searchlib/query/streaming/queryterm.h> -#include <vespa/searchlib/query/streaming/query.h> -#include <vespa/vsm/searcher/fieldsearcher.h> -#include <vespa/vsm/searcher/utf8strchrfieldsearcher.h> -#include <vespa/vsm/searcher/intfieldsearcher.h> #include <vespa/searchvisitor/hitcollector.h> #include <vespa/searchvisitor/matching_elements_filler.h> #include <vespa/vdslib/container/searchresult.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vsm/searcher/fieldsearcher.h> +#include <vespa/vsm/searcher/intfieldsearcher.h> +#include <vespa/vsm/searcher/utf8strchrfieldsearcher.h> #include <iostream> using document::ArrayDataType; @@ -38,7 +38,7 @@ using document::StringFieldValue; using document::StructDataType; using document::StructFieldValue; using search::MatchingElements; -using search::StructFieldMapper; +using search::MatchingElementsFields; using search::fef::MatchData; using search::query::StackDumpCreator; using search::query::Weight; @@ -256,24 +256,23 @@ vsm::DocumentTypeIndexFieldMapT make_index_to_field_ids() { return ret; } -StructFieldMapper make_struct_field_mapper() { - StructFieldMapper mapper; - mapper.add_mapping("elem_array", "elem_array.name"); - mapper.add_mapping("elem_array", "elem_array.weight"); - mapper.add_mapping("elem_map", "elem_map.key"); - mapper.add_mapping("elem_map", "elem_map.value.name"); - mapper.add_mapping("elem_map", "elem_map.value.weight"); - mapper.add_mapping("str_int_map", "str_int_map.key"); - mapper.add_mapping("str_int_map", "str_int_map.value"); - return mapper; - +MatchingElementsFields make_matching_elements_fields() { + MatchingElementsFields fields; + fields.add_mapping("elem_array", "elem_array.name"); + fields.add_mapping("elem_array", "elem_array.weight"); + fields.add_mapping("elem_map", "elem_map.key"); + fields.add_mapping("elem_map", "elem_map.value.name"); + fields.add_mapping("elem_map", "elem_map.value.weight"); + fields.add_mapping("str_int_map", "str_int_map.key"); + fields.add_mapping("str_int_map", "str_int_map.value"); + return fields; } } class MatchingElementsFillerTest : public ::testing::Test { const MyDocType _doc_type; - StructFieldMapper _struct_field_mapper; + MatchingElementsFields _matching_elems_fields; vsm::SharedFieldPathMap _field_path_map; vsm::FieldIdTSearcherMap _field_searcher_map; vsm::DocumentTypeIndexFieldMapT _index_to_field_ids; @@ -296,7 +295,7 @@ public: MatchingElementsFillerTest::MatchingElementsFillerTest() : ::testing::Test(), _doc_type(), - _struct_field_mapper(make_struct_field_mapper()), + _matching_elems_fields(make_matching_elements_fields()), _field_path_map(make_field_path_map(_doc_type)), _field_searcher_map(make_field_searcher_map()), _index_to_field_ids(make_index_to_field_ids()), @@ -325,7 +324,7 @@ MatchingElementsFillerTest::fill_matching_elements(Query &&query) _query = std::move(query); _field_searcher_map.prepare(_index_to_field_ids, _shared_searcher_buf, _query); _matching_elements_filler = std::make_unique<MatchingElementsFiller>(_field_searcher_map, _query, _hit_collector, _search_result); - _matching_elements = _matching_elements_filler->fill_matching_elements(_struct_field_mapper); + _matching_elements = _matching_elements_filler->fill_matching_elements(_matching_elems_fields); } void diff --git a/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp b/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp index 994a7993666..fe717313ca4 100644 --- a/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.cpp @@ -2,14 +2,14 @@ #include "matching_elements_filler.h" #include <vespa/searchlib/common/matching_elements.h> -#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vsm/searcher/fieldsearcher.h> #include <vespa/vdslib/container/searchresult.h> #include "hitcollector.h" #include <algorithm> using search::MatchingElements; -using search::StructFieldMapper; +using search::MatchingElementsFields; using search::streaming::AndNotQueryNode; using search::streaming::HitList; using search::streaming::Query; @@ -27,15 +27,15 @@ namespace { struct SubFieldTerm { - vespalib::string _struct_field_name; + vespalib::string _field_name; const QueryTerm* _term; public: - SubFieldTerm(vespalib::string struct_field_name, const QueryTerm* term) - : _struct_field_name(std::move(struct_field_name)), + SubFieldTerm(vespalib::string field_name, const QueryTerm* term) + : _field_name(std::move(field_name)), _term(term) { } - const vespalib::string& get_struct_field_name() const { return _struct_field_name; } + const vespalib::string& get_field_name() const { return _field_name; } const QueryTerm& get_term() const { return *_term; } }; @@ -47,12 +47,12 @@ class Matcher HitList _hit_list; std::vector<uint32_t> _elements; - void select_query_nodes(const StructFieldMapper& mapper, const QueryNode& query_node); - void add_matching_elements(const vespalib::string& struct_field_name, uint32_t doc_lid, const HitList& hit_list, MatchingElements& matching_elements); + void select_query_nodes(const MatchingElementsFields& fields, const QueryNode& query_node); + void add_matching_elements(const vespalib::string& field_name, uint32_t doc_lid, const HitList& hit_list, MatchingElements& matching_elements); void find_matching_elements(const SameElementQueryNode& same_element, uint32_t doc_lid, MatchingElements& matching_elements); void find_matching_elements(const SubFieldTerm& sub_field_term, uint32_t doc_lid, MatchingElements& matching_elements); public: - Matcher(vsm::FieldIdTSearcherMap& field_searcher_map, const StructFieldMapper& mapper, const Query& query); + Matcher(vsm::FieldIdTSearcherMap& field_searcher_map, const MatchingElementsFields& fields, const Query& query); ~Matcher(); bool empty() const { return _same_element_nodes.empty() && _sub_field_terms.empty(); } void find_matching_elements(const vsm::StorageDocument& doc, uint32_t doc_lid, MatchingElements& matching_elements); @@ -61,39 +61,39 @@ public: template<typename T> const T* as(const QueryNode& query_node) { return dynamic_cast<const T*>(&query_node); } -Matcher::Matcher(FieldIdTSearcherMap& field_searcher_map, const StructFieldMapper& mapper, const Query& query) +Matcher::Matcher(FieldIdTSearcherMap& field_searcher_map, const MatchingElementsFields& fields, const Query& query) : _same_element_nodes(), _sub_field_terms(), _field_searcher_map(field_searcher_map), _hit_list() { - select_query_nodes(mapper, query.getRoot()); + select_query_nodes(fields, query.getRoot()); } Matcher::~Matcher() = default; void -Matcher::select_query_nodes(const StructFieldMapper& mapper, const QueryNode& query_node) +Matcher::select_query_nodes(const MatchingElementsFields& fields, const QueryNode& query_node) { if (auto same_element = as<SameElementQueryNode>(query_node)) { - if (mapper.is_struct_field(same_element->getIndex())) { + if (fields.has_field(same_element->getIndex())) { _same_element_nodes.emplace_back(same_element); } } else if (auto query_term = as<QueryTerm>(query_node)) { - if (mapper.is_struct_subfield(query_term->getIndex())) { - _sub_field_terms.emplace_back(mapper.get_struct_field(query_term->getIndex()), query_term); + if (fields.has_struct_field(query_term->getIndex())) { + _sub_field_terms.emplace_back(fields.get_enclosing_field(query_term->getIndex()), query_term); } } else if (auto and_not = as<AndNotQueryNode>(query_node)) { - select_query_nodes(mapper, *(*and_not)[0]); + select_query_nodes(fields, *(*and_not)[0]); } else if (auto intermediate = as<QueryConnector>(query_node)) { for (size_t i = 0; i < intermediate->size(); ++i) { - select_query_nodes(mapper, *(*intermediate)[i]); + select_query_nodes(fields, *(*intermediate)[i]); } } } void -Matcher::add_matching_elements(const vespalib::string& struct_field_name, uint32_t doc_lid, const HitList& hit_list, MatchingElements& matching_elements) +Matcher::add_matching_elements(const vespalib::string& field_name, uint32_t doc_lid, const HitList& hit_list, MatchingElements& matching_elements) { _elements.clear(); for (auto& hit : hit_list) { @@ -104,7 +104,7 @@ Matcher::add_matching_elements(const vespalib::string& struct_field_name, uint32 auto last = std::unique(_elements.begin(), _elements.end()); _elements.erase(last, _elements.end()); } - matching_elements.add_matching_elements(doc_lid, struct_field_name, _elements); + matching_elements.add_matching_elements(doc_lid, field_name, _elements); } void @@ -121,7 +121,7 @@ Matcher::find_matching_elements(const SubFieldTerm& sub_field_term, uint32_t doc { const HitList& hit_list = sub_field_term.get_term().evaluateHits(_hit_list); if (!hit_list.empty()) { - add_matching_elements(sub_field_term.get_struct_field_name(), doc_lid, hit_list, matching_elements); + add_matching_elements(sub_field_term.get_field_name(), doc_lid, hit_list, matching_elements); } } @@ -154,13 +154,13 @@ MatchingElementsFiller::MatchingElementsFiller(FieldIdTSearcherMap& field_search MatchingElementsFiller::~MatchingElementsFiller() = default; std::unique_ptr<MatchingElements> -MatchingElementsFiller::fill_matching_elements(const StructFieldMapper& struct_field_mapper) +MatchingElementsFiller::fill_matching_elements(const MatchingElementsFields& fields) { auto result = std::make_unique<MatchingElements>(); - if (struct_field_mapper.empty()) { + if (fields.empty()) { return result; } - Matcher matcher(_field_searcher_map, struct_field_mapper, _query); + Matcher matcher(_field_searcher_map, fields, _query); if (matcher.empty()) { return result; } diff --git a/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.h b/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.h index c93f89ac43d..244d3cea6bc 100644 --- a/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.h +++ b/streamingvisitors/src/vespa/searchvisitor/matching_elements_filler.h @@ -29,7 +29,7 @@ public: MatchingElementsFiller(vsm::FieldIdTSearcherMap& field_searcher_map, search::streaming::Query& query, HitCollector& hit_collector, vdslib::SearchResult& search_result); virtual ~MatchingElementsFiller(); - std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::StructFieldMapper& struct_field_mapper) override; + std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields& fields) override; }; } diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index a3fac22a93d..767119d2a02 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -93,23 +93,29 @@ <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-cd</artifactId> + <artifactId>tenant-cd-api</artifactId> <version>${test-framework.version}</version> <scope>test</scope> - <exclusions> - <exclusion> - <groupId>net.java.dev.jna</groupId> - <artifactId>jna</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </exclusion> - <exclusion> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </exclusion> - </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>cloud-tenant-cd</artifactId> + <version>${test-framework.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> </dependency> </dependencies> diff --git a/tenant-cd/OWNERS b/tenant-cd-api/OWNERS index d0a102ecbf4..d0a102ecbf4 100644 --- a/tenant-cd/OWNERS +++ b/tenant-cd-api/OWNERS diff --git a/tenant-cd/README b/tenant-cd-api/README index a3803b81d53..a3803b81d53 100644 --- a/tenant-cd/README +++ b/tenant-cd-api/README diff --git a/tenant-cd-api/abi-spec.json b/tenant-cd-api/abi-spec.json new file mode 100644 index 00000000000..677a18c74e4 --- /dev/null +++ b/tenant-cd-api/abi-spec.json @@ -0,0 +1,117 @@ +{ + "ai.vespa.hosted.cd.Deployment": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract ai.vespa.hosted.cd.Endpoint endpoint(java.lang.String)" + ], + "fields": [] + }, + "ai.vespa.hosted.cd.Endpoint": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract java.net.URI uri()", + "public abstract java.net.http.HttpResponse send(java.net.http.HttpRequest$Builder, java.net.http.HttpResponse$BodyHandler)", + "public java.net.http.HttpResponse send(java.net.http.HttpRequest$Builder)", + "public abstract java.net.http.HttpRequest$Builder request(java.lang.String, java.util.Map)", + "public java.net.http.HttpRequest$Builder request(java.lang.String)" + ], + "fields": [] + }, + "ai.vespa.hosted.cd.IntegrationTest": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [], + "fields": [] + }, + "ai.vespa.hosted.cd.ProductionTest": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [], + "fields": [] + }, + "ai.vespa.hosted.cd.StagingSetup": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [], + "fields": [] + }, + "ai.vespa.hosted.cd.StagingTest": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [], + "fields": [] + }, + "ai.vespa.hosted.cd.SystemTest": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.annotation.Annotation" + ], + "attributes": [ + "public", + "interface", + "abstract", + "annotation" + ], + "methods": [], + "fields": [] + }, + "ai.vespa.hosted.cd.TestRuntime": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public static ai.vespa.hosted.cd.TestRuntime get()", + "public abstract ai.vespa.hosted.cd.Deployment deploymentToTest()", + "public abstract ai.vespa.cloud.Zone zone()" + ], + "fields": [] + } +}
\ No newline at end of file diff --git a/tenant-cd-api/pom.xml b/tenant-cd-api/pom.xml new file mode 100644 index 00000000000..b19d42d094f --- /dev/null +++ b/tenant-cd-api/pom.xml @@ -0,0 +1,76 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-cd-api</artifactId> + <name>Hosted Vespa tenant CD API</name> + <description>Test API library for hosted Vespa applications.</description> + <url>https://github.com/vespa-engine</url> + <packaging>container-plugin</packaging> + + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent</relativePath> + </parent> + + <dependencies> + <!-- provided --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-zone-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <!-- required for bundle-plugin to generate import-package statements for Java's standard library --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <!-- compile --> + <dependency> <!-- TODO(bjorncs): share junit version number with test-runner implementation --> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <version>5.6.2</version> <!-- NOTE: This version must match the string in all ExportPackage annotations --> + <scope>compile</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <attachBundleArtifact>true</attachBundleArtifact> + <bundleClassifierName>deploy</bundleClassifierName> + <useCommonAssemblyIds>false</useCommonAssemblyIds> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>abi-check-plugin</artifactId> + </plugin> + </plugins> + </build> + +</project> diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Deployment.java index 8327916b41d..7d7b2f74981 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Deployment.java @@ -2,7 +2,7 @@ package ai.vespa.hosted.cd; /** - * A deployment of a Vespa application, which contains endpoints for document and metrics retrieval. + * A deployment of a Vespa application, which contains endpoints for document retrieval. * * @author jonmv */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Endpoint.java index 46f5f8ef5fd..afc6aa1b519 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Endpoint.java @@ -1,23 +1,15 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; -import ai.vespa.hosted.api.EndpointAuthenticator; -import ai.vespa.hosted.cd.metric.Metrics; - import java.net.URI; -import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; -import java.util.stream.Collectors; - -import static java.net.URLEncoder.encode; -import static java.nio.charset.StandardCharsets.UTF_8; /** - * An endpoint in a Vespa application {@link Deployment}, which allows document and metrics retrieval. + * An endpoint in a Vespa application {@link Deployment}, which allows document retrieval. * * @author jonmv */ @@ -26,7 +18,7 @@ public interface Endpoint { /** Returns the URI of the endpoint, with scheme, host and port. */ URI uri(); - /** Sends the given request with required authentication. See {@link EndpointAuthenticator#authenticated} and {@link HttpClient#send}. */ + /** Sends the given request with required authentication. */ <T> HttpResponse<T> send(HttpRequest.Builder request, HttpResponse.BodyHandler<T> handler); /** Sends the given request with required authentication. */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java index f9dc15df32e..f9dc15df32e 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java index 220d0cbd6bb..b9054689b00 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java @@ -2,7 +2,6 @@ package ai.vespa.hosted.cd; import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; import java.lang.annotation.Retention; import java.lang.annotation.Target; @@ -15,7 +14,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * Tests that verify the health of production deployments of Vespa applications. * * Test classes annotated with this annotation are run during declared production tests. - * See <a href="https://cloud.vespa.ai/automated-deployments.html#production-tests">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingSetup.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java index 6853ebfc008..bef3eabcef6 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingSetup.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java @@ -15,7 +15,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * * Test classes annotated with this annotation are run in the first phase of automated staging tests, * to make the initial deployment similar to a production one. - * See <a href="https://cloud.vespa.ai/automated-deployments.html#staging-tests">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java index 4ecc3ce5722..59360b2753c 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java @@ -16,7 +16,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * * Test classes annotated with this annotation are run in the second phase of automated staging tests, * to verify the upgraded deployment. - * See <a href="https://cloud.vespa.ai/automated-deployments.html#staging-tests">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java index 4712c1b41ef..f01f2ca6c90 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java @@ -17,7 +17,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * Tests that compare the behaviour of a Vespa application deployment against a fixed specification. * * Test classes annotated with this annotation are run against a fresh deployment during automated system tests. - * See <a href="https://cloud.vespa.ai/automated-deployments.html#system-tests">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/TestRuntime.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/TestRuntime.java new file mode 100644 index 00000000000..08cc0467b71 --- /dev/null +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/TestRuntime.java @@ -0,0 +1,24 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.hosted.cd; + +import ai.vespa.cloud.Zone; + +import java.util.ServiceLoader; + +/** + * The place to obtain environment-dependent configuration for test of a Vespa deployment. + * + * @author jvenstad + * @author mortent + */ +public interface TestRuntime { + static TestRuntime get() { + ServiceLoader<TestRuntime> serviceLoader = ServiceLoader.load(TestRuntime.class); + return serviceLoader.findFirst().orElseThrow(() -> new RuntimeException("No TestRuntime implementation found")); + } + + Deployment deploymentToTest(); + + Zone zone(); + +} diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/package-info.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/package-info.java new file mode 100644 index 00000000000..fc10fb82c5c --- /dev/null +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/package-info.java @@ -0,0 +1,10 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +@PublicApi +package ai.vespa.hosted.cd; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/condition/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/condition/package-info.java new file mode 100644 index 00000000000..bc0684f0c76 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/condition/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api.condition; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/package-info.java new file mode 100644 index 00000000000..bcb46dbe671 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api.extension; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/function/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/function/package-info.java new file mode 100644 index 00000000000..8d62bceeae7 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/function/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api.function; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/io/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/io/package-info.java new file mode 100644 index 00000000000..7fc2e15c716 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/io/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api.io; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/package-info.java new file mode 100644 index 00000000000..dd82f705bd3 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/parallel/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/parallel/package-info.java new file mode 100644 index 00000000000..dc88b0d33bf --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/parallel/package-info.java @@ -0,0 +1,9 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage(version = @Version(major = 5, minor = 4, micro = 2)) +package org.junit.jupiter.api.parallel; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version;
\ No newline at end of file diff --git a/tenant-cd/pom.xml b/tenant-cd/pom.xml deleted file mode 100644 index 829b1de457b..00000000000 --- a/tenant-cd/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - - <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-cd</artifactId> - <name>Hosted Vespa tenant CD</name> - <description>Test library for hosted Vespa applications.</description> - <url>https://github.com/vespa-engine</url> - <packaging>jar</packaging> - - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent</relativePath> - </parent> - - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>security-utils</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-provisioning</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-auth</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>hosted-api</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-engine</artifactId> - </dependency> - </dependencies> - -</project> diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java deleted file mode 100644 index c479bab6e13..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd; - -import ai.vespa.hosted.api.ControllerHttpClient; -import ai.vespa.hosted.api.EndpointAuthenticator; -import ai.vespa.hosted.api.Properties; -import ai.vespa.hosted.api.TestConfig; -import ai.vespa.hosted.cd.http.HttpDeployment; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.zone.ZoneId; - -import java.nio.file.Files; -import java.nio.file.Paths; - -/** - * The place to obtain environment-dependent configuration for test of a Vespa deployment. - * - * @author jvenstad - */ -public class TestRuntime { - - private static TestRuntime theRuntime; - - private final TestConfig config; - private final Deployment deploymentToTest; - - private TestRuntime(TestConfig config, EndpointAuthenticator authenticator) { - this.config = config; - this.deploymentToTest = new HttpDeployment(config.deployments().get(config.zone()), authenticator); - } - - /** - * Returns the config and authenticator to use when running integration tests. - * - * If the system property {@code "vespa.test.config"} is set (to a file path), a file at that location - * is attempted read, and config parsed from it. - * Otherwise, config is fetched over HTTP from the hosted Vespa API, assuming the deployment indicated - * by the optional {@code "environment"} and {@code "region"} system properties exists. - * When environment is not specified, it defaults to {@link Environment#dev}, - * while region must be set unless the environment is {@link Environment#dev} or {@link Environment#perf}. - */ - public static synchronized TestRuntime get() { - if (theRuntime == null) { - String configPath = System.getProperty("vespa.test.config"); - TestConfig config = configPath != null ? fromFile(configPath) : fromController(); - theRuntime = new TestRuntime(config, - new ai.vespa.hosted.auth.EndpointAuthenticator(config.system())); - } - return theRuntime; - } - - /** Returns a copy of this runtime, with the given endpoint authenticator. */ - public TestRuntime with(EndpointAuthenticator authenticator) { - return new TestRuntime(config, authenticator); - } - - /** Returns the full id of the application this is testing. */ - public ApplicationId application() { return config.application(); } - - /** Returns the zone of the deployment this is testing. */ - public ZoneId zone() { return config.zone(); } - - /** Returns the deployment this is testing. */ - public Deployment deploymentToTest() { return deploymentToTest; } - - private static TestConfig fromFile(String path) { - try { - return TestConfig.fromJson(Files.readAllBytes(Paths.get(path))); - } - catch (Exception e) { - throw new IllegalArgumentException("Failed reading config from '" + path + "'!", e); - } - } - - private static TestConfig fromController() { - ControllerHttpClient controller = new ai.vespa.hosted.auth.ApiAuthenticator().controller(); - ApplicationId id = Properties.application(); - Environment environment = Properties.environment().orElse(Environment.dev); - ZoneId zone = Properties.region().map(region -> ZoneId.from(environment, region)) - .orElseGet(() -> controller.defaultZone(environment)); - return controller.testConfig(id, zone); - } - -} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java deleted file mode 100644 index 39e9cd5bc75..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.metric; - -import java.util.HashMap; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.StringJoiner; - -import static java.util.Map.copyOf; -import static java.util.stream.Collectors.reducing; -import static java.util.stream.Collectors.toUnmodifiableMap; - -/** - * A set of statistics for a metric, for points over a space with named dimensions of arbitrary type. - * - * @author jonmv - */ -public class Metric { - - private final Map<Map<String, ?>, Statistic> statistics; - - private Metric(Map<Map<String, ?>, Statistic> statistics) { - this.statistics = statistics; - } - - /** Creates a new Metric with a copy of the given data. */ - public static Metric of(Map<Map<String, ?>, Statistic> data) { - if (data.isEmpty()) - throw new IllegalArgumentException("No data given."); - - Map<Map<String, ?>, Statistic> copies = new HashMap<>(); - Set<String> dimensions = data.keySet().iterator().next().keySet(); - data.forEach((point, statistic) -> { - if ( ! point.keySet().equals(dimensions)) - throw new IllegalArgumentException("Given data has inconsistent dimensions: '" + dimensions + "' vs '" + point.keySet() + "'."); - - copies.put(copyOf(point), statistic); - }); - - return new Metric(copyOf(copies)); - } - - /** Returns a Metric view of the subset of points in the given hyperplane; its dimensions must be a subset of those of this Metric. */ - public Metric at(Map<String, ?> hyperplane) { - return new Metric(statistics.keySet().stream() - .filter(point -> point.entrySet().containsAll(hyperplane.entrySet())) - .collect(toUnmodifiableMap(point -> point, statistics::get))); - } - - /** Returns a version of this where statistics along the given hyperspace are aggregated. This does not preserve last, 95 and 99 percentile values. */ - public Metric collapse(Set<String> hyperspace) { - return new Metric(statistics.keySet().stream() - .collect(toUnmodifiableMap(point -> point.keySet().stream() - .filter(dimension -> ! hyperspace.contains(dimension)) - .collect(toUnmodifiableMap(dimension -> dimension, point::get)), - statistics::get, - Statistic::mergedWith))); - } - - /** Returns a collapsed version of this, with all statistics aggregated. This does not preserve last, 95 and 99 percentile values. */ - public Metric collapse() { - Map<String, ?> firstStatistic = statistics.keySet().iterator().next(); - return firstStatistic == null ? this : collapse(firstStatistic.keySet()); - } - - /** If this Metric contains a single point, returns the Statistic of that point; otherwise, throws an exception. */ - public Statistic statistic() { - if (statistics.size() == 1) - return statistics.values().iterator().next(); - - if (statistics.isEmpty()) - throw new NoSuchElementException("This Metric has no data."); - - throw new IllegalStateException("This Metric has more than one point of data."); - } - - /** Returns the underlying, unmodifiable Map. */ - public Map<Map<String, ?>, Statistic> asMap() { - return statistics; - } - - @Override - public String toString() { - return new StringJoiner(", ", Metric.class.getSimpleName() + "[", "]") - .add("statistics=" + statistics) - .toString(); - } - -} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java deleted file mode 100644 index f1c5fbb0ba5..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.metric; - -import ai.vespa.hosted.cd.Endpoint; - -import java.time.Instant; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.StringJoiner; - -import static java.util.Map.copyOf; - -/** - * Metrics from a Vespa application {@link Endpoint}, indexed by their names, and optionally by a set of custom dimensions. - * - * Metrics are collected from the <a href="https://docs.vespa.ai/documentation/reference/metrics-health-format.html">metrics</a> - * API of a Vespa endpoint, and contain the current health status of the endpoint, values for all configured metrics in - * that endpoint, and the time interval from which these metrics were sampled. - * - * Each metric is indexed by a name, and, optionally, along a custom set of dimensions, given by a {@code Map<String, String>}. - * - * @author jonmv - */ -public class Metrics { - - private final Instant start, end; - private final Map<String, Metric> metrics; - - private Metrics(Instant start, Instant end, Map<String, Metric> metrics) { - this.start = start; - this.end = end; - this.metrics = metrics; - } - - public static Metrics of(Instant start, Instant end, Map<String, Metric> metrics) { - if ( ! start.isBefore(end)) - throw new IllegalArgumentException("Given time interval must be positive: '" + start + "' to '" + end + "'."); - - return new Metrics(start, end, copyOf(metrics)); - } - - /** Returns the start of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */ - public Instant start() { - return start; - } - - /** Returns the end of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */ - public Instant end() { - return end; - } - - /** Returns the metric with the given name, or throws a NoSuchElementException if no such Metric is known. */ - public Metric get(String name) { - if ( ! metrics.containsKey(name)) - throw new NoSuchElementException("No metric with name '" + name + "'."); - - return metrics.get(name); - } - - /** Returns the underlying, unmodifiable Map. */ - public Map<String, Metric> asMap() { - return metrics; - } - - @Override - public String toString() { - return new StringJoiner(", ", Metrics.class.getSimpleName() + "[", "]") - .add("start=" + start) - .add("end=" + end) - .add("metrics=" + metrics) - .toString(); - } - -} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java deleted file mode 100644 index 561c0f9dee3..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.metric; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.IntStream; - -import static java.util.stream.Collectors.toUnmodifiableMap; - -/** - * Used to easily generate points (Map<String, ?>) for a space defined here by its dimension names. - * - * @author jonmv - */ -public class Space { - - private final List<String> dimensions; - - private Space(List<String> dimensions) { - this.dimensions = dimensions; - } - - /** Creates a new space with the given named dimensions, in order. */ - public static Space of(List<String> dimensions) { - if (Set.copyOf(dimensions).size() != dimensions.size()) - throw new IllegalArgumentException("Duplicated dimension names in '" + dimensions + "'."); - - return new Space(List.copyOf(dimensions)); - } - - /** Returns a point in this space, with the given values along each dimensions, in order. */ - public Map<String, ?> at(List<?> values) { - if (dimensions.size() != values.size()) - throw new IllegalArgumentException("This space has " + dimensions.size() + " dimensions, but " + values.size() + " were given."); - - return IntStream.range(0, dimensions.size()).boxed().collect(toUnmodifiableMap(dimensions::get, values::get)); - } - - /** Returns a point in this space, with the given values along each dimensions, in order. */ - public Map<String, ?> at(Object... values) { - return at(List.of(values)); - } - -} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java deleted file mode 100644 index 62b2528e0a4..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.metric; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.StringJoiner; - -import static java.util.Map.copyOf; - -/** - * Known statistic about a metric, at a certain point. - * - * @author jonmv - */ -public class Statistic { - - private final Map<Type, Double> data; - - /** Creates a new Statistic with a copy of the given data. */ - private Statistic(Map<Type, Double> data) { - this.data = data; - } - - public static Statistic of(Map<Type, Double> data) { - for (Type type : List.of(Type.count, Type.rate, Type.average)) - if ( ! data.containsKey(type)) - throw new IllegalArgumentException("Required data type '" + type + "' not present in '" + data + "'"); - - return new Statistic(copyOf(data)); - } - - /** Returns the value of the given type, or throws a NoSuchElementException if this isn't known. */ - public double get(Type key) { - if ( ! data.containsKey(key)) - throw new NoSuchElementException("No value with key '" + key + "' is known."); - - return data.get(key); - } - - /** Returns the underlying, unmodifiable Map. */ - public Map<Type, Double> asMap() { - return data; - } - - Statistic mergedWith(Statistic other) { - if (data.keySet().equals(other.data.keySet())) - throw new IllegalArgumentException("Unequal key sets '" + data.keySet() + "' and '" + other.data.keySet() + "'."); - - Map<Type, Double> merged = new HashMap<>(); - double n1 = get(Type.count), n2 = other.get(Type.count); - for (Type type : data.keySet()) switch (type) { - case count: merged.put(type, n1 + n2); break; - case rate: merged.put(type, get(Type.rate) + other.get(Type.rate)); break; - case max: merged.put(type, Math.max(get(Type.max), other.get(Type.max))); break; - case min: merged.put(type, Math.min(get(Type.min), other.get(Type.min))); break; - case average: merged.put(type, (n1 * get(Type.average) + n2 * other.get(Type.average)) / (n1 + n2)); break; - case last: - case percentile95: - case percentile99: break; - default: throw new IllegalArgumentException("Unexpected type '" + type + "'."); - } - return of(merged); - } - - @Override - public String toString() { - return new StringJoiner(", ", Statistic.class.getSimpleName() + "[", "]") - .add("data=" + data) - .toString(); - } - -} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java deleted file mode 100644 index d02593e5eb3..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.metric; - -/** - * Known statistic types. - */ -public enum Type { - - /** 95th percentile measurement. */ - percentile95, - - /** 99th percentile measurement. */ - percentile99, - - /** Average over all measurements. */ - average, - - /** Number of measurements. */ - count, - - /** Last measurement. */ - last, - - /** Maximum measurement. */ - max, - - /** Minimum measurement. */ - min, - - /** Number of measurements per second. */ - rate; - -} diff --git a/travis/travis-build-full.sh b/travis/travis-build-full.sh index 2f565e43605..46268a964c2 100755 --- a/travis/travis-build-full.sh +++ b/travis/travis-build-full.sh @@ -9,7 +9,7 @@ export MAVEN_OPTS="-Xss1m -Xms128m -Xmx2g" source /etc/profile.d/enable-devtoolset-9.sh source /etc/profile.d/enable-rh-maven35.sh -ccache --max-size=1250M +ccache --max-size=1600M ccache --set-config=compression=true ccache -p diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java index e5ed885b316..f6888acb018 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java @@ -2,18 +2,56 @@ package com.yahoo.vespa.athenz.identity; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; +import com.yahoo.security.X509CertificateWithKey; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; import javax.net.ssl.SSLContext; +import java.nio.file.Path; /** - * A interface for types that provides a service identity. - * Some similarities to {@link AthenzIdentityProvider}, but this type is not public api and intended for internal use. + * A interface for types that provides the Athenz service identity (SIA) from the environment. + * Some similarities to {@link AthenzIdentityProvider}, but this type is not public API and intended for internal use. * * @author bjorncs */ public interface ServiceIdentityProvider { + /** + * + * @return The Athenz identity of the environment + */ AthenzIdentity identity(); + + /** + * @return {@link SSLContext} that is automatically updated. + */ SSLContext getIdentitySslContext(); + + /** + * @return Current certificate and private key. Unlike {@link #getIdentitySslContext()} underlying credentials are not automatically updated. + */ + X509CertificateWithKey getIdentityCertificateWithKey(); + + /** + * @return Path to X.509 certificate in PEM format + */ + Path certificatePath(); + + /** + * @return Path to private key in PEM format + */ + Path privateKeyPath(); + + /** + * @return Path to Athenz truststore in PEM format + */ + Path athenzTruststorePath(); + + /** + * The client truststore contains the Athenz certificates from {@link #athenzTruststorePath()} + * and additional certificate authorities that issues trusted server certificates. + * + * @return Path to client truststore in PEM format + */ + Path clientTruststorePath(); + } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index 4981b80998f..9f9bd8be955 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -3,15 +3,14 @@ package com.yahoo.vespa.athenz.identity; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.security.KeyStoreType; import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.X509CertificateWithKey; import com.yahoo.security.tls.AutoReloadingX509KeyManager; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.utils.SiaUtils; import javax.net.ssl.SSLContext; -import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; @@ -26,34 +25,43 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde private final AutoReloadingX509KeyManager keyManager; private final SSLContext sslContext; private final AthenzIdentity service; + private final Path certificateFile; + private final Path privateKeyFile; + private final Path clientTruststoreFile; + private final Path athenzTruststoreFile; @Inject public SiaIdentityProvider(SiaProviderConfig config) { this(new AthenzService(config.athenzDomain(), config.athenzService()), - SiaUtils.getPrivateKeyFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())).toFile(), - SiaUtils.getCertificateFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())).toFile(), - new File(config.trustStorePath()), - config.trustStoreType()); + SiaUtils.getPrivateKeyFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())), + SiaUtils.getCertificateFile(Paths.get(config.keyPathPrefix()), new AthenzService(config.athenzDomain(), config.athenzService())), + Paths.get(config.athenzTruststorePath()), + Paths.get(config.trustStorePath())); } public SiaIdentityProvider(AthenzIdentity service, Path siaPath, - File trustStoreFile) { + Path athenzTruststoreFile, + Path clientTruststoreFile) { this(service, - SiaUtils.getPrivateKeyFile(siaPath, service).toFile(), - SiaUtils.getCertificateFile(siaPath, service).toFile(), - trustStoreFile, - SiaProviderConfig.TrustStoreType.Enum.jks); + SiaUtils.getPrivateKeyFile(siaPath, service), + SiaUtils.getCertificateFile(siaPath, service), + athenzTruststoreFile, + clientTruststoreFile); } public SiaIdentityProvider(AthenzIdentity service, - File privateKeyFile, - File certificateFile, - File trustStoreFile, - SiaProviderConfig.TrustStoreType.Enum trustStoreType) { + Path privateKeyFile, + Path certificateFile, + Path athenzTruststoreFile, + Path clientTruststoreFile) { this.service = service; - this.keyManager = AutoReloadingX509KeyManager.fromPemFiles(privateKeyFile.toPath(), certificateFile.toPath()); - this.sslContext = createIdentitySslContext(keyManager, trustStoreFile.toPath(), trustStoreType); + this.keyManager = AutoReloadingX509KeyManager.fromPemFiles(privateKeyFile, certificateFile); + this.sslContext = createIdentitySslContext(keyManager, clientTruststoreFile); + this.certificateFile = certificateFile; + this.privateKeyFile = privateKeyFile; + this.athenzTruststoreFile = athenzTruststoreFile; + this.clientTruststoreFile = clientTruststoreFile; } @Override @@ -66,17 +74,17 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde return sslContext; } - private static SSLContext createIdentitySslContext(AutoReloadingX509KeyManager keyManager, Path trustStoreFile, - SiaProviderConfig.TrustStoreType.Enum trustStoreType) { - var builder = new SslContextBuilder(); - if (trustStoreType == SiaProviderConfig.TrustStoreType.Enum.pem) { - builder = builder.withTrustStore(trustStoreFile); - } else if (trustStoreType == SiaProviderConfig.TrustStoreType.Enum.jks) { - builder = builder.withTrustStore(trustStoreFile, KeyStoreType.JKS); - } else { - throw new IllegalArgumentException("Unsupported trust store type: " + trustStoreType); - } - return builder.withKeyManager(keyManager).build(); + @Override public X509CertificateWithKey getIdentityCertificateWithKey() { return keyManager.getCurrentCertificateWithKey(); } + @Override public Path certificatePath() { return certificateFile; } + @Override public Path privateKeyPath() { return privateKeyFile; } + @Override public Path athenzTruststorePath() { return athenzTruststoreFile; } + @Override public Path clientTruststorePath() { return clientTruststoreFile; } + + private static SSLContext createIdentitySslContext(AutoReloadingX509KeyManager keyManager, Path trustStoreFile) { + return new SslContextBuilder() + .withTrustStore(trustStoreFile) + .withKeyManager(keyManager) + .build(); } @Override diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java index 3b733e05708..8e029906c30 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java @@ -64,6 +64,9 @@ class AthenzCredentialsService { this.clock = clock; } + Path certificatePath() { return SiaUtils.getCertificateFile(VESPA_SIA_DIRECTORY, tenantIdentity); } + Path privateKeyPath() { return SiaUtils.getPrivateKeyFile(VESPA_SIA_DIRECTORY, tenantIdentity); } + AthenzCredentials registerInstance() { Optional<AthenzCredentials> athenzCredentialsFromDisk = tryReadCredentialsFromDisk(); if (athenzCredentialsFromDisk.isPresent()) { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index 71a4c1a9954..65574d7583e 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -10,11 +10,10 @@ import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; -import java.util.logging.Level; import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.X509CertificateWithKey; import com.yahoo.security.tls.MutableX509KeyManager; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -44,9 +43,10 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Function; +import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; -import static com.yahoo.security.KeyStoreType.JKS; import static com.yahoo.security.KeyStoreType.PKCS12; /** @@ -55,6 +55,8 @@ import static com.yahoo.security.KeyStoreType.PKCS12; * @author mortent * @author bjorncs */ +// This class should probably not implement ServiceIdentityProvider, +// as that interface is intended for providing the node's identity, not the tenant's application identity. public final class AthenzIdentityProviderImpl extends AbstractComponent implements AthenzIdentityProvider, ServiceIdentityProvider { private static final Logger log = Logger.getLogger(AthenzIdentityProviderImpl.class.getName()); @@ -66,8 +68,9 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private final static Duration ROLE_SSL_CONTEXT_EXPIRY = Duration.ofHours(24); private final static Duration ROLE_TOKEN_EXPIRY = Duration.ofMinutes(30); - // TODO Make path to trust store config - private static final Path DEFAULT_TRUST_STORE = Paths.get("/opt/yahoo/share/ssl/certs/yahoo_certificate_bundle.jks"); + // TODO Make path to trust store paths config + private static final Path CLIENT_TRUST_STORE = Paths.get("/opt/yahoo/share/ssl/certs/yahoo_certificate_bundle.pem"); + private static final Path ATHENZ_TRUST_STORE = Paths.get("/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"); public static final String CERTIFICATE_EXPIRY_METRIC_NAME = "athenz-tenant-cert.expiry.seconds"; @@ -93,9 +96,9 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen public AthenzIdentityProviderImpl(IdentityConfig config, Metric metric) { this(config, metric, - DEFAULT_TRUST_STORE, + CLIENT_TRUST_STORE, new AthenzCredentialsService(config, - createNodeIdentityProvider(config, DEFAULT_TRUST_STORE), + createNodeIdentityProvider(config), Defaults.getDefaults().vespaHostname(), Clock.systemUTC()), new ScheduledThreadPoolExecutor(1), @@ -141,7 +144,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private static SSLContext createIdentitySslContext(X509ExtendedKeyManager keyManager, Path trustStore) { return new SslContextBuilder() .withKeyManager(keyManager) - .withTrustStore(trustStore, JKS) + .withTrustStore(trustStore) .build(); } @@ -176,6 +179,20 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } @Override + public X509CertificateWithKey getIdentityCertificateWithKey() { + AthenzCredentials copy = this.credentials; + return new X509CertificateWithKey(copy.getCertificate(), copy.getKeyPair().getPrivate()); + } + + @Override public Path certificatePath() { return athenzCredentialsService.certificatePath(); } + + @Override public Path privateKeyPath() { return athenzCredentialsService.privateKeyPath(); } + + @Override public Path athenzTruststorePath() { return ATHENZ_TRUST_STORE; } + + @Override public Path clientTruststorePath() { return CLIENT_TRUST_STORE; } + + @Override public SSLContext getRoleSslContext(String domain, String role) { // This ssl context should ideally be cached as it is quite expensive to create. try { @@ -205,12 +222,23 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen @Override public String getAccessToken(String domain) { - return null; + try { + return domainSpecificAccessTokenCache.get(new AthenzDomain(domain)).value(); + } catch (Exception e) { + throw new AthenzIdentityProviderException("Could not retrieve access token: " + e.getMessage(), e); + } } @Override public String getAccessToken(String domain, List<String> roles) { - return null; + try { + List<AthenzRole> roleList = roles.stream() + .map(roleName -> new AthenzRole(domain, roleName)) + .collect(Collectors.toList()); + return roleSpecificAccessTokenCache.get(roleList).value(); + } catch (Exception e) { + throw new AthenzIdentityProviderException("Could not retrieve access token: " + e.getMessage(), e); + } } @Override @@ -243,7 +271,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen X509Certificate roleCertificate = client.getRoleCertificate(role, csr); return new SslContextBuilder() .withKeyStore(credentials.getKeyPair().getPrivate(), roleCertificate) - .withTrustStore(trustStore, KeyStoreType.JKS) + .withTrustStore(trustStore) .build(); } } @@ -286,9 +314,9 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } } - private static SiaIdentityProvider createNodeIdentityProvider(IdentityConfig config, Path trustStore) { + private static SiaIdentityProvider createNodeIdentityProvider(IdentityConfig config) { return new SiaIdentityProvider( - new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, trustStore.toFile()); + new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, ATHENZ_TRUST_STORE, CLIENT_TRUST_STORE); } private boolean isExpired(AthenzCredentials credentials) { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java index 12a8c3f911e..894c04df233 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -53,10 +53,11 @@ public class SiaUtils { } public static Path getCaCertificatesFile() { - // The contents of this is the same as /opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem installed - // by the yahoo_certificates_bundle RPM package, except the latter also contains a textual description - // (decoded) of the certificates. - return DEFAULT_SIA_DIRECTORY.resolve("certs").resolve("ca.cert.pem"); + return getCaCertificatesFile(DEFAULT_SIA_DIRECTORY); + } + + public static Path getCaCertificatesFile(Path root) { + return root.resolve("certs").resolve("ca.cert.pem"); } public static Optional<PrivateKey> readPrivateKeyFile(AthenzIdentity service) { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java index 47ae45a69ca..c010649ed24 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.zpe; +import com.yahoo.athenz.auth.token.AccessToken; import com.yahoo.athenz.zpe.AuthZpeClient; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzResourceName; @@ -20,6 +21,8 @@ public class DefaultZpe implements Zpe { public DefaultZpe() { AuthZpeClient.init(); + // Disable access token cert offset validation to allow existing tokens with refreshed certs + AccessToken.setAccessTokenCertOffset(-1); } @Override diff --git a/vespa-athenz/src/main/resources/configdefinitions/sia-provider.def b/vespa-athenz/src/main/resources/configdefinitions/sia-provider.def index b1145a9a4fc..77310d0c91e 100644 --- a/vespa-athenz/src/main/resources/configdefinitions/sia-provider.def +++ b/vespa-athenz/src/main/resources/configdefinitions/sia-provider.def @@ -6,3 +6,4 @@ athenzService string keyPathPrefix string trustStorePath string trustStoreType enum {pem, jks} default=jks +athenzTruststorePath string default="/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"
\ No newline at end of file diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java index ce02860cc78..b7db502b1d0 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identity/SiaIdentityProviderTest.java @@ -2,15 +2,11 @@ package com.yahoo.vespa.athenz.identity; import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; -import com.yahoo.security.KeyStoreUtils; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.yolean.Exceptions; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -21,11 +17,11 @@ import java.io.IOException; import java.math.BigInteger; import java.nio.file.Files; import java.security.KeyPair; -import java.security.KeyStore; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; +import static com.yahoo.yolean.Exceptions.uncheck; import static org.junit.Assert.assertNotNull; /** @@ -52,10 +48,10 @@ public class SiaIdentityProviderTest { SiaIdentityProvider provider = new SiaIdentityProvider( new AthenzService("domain", "service-name"), - keyFile, - certificateFile, - trustStoreFile, - SiaProviderConfig.TrustStoreType.Enum.jks); + keyFile.toPath(), + certificateFile.toPath(), + trustStoreFile.toPath(), + trustStoreFile.toPath()); assertNotNull(provider.getIdentitySslContext()); } @@ -76,10 +72,10 @@ public class SiaIdentityProviderTest { SiaIdentityProvider provider = new SiaIdentityProvider( new AthenzService("domain", "service-name"), - keyFile, - certificateFile, - trustStoreFile, - SiaProviderConfig.TrustStoreType.Enum.pem); + keyFile.toPath(), + certificateFile.toPath(), + trustStoreFile.toPath(), + trustStoreFile.toPath()); assertNotNull(provider.getIdentitySslContext()); } @@ -109,14 +105,11 @@ public class SiaIdentityProviderTest { private void createPemTrustStoreFile(X509Certificate certificate, File trustStoreFile) { var pemEncoded = X509CertificateUtils.toPem(certificate); - Exceptions.uncheck(() -> Files.writeString(trustStoreFile.toPath(), pemEncoded)); + uncheck(() -> Files.writeString(trustStoreFile.toPath(), pemEncoded)); } private void createTrustStoreFile(X509Certificate certificate, File trustStoreFile) { - KeyStore keystore = KeyStoreBuilder.withType(KeyStoreType.JKS) - .withCertificateEntry("dummy-cert", certificate) - .build(); - KeyStoreUtils.writeKeyStoreToFile(keystore, trustStoreFile.toPath()); + uncheck(() -> Files.writeString(trustStoreFile.toPath(), X509CertificateUtils.toPem(certificate))); } } diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 95b6528bd77..3faf47ccfa9 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -441,7 +441,6 @@ public class DocumentGenMojo extends AbstractMojo { ind(1)+"/** The doc type of this.*/\n" + ind(1)+"public static final com.yahoo.document.DocumentType type = getDocumentType();\n\n"+ ind(1)+"/** Struct type view of the type of the body of this.*/\n" + - ind(1)+"private static final com.yahoo.document.StructDataType bodyStructType = getBodyStructType();\n\n" + ind(1)+"/** Struct type view of the type of the header of this.*/\n" + ind(1)+"private static final com.yahoo.document.StructDataType headerStructType = getHeaderStructType();\n\n"); @@ -460,9 +459,7 @@ public class DocumentGenMojo extends AbstractMojo { // Mimic header and body to make serialization work. // This can be improved by generating a method to serialize the document _here_, and use that in serialization. exportOverriddenStructGetter(docType.allHeader().getFields(), out, 1, "getHeader", className+".headerStructType"); - exportOverriddenStructGetter(docType.allBody().getFields(), out, 1, "getBody", className+".bodyStructType"); exportStructTypeGetter(docType.getName()+".header", docType.allHeader().getFields(), out, 1, "getHeaderStructType", "com.yahoo.document.StructDataType"); - exportStructTypeGetter(docType.getName()+".body", docType.allBody().getFields(), out, 1, "getBodyStructType", "com.yahoo.document.StructDataType"); Collection<Field> allUniqueFields = getAllUniqueFields(multiExtends, docType.getAllFields()); exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(), diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 1224e668bc0..8f0e70f554c 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -72,6 +72,7 @@ public class OperationHandlerImpl implements OperationHandler { public static final int VISIT_TIMEOUT_MS = 120000; public static final int WANTED_DOCUMENT_COUNT_UPPER_BOUND = 1000; // Approximates the max default size of a bucket + public static final int CONCURRENCY_UPPER_BOUND = 100; private final DocumentAccess documentAccess; private final DocumentApiMetrics metricsHelper; private final ClusterEnumerator clusterEnumerator; @@ -109,14 +110,14 @@ public class OperationHandlerImpl implements OperationHandler { private static final int HTTP_STATUS_BAD_REQUEST = 400; private static final int HTTP_STATUS_INSUFFICIENT_STORAGE = 507; - private static final int HTTP_PRE_CONDIDTION_FAILED = 412; + private static final int HTTP_PRECONDITION_FAILED = 412; public static int getHTTPStatusCode(Set<Integer> errorCodes) { if (errorCodes.size() == 1 && errorCodes.contains(DocumentProtocol.ERROR_NO_SPACE)) { return HTTP_STATUS_INSUFFICIENT_STORAGE; } if (errorCodes.contains(DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED)) { - return HTTP_PRE_CONDIDTION_FAILED; + return HTTP_PRECONDITION_FAILED; } return HTTP_STATUS_BAD_REQUEST; } @@ -399,6 +400,11 @@ public class OperationHandlerImpl implements OperationHandler { return selection.toString(); } + private static int computeEffectiveConcurrency(Optional<Integer> requestConcurrency) { + int wantedConcurrency = requestConcurrency.orElse(1); + return Math.min(Math.max(wantedConcurrency, 1), CONCURRENCY_UPPER_BOUND); + } + private VisitorParameters createVisitorParameters( RestUri restUri, String documentSelection, @@ -425,7 +431,7 @@ public class OperationHandlerImpl implements OperationHandler { params.setMaxTotalHits(options.wantedDocumentCount .map(n -> Math.min(Math.max(n, 1), WANTED_DOCUMENT_COUNT_UPPER_BOUND)) .orElse(1)); - params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(options.concurrency.orElse(1))); + params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(computeEffectiveConcurrency(options.concurrency))); params.setToTimestamp(0L); params.setFromTimestamp(0L); params.setSessionTimeoutMs(VISIT_TIMEOUT_MS); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index ccd2d80efb2..cf33b6033cd 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -99,10 +99,7 @@ public class RestApi extends LoggingRequestHandler { } // For testing and development - public RestApi(Executor executor, - AccessLog accessLog, - OperationHandler operationHandler, - int threadsAvailable) { + RestApi(Executor executor, AccessLog accessLog, OperationHandler operationHandler, int threadsAvailable) { super(executor, accessLog, null); this.operationHandler = operationHandler; this.threadsAvailableForApi = new AtomicInteger(threadsAvailable); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index 5c9669e5258..a932ca935e0 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -45,9 +45,6 @@ public class FeedHandlerV3 extends LoggingRequestHandler { protected final ReplyHandler feedReplyHandler; private final Metric metric; private final Object monitor = new Object(); - private int remainingThreadsForFeedingAllowance; - private final Duration timeBetweenBumpingMaxThreads; - private Instant nextTimeToAllocateAnotherThread = Instant.now(); private final AtomicInteger threadsAvailableForFeeding; private static final Logger log = Logger.getLogger(FeedHandlerV3.class.getName()); @@ -65,20 +62,10 @@ public class FeedHandlerV3 extends LoggingRequestHandler { this.metric = parentCtx.getMetric(); // 40% of the threads can be blocking on feeding before we deny requests. if (threadpoolConfig != null) { - remainingThreadsForFeedingAllowance = Math.max((int) (0.4 * threadpoolConfig.maxthreads()), 1); - if (threadpoolConfig.softStartSeconds() > 0.0) { - threadsAvailableForFeeding = new AtomicInteger(0); - timeBetweenBumpingMaxThreads = Duration.ofMillis((long)(threadpoolConfig.softStartSeconds() * 1000) / remainingThreadsForFeedingAllowance); - } else { - threadsAvailableForFeeding = new AtomicInteger(remainingThreadsForFeedingAllowance); - remainingThreadsForFeedingAllowance = 0; - timeBetweenBumpingMaxThreads = null; - } + threadsAvailableForFeeding = new AtomicInteger(Math.max((int) (0.4 * threadpoolConfig.maxthreads()), 1)); } else { log.warning("No config for threadpool, using 200 for max blocking threads for feeding."); threadsAvailableForFeeding = new AtomicInteger(200); - remainingThreadsForFeedingAllowance = 0; - timeBetweenBumpingMaxThreads = null; } } @@ -93,12 +80,6 @@ public class FeedHandlerV3 extends LoggingRequestHandler { String clientId = clientId(request); ClientFeederV3 clientFeederV3; synchronized (monitor) { - Instant now = Instant.now(); - if ((remainingThreadsForFeedingAllowance > 0) && (now.isAfter(nextTimeToAllocateAnotherThread))) { - threadsAvailableForFeeding.incrementAndGet(); - remainingThreadsForFeedingAllowance --; - nextTimeToAllocateAnotherThread = now.plus(timeBetweenBumpingMaxThreads); - } if (! clientFeederByClientId.containsKey(clientId)) { SourceSessionParams sourceSessionParams = sourceSessionParams(request); clientFeederByClientId.put(clientId, diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java index bda49ecd3f5..91c52c4b98b 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java @@ -290,18 +290,34 @@ public class OperationHandlerImplTest { assertThat(params.fieldSet(), equalTo("document-type:bjarne")); } + private void assertConcurrencyPropagated(VisitorParameters params, int expectedConcurrency) { + assertThat(params.getThrottlePolicy(), instanceOf(StaticThrottlePolicy.class)); + assertThat(((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount(), is(expectedConcurrency)); + } + @Test public void visit_concurrency_is_1_by_default() throws Exception { VisitorParameters params = generatedParametersFromVisitOptions(emptyVisitOptions()); - assertThat(params.getThrottlePolicy(), instanceOf(StaticThrottlePolicy.class)); - assertThat(((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount(), is((int)1)); + assertConcurrencyPropagated(params, 1); } @Test public void visit_concurrency_is_propagated_to_visitor_parameters() throws Exception { VisitorParameters params = generatedParametersFromVisitOptions(optionsBuilder().concurrency(3).build()); - assertThat(params.getThrottlePolicy(), instanceOf(StaticThrottlePolicy.class)); - assertThat(((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount(), is((int)3)); + assertConcurrencyPropagated(params, 3); + } + + @Test + public void too_low_visit_concurrency_is_capped_to_1() throws Exception { + VisitorParameters params = generatedParametersFromVisitOptions(optionsBuilder().concurrency(0).build()); + assertConcurrencyPropagated(params, 1); + } + + @Test + public void too_high_visit_concurrency_is_capped_to_max() throws Exception { + VisitorParameters params = generatedParametersFromVisitOptions( + optionsBuilder().concurrency(OperationHandlerImpl.CONCURRENCY_UPPER_BOUND + 1).build()); + assertConcurrencyPropagated(params, OperationHandlerImpl.CONCURRENCY_UPPER_BOUND); } @Test diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java index 0b3ee6d0792..eb6bb609970 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java @@ -33,7 +33,7 @@ public class MockedOperationHandler implements OperationHandler { @SuppressWarnings("deprecation") public void put(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException { log.append("PUT: " + data.getDocument().getId()); - log.append(data.getDocument().getBody().toString()); + log.append(data.getDocument().getHeader().toString()); } @Override diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java index 0bb42851347..1efa8129cdb 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java @@ -77,20 +77,6 @@ public class FeedTesterV3 { assertThat(Splitter.on("\n").splitToList(result).size(), is(101)); } - @Test - public void softRestart() throws Exception { - ThreadpoolConfig.Builder builder = new ThreadpoolConfig.Builder().softStartSeconds(5); - final FeedHandlerV3 feedHandlerV3 = setupFeederHandler(builder.build()); - for (int i= 0; i < 100; i++) { - HttpResponse httpResponse = feedHandlerV3.handle(createRequest(100)); - ByteArrayOutputStream outStream = new ByteArrayOutputStream(); - httpResponse.render(outStream); - assertThat(httpResponse.getContentType(), is("text/plain")); - String result = Utf8.toString(outStream.toByteArray()); - assertThat(Splitter.on("\n").splitToList(result).size(), is(101)); - } - } - private static DocumentTypeManager createDoctypeManager() { DocumentTypeManager docTypeManager = new DocumentTypeManager(); DocumentType documentType = new DocumentType("testdocument"); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java index ff84f5117dd..e652dc86ab2 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java @@ -19,9 +19,10 @@ import java.text.SimpleDateFormat; /** * An abstract class that can be subclassed by different visitor handlers. * - * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + * @author Thomas Gundersen */ public abstract class VdsVisitHandler { + boolean showProgress; boolean showStatistics; boolean abortOnClusterDown; @@ -33,8 +34,7 @@ public abstract class VdsVisitHandler { final VisitorControlHandler controlHandler = new ControlHandler(); - public VdsVisitHandler(boolean showProgress, boolean showStatistics, boolean abortOnClusterDown) - { + public VdsVisitHandler(boolean showProgress, boolean showStatistics, boolean abortOnClusterDown) { this.showProgress = showProgress; this.showStatistics = showStatistics; this.abortOnClusterDown = abortOnClusterDown; @@ -72,8 +72,7 @@ public abstract class VdsVisitHandler { return printLock; } - public void onDone() { - } + public void onDone() { } public String getProgressFileName() { return progressFileName; @@ -127,8 +126,7 @@ public abstract class VdsVisitHandler { } private String getDateTime() { - DateFormat dateFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss zzz"); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss zzz"); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); Date date = new Date(); return dateFormat.format(date); @@ -140,14 +138,10 @@ public abstract class VdsVisitHandler { System.err.print('\r'); lastLineIsProgress = false; } - System.err.println("Visitor error (" + getDateTime() + "): " + - message); - if (abortOnClusterDown && - !isDone() && - (message.lastIndexOf("Could not resolve")>=0 || - message.lastIndexOf("don't allow external load")>=0)) { - System.err.println("Aborting visitor as " + - "--abortonclusterdown flag is set."); + System.err.println("Visitor error (" + getDateTime() + "): " + message); + if (abortOnClusterDown && !isDone() && (message.lastIndexOf("Could not resolve")>=0 || + message.lastIndexOf("don't allow external load")>=0)) { + System.err.println("Aborting visitor as --abortonclusterdown flag is set."); abort(); } } @@ -160,11 +154,12 @@ public abstract class VdsVisitHandler { if (code != CompletionCode.SUCCESS) { if (code == CompletionCode.ABORTED) { System.err.println("Visitor aborted: " + message); - } else if (code == CompletionCode.TIMEOUT) { + } + else if (code == CompletionCode.TIMEOUT) { System.err.println("Visitor timed out: " + message); - } else { - System.err.println("Visitor aborted due to unknown issue " - + code + ": " + message); + } + else { + System.err.println("Visitor aborted due to unknown issue " + code + ": " + message); } } else { if (showProgress) { diff --git a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java index 176e5044bc2..de1040852f5 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java @@ -1,12 +1,15 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.collections; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.Random; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Predicate; @@ -88,4 +91,11 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte return items.iterator(); } + /** Returns the items in this shuffled using random as source of randomness */ + public final ListType shuffle(Random random) { + ArrayList<Type> shuffled = new ArrayList<>(items); + Collections.shuffle(shuffled, random); + return constructor.apply(shuffled, false); + } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java index 2ed7331a60c..5084e6554cb 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java @@ -4,7 +4,12 @@ package com.yahoo.slime; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Iterator; import java.util.Optional; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; /** * Extra utilities/operations on slime trees. @@ -118,4 +123,20 @@ public class SlimeUtils { public static Optional<String> optionalString(Inspector inspector) { return Optional.of(inspector.asString()).filter(s -> !s.isEmpty()); } + + public static Iterator<Inspector> entriesIterator(Inspector inspector) { + return new Iterator<>() { + private int current = 0; + @Override public boolean hasNext() { return current < inspector.entries(); } + @Override public Inspector next() { return inspector.entry(current++); } + }; + } + + /** Returns stream of entries for given inspector. If the inspector is not an array, empty stream is returned */ + public static Stream<Inspector> entriesStream(Inspector inspector) { + int characteristics = Spliterator.NONNULL | Spliterator.SIZED | Spliterator.ORDERED; + return StreamSupport.stream( + Spliterators.spliteratorUnknownSize(entriesIterator(inspector), characteristics), + false); + } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java index 10995d3dd2d..c78be98e11e 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java @@ -20,8 +20,8 @@ import java.util.Iterator; /** * Writes tensors on the JSON format used in Vespa tensor document fields: * A JSON map containing a 'cells' or 'values' array. - * See <a href="http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor"> - * http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor</a> + * See <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html"> + * https://docs.vespa.ai/documentation/reference/document-json-format.html</a> * * @author bratseth */ diff --git a/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java b/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java index 237b1575bfb..5757202aaf1 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java +++ b/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java @@ -5,6 +5,8 @@ import com.yahoo.text.Utf8; import org.junit.Test; import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; @@ -97,4 +99,15 @@ public class SlimeUtilsTest { } } + @Test + public void test_stream() { + String json = "{\"constant\":0,\"list\":[1,2,4,3,0],\"object\":{\"a\":1,\"c\":3,\"b\":2}}"; + Inspector inspector = SlimeUtils.jsonToSlimeOrThrow(json).get(); + assertEquals(0, SlimeUtils.entriesStream(inspector.field("constant")).count()); + assertEquals(0, SlimeUtils.entriesStream(inspector.field("object")).count()); + + assertEquals(List.of(1L, 2L, 4L, 3L, 0L), + SlimeUtils.entriesStream(inspector.field("list")).map(Inspector::asLong).collect(Collectors.toList())); + } + } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 2675bc16bf2..1ca9816a921 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -24,8 +24,8 @@ vespa_define_module( src/tests/assert src/tests/barrier src/tests/benchmark_timer - src/tests/btree src/tests/box + src/tests/btree src/tests/closure src/tests/component src/tests/compress @@ -128,6 +128,7 @@ vespa_define_module( src/tests/tutorial/minimal src/tests/tutorial/simple src/tests/tutorial/threads + src/tests/typify src/tests/util/generationhandler src/tests/util/generationhandler_stress src/tests/util/md5 diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index 848c8a37125..63afd8b770f 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -36,6 +36,54 @@ toStr(const T & v) return ss.str(); } +class SequenceValidator +{ + int _wanted_count; + int _prev_key; + int _count; + bool _failed; + +public: + SequenceValidator(int start, int wanted_count) + : _wanted_count(wanted_count), + _prev_key(start - 1), + _count(0), + _failed(false) + { + } + + bool failed() const { + return _failed || _wanted_count != _count; + } + + void operator()(int key) { + if (key != _prev_key + 1) { + _failed = true; + } + _prev_key = key; + ++_count; + } +}; + +class ForeachKeyValidator +{ + SequenceValidator & _validator; +public: + ForeachKeyValidator(SequenceValidator &validator) + : _validator(validator) + { + } + void operator()(int key) { + _validator(key); + } +}; + +template <typename Iterator> +void validate_subrange(Iterator &start, Iterator &end, SequenceValidator &validator) { + start.foreach_key_range(end, ForeachKeyValidator(validator)); + EXPECT_FALSE(validator.failed()); +} + } typedef BTreeTraits<4, 4, 31, false> MyTraits; @@ -210,6 +258,8 @@ private: void requireThatIteratorDistanceWorks(); + + void requireThatForeachKeyWorks(); public: int Main() override; }; @@ -1489,6 +1539,32 @@ Test::requireThatIteratorDistanceWorks() requireThatIteratorDistanceWorks(400); } +void +Test::requireThatForeachKeyWorks() +{ + using Tree = BTree<int, int, btree::NoAggregated, MyComp, MyTraits>; + using Iterator = typename Tree::ConstIterator; + Tree t; + populateTree(t, 256, 1); + + { + // Whole range + SequenceValidator validator(1, 256); + t.foreach_key(ForeachKeyValidator(validator)); + EXPECT_FALSE(validator.failed()); + } + { + // Subranges + for (int startval = 1; startval < 259; ++startval) { + for (int endval = 1; endval < 259; ++endval) { + SequenceValidator validator(startval, std::max(0, std::min(endval,257) - std::min(startval, 257))); + Iterator start = t.lowerBound(startval); + Iterator end = t.lowerBound(endval); + validate_subrange(start, end, validator); + } + } + } +}; int Test::Main() @@ -1515,6 +1591,7 @@ Test::Main() requireThatSmallNodesWorks(); requireThatApplyWorks(); requireThatIteratorDistanceWorks(); + requireThatForeachKeyWorks(); TEST_DONE(); } diff --git a/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp b/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp index d6e1aef9394..e95e8a5c58b 100644 --- a/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp +++ b/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp @@ -60,7 +60,7 @@ template <typename T> FullBenchmark<T>::FullBenchmark(size_t numDocs, size_t numValues) : _values(numDocs*numValues), _query(numValues), - _dp(IAccelrated::getAccelrator()) + _dp(IAccelrated::getAccelerator()) { for (size_t i(0); i < numDocs; i++) { for (size_t j(0); j < numValues; j++) { diff --git a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp index f94a298d19f..8d81d9e0821 100644 --- a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp +++ b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp @@ -5,6 +5,14 @@ using namespace vespalib; +#ifndef __SANITIZE_ADDRESS__ +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define __SANITIZE_ADDRESS__ +#endif +#endif +#endif + TEST("that uncaught exception causes negative exitcode.") { SlaveProc proc("ulimit -c 0 && exec ./vespalib_caught_uncaught_app uncaught"); proc.wait(); @@ -29,6 +37,7 @@ TEST("that caught silenced exception causes exitcode 0") { EXPECT_EQUAL(proc.getExitCode(), 0); } +#ifndef __SANITIZE_ADDRESS__ TEST("that mmap within limits are fine cause exitcode 0") { SlaveProc proc("exec ./vespalib_mmap_app 150000000 10485760 1"); proc.wait(); @@ -50,5 +59,6 @@ TEST("that mmap beyond limits with set VESPA_SILENCE_CORE_ON_OOM cause exitcode EXPECT_EQUAL(proc.getExitCode(), 66); } #endif +#endif TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp index d23c2c6b68c..e86a9ad020a 100644 --- a/vespalib/src/tests/stllike/hash_test.cpp +++ b/vespalib/src/tests/stllike/hash_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/stllike/hash_map_equal.hpp> +#include <vespa/vespalib/stllike/allocator.h> #include <cstddef> #include <algorithm> @@ -553,4 +554,17 @@ TEST("test that begin and end are identical with empty hashtables") { EXPECT_TRUE(empty_but_reserved.begin() == empty_but_reserved.end()); } +TEST ("test that large_allocator works fine with std::vector") { + using V = std::vector<uint64_t, allocator_large<uint64_t>>; + V a; + a.push_back(1); + a.reserve(14); + for (size_t i(0); i < 400000; i++) { + a.push_back(i); + } + V b = std::move(a); + V c = b; + ASSERT_EQUAL(b.size(), c.size()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/traits/traits_test.cpp b/vespalib/src/tests/traits/traits_test.cpp index 0a29721df1d..7751327df75 100644 --- a/vespalib/src/tests/traits/traits_test.cpp +++ b/vespalib/src/tests/traits/traits_test.cpp @@ -42,4 +42,14 @@ TEST("require that can_skip_destruction works") { EXPECT_EQUAL(can_skip_destruction<Child2>::value, true); } +struct NoType {}; +struct TypeType { using type = NoType; }; +struct NoTypeType { static constexpr int type = 3; }; + +TEST("require that type type member can be detected") { + EXPECT_FALSE(has_type_type_v<NoType>); + EXPECT_TRUE(has_type_type_v<TypeType>); + EXPECT_FALSE(has_type_type_v<NoTypeType>); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/typify/CMakeLists.txt b/vespalib/src/tests/typify/CMakeLists.txt new file mode 100644 index 00000000000..29e95af1988 --- /dev/null +++ b/vespalib/src/tests/typify/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_typify_test_app TEST + SOURCES + typify_test.cpp + DEPENDS + vespalib + gtest +) +vespa_add_test(NAME vespalib_typify_test_app COMMAND vespalib_typify_test_app) diff --git a/vespalib/src/tests/typify/typify_test.cpp b/vespalib/src/tests/typify/typify_test.cpp new file mode 100644 index 00000000000..4c3f1c512ca --- /dev/null +++ b/vespalib/src/tests/typify/typify_test.cpp @@ -0,0 +1,124 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/typify.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; + +struct A { static constexpr int value_from_type = 1; }; +struct B { static constexpr int value_from_type = 2; }; + +struct MyIntA { int value; }; +struct MyIntB { int value; }; +struct MyIntC { int value; }; // no typifier for this type + +// MyIntA -> A or B +struct TypifyMyIntA { + template <typename T> using Result = TypifyResultType<T>; + template <typename F> static decltype(auto) resolve(MyIntA value, F &&f) { + if (value.value == 1) { + return f(Result<A>()); + } else if (value.value == 2) { + return f(Result<B>()); + } + abort(); + } +}; + +// MyIntB -> TypifyResultValue<int,1> or TypifyResultValue<int,2> +struct TypifyMyIntB { + template <int VALUE> using Result = TypifyResultValue<int,VALUE>; + template <typename F> static decltype(auto) resolve(MyIntB value, F &&f) { + if (value.value == 1) { + return f(Result<1>()); + } else if (value.value == 2) { + return f(Result<2>()); + } + abort(); + } +}; + +using TX = TypifyValue<TypifyBool, TypifyMyIntA, TypifyMyIntB>; + +//----------------------------------------------------------------------------- + +struct GetFromType { + template <typename T> static int invoke() { return T::value_from_type; } +}; + +TEST(TypifyTest, simple_type_typification_works) { + auto res1 = typify_invoke<1,TX,GetFromType>(MyIntA{1}); + auto res2 = typify_invoke<1,TX,GetFromType>(MyIntA{2}); + EXPECT_EQ(res1, 1); + EXPECT_EQ(res2, 2); +} + +struct GetFromValue { + template <typename R> static int invoke() { return R::value; } +}; + +TEST(TypifyTest, simple_value_typification_works) { + auto res1 = typify_invoke<1,TX,GetFromValue>(MyIntB{1}); + auto res2 = typify_invoke<1,TX,GetFromValue>(MyIntB{2}); + EXPECT_EQ(res1, 1); + EXPECT_EQ(res2, 2); +} + +struct MaybeSum { + template <typename F1, typename V1, typename F2, typename V2> static int invoke(MyIntC v3) { + int res = 0; + if (F1::value) { + res += V1::value_from_type; + } + if (F2::value) { + res += V2::value; + } + res += v3.value; + return res; + } +}; + +TEST(TypifyTest, complex_typification_works) { + auto res1 = typify_invoke<4,TX,MaybeSum>(false, MyIntA{2}, false, MyIntB{1}, MyIntC{4}); + auto res2 = typify_invoke<4,TX,MaybeSum>(false, MyIntA{2}, true, MyIntB{1}, MyIntC{4}); + auto res3 = typify_invoke<4,TX,MaybeSum>(true, MyIntA{2}, false, MyIntB{1}, MyIntC{4}); + auto res4 = typify_invoke<4,TX,MaybeSum>(true, MyIntA{2}, true, MyIntB{1}, MyIntC{4}); + EXPECT_EQ(res1, 4); + EXPECT_EQ(res2, 5); + EXPECT_EQ(res3, 6); + EXPECT_EQ(res4, 7); +} + +struct Singleton { + virtual int get() const = 0; + virtual ~Singleton() {} +}; + +template <int A, int B> +struct MySingleton : Singleton { + MySingleton() = default; + MySingleton(const MySingleton &) = delete; + MySingleton &operator=(const MySingleton &) = delete; + int get() const override { return A + B; } +}; + +struct GetSingleton { + template <typename A, typename B> + static const Singleton &invoke() { + static MySingleton<A::value, B::value> obj; + return obj; + } +}; + +TEST(TypifyTest, typify_invoke_can_return_object_reference) { + const Singleton &s1 = typify_invoke<2,TX,GetSingleton>(MyIntB{1}, MyIntB{1}); + const Singleton &s2 = typify_invoke<2,TX,GetSingleton>(MyIntB{2}, MyIntB{2}); + const Singleton &s3 = typify_invoke<2,TX,GetSingleton>(MyIntB{2}, MyIntB{2}); + EXPECT_EQ(s1.get(), 2); + EXPECT_EQ(s2.get(), 4); + EXPECT_EQ(s3.get(), 4); + EXPECT_NE(&s1, &s2); + EXPECT_EQ(&s2, &s3); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.h b/vespalib/src/vespa/vespalib/btree/btreeiterator.h index 55ab37759ad..6933fc1c2d0 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.h +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.h @@ -303,6 +303,47 @@ protected: * @param pathSize New tree height (number of levels of internal nodes) */ VESPA_DLL_LOCAL void clearPath(uint32_t pathSize); + + /** + * Call func with leaf entry key value as argument for all leaf entries in subtree + * from this iterator position to end of subtree. + */ + template <typename FunctionType> + void + foreach_key_range_start(uint32_t level, FunctionType func) const + { + if (level > 0u) { + --level; + foreach_key_range_start(level, func); + auto &store = _allocator->getNodeStore(); + auto node = _path[level].getNode(); + uint32_t idx = _path[level].getIdx(); + node->foreach_key_range(store, idx + 1, node->validSlots(), func); + } else { + _leaf.getNode()->foreach_key_range(_leaf.getIdx(), _leaf.getNode()->validSlots(), func); + } + } + + /** + * Call func with leaf entry key value as argument for all leaf entries in subtree + * from start of subtree until this iterator position is reached (i.e. entries in + * subtree before this iterator position). + */ + template <typename FunctionType> + void + foreach_key_range_end(uint32_t level, FunctionType func) const + { + if (level > 0u) { + --level; + auto &store = _allocator->getNodeStore(); + auto node = _path[level].getNode(); + uint32_t eidx = _path[level].getIdx(); + node->foreach_key_range(store, 0, eidx, func); + foreach_key_range_end(level, func); + } else { + _leaf.getNode()->foreach_key_range(0, _leaf.getIdx(), func); + } + } public: bool @@ -451,6 +492,68 @@ public: _leafRoot->foreach_key(func); } } + + /** + * Call func with leaf entry key value as argument for all leaf entries in tree from + * this iterator position until end_itr position is reached (i.e. entries in + * range [this iterator, end_itr)). + */ + template <typename FunctionType> + void + foreach_key_range(const BTreeIteratorBase &end_itr, FunctionType func) const + { + if (!valid()) { + return; + } + if (!end_itr.valid()) { + foreach_key_range_start(_pathSize, func); + return; + } + assert(_pathSize == end_itr._pathSize); + assert(_allocator == end_itr._allocator); + uint32_t level = _pathSize; + if (level > 0u) { + /** + * Tree has intermediate nodes. Detect lowest shared tree node for this + * iterator and end_itr. + */ + uint32_t idx; + uint32_t eidx; + do { + --level; + assert(_path[level].getNode() == end_itr._path[level].getNode()); + idx = _path[level].getIdx(); + eidx = end_itr._path[level].getIdx(); + if (idx > eidx) { + return; + } + if (idx != eidx) { + ++level; + break; + } + } while (level != 0); + if (level > 0u) { + // Lowest shared node is an intermediate node. + // Left subtree for child [idx], from this iterator position to end of subtree. + foreach_key_range_start(level - 1, func); + auto &store = _allocator->getNodeStore(); + auto node = _path[level - 1].getNode(); + // Any intermediate subtrees for children [idx + 1, eidx). + node->foreach_key_range(store, idx + 1, eidx, func); + // Right subtree for child [eidx], from start of subtree to end_itr position. + end_itr.foreach_key_range_end(level - 1, func); + return; + } else { + // Lowest shared node is a leaf node. + assert(_leaf.getNode() == end_itr._leaf.getNode()); + } + } + uint32_t idx = _leaf.getIdx(); + uint32_t eidx = end_itr._leaf.getIdx(); + if (idx < eidx) { + _leaf.getNode()->foreach_key_range(idx, eidx, func); + } + } }; diff --git a/vespalib/src/vespa/vespalib/btree/btreenode.h b/vespalib/src/vespa/vespalib/btree/btreenode.h index b34be33ccf5..0c70e70bc6a 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenode.h +++ b/vespalib/src/vespa/vespalib/btree/btreenode.h @@ -370,6 +370,26 @@ public: } } + /** + * Call func with leaf entry key value as argument for all leaf entries in subtrees + * for children [start_idx, end_idx). + */ + template <typename NodeStoreType, typename FunctionType> + void foreach_key_range(NodeStoreType &store, uint32_t start_idx, uint32_t end_idx, FunctionType func) const { + const BTreeNode::Ref *it = this->_data; + const BTreeNode::Ref *ite = it + end_idx; + it += start_idx; + if (this->getLevel() > 1u) { + for (; it != ite; ++it) { + store.mapInternalRef(*it)->foreach_key(store, func); + } + } else { + for (; it != ite; ++it) { + store.mapLeafRef(*it)->foreach_key(func); + } + } + } + template <typename NodeStoreType, typename FunctionType> void foreach(NodeStoreType &store, FunctionType func) const { const BTreeNode::Ref *it = this->_data; @@ -459,6 +479,19 @@ public: } } + /** + * Call func with leaf entry key value as argument for leaf entries [start_idx, end_idx). + */ + template <typename FunctionType> + void foreach_key_range(uint32_t start_idx, uint32_t end_idx, FunctionType func) const { + const KeyT *it = _keys; + const KeyT *ite = it + end_idx; + it += start_idx; + for (; it != ite; ++it) { + func(*it); + } + } + template <typename FunctionType> void foreach(FunctionType func) const { const KeyT *it = _keys; diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h index 6b9a4f7201e..2c495ca9a1e 100644 --- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h @@ -44,9 +44,9 @@ public: virtual void move_entries(ICompactable& compactable) = 0; virtual uint32_t get_num_uniques() const = 0; virtual vespalib::MemoryUsage get_memory_usage() const = 0; - virtual void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) = 0; + virtual void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) = 0; virtual void build(vespalib::ConstArrayRef<EntryRef> refs) = 0; - virtual void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) = 0; + virtual void build_with_payload(vespalib::ConstArrayRef<EntryRef> refs, vespalib::ConstArrayRef<uint32_t> payloads) = 0; virtual std::unique_ptr<ReadSnapshot> get_read_snapshot() const = 0; }; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h index f2be4e1237b..cbafdbafd4f 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.h @@ -3,6 +3,7 @@ #pragma once #include "unique_store_allocator.h" +#include <vespa/vespalib/stllike/allocator.h> namespace vespalib::datastore { @@ -21,8 +22,8 @@ class UniqueStoreBuilder { Allocator& _allocator; IUniqueStoreDictionary& _dict; - std::vector<EntryRef> _refs; - std::vector<uint32_t> _refCounts; + std::vector<EntryRef, allocator_large<EntryRef>> _refs; + std::vector<uint32_t, allocator_large<uint32_t>> _refCounts; public: UniqueStoreBuilder(Allocator& allocator, IUniqueStoreDictionary& dict, uint32_t uniqueValuesHint); diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp index 0925702ddcb..d54b86558be 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_builder.hpp @@ -20,9 +20,7 @@ UniqueStoreBuilder<Allocator>::UniqueStoreBuilder(Allocator& allocator, IUniqueS } template <typename Allocator> -UniqueStoreBuilder<Allocator>::~UniqueStoreBuilder() -{ -} +UniqueStoreBuilder<Allocator>::~UniqueStoreBuilder() = default; template <typename Allocator> void diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h index 4ec98dc2100..8f7a344a99f 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h @@ -46,9 +46,9 @@ public: void move_entries(ICompactable& compactable) override; uint32_t get_num_uniques() const override; vespalib::MemoryUsage get_memory_usage() const override; - void build(const std::vector<EntryRef> &refs, const std::vector<uint32_t> &ref_counts, std::function<void(EntryRef)> hold) override; + void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) override; void build(vespalib::ConstArrayRef<EntryRef> refs) override; - void build_with_payload(const std::vector<EntryRef>& refs, const std::vector<uint32_t>& payloads) override; + void build_with_payload(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> payloads) override; std::unique_ptr<ReadSnapshot> get_read_snapshot() const override; }; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp index bce11591970..8ecf71d08c7 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -158,8 +158,8 @@ UniqueStoreDictionary<DictionaryT, ParentT>::get_memory_usage() const template <typename DictionaryT, typename ParentT> void -UniqueStoreDictionary<DictionaryT, ParentT>::build(const std::vector<EntryRef> &refs, - const std::vector<uint32_t> &ref_counts, +UniqueStoreDictionary<DictionaryT, ParentT>::build(vespalib::ConstArrayRef<EntryRef> refs, + vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) { assert(refs.size() == ref_counts.size()); @@ -188,8 +188,8 @@ UniqueStoreDictionary<DictionaryT, ParentT>::build(vespalib::ConstArrayRef<Entry template <typename DictionaryT, typename ParentT> void -UniqueStoreDictionary<DictionaryT, ParentT>::build_with_payload(const std::vector<EntryRef>& refs, - const std::vector<uint32_t>& payloads) +UniqueStoreDictionary<DictionaryT, ParentT>::build_with_payload(vespalib::ConstArrayRef<EntryRef> refs, + vespalib::ConstArrayRef<uint32_t> payloads) { assert(refs.size() == payloads.size()); typename DictionaryType::Builder builder(_dict.getAllocator()); diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp index 7ff393c87f8..8588a5510f7 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp @@ -20,4 +20,14 @@ Avx2Accelrator::squaredEuclideanDistance(const double * a, const double * b, siz return avx::euclideanDistanceSelectAlignment<double, 32>(a, b, sz); } +void +Avx2Accelrator::and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::andChunks<32u, 2u>(offset, src, dest); +} + +void +Avx2Accelrator::or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::orChunks<32u, 2u>(offset, src, dest); +} + } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h index 3e0dbb28110..b6f3d299748 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h @@ -15,6 +15,8 @@ public: size_t populationCount(const uint64_t *a, size_t sz) const override; double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override; double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override; + void and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; + void or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp index 0941e6d6ad8..4dade08e77a 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp @@ -32,4 +32,14 @@ Avx512Accelrator::squaredEuclideanDistance(const double * a, const double * b, s return avx::euclideanDistanceSelectAlignment<double, 64>(a, b, sz); } +void +Avx512Accelrator::and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::andChunks<64, 1>(offset, src, dest); +} + +void +Avx512Accelrator::or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::orChunks<64, 1>(offset, src, dest); +} + } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h index 209ec06c857..a54d57407b2 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h @@ -17,6 +17,8 @@ public: size_t populationCount(const uint64_t *a, size_t sz) const override; double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override; double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override; + void and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; + void or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp index 087729a23b2..04859a9e565 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp @@ -123,7 +123,7 @@ double euclideanDistanceSelectAlignment(const T * af, const T * bf, size_t sz) if (validAlignment(bf, ALIGN)) { return euclideanDistanceT<T, VLEN, ALIGN, ALIGN>(af, bf, sz); } else { - return euclideanDistanceT<T, ALIGN, ALIGN, 1>(af, bf, sz); + return euclideanDistanceT<T, VLEN, ALIGN, 1>(af, bf, sz); } } else { if (validAlignment(bf, ALIGN)) { diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp index f9684e88c63..f9dfaacf626 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp @@ -165,4 +165,14 @@ GenericAccelrator::squaredEuclideanDistance(const double * a, const double * b, return euclideanDistanceT<double, 4>(a, b, sz); } +void +GenericAccelrator::and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::andChunks<16, 4>(offset, src, dest); +} + +void +GenericAccelrator::or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const { + helper::orChunks<16,4>(offset, src, dest); +} + } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h index 50a3d59d49d..2335b40fe85 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h @@ -25,6 +25,8 @@ public: size_t populationCount(const uint64_t *a, size_t sz) const override; double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override; double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override; + void and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; + void or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp index bb132165e53..de917c5f065 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp @@ -46,7 +46,8 @@ std::vector<T> createAndFill(size_t sz) { } template<typename T> -void verifyDotproduct(const IAccelrated & accel) +void +verifyDotproduct(const IAccelrated & accel) { const size_t testLength(255); srand(1); @@ -66,7 +67,8 @@ void verifyDotproduct(const IAccelrated & accel) } template<typename T> -void verifyEuclideanDistance(const IAccelrated & accel) { +void +verifyEuclideanDistance(const IAccelrated & accel) { const size_t testLength(255); srand(1); std::vector<T> a = createAndFill<T>(testLength); @@ -84,7 +86,8 @@ void verifyEuclideanDistance(const IAccelrated & accel) { } } -void verifyPopulationCount(const IAccelrated & accel) +void +verifyPopulationCount(const IAccelrated & accel) { const uint64_t words[7] = {0x123456789abcdef0L, // 32 0x0000000000000000L, // 0 @@ -101,6 +104,118 @@ void verifyPopulationCount(const IAccelrated & accel) } } +void +fill(std::vector<uint64_t> & v, size_t n) { + v.reserve(n); + for (size_t i(0); i < n; i++) { + v.emplace_back(random()); + } +} + +void +simpleAndWith(std::vector<uint64_t> & dest, const std::vector<uint64_t> & src) { + for (size_t i(0); i < dest.size(); i++) { + dest[i] &= src[i]; + } +} + +void +simpleOrWith(std::vector<uint64_t> & dest, const std::vector<uint64_t> & src) { + for (size_t i(0); i < dest.size(); i++) { + dest[i] |= src[i]; + } +} + +std::vector<uint64_t> +simpleInvert(const std::vector<uint64_t> & src) { + std::vector<uint64_t> inverted; + inverted.reserve(src.size()); + for (size_t i(0); i < src.size(); i++) { + inverted.push_back(~src[i]); + } + return inverted; +} + +std::vector<uint64_t> +optionallyInvert(bool invert, std::vector<uint64_t> v) { + return invert ? simpleInvert(std::move(v)) : std::move(v); +} + +bool shouldInvert(bool invertSome) { + return invertSome ? (random() & 1) : false; +} + +void +verifyOr64(const IAccelrated & accel, const std::vector<std::vector<uint64_t>> & vectors, + size_t offset, size_t num_vectors, bool invertSome) +{ + std::vector<std::pair<const void *, bool>> vRefs; + for (size_t j(0); j < num_vectors; j++) { + vRefs.emplace_back(&vectors[j][0], shouldInvert(invertSome)); + } + + std::vector<uint64_t> expected = optionallyInvert(vRefs[0].second, vectors[0]); + for (size_t j = 1; j < num_vectors; j++) { + simpleOrWith(expected, optionallyInvert(vRefs[j].second, vectors[j])); + } + + uint64_t dest[8] __attribute((aligned(64))); + accel.or64(offset*sizeof(uint64_t), vRefs, dest); + int diff = memcmp(&expected[offset], dest, sizeof(dest)); + if (diff != 0) { + LOG_ABORT("Accelerator fails to compute correct 64 bytes OR"); + } +} + +void +verifyAnd64(const IAccelrated & accel, const std::vector<std::vector<uint64_t>> & vectors, + size_t offset, size_t num_vectors, bool invertSome) +{ + std::vector<std::pair<const void *, bool>> vRefs; + for (size_t j(0); j < num_vectors; j++) { + vRefs.emplace_back(&vectors[j][0], shouldInvert(invertSome)); + } + std::vector<uint64_t> expected = optionallyInvert(vRefs[0].second, vectors[0]); + for (size_t j = 1; j < num_vectors; j++) { + simpleAndWith(expected, optionallyInvert(vRefs[j].second, vectors[j])); + } + + uint64_t dest[8] __attribute((aligned(64))); + accel.and64(offset*sizeof(uint64_t), vRefs, dest); + int diff = memcmp(&expected[offset], dest, sizeof(dest)); + if (diff != 0) { + LOG_ABORT("Accelerator fails to compute correct 64 bytes AND"); + } +} + +void +verifyOr64(const IAccelrated & accel) { + std::vector<std::vector<uint64_t>> vectors(3) ; + for (auto & v : vectors) { + fill(v, 16); + } + for (size_t offset = 0; offset < 8; offset++) { + for (size_t i = 1; i < vectors.size(); i++) { + verifyOr64(accel, vectors, offset, i, false); + verifyOr64(accel, vectors, offset, i, true); + } + } +} + +void +verifyAnd64(const IAccelrated & accel) { + std::vector<std::vector<uint64_t>> vectors(3); + for (auto & v : vectors) { + fill(v, 16); + } + for (size_t offset = 0; offset < 8; offset++) { + for (size_t i = 1; i < vectors.size(); i++) { + verifyAnd64(accel, vectors, offset, i, false); + verifyAnd64(accel, vectors, offset, i, true); + } + } +} + class RuntimeVerificator { public: @@ -114,6 +229,8 @@ private: verifyEuclideanDistance<float>(accelrated); verifyEuclideanDistance<double>(accelrated); verifyPopulationCount(accelrated); + verifyAnd64(accelrated); + verifyOr64(accelrated); } }; @@ -122,7 +239,7 @@ RuntimeVerificator::RuntimeVerificator() GenericAccelrator generic; verify(generic); - const IAccelrated & thisCpu(IAccelrated::getAccelrator()); + const IAccelrated & thisCpu(IAccelrated::getAccelerator()); verify(thisCpu); } @@ -155,7 +272,7 @@ static Selector _G_selector; RuntimeVerificator _G_verifyAccelrator; const IAccelrated & -IAccelrated::getAccelrator() +IAccelrated::getAccelerator() { static IAccelrated::UP accelrator = _G_selector.create(); return *accelrator; diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h index 0292ad14643..2594a48dd33 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h @@ -4,6 +4,7 @@ #include <memory> #include <cstdint> +#include <vector> namespace vespalib::hwaccelrated { @@ -29,8 +30,12 @@ public: virtual size_t populationCount(const uint64_t *a, size_t sz) const = 0; virtual double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const = 0; virtual double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const = 0; + // AND 64 bytes from multiple, optionally inverted sources + virtual void and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const = 0; + // OR 64 bytes from multiple, optionally inverted sources + virtual void or64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const = 0; - static const IAccelrated & getAccelrator() __attribute__((noinline)); + static const IAccelrated & getAccelerator() __attribute__((noinline)); }; } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp index f5daf2b9081..65b4c717681 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp @@ -24,5 +24,55 @@ populationCount(const uint64_t *a, size_t sz) { return count; } +template<typename T> +T get(const void * base, bool invert) { + T v; + memcpy(&v, base, sizeof(T)); + return __builtin_expect(invert, false) ? ~v : v; +} + +template <typename T> +const T * cast(const void * ptr, size_t offsetBytes) { + return static_cast<const T *>(static_cast<const void *>(static_cast<const char *>(ptr) + offsetBytes)); +} + +template<unsigned ChunkSize, unsigned Chunks> +void +andChunks(size_t offset, const std::vector<std::pair<const void *, bool>> & src, void * dest) { + typedef uint64_t Chunk __attribute__ ((vector_size (ChunkSize))); + static_assert(sizeof(Chunk) == ChunkSize, "sizeof(Chunk) == ChunkSize"); + static_assert(ChunkSize*Chunks == 64, "ChunkSize*Chunks == 64"); + Chunk * chunk = static_cast<Chunk *>(dest); + const Chunk * tmp = cast<Chunk>(src[0].first, offset); + for (size_t n=0; n < Chunks; n++) { + chunk[n] = get<Chunk>(tmp+n, src[0].second); + } + for (size_t i(1); i < src.size(); i++) { + tmp = cast<Chunk>(src[i].first, offset); + for (size_t n=0; n < Chunks; n++) { + chunk[n] &= get<Chunk>(tmp+n, src[i].second); + } + } +} + +template<unsigned ChunkSize, unsigned Chunks> +void +orChunks(size_t offset, const std::vector<std::pair<const void *, bool>> & src, void * dest) { + typedef uint64_t Chunk __attribute__ ((vector_size (ChunkSize))); + static_assert(sizeof(Chunk) == ChunkSize, "sizeof(Chunk) == ChunkSize"); + static_assert(ChunkSize*Chunks == 64, "ChunkSize*Chunks == 64"); + Chunk * chunk = static_cast<Chunk *>(dest); + const Chunk * tmp = cast<Chunk>(src[0].first, offset); + for (size_t n=0; n < Chunks; n++) { + chunk[n] = get<Chunk>(tmp+n, src[0].second); + } + for (size_t i(1); i < src.size(); i++) { + tmp = cast<Chunk>(src[i].first, offset); + for (size_t n=0; n < Chunks; n++) { + chunk[n] |= get<Chunk>(tmp+n, src[i].second); + } + } +} + } } diff --git a/vespalib/src/vespa/vespalib/regex/regex.cpp b/vespalib/src/vespa/vespalib/regex/regex.cpp index 3229365b753..ebdbb256d19 100644 --- a/vespalib/src/vespa/vespalib/regex/regex.cpp +++ b/vespalib/src/vespa/vespalib/regex/regex.cpp @@ -58,6 +58,9 @@ Regex Regex::from_pattern(std::string_view pattern, uint32_t opt_mask) { if ((opt_mask & Options::IgnoreCase) != 0) { opts.set_case_sensitive(false); } + if ((opt_mask & Options::DotMatchesNewline) != 0) { + opts.set_dot_nl(true); + } return Regex(std::make_shared<const Impl>(pattern, opts)); } diff --git a/vespalib/src/vespa/vespalib/regex/regex.h b/vespalib/src/vespa/vespalib/regex/regex.h index 4382d057252..0c80f4e5d3a 100644 --- a/vespalib/src/vespa/vespalib/regex/regex.h +++ b/vespalib/src/vespa/vespalib/regex/regex.h @@ -43,8 +43,9 @@ class Regex { public: // TODO consider using type-safe parameter instead. enum Options { - None = 0, - IgnoreCase = 1 + None = 0, + IgnoreCase = 1, + DotMatchesNewline = 2 }; ~Regex(); diff --git a/vespalib/src/vespa/vespalib/stllike/allocator.h b/vespalib/src/vespa/vespalib/stllike/allocator.h new file mode 100644 index 00000000000..efe885f5c96 --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/allocator.h @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/alloc.h> + +namespace vespalib { + +/** + * std compliant allocator that will use a smart allocator + * that uses mmap prefering huge pages for large allocations. + * This is a good fit for use with std::vector and std::deque. + */ +template <typename T> +class allocator_large { + using PtrAndSize = alloc::MemoryAllocator::PtrAndSize; +public: + allocator_large() noexcept : _allocator(alloc::MemoryAllocator::select_allocator()) {} + using value_type = T; + constexpr T * allocate(std::size_t n) { + return static_cast<T *>(_allocator->alloc(n*sizeof(T)).first); + } + void deallocate(T * p, std::size_t n) { + _allocator->free(p, n*sizeof(T)); + } + const alloc::MemoryAllocator * allocator() const { return _allocator; } +private: + const alloc::MemoryAllocator * _allocator; +}; + +template< class T1, class T2 > +constexpr bool operator==( const allocator_large<T1>& lhs, const allocator_large<T2>& rhs ) noexcept { + return lhs.allocator() == rhs.allocator(); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index 4c0675198bb..c550964a19b 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -162,6 +162,7 @@ public: AutoAllocator(size_t mmapLimit, size_t alignment) : _mmapLimit(mmapLimit), _alignment(alignment) { } PtrAndSize alloc(size_t sz) const override; void free(PtrAndSize alloc) const override; + void free(void * ptr, size_t sz) const override; size_t resize_inplace(PtrAndSize current, size_t newSize) const override; static MemoryAllocator & getDefault(); static MemoryAllocator & getAllocator(size_t mmapLimit, size_t alignment); @@ -171,13 +172,11 @@ private: ? MMapAllocator::roundUpToHugePages(sz) : sz; } - bool isMMapped(size_t sz) const { return (sz >= _mmapLimit); } bool useMMap(size_t sz) const { - if (_mmapLimit >= HUGEPAGE_SIZE) { - return (sz + (HUGEPAGE_SIZE >> 1) - 1) >= _mmapLimit; - } else { - return (sz >= _mmapLimit); - } + return (sz + (HUGEPAGE_SIZE >> 1) - 1) >= _mmapLimit; + } + bool isMMapped(size_t sz) const { + return sz >= _mmapLimit; } size_t _mmapLimit; size_t _alignment; @@ -424,7 +423,7 @@ void MMapAllocator::sfree(PtrAndSize alloc) size_t AutoAllocator::resize_inplace(PtrAndSize current, size_t newSize) const { - if (isMMapped(current.second) && useMMap(newSize)) { + if (useMMap(current.second) && useMMap(newSize)) { newSize = roundUpToHugePages(newSize); return MMapAllocator::sresize_inplace(current, newSize); } else { @@ -455,8 +454,21 @@ AutoAllocator::free(PtrAndSize alloc) const { } } +void +AutoAllocator::free(void * ptr, size_t sz) const { + if (useMMap(sz)) { + return MMapAllocator::sfree(PtrAndSize(ptr, roundUpToHugePages(sz))); + } else { + return HeapAllocator::sfree(PtrAndSize(ptr, sz)); + } +} + } +const MemoryAllocator * +MemoryAllocator::select_allocator(size_t mmapLimit, size_t alignment) { + return & AutoAllocator::getAllocator(mmapLimit, alignment); +} Alloc Alloc::allocHeap(size_t sz) diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index 03ebc2807f9..449cdde5fc7 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -17,6 +17,10 @@ public: virtual ~MemoryAllocator() { } virtual PtrAndSize alloc(size_t sz) const = 0; virtual void free(PtrAndSize alloc) const = 0; + // Allow for freeing memory there size is the size requested, and not the size allocated. + virtual void free(void * ptr, size_t sz) const { + free(PtrAndSize(ptr, sz)); + } /* * If possible the allocations will be resized. If it was possible it will return the real size, * if not it shall return 0. @@ -30,6 +34,7 @@ public: static size_t roundUpToHugePages(size_t sz) { return (sz+(HUGEPAGE_SIZE-1)) & ~(HUGEPAGE_SIZE-1); } + static const MemoryAllocator * select_allocator(size_t mmapLimit = MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0); }; /** diff --git a/vespalib/src/vespa/vespalib/util/arrayref.h b/vespalib/src/vespa/vespalib/util/arrayref.h index 186316c10d3..749395ff574 100644 --- a/vespalib/src/vespa/vespalib/util/arrayref.h +++ b/vespalib/src/vespa/vespalib/util/arrayref.h @@ -15,11 +15,13 @@ class ArrayRef { public: ArrayRef() : _v(nullptr), _sz(0) { } ArrayRef(T * v, size_t sz) : _v(v), _sz(sz) { } - ArrayRef(std::vector<T> & v) : _v(&v[0]), _sz(v.size()) { } + template<typename A=std::allocator<T>> + ArrayRef(std::vector<T, A> & v) : _v(&v[0]), _sz(v.size()) { } ArrayRef(Array<T> &v) : _v(&v[0]), _sz(v.size()) { } T & operator [] (size_t i) { return _v[i]; } const T & operator [] (size_t i) const { return _v[i]; } size_t size() const { return _sz; } + bool empty() const { return _sz == 0; } T *begin() { return _v; } T *end() { return _v + _sz; } private: @@ -31,12 +33,14 @@ template <typename T> class ConstArrayRef { public: ConstArrayRef(const T *v, size_t sz) : _v(v), _sz(sz) { } - ConstArrayRef(const std::vector<T> & v) : _v(&v[0]), _sz(v.size()) { } + template<typename A=std::allocator<T>> + ConstArrayRef(const std::vector<T, A> & v) : _v(&v[0]), _sz(v.size()) { } ConstArrayRef(const ArrayRef<T> & v) : _v(&v[0]), _sz(v.size()) { } ConstArrayRef(const Array<T> &v) : _v(&v[0]), _sz(v.size()) { } ConstArrayRef() : _v(nullptr), _sz(0) {} const T & operator [] (size_t i) const { return _v[i]; } size_t size() const { return _sz; } + bool empty() const { return _sz == 0; } const T *cbegin() const { return _v; } const T *cend() const { return _v + _sz; } const T *begin() const { return _v; } diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp index efb1dbf4054..ad5d78d5ab6 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp @@ -87,6 +87,7 @@ ThreadStackExecutorBase::obtainTask(Worker &worker) if (!worker.idle) { assert(_taskCount != 0); --_taskCount; + wakeup(monitor); _barrier.completeEvent(worker.task.token); worker.idle = true; } @@ -96,7 +97,6 @@ ThreadStackExecutorBase::obtainTask(Worker &worker) worker.task = std::move(_tasks.front()); worker.idle = false; _tasks.pop(); - wakeup(monitor); return true; } if (_closed) { diff --git a/vespalib/src/vespa/vespalib/util/traits.h b/vespalib/src/vespa/vespalib/util/traits.h index eb0385abc72..7f8945954a8 100644 --- a/vespalib/src/vespa/vespalib/util/traits.h +++ b/vespalib/src/vespa/vespalib/util/traits.h @@ -39,4 +39,10 @@ struct can_skip_destruction : std::is_trivially_destructible<T> {}; //----------------------------------------------------------------------------- +template <typename, typename = std::void_t<>> struct has_type_type : std::false_type {}; +template <typename T> struct has_type_type<T, std::void_t<typename T::type>> : std::true_type {}; +template <typename T> constexpr bool has_type_type_v = has_type_type<T>::value; + +//----------------------------------------------------------------------------- + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/typify.h b/vespalib/src/vespa/vespalib/util/typify.h new file mode 100644 index 00000000000..0d84d1756a5 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/typify.h @@ -0,0 +1,110 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "traits.h" +#include <stddef.h> +#include <utility> + +namespace vespalib { + +//----------------------------------------------------------------------------- + +/** + * Typification result for values resolving into actual types. Using + * this exact template is not required, but the result type must have + * the name 'type' for the auto-unwrapping performed by typify_invoke + * to work. + **/ +template <typename T> struct TypifyResultType { + using type = T; +}; + +/** + * Typification result for values resolving into non-types. Using this + * exact template is not required, but is supplied for + * convenience. The resolved compile-time value should be called + * 'value' for consistency across typifiers. + **/ +template <typename T, T VALUE> struct TypifyResultValue { + static constexpr T value = VALUE; +}; + +/** + * Typification result for values resolving into simple templates + * (templated on one type). Using this exact template is not required, + * but is supplied for convenience and as example. The resolved + * template should be called 'templ' for consistency across typifiers. + **/ +template <template<typename> typename TT> struct TypifyResultSimpleTemplate { + template <typename T> using templ = TT<T>; +}; + +/** + * A Typifier is able to take a run-time value and resolve it into a + * type, non-type constant value or a template. The resolve result is + * passed to the specified function in the form of a thin result + * wrapper. + **/ +struct TypifyBool { + template <bool VALUE> using Result = TypifyResultValue<bool, VALUE>; + template <typename F> static decltype(auto) resolve(bool value, F &&f) { + if (value) { + return f(Result<true>()); + } else { + return f(Result<false>()); + } + } +}; + +//----------------------------------------------------------------------------- + +/** + * Template used to combine individual typifiers into a typifier able + * to resolve multiple types. + **/ +template <typename ...Ts> struct TypifyValue : Ts... { using Ts::resolve...; }; + +//----------------------------------------------------------------------------- + +template <size_t N, typename Typifier, typename Target, typename ...Rs> struct TypifyInvokeImpl { + static decltype(auto) select() { + static_assert(sizeof...(Rs) == N); + return Target::template invoke<Rs...>(); + } + template <typename T, typename ...Args> static decltype(auto) select(T &&value, Args &&...args) { + if constexpr (N == sizeof...(Rs)) { + return Target::template invoke<Rs...>(std::forward<T>(value), std::forward<Args>(args)...); + } else { + return Typifier::resolve(value, [&](auto t)->decltype(auto) + { + using X = decltype(t); + if constexpr (has_type_type_v<X>) { + return TypifyInvokeImpl<N, Typifier, Target, Rs..., typename X::type>::select(std::forward<Args>(args)...); + } else { + return TypifyInvokeImpl<N, Typifier, Target, Rs..., X>::select(std::forward<Args>(args)...); + } + }); + } + } +}; + +/** + * Typify the N first parameters using 'Typifier' (typically an + * instantiation of the TypifyValue template) and forward the + * remaining parameters to the Target::invoke template function with + * the typification results from the N first parameters as template + * parameters. Note that typification results that are types are + * unwrapped before being used as template parameters while + * typification results that are non-types or templates are kept in + * their wrappers when passed as template parameters. Please refer to + * the unit test for examples. + **/ +template <size_t N, typename Typifier, typename Target, typename ...Args> decltype(auto) typify_invoke(Args && ...args) { + static_assert(N > 0); + return TypifyInvokeImpl<N,Typifier,Target>::select(std::forward<Args>(args)...); +} + +//----------------------------------------------------------------------------- + +} diff --git a/vespamalloc/src/tests/overwrite/.gitignore b/vespamalloc/src/tests/overwrite/.gitignore index 5a8760f913d..537a606f87f 100644 --- a/vespamalloc/src/tests/overwrite/.gitignore +++ b/vespamalloc/src/tests/overwrite/.gitignore @@ -1,8 +1,5 @@ .depend Makefile -expectsignal -overwrite_test -overwrite_testd -/expectsignal-overwrite vespamalloc_overwrite_test_app vespamalloc_expectsignal-overwrite_app +vespamalloc_overwrite_test_with_vespamallocd_app diff --git a/vespamalloc/src/tests/overwrite/CMakeLists.txt b/vespamalloc/src/tests/overwrite/CMakeLists.txt index 29b6ac46eb4..9f8274ea2ce 100644 --- a/vespamalloc/src/tests/overwrite/CMakeLists.txt +++ b/vespamalloc/src/tests/overwrite/CMakeLists.txt @@ -13,3 +13,11 @@ vespa_add_executable(vespamalloc_expectsignal-overwrite_app vespa_add_test(NAME vespamalloc_overwrite_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/overwrite_test.sh DEPENDS vespamalloc_overwrite_test_app vespamalloc_expectsignal-overwrite_app vespamalloc vespamallocd) + +vespa_add_executable(vespamalloc_overwrite_test_with_vespamallocd_app TEST + SOURCES + overwrite.cpp + DEPENDS + vespamallocd +) +vespa_add_test(NAME vespamalloc_overwrite_test_with_vespamallocd_app NO_VALGRIND COMMAND vespamalloc_overwrite_test_with_vespamallocd_app testmemoryfill) diff --git a/vespamalloc/src/tests/overwrite/overwrite.cpp b/vespamalloc/src/tests/overwrite/overwrite.cpp index 84f96fbbb3e..1919a75ab00 100644 --- a/vespamalloc/src/tests/overwrite/overwrite.cpp +++ b/vespamalloc/src/tests/overwrite/overwrite.cpp @@ -29,9 +29,7 @@ private: void verifyWriteAfterFreeDetection(); // Should abort }; -Test::~Test() -{ -} +Test::~Test() = default; void Test::testFillValue(char *a) { diff --git a/vespamalloc/src/tests/test1/.gitignore b/vespamalloc/src/tests/test1/.gitignore index b7fab5d205c..45356cc94e7 100644 --- a/vespamalloc/src/tests/test1/.gitignore +++ b/vespamalloc/src/tests/test1/.gitignore @@ -2,3 +2,5 @@ Makefile testatomic vespamalloc_testatomic_app +vespamalloc_new_test_with_vespamalloc_app +vespamalloc_new_test_with_vespamallocd_app diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index cade2e092b4..15680f22595 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -6,3 +6,25 @@ vespa_add_executable(vespamalloc_testatomic_app TEST ${VESPA_ATOMIC_LIB} ) vespa_add_test(NAME vespamalloc_testatomic_app NO_VALGRIND COMMAND vespamalloc_testatomic_app) + +vespa_add_executable(vespamalloc_new_test_app TEST + SOURCES + new_test.cpp +) +vespa_add_test(NAME vespamalloc_new_test_app NO_VALGRIND COMMAND vespamalloc_new_test_app) + +vespa_add_executable(vespamalloc_new_test_with_vespamalloc_app TEST + SOURCES + new_test.cpp + DEPENDS + vespamalloc +) +vespa_add_test(NAME vespamalloc_new_test_with_vespamalloc_app NO_VALGRIND COMMAND vespamalloc_new_test_with_vespamalloc_app) + +vespa_add_executable(vespamalloc_new_test_with_vespamallocd_app TEST + SOURCES + new_test.cpp + DEPENDS + vespamallocd +) +vespa_add_test(NAME vespamalloc_new_test_with_vespamallocd_app NO_VALGRIND COMMAND vespamalloc_new_test_with_vespamallocd_app) diff --git a/vespamalloc/src/tests/test1/new_test.cpp b/vespamalloc/src/tests/test1/new_test.cpp new file mode 100644 index 00000000000..2400e41a1d9 --- /dev/null +++ b/vespamalloc/src/tests/test1/new_test.cpp @@ -0,0 +1,109 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/log/log.h> + +LOG_SETUP("new_test"); + +void cmp(const void *a, const void *b) { + EXPECT_EQUAL(a, b); +} +void cmp(const void *base, size_t offset, const void *p) { + cmp((static_cast<const char *>(base) + offset), p); +} + +template <typename S> +void veryfy_aligned(S * p) { + EXPECT_TRUE((uintptr_t(p) % alignof(S)) == 0); + memset(p, 0, sizeof(S)); +} + +TEST("verify new with normal alignment") { + struct S { + int a; + long b; + int c; + }; + static_assert(sizeof(S) == 24); + static_assert(alignof(S) == 8); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + cmp(s.get(), 8, &s->b); + cmp(s.get(), 16, &s->c); + LOG(info, "&s=%p &s.b=%p &s.c=%p", s.get(), &s->b, &s->c); +} + +TEST("verify new with alignment = 16") { + struct S { + int a; + alignas(16) long b; + int c; + }; + static_assert(sizeof(S) == 32); + static_assert(alignof(S) == 16); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + cmp(s.get(), 16, &s->b); + cmp(s.get(), 24, &s->c); + LOG(info, "&s=%p &s.b=%p &s.c=%p", s.get(), &s->b, &s->c); +} + +TEST("verify new with alignment = 32") { + struct S { + int a; + alignas(32) long b; + int c; + }; + static_assert(sizeof(S) == 64); + static_assert(alignof(S) == 32); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + cmp(s.get(), 32, &s->b); + cmp(s.get(), 40, &s->c); + LOG(info, "&s=%p &s.b=%p &s.c=%p", s.get(), &s->b, &s->c); +} + +TEST("verify new with alignment = 64") { + struct S { + int a; + alignas(64) long b; + int c; + }; + static_assert(sizeof(S) == 128); + static_assert(alignof(S) == 64); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + cmp(s.get(), 64, &s->b); + cmp(s.get(), 72, &s->c); + LOG(info, "&s=%p &s.b=%p &s.c=%p", s.get(), &s->b, &s->c); +} + +TEST("verify new with alignment = 64 with single element") { + struct S { + alignas(64) long a; + }; + static_assert(sizeof(S) == 64); + static_assert(alignof(S) == 64); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + LOG(info, "&s=%p", s.get()); +} + +TEST("verify new with alignment = 64 with single element") { + struct alignas(64) S { + long a; + }; + static_assert(sizeof(S) == 64); + static_assert(alignof(S) == 64); + auto s = std::make_unique<S>(); + veryfy_aligned(s.get()); + cmp(s.get(), &s->a); + LOG(info, "&s=%p", s.get()); +} + + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespamalloc/src/vespamalloc/malloc/common.h b/vespamalloc/src/vespamalloc/malloc/common.h index b21a2f63ed5..36f1bd0521d 100644 --- a/vespamalloc/src/vespamalloc/malloc/common.h +++ b/vespamalloc/src/vespamalloc/malloc/common.h @@ -66,6 +66,7 @@ template <size_t MinClassSizeC> class CommonT { public: + static constexpr size_t MAX_ALIGN = 0x200000ul; enum {MinClassSize = MinClassSizeC}; static inline SizeClassT sizeClass(size_t sz) { SizeClassT tmp(msbIdx(sz - 1) - (MinClassSizeC - 1)); diff --git a/vespamalloc/src/vespamalloc/malloc/malloc.h b/vespamalloc/src/vespamalloc/malloc/malloc.h index b3131ac5cbb..df40197bbbd 100644 --- a/vespamalloc/src/vespamalloc/malloc/malloc.h +++ b/vespamalloc/src/vespamalloc/malloc/malloc.h @@ -16,7 +16,7 @@ class MemoryManager : public IAllocator { public: MemoryManager(size_t logLimitAtStart); - ~MemoryManager(); + ~MemoryManager() override; bool initThisThread() override; bool quitThisThread() override; void enableThreadSupport() override; @@ -26,9 +26,17 @@ public: size_t getMaxNumThreads() const override { return _threadList.getMaxNumThreads(); } void *malloc(size_t sz); + void *malloc(size_t sz, std::align_val_t); void *realloc(void *oldPtr, size_t sz); - void free(void *ptr) { freeSC(ptr, _segment.sizeClass(ptr)); } - void free(void *ptr, size_t sz) { freeSC(ptr, MemBlockPtrT::sizeClass(sz)); } + void free(void *ptr) { + freeSC(ptr, _segment.sizeClass(ptr)); + } + void free(void *ptr, size_t sz) { + freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz))); + } + void free(void *ptr, size_t sz, std::align_val_t alignment) { + freeSC(ptr, MemBlockPtrT::sizeClass(MemBlockPtrT::adjustSize(sz, alignment))); + } size_t getMinSizeForAlignment(size_t align, size_t sz) const { return MemBlockPtrT::getMinSizeForAlignment(align, sz); } size_t sizeClass(const void *ptr) const { return _segment.sizeClass(ptr); } @@ -88,9 +96,7 @@ MemoryManager<MemBlockPtrT, ThreadListT>::MemoryManager(size_t logLimitAtStart) } template <typename MemBlockPtrT, typename ThreadListT> -MemoryManager<MemBlockPtrT, ThreadListT>::~MemoryManager() -{ -} +MemoryManager<MemBlockPtrT, ThreadListT>::~MemoryManager() = default; template <typename MemBlockPtrT, typename ThreadListT> bool MemoryManager<MemBlockPtrT, ThreadListT>::initThisThread() @@ -159,13 +165,27 @@ void * MemoryManager<MemBlockPtrT, ThreadListT>::malloc(size_t sz) fprintf(stderr, "Memory %p(%ld) has been tampered with after free.\n", mem.ptr(), mem.size()); crash(); } - PARANOID_CHECK2(if (!mem.validFree() && mem.ptr()) { crash(); } ); mem.setExact(sz); mem.alloc(_prAllocLimit<=mem.adjustSize(sz)); return mem.ptr(); } template <typename MemBlockPtrT, typename ThreadListT> +void * MemoryManager<MemBlockPtrT, ThreadListT>::malloc(size_t sz, std::align_val_t alignment) +{ + MemBlockPtrT mem; + ThreadPool & tp = _threadList.getCurrent(); + tp.malloc(mem.adjustSize(sz, alignment), mem); + if (!mem.validFree()) { + fprintf(stderr, "Memory %p(%ld) has been tampered with after free.\n", mem.ptr(), mem.size()); + crash(); + } + mem.setExact(sz, alignment); + mem.alloc(_prAllocLimit<=mem.adjustSize(sz, alignment)); + return mem.ptr(); +} + +template <typename MemBlockPtrT, typename ThreadListT> void MemoryManager<MemBlockPtrT, ThreadListT>::freeSC(void *ptr, SizeClassT sc) { if (MemBlockPtrT::verifySizeClass(sc)) { diff --git a/vespamalloc/src/vespamalloc/malloc/mallocd.cpp b/vespamalloc/src/vespamalloc/malloc/mallocd.cpp index 8e8bb642efc..47c12b4f186 100644 --- a/vespamalloc/src/vespamalloc/malloc/mallocd.cpp +++ b/vespamalloc/src/vespamalloc/malloc/mallocd.cpp @@ -8,11 +8,11 @@ typedef ThreadListT<MemBlockBoundsCheck, Stat> ThreadList; typedef MemoryWatcher<MemBlockBoundsCheck, ThreadList> Allocator; static char _Gmem[sizeof(Allocator)]; -static Allocator *_GmemP = NULL; +static Allocator *_GmemP = nullptr; static Allocator * createAllocator() { - if (_GmemP == NULL) { + if (_GmemP == nullptr) { _GmemP = new (_Gmem) Allocator(-1, 0x7fffffffffffffffl); } return _GmemP; diff --git a/vespamalloc/src/vespamalloc/malloc/memblock.h b/vespamalloc/src/vespamalloc/malloc/memblock.h index 118fb0e046c..e8d8e274678 100644 --- a/vespamalloc/src/vespamalloc/malloc/memblock.h +++ b/vespamalloc/src/vespamalloc/malloc/memblock.h @@ -10,14 +10,14 @@ namespace vespamalloc { template <size_t MinSizeClassC, size_t MaxSizeClassMultiAllocC> class MemBlockT : public CommonT<MinSizeClassC> { - static const size_t MAX_ALIGN= 0x200000ul; public: - typedef StackEntry<StackReturnEntry> Stack; + using Parent = CommonT<MinSizeClassC>; + using Stack = StackEntry<StackReturnEntry>; enum { MaxSizeClassMultiAlloc = MaxSizeClassMultiAllocC, SizeClassSpan = (MaxSizeClassMultiAllocC-MinSizeClassC) }; - MemBlockT() : _ptr(NULL) { } + MemBlockT() : _ptr(nullptr) { } MemBlockT(void * p) : _ptr(p) { } MemBlockT(void * p, size_t /*sz*/) : _ptr(p) { } MemBlockT(void * p, size_t, bool) : _ptr(p) { } @@ -28,7 +28,8 @@ public: const void *ptr() const { return _ptr; } bool validAlloc() const { return true; } bool validFree() const { return true; } - void setExact(size_t ) { } + void setExact(size_t) { } + void setExact(size_t, std::align_val_t ) { } void alloc(bool ) { } void setThreadId(int ) { } void free() { } @@ -36,12 +37,13 @@ public: bool allocated() const { return false; } int threadId() const { return 0; } void info(FILE *, unsigned level=0) const { (void) level; } - Stack * callStack() { return NULL; } + Stack * callStack() { return nullptr; } size_t callStackLen() const { return 0; } void fillMemory(size_t) { } void logBigBlock(size_t exact, size_t adjusted, size_t gross) const __attribute__((noinline)); static size_t adjustSize(size_t sz) { return sz; } + static size_t adjustSize(size_t sz, std::align_val_t) { return sz; } static size_t unAdjustSize(size_t sz) { return sz; } static void dumpInfo(size_t level); static void dumpFile(FILE * fp) { _logFile = fp; } @@ -49,9 +51,9 @@ public: static void setFill(uint8_t ) { } static bool verifySizeClass(int sc) { (void) sc; return true; } static size_t getMinSizeForAlignment(size_t align, size_t sz) { - return (sz < MAX_ALIGN) + return (sz < Parent::MAX_ALIGN) ? std::max(sz, align) - : (align < MAX_ALIGN) ? sz : sz + align; + : (align < Parent::MAX_ALIGN) ? sz : sz + align; } private: void * _ptr; @@ -64,4 +66,3 @@ template <> void MemBlock::dumpInfo(size_t level); extern template class MemBlockT<5, 20>; } - diff --git a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.cpp b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.cpp index 0c608fed5d5..d147bd5ba41 100644 --- a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.cpp +++ b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.cpp @@ -15,7 +15,8 @@ void MemBlockBoundsCheckBaseTBase::verifyFill() const const uint8_t *c(static_cast<const uint8_t *>(ptr())), *e(c+size()); for(;(c < e) && (*c == _fillValue); c++) { } if (c != e) { - fprintf(_logFile, "Incorrect fillvalue (%2x) instead of (%2x) at position %ld of %ld\n", *c, _fillValue, c - static_cast<const uint8_t *>(ptr()), size()); + fprintf(_logFile, "Incorrect fillvalue (%2x) instead of (%2x) at position %ld(%p) of %ld(%p - %p)\n", + *c, _fillValue, c - static_cast<const uint8_t *>(ptr()), c, size(), ptr(), e); abort(); } } diff --git a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h index 21e9d74c0d2..1860f2f36d3 100644 --- a/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h +++ b/vespamalloc/src/vespamalloc/malloc/memblockboundscheck.h @@ -11,15 +11,22 @@ class MemBlockBoundsCheckBaseTBase : public CommonT<5> public: typedef StackEntry<StackReturnEntry> Stack; void * rawPtr() { return _ptr; } - void *ptr() { unsigned *p((unsigned*)_ptr); return p ? (p+4) : NULL; } - const void *ptr() const { unsigned *p((unsigned*)_ptr); return p ? (p+4) : NULL; } + void *ptr() { + char *p((char*)_ptr); + return p ? (p+alignment()) : nullptr; + } + const void *ptr() const { + const char *p((const char*)_ptr); + return p ? (p+alignment()) : nullptr; + } void setThreadId(int th) { if (_ptr) { static_cast<uint32_t*>(_ptr)[2] = th; } } - bool allocated() const { return (static_cast<unsigned*>(_ptr)[3] == ALLOC_MAGIC); } - size_t size() const { return static_cast<const uint64_t *>(_ptr)[0]; } + bool allocated() const { return (static_cast<uint32_t*>(_ptr)[3] == ALLOC_MAGIC); } + size_t size() const { return static_cast<const uint32_t *>(_ptr)[0]; } + size_t alignment() const { return static_cast<const uint32_t *>(_ptr)[1]; } int threadId() const { return static_cast<int*>(_ptr)[2]; } - Stack * callStack() { return reinterpret_cast<Stack *>((char *)_ptr + size() + 4*sizeof(unsigned)); } - const Stack * callStack() const { return reinterpret_cast<const Stack *>((const char *)_ptr + size() + 4*sizeof(unsigned)); } + Stack * callStack() { return reinterpret_cast<Stack *>((char *)_ptr + size() + alignment()); } + const Stack * callStack() const { return reinterpret_cast<const Stack *>((const char *)_ptr + size() + alignment()); } void fillMemory(size_t sz) { if (_fillValue != NO_FILL) { memset(ptr(), _fillValue, sz); @@ -44,7 +51,17 @@ protected: MemBlockBoundsCheckBaseTBase(void * p) : _ptr(p) { } void verifyFill() const __attribute__((noinline)); - void setSize(size_t sz) { static_cast<uint64_t *>(_ptr)[0] = sz; } + void setSize(size_t sz) { + assert(sz < 0x100000000ul); + static_cast<uint32_t *>(_ptr)[0] = sz; + } + void setAlignment(size_t alignment) { static_cast<uint32_t *>(_ptr)[1] = alignment; } + static constexpr size_t preambleOverhead(std::align_val_t alignment) { + return std::max(preambleOverhead(), size_t(alignment)); + } + static constexpr size_t preambleOverhead() { + return 4*sizeof(unsigned); + } enum { ALLOC_MAGIC = 0xF1E2D3C4, @@ -69,15 +86,22 @@ public: MaxSizeClassMultiAlloc = MaxSizeClassMultiAllocC, SizeClassSpan = (MaxSizeClassMultiAllocC-5) }; - MemBlockBoundsCheckBaseT() : MemBlockBoundsCheckBaseTBase(NULL) { } - MemBlockBoundsCheckBaseT(void * p) : MemBlockBoundsCheckBaseTBase(p ? (unsigned *)p-4 : NULL) { } - MemBlockBoundsCheckBaseT(void * p, size_t sz) : MemBlockBoundsCheckBaseTBase(p) { setSize(sz); } + MemBlockBoundsCheckBaseT() : MemBlockBoundsCheckBaseTBase(nullptr) { } + MemBlockBoundsCheckBaseT(void * p) + : MemBlockBoundsCheckBaseTBase(p ? static_cast<char *>(p) - preambleOverhead() : nullptr) + { } + MemBlockBoundsCheckBaseT(void * p, size_t sz) + : MemBlockBoundsCheckBaseTBase(p) + { + setSize(sz); + setAlignment(preambleOverhead()); + } MemBlockBoundsCheckBaseT(void * p, size_t, bool) : MemBlockBoundsCheckBaseTBase(p) { } bool validCommon() const { const unsigned *p(reinterpret_cast<const unsigned*>(_ptr)); return p && ((p[3] == ALLOC_MAGIC) || (p[3] == FREE_MAGIC)) - && *(reinterpret_cast<const unsigned *> ((const char*)_ptr + size() + 4*sizeof(unsigned) + StackTraceLen*sizeof(void *))) == TAIL_MAGIC; + && *(reinterpret_cast<const unsigned *> ((const char*)_ptr + size() + alignment() + StackTraceLen*sizeof(void *))) == TAIL_MAGIC; } bool validAlloc1() const { unsigned *p((unsigned*)_ptr); @@ -110,7 +134,12 @@ public: fillMemory(size()); setTailMagic(); } - void setExact(size_t sz) { init(sz); } + void setExact(size_t sz) { + init(sz, preambleOverhead()); + } + void setExact(size_t sz, std::align_val_t alignment) { + init(sz, preambleOverhead(alignment)); + } size_t callStackLen() const { const Stack * stack = callStack(); // Use int to avoid compiler warning about always true. @@ -121,17 +150,30 @@ public: } return StackTraceLen; } - static size_t adjustSize(size_t sz) { return sz + ((4+1)*sizeof(unsigned) + StackTraceLen*sizeof(void *)); } - static size_t unAdjustSize(size_t sz) { return sz - ((4+1)*sizeof(unsigned) + StackTraceLen*sizeof(void *)); } + static constexpr size_t adjustSize(size_t sz) { return sz + overhead(); } + static constexpr size_t adjustSize(size_t sz, std::align_val_t alignment) { return sz + overhead(alignment); } + static constexpr size_t unAdjustSize(size_t sz) { return sz - overhead(); } static void dumpInfo(size_t level) __attribute__((noinline)); - static size_t getMinSizeForAlignment(size_t align, size_t sz) { return sz + align; } + static constexpr size_t getMinSizeForAlignment(size_t align, size_t sz) { return sz + align; } void info(FILE * os, unsigned level=0) const __attribute__((noinline)); protected: - void setTailMagic() { *(reinterpret_cast<unsigned *> ((char*)_ptr + size() + 4*sizeof(unsigned) + StackTraceLen*sizeof(void *))) = TAIL_MAGIC; } - void init(size_t sz) { + static constexpr size_t postambleOverhead() { + return sizeof(unsigned) + StackTraceLen*sizeof(void *); + } + static constexpr size_t overhead() { + return preambleOverhead() + postambleOverhead(); + } + static constexpr size_t overhead(std::align_val_t alignment) { + return preambleOverhead(alignment) + postambleOverhead(); + } + void setTailMagic() { + *(reinterpret_cast<unsigned *> ((char*)_ptr + size() + alignment() + StackTraceLen*sizeof(void *))) = TAIL_MAGIC; + } + void init(size_t sz, size_t alignment) { if (_ptr) { setSize(sz); + setAlignment(alignment); setTailMagic(); } } diff --git a/vespamalloc/src/vespamalloc/malloc/overload.h b/vespamalloc/src/vespamalloc/malloc/overload.h index 7883578cc28..56cd8101731 100644 --- a/vespamalloc/src/vespamalloc/malloc/overload.h +++ b/vespamalloc/src/vespamalloc/malloc/overload.h @@ -21,13 +21,6 @@ private: static CreateAllocator _CreateAllocator __attribute__ ((init_priority (543))); -#if 1 // Only until we get on to a new C++14 compiler -void operator delete(void* ptr, std::size_t sz) noexcept __attribute__((visibility ("default"))); -void operator delete[](void* ptr, std::size_t sz) noexcept __attribute__((visibility ("default"))); -void operator delete(void* ptr, std::size_t sz, const std::nothrow_t&) noexcept __attribute__((visibility ("default"))); -void operator delete[](void* ptr, std::size_t sz, const std::nothrow_t&) noexcept __attribute__((visibility ("default"))); -#endif - void* operator new(std::size_t sz) { void * ptr(vespamalloc::createAllocator()->malloc(sz)); @@ -74,6 +67,42 @@ void operator delete[](void* ptr, std::size_t sz, const std::nothrow_t&) noexcep if (ptr) { vespamalloc::_GmemP->free(ptr, sz); } } +/* + * Below are overloads taking alignment into account too. + * Due to allocation being power of 2 up to huge page size (2M) + * alignment will always be satisfied. size will always be larger or equal to alignment. + */ +void* operator new(std::size_t sz, std::align_val_t alignment) { + return vespamalloc::_GmemP->malloc(sz, alignment); +} +void* operator new(std::size_t sz, std::align_val_t alignment, const std::nothrow_t&) noexcept { + return vespamalloc::_GmemP->malloc(sz, alignment); +} +void operator delete(void* ptr , std::align_val_t) noexcept { + return vespamalloc::_GmemP->free(ptr); +} +void operator delete(void* ptr, std::align_val_t, const std::nothrow_t&) noexcept { + return vespamalloc::_GmemP->free(ptr); +} +void* operator new[](std::size_t sz, std::align_val_t alignment) { + return vespamalloc::_GmemP->malloc(sz, alignment); +} +void* operator new[](std::size_t sz, std::align_val_t alignment, const std::nothrow_t&) noexcept { + return vespamalloc::_GmemP->malloc(sz, alignment); +} +void operator delete[](void* ptr, std::align_val_t) noexcept { + return vespamalloc::_GmemP->free(ptr); +} +void operator delete[](void* ptr, std::align_val_t, const std::nothrow_t&) noexcept { + return vespamalloc::_GmemP->free(ptr); +} +void operator delete(void* ptr, std::size_t sz, std::align_val_t alignment) noexcept { + return vespamalloc::_GmemP->free(ptr, sz, alignment); +} +void operator delete[](void* ptr, std::size_t sz, std::align_val_t alignment) noexcept { + return vespamalloc::_GmemP->free(ptr, sz, alignment); +} + extern "C" { void * malloc(size_t sz) { diff --git a/vsm/src/vespa/vsm/vsm/docsumconfig.cpp b/vsm/src/vespa/vsm/vsm/docsumconfig.cpp index 7402a45fa4a..2512bea26df 100644 --- a/vsm/src/vespa/vsm/vsm/docsumconfig.cpp +++ b/vsm/src/vespa/vsm/vsm/docsumconfig.cpp @@ -3,11 +3,11 @@ #include <vespa/vsm/vsm/docsumconfig.h> #include <vespa/searchsummary/docsummary/docsumfieldwriter.h> #include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h> -#include <vespa/searchlib/common/struct_field_mapper.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vsm/config/config-vsmfields.h> #include <vespa/vsm/config/config-vsmsummary.h> -using search::StructFieldMapper; +using search::MatchingElementsFields; using search::docsummary::IDocsumFieldWriter; using search::docsummary::EmptyDFW; using search::docsummary::MatchedElementsFilterDFW; @@ -19,12 +19,12 @@ namespace vsm { namespace { -void populate_mapper(StructFieldMapper& mapper, VsmfieldsConfig& fields_config, const vespalib::string& field_name) +void populate_fields(MatchingElementsFields& fields, VsmfieldsConfig& fields_config, const vespalib::string& field_name) { vespalib::string prefix = field_name + "."; for (const auto& spec : fields_config.fieldspec) { if (spec.name.substr(0, prefix.size()) == prefix) { - mapper.add_mapping(field_name, spec.name); + fields.add_mapping(field_name, spec.name); } } } @@ -38,7 +38,7 @@ DynamicDocsumConfig::DynamicDocsumConfig(search::docsummary::IDocsumEnvironment* } IDocsumFieldWriter::UP -DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<search::StructFieldMapper> struct_field_mapper) +DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<search::MatchingElementsFields> matching_elems_fields) { IDocsumFieldWriter::UP fieldWriter; if ((overrideName == "staticrank") || @@ -60,11 +60,11 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & string source_field = argument.empty() ? fieldName : argument; const ResultConfig& resultConfig = getResultConfig(); int source_field_enum = resultConfig.GetFieldNameEnum().Lookup(source_field.c_str()); - populate_mapper(*struct_field_mapper, *_vsm_fields_config, source_field); - fieldWriter = MatchedElementsFilterDFW::create(source_field, source_field_enum, struct_field_mapper); + populate_fields(*matching_elems_fields, *_vsm_fields_config, source_field); + fieldWriter = MatchedElementsFilterDFW::create(source_field, source_field_enum, matching_elems_fields); rc = static_cast<bool>(fieldWriter); } else { - fieldWriter = search::docsummary::DynamicDocsumConfig::createFieldWriter(fieldName, overrideName, argument, rc, struct_field_mapper); + fieldWriter = search::docsummary::DynamicDocsumConfig::createFieldWriter(fieldName, overrideName, argument, rc, matching_elems_fields); } return fieldWriter; } diff --git a/vsm/src/vespa/vsm/vsm/docsumconfig.h b/vsm/src/vespa/vsm/vsm/docsumconfig.h index 17128798ef2..8be9324b9d6 100644 --- a/vsm/src/vespa/vsm/vsm/docsumconfig.h +++ b/vsm/src/vespa/vsm/vsm/docsumconfig.h @@ -22,7 +22,7 @@ public: private: std::unique_ptr<search::docsummary::IDocsumFieldWriter> createFieldWriter(const string & fieldName, const string & overrideName, - const string & cf, bool & rc, std::shared_ptr<search::StructFieldMapper> struct_field_mapper) override; + const string & cf, bool & rc, std::shared_ptr<search::MatchingElementsFields> matching_elems_fields) override; }; } diff --git a/vsm/src/vespa/vsm/vsm/i_matching_elements_filler.h b/vsm/src/vespa/vsm/vsm/i_matching_elements_filler.h index a30dcbf2a5b..45233818452 100644 --- a/vsm/src/vespa/vsm/vsm/i_matching_elements_filler.h +++ b/vsm/src/vespa/vsm/vsm/i_matching_elements_filler.h @@ -6,7 +6,7 @@ namespace search { class MatchingElements; -class StructFieldMapper; +class MatchingElementsFields; } namespace vsm { @@ -17,7 +17,7 @@ namespace vsm { */ class IMatchingElementsFiller { public: - virtual std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::StructFieldMapper& struct_field_mapper) = 0; + virtual std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields& fields) = 0; virtual ~IMatchingElementsFiller() = default; }; diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp index dd7b36015e4..5d8c7735c0e 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -50,10 +50,10 @@ void GetDocsumsStateCallback::FillDocumentLocations(GetDocsumsState *state, IDoc } std::unique_ptr<MatchingElements> -GetDocsumsStateCallback::fill_matching_elements(const search::StructFieldMapper& struct_field_mapper) +GetDocsumsStateCallback::fill_matching_elements(const search::MatchingElementsFields& fields) { if (_matching_elements_filler) { - return _matching_elements_filler->fill_matching_elements(struct_field_mapper); + return _matching_elements_filler->fill_matching_elements(fields); } return std::make_unique<MatchingElements>(); } diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.h b/vsm/src/vespa/vsm/vsm/vsm-adapter.h index 96d12e23db6..cffae318586 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.h +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.h @@ -42,7 +42,7 @@ public: void FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) override; void ParseLocation(GetDocsumsState * state) override; virtual void FillDocumentLocations(GetDocsumsState * state, IDocsumEnvironment * env); - virtual std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::StructFieldMapper& struct_field_mapper) override; + virtual std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::MatchingElementsFields& fields) override; void setSummaryFeatures(const search::FeatureSet::SP & sf) { _summaryFeatures = sf; } void setRankFeatures(const search::FeatureSet::SP & rf) { _rankFeatures = rf; } void set_matching_elements_filler(std::unique_ptr<IMatchingElementsFiller> matching_elements_filler); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java index 537ec2ae751..05582d6bbda 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java @@ -62,13 +62,8 @@ class CuratorCompletionWaiter implements Curator.CompletionWaiter { log.log(Level.FINE, barrierCompletedMessage(respondents, startTime)); break; } - // Then, if some are missing after 2 seconds, allow if the server this code is running on is one of the repondents - if (usedMoreTimeThan(Duration.ofSeconds(2), startTime) && respondents.contains(myId) && respondents.size() >= barrierMemberCount()) { - log.log(Level.INFO, barrierCompletedMessage(respondents, startTime)); - break; - } - // If some are still missing after 4 seconds, quorum is enough - if (usedMoreTimeThan(Duration.ofSeconds(4), startTime) && respondents.size() >= barrierMemberCount()) { + // If some are missing, quorum is enough + if (respondents.size() >= barrierMemberCount()) { log.log(Level.INFO, barrierCompletedMessage(respondents, startTime)); break; } @@ -79,10 +74,6 @@ class CuratorCompletionWaiter implements Curator.CompletionWaiter { return respondents; } - private boolean usedMoreTimeThan(Duration waitTime, Instant startTime) { - return clock.instant().isAfter(startTime.plus(waitTime)); - } - private String barrierCompletedMessage(List<String> respondents, Instant startTime) { return barrierPath + " completed in " + Duration.between(startTime, Instant.now()).toString() + ", " + respondents.size() + "/" + curator.zooKeeperEnsembleCount() + " responded: " + respondents; diff --git a/zookeeper-command-line-client/pom.xml b/zookeeper-command-line-client/pom.xml index 6d17c28b621..748a156b30c 100644 --- a/zookeeper-command-line-client/pom.xml +++ b/zookeeper-command-line-client/pom.xml @@ -14,8 +14,7 @@ <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> - <!-- TODO: Testing out 3.5 version, set to zookeeper.client.version when that has been set to 3.5.x --> - <version>3.5.6</version> + <version>${zookeeper.client.version}</version> </dependency> <dependency> <!-- Needed by vespa-zkcli --> diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index 6e8c82bc66e..b146390046c 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,3 +1,4 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) -add_subdirectory(zookeeper-server-3.5) +add_subdirectory(zookeeper-server-3.5.6) +add_subdirectory(zookeeper-server-3.5.8) diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index edfbdbad02e..28f18000c2d 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -13,7 +13,8 @@ <version>7-SNAPSHOT</version> <modules> <module>zookeeper-server-common</module> - <module>zookeeper-server-3.5</module> + <module>zookeeper-server-3.5.6</module> + <module>zookeeper-server-3.5.8</module> </modules> <dependencies> <dependency> diff --git a/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt new file mode 100644 index 00000000000..b68994d32e0 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt @@ -0,0 +1,3 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(zookeeper-server-3.5.6) +install_symlink(lib/jars/zookeeper-server-3.5.6-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-3.5/pom.xml b/zookeeper-server/zookeeper-server-3.5.6/pom.xml index d283221457c..e50324a0488 100644 --- a/zookeeper-server/zookeeper-server-3.5/pom.xml +++ b/zookeeper-server/zookeeper-server-3.5.6/pom.xml @@ -8,7 +8,7 @@ <version>7-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>zookeeper-server-3.5</artifactId> + <artifactId>zookeeper-server-3.5.6</artifactId> <packaging>container-plugin</packaging> <version>7-SNAPSHOT</version> <dependencies> @@ -20,7 +20,7 @@ <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> - <version>3.5.6</version> + <version>${zookeeper.server.version}</version> </dependency> <dependency> <groupId>org.slf4j</groupId> diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java new file mode 100644 index 00000000000..ee3695b02f8 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.security.tls.TransportSecurityUtils; + +import java.util.logging.Level; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static com.yahoo.vespa.zookeeper.Configurator.zookeeperServerHostnames; + +/** + * Writes zookeeper config and starts zookeeper server. + * + * @author Ulf Lilleengen + * @author Harald Musum + */ +public class VespaZooKeeperServerImpl extends AbstractComponent implements Runnable, VespaZooKeeperServer { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperServerImpl.class.getName()); + private final Thread zkServerThread; + private final ZookeeperServerConfig zookeeperServerConfig; + + @Inject + public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { + this.zookeeperServerConfig = zookeeperServerConfig; + new Configurator(zookeeperServerConfig).writeConfigToDisk(TransportSecurityUtils.getOptions()); + zkServerThread = new Thread(this, "zookeeper server"); + zkServerThread.start(); + } + + private void shutdown() { + zkServerThread.interrupt(); + try { + zkServerThread.join(); + } catch (InterruptedException e) { + log.log(Level.WARNING, "Error joining server thread on shutdown", e); + } + } + + @Override + public void run() { + String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; + log.log(Level.INFO, "Starting ZooKeeper server with config file " + args[0] + + ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); + org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); + } + + @Override + public void deconstruct() { + shutdown(); + super.deconstruct(); + } + +} diff --git a/zookeeper-server/zookeeper-server-3.5.8/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.5.8/CMakeLists.txt new file mode 100644 index 00000000000..66765e34997 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.5.8/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(zookeeper-server-3.5.8) +# TODO: Needs to be included when this is the wanted default version (and other symlinks need to be removed) +#install_symlink(lib/jars/zookeeper-server-3.5.8-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-3.5.8/pom.xml b/zookeeper-server/zookeeper-server-3.5.8/pom.xml new file mode 100644 index 00000000000..e4000285ffa --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.5.8/pom.xml @@ -0,0 +1,78 @@ +<?xml version="1.0"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-server</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <artifactId>zookeeper-server-3.5.8</artifactId> + <packaging>container-plugin</packaging> + <version>7-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-server-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>3.5.8</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-jdk14</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <version>1.7.5</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> + <forkMode>once</forkMode> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <importPackage>com.sun.management</importPackage> + <bundleSymbolicName>zookeeper-server</bundleSymbolicName> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/zookeeper-server/zookeeper-server-3.5.8/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5.8/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java new file mode 100644 index 00000000000..7f5b6170947 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.5.8/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.security.tls.TransportSecurityUtils; + +import java.util.logging.Level; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static com.yahoo.vespa.zookeeper.Configurator.zookeeperServerHostnames; + +/** + * Writes zookeeper config and starts zookeeper server. + * + * @author Ulf Lilleengen + * @author Harald Musum + */ +public class VespaZooKeeperServerImpl extends AbstractComponent implements Runnable, VespaZooKeeperServer { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperServerImpl.class.getName()); + private final Thread zkServerThread; + private final ZookeeperServerConfig zookeeperServerConfig; + + @Inject + public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { + this.zookeeperServerConfig = zookeeperServerConfig; + new Configurator(zookeeperServerConfig).writeConfigToDisk(TransportSecurityUtils.getOptions()); + zkServerThread = new Thread(this, "zookeeper server"); + zkServerThread.start(); + } + + private void shutdown() { + zkServerThread.interrupt(); + try { + zkServerThread.join(); + } catch (InterruptedException e) { + log.log(Level.WARNING, "Error joining server thread on shutdown", e); + } + } + + @Override + public void run() { + String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; + log.log(Level.INFO, "Starting ZooKeeper server with config file " + args[0] + + ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); + org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); + } + + @Override + public void deconstruct() { + shutdown(); + super.deconstruct(); + } + +} diff --git a/zookeeper-server/zookeeper-server-3.5/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.5/CMakeLists.txt deleted file mode 100644 index 782c5f07b83..00000000000 --- a/zookeeper-server/zookeeper-server-3.5/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(zookeeper-server-3.5) -install_symlink(lib/jars/zookeeper-server-3.5-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-common/pom.xml b/zookeeper-server/zookeeper-server-common/pom.xml index 66e5bc2075c..b05136cd151 100644 --- a/zookeeper-server/zookeeper-server-common/pom.xml +++ b/zookeeper-server/zookeeper-server-common/pom.xml @@ -11,6 +11,13 @@ <artifactId>zookeeper-server-common</artifactId> <packaging>container-plugin</packaging> <version>7-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> <build> <plugins> <plugin> diff --git a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java index 1f4be8852a3..44ea8cece34 100644 --- a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java @@ -1,10 +1,8 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.vespa.zookeeper; -import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; -import java.util.logging.Level; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyStoreUtils; @@ -13,11 +11,10 @@ import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.TransportSecurityOptions; -import com.yahoo.security.tls.TransportSecurityUtils; import com.yahoo.text.Utf8; +import com.yahoo.vespa.defaults.Defaults; import javax.net.ssl.SSLContext; -import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.nio.file.Files; @@ -29,62 +26,46 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.logging.Level; import java.util.stream.Collectors; import static com.yahoo.vespa.defaults.Defaults.getDefaults; -/** - * Writes zookeeper config and starts zookeeper server. - * - * @author Ulf Lilleengen - * @author Harald Musum - */ -public class VespaZooKeeperServerImpl extends AbstractComponent implements Runnable, VespaZooKeeperServer { +public class Configurator { - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperServerImpl.class.getName()); + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(Configurator.class.getName()); private static final String ZOOKEEPER_JMX_LOG4J_DISABLE = "zookeeper.jmx.log4j.disable"; static final String ZOOKEEPER_JUTE_MAX_BUFFER = "jute.maxbuffer"; - private final Thread zkServerThread; + private final ZookeeperServerConfig zookeeperServerConfig; - private final String configFilePath; - private final String jksKeyStoreFilePath; + private final Path configFilePath; + private final Path jksKeyStoreFilePath; - VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig, boolean startServer, - Optional<TransportSecurityOptions> transportSecurityOptions) { + public Configurator(ZookeeperServerConfig zookeeperServerConfig) { + log.log(Level.FINE, zookeeperServerConfig.toString()); this.zookeeperServerConfig = zookeeperServerConfig; - System.setProperty("zookeeper.jmx.log4j.disable", "true"); + this.configFilePath = makeAbsolutePath(zookeeperServerConfig.zooKeeperConfigFile()); + this.jksKeyStoreFilePath = makeAbsolutePath(zookeeperServerConfig.jksKeyStoreFile()); + System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true"); System.setProperty("zookeeper.snapshot.trust.empty", Boolean.valueOf(zookeeperServerConfig.trustEmptySnapshot()).toString()); System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, Integer.valueOf(zookeeperServerConfig.juteMaxBuffer()).toString()); - - configFilePath = getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile()); - jksKeyStoreFilePath = getDefaults().underVespaHome(zookeeperServerConfig.jksKeyStoreFile()); - writeConfigToDisk(zookeeperServerConfig, transportSecurityOptions); - zkServerThread = new Thread(this, "zookeeper server"); - if (startServer) { - zkServerThread.start(); - } } - @Inject - public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { - this(zookeeperServerConfig, true, TransportSecurityUtils.getOptions()); - } - - private void writeConfigToDisk(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) { - new File(configFilePath).getParentFile().mkdirs(); + void writeConfigToDisk(Optional<TransportSecurityOptions> transportSecurityOptions) { + configFilePath.toFile().getParentFile().mkdirs(); try { writeZooKeeperConfigFile(zookeeperServerConfig, transportSecurityOptions); - writeMyIdFile(config); + writeMyIdFile(zookeeperServerConfig); transportSecurityOptions.ifPresent(this::writeJksKeystore); } catch (IOException e) { throw new RuntimeException("Error writing zookeeper config", e); } - } + } private void writeZooKeeperConfigFile(ZookeeperServerConfig config, Optional<TransportSecurityOptions> transportSecurityOptions) throws IOException { - try (FileWriter writer = new FileWriter(configFilePath)) { + try (FileWriter writer = new FileWriter(configFilePath.toFile())) { writer.write(transformConfigToString(config, transportSecurityOptions)); } } @@ -108,6 +89,7 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna sb.append("admin.enableServer=false").append("\n"); // Need NettyServerCnxnFactory to be able to use TLS for communication sb.append("serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory").append("\n"); + sb.append("quorumListenOnAllIPs=true").append("\n"); ensureThisServerIsRepresented(config.myid(), config.server()); config.server().forEach(server -> addServerToCfg(sb, server)); SSLContext sslContext = new SslContextBuilder().build(); @@ -140,7 +122,7 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna .withType(KeyStoreType.JKS) .withKeyEntry("foo", privateKey, certificates); - KeyStoreUtils.writeKeyStoreToFile(keyStoreBuilder.build(), Paths.get(jksKeyStoreFilePath)); + KeyStoreUtils.writeKeyStoreToFile(keyStoreBuilder.build(), jksKeyStoreFilePath); } private void ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) { @@ -160,32 +142,16 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna sb.append("server.").append(server.id()).append("=").append(server.hostname()).append(":").append(server.quorumPort()).append(":").append(server.electionPort()).append("\n"); } - private void shutdown() { - zkServerThread.interrupt(); - try { - zkServerThread.join(); - } catch (InterruptedException e) { - log.log(Level.WARNING, "Error joining server thread on shutdown", e); - } - } - - @Override - public void run() { - System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true"); - String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; - log.log(Level.INFO, "Starting ZooKeeper server with config file " + args[0] + - ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); - org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); - } - - @Override - public void deconstruct() { - shutdown(); - super.deconstruct(); + static Set<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) { + return zookeeperServerConfig.server().stream().map(ZookeeperServerConfig.Server::hostname).collect(Collectors.toSet()); } - private static Set<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) { - return zookeeperServerConfig.server().stream().map(ZookeeperServerConfig.Server::hostname).collect(Collectors.toSet()); + Path makeAbsolutePath(String filename) { + Path path = Paths.get(filename); + if (path.isAbsolute()) + return path; + else + return Paths.get(Defaults.getDefaults().underVespaHome(filename)); } private interface TlsConfig { @@ -205,7 +171,7 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna String configFieldPrefix(); - String jksKeyStoreFilePath(); + Path jksKeyStoreFilePath(); SSLContext sslContext(); @@ -238,9 +204,9 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna static class TlsClientServerConfig implements TlsConfig { private final SSLContext sslContext; - private final String jksKeyStoreFilePath; + private final Path jksKeyStoreFilePath; - TlsClientServerConfig(SSLContext sslContext, String jksKeyStoreFilePath) { + TlsClientServerConfig(SSLContext sslContext, Path jksKeyStoreFilePath) { this.sslContext = sslContext; this.jksKeyStoreFilePath = jksKeyStoreFilePath; } @@ -276,7 +242,7 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna } @Override - public String jksKeyStoreFilePath() { + public Path jksKeyStoreFilePath() { return jksKeyStoreFilePath; } @@ -289,9 +255,9 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna static class TlsQuorumConfig implements TlsConfig { private final SSLContext sslContext; - private final String jksKeyStoreFilePath; + private final Path jksKeyStoreFilePath; - TlsQuorumConfig(SSLContext sslContext, String jksKeyStoreFilePath) { + TlsQuorumConfig(SSLContext sslContext, Path jksKeyStoreFilePath) { this.sslContext = sslContext; this.jksKeyStoreFilePath = jksKeyStoreFilePath; } @@ -336,7 +302,7 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna } @Override - public String jksKeyStoreFilePath() { + public Path jksKeyStoreFilePath() { return jksKeyStoreFilePath; } diff --git a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java index 863d2dba708..74e21339022 100644 --- a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java +++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java @@ -26,6 +26,7 @@ import static com.yahoo.cloud.config.ZookeeperServerConfig.TlsForQuorumCommunica import static com.yahoo.cloud.config.ZookeeperServerConfig.TlsForClientServerCommunication; import static com.yahoo.security.KeyAlgorithm.EC; import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; +import static com.yahoo.vespa.zookeeper.Configurator.ZOOKEEPER_JUTE_MAX_BUFFER; import static java.time.Instant.EPOCH; import static java.time.temporal.ChronoUnit.DAYS; import static org.hamcrest.CoreMatchers.is; @@ -36,7 +37,7 @@ import static org.junit.Assert.assertTrue; /** * Tests the zookeeper server. */ -public class VespaZooKeeperServerImplTest { +public class ConfiguratorTest { private File cfgFile; private File idFile; @@ -55,7 +56,7 @@ public class VespaZooKeeperServerImplTest { @Test public void config_is_written_correctly_when_one_server() throws IOException { ZookeeperServerConfig.Builder builder = createConfigBuilderForSingleHost(cfgFile, idFile, jksKeyStoreFile); - createServer(builder); + new Configurator(builder.build()).writeConfigToDisk(Optional.empty()); validateConfigFileSingleHost(cfgFile); validateIdFile(idFile, ""); } @@ -69,7 +70,7 @@ public class VespaZooKeeperServerImplTest { builder.server(newServer(2, "baz", 345, 543)); builder.myidFile(idFile.getAbsolutePath()); builder.myid(1); - createServer(builder); + new Configurator(builder.build()).writeConfigToDisk(Optional.empty()); validateConfigFileMultipleHosts(cfgFile); validateIdFile(idFile, "1\n"); } @@ -80,7 +81,7 @@ public class VespaZooKeeperServerImplTest { builder.tlsForQuorumCommunication(TlsForQuorumCommunication.PORT_UNIFICATION); builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.PORT_UNIFICATION); Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions(); - createServer(builder, transportSecurityOptions); + new Configurator(builder.build()).writeConfigToDisk(transportSecurityOptions); validateConfigFilePortUnification(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile()); validateThatJksKeyStoreFileExists(jksKeyStoreFile); } @@ -91,7 +92,7 @@ public class VespaZooKeeperServerImplTest { builder.tlsForQuorumCommunication(TlsForQuorumCommunication.TLS_WITH_PORT_UNIFICATION); builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.TLS_WITH_PORT_UNIFICATION); Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions(); - createServer(builder, transportSecurityOptions); + new Configurator(builder.build()).writeConfigToDisk(transportSecurityOptions); validateConfigFileTlsWithPortUnification(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile()); validateThatJksKeyStoreFileExists(jksKeyStoreFile); } @@ -102,7 +103,7 @@ public class VespaZooKeeperServerImplTest { builder.tlsForQuorumCommunication(TlsForQuorumCommunication.TLS_ONLY); builder.tlsForClientServerCommunication(TlsForClientServerCommunication.Enum.TLS_ONLY); Optional<TransportSecurityOptions> transportSecurityOptions = createTransportSecurityOptions(); - createServer(builder, transportSecurityOptions); + new Configurator(builder.build()).writeConfigToDisk(transportSecurityOptions); validateConfigFileTlsOnly(cfgFile, jksKeyStoreFile, transportSecurityOptions.get().getCaCertificatesFile().get().toFile()); validateThatJksKeyStoreFileExists(jksKeyStoreFile); } @@ -117,21 +118,14 @@ public class VespaZooKeeperServerImplTest { return builder; } - private void createServer(ZookeeperServerConfig.Builder builder) { - createServer(builder, Optional.empty()); - } - - private void createServer(ZookeeperServerConfig.Builder builder, Optional<TransportSecurityOptions> options) { - new VespaZooKeeperServerImpl(new ZookeeperServerConfig(builder), false, options); - } - @Test(expected = RuntimeException.class) public void require_that_this_id_must_be_present_amongst_servers() { ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); + builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); builder.server(newServer(1, "bar", 234, 432)); builder.server(newServer(2, "baz", 345, 543)); builder.myid(0); - createServer(builder); + new Configurator(builder.build()).writeConfigToDisk(Optional.empty()); } @Test @@ -145,13 +139,13 @@ public class VespaZooKeeperServerImplTest { builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); builder.myidFile(idFile.getAbsolutePath()); - createServer(builder); - assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + new ZookeeperServerConfig(builder).juteMaxBuffer())); + new Configurator(builder.build()).writeConfigToDisk(Optional.empty()); + assertThat(System.getProperty(ZOOKEEPER_JUTE_MAX_BUFFER), is("" + new ZookeeperServerConfig(builder).juteMaxBuffer())); final int max_buffer = 1; builder.juteMaxBuffer(max_buffer); - createServer(builder); - assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + max_buffer)); + new Configurator(builder.build()).writeConfigToDisk(Optional.empty()); + assertThat(System.getProperty(ZOOKEEPER_JUTE_MAX_BUFFER), is("" + max_buffer)); } private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName, int electionPort, int quorumPort) { @@ -181,7 +175,8 @@ public class VespaZooKeeperServerImplTest { "autopurge.snapRetainCount=15\n" + "4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" + "admin.enableServer=false\n" + - "serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory\n"; + "serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory\n" + + "quorumListenOnAllIPs=true\n"; } private String quorumKeyStoreAndTrustStoreConfig(File jksKeyStoreFilePath, File caCertificatesFilePath) { |
