diff options
33 files changed, 1017 insertions, 148 deletions
diff --git a/cloud-tenant-base/OWNERS b/cloud-tenant-base/OWNERS new file mode 100644 index 00000000000..ff9741f2060 --- /dev/null +++ b/cloud-tenant-base/OWNERS @@ -0,0 +1,2 @@ +mortent +bjorncs
\ No newline at end of file diff --git a/cloud-tenant-base/README b/cloud-tenant-base/README new file mode 100644 index 00000000000..3863ef33b12 --- /dev/null +++ b/cloud-tenant-base/README @@ -0,0 +1 @@ +Parent pom for Vespa Cloud applications diff --git a/cloud-tenant-base/pom.xml b/cloud-tenant-base/pom.xml new file mode 100644 index 00000000000..8ffa6be4f9c --- /dev/null +++ b/cloud-tenant-base/pom.xml @@ -0,0 +1,383 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <artifactId>cloud-tenant-base</artifactId> + <name>Vespa Cloud tenant base</name> + <version>7-SNAPSHOT</version> + <description>Parent POM for all Vespa Cloud applications.</description> + <url>https://github.com/vespa-engine</url> + <packaging>pom</packaging> + + <parent> + <artifactId>hosted-tenant-base</artifactId> + <groupId>com.yahoo.vespa</groupId> + <version>7-SNAPSHOT</version> + <relativePath>../hosted-tenant-base/pom.xml</relativePath> + </parent> + + <licenses> + <license> + <name>The Apache License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + </license> + </licenses> + <developers> + <developer> + <name>Vespa</name> + <url>https://github.com/vespa-engine</url> + </developer> + </developers> + <scm> + <connection>scm:git:git@github.com:vespa-engine/vespa.git</connection> + <developerConnection>scm:git:git@github.com:vespa-engine/vespa.git</developerConnection> + <url>git@github.com:vespa-engine/vespa.git</url> + </scm> + + <properties> + <endpoint>https://api.vespa-external.aws.oath.cloud:4443</endpoint> + </properties> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dependency-versions</artifactId> + <version>${vespaversion}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <version>${junit.version}</version> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <version>${junit.version}</version> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container</artifactId> + <version>${vespaversion}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${vespaversion}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>cloud-tenant-cd</artifactId> + <version>${test-framework.version}</version> + <scope>runtime</scope> + </dependency> + </dependencies> + + <profiles> + <profile> + <!-- Build *-fat-test.jar file that includes all non-test classes and resources + that are part of the class path during test and and test.jar that includes + all test classes and resources, and put it inside a zip: + 1. application classes and resources + 2. test classes and resources + 3. classes and resources in all dependencies of both (1) and (2) + 4. copy the fat-test-jar and test-jar to application-test/artifacts directory + 5. zip application-test --> + <id>fat-test-application</id> + <build> + <plugins> + <plugin> + <!-- dependencies, see (3) above --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <!-- JAR-like dependencies --> + <id>unpack-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <configuration> + <includeTypes>jar,test-jar</includeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + <!-- WARNING(2018-06-27): bcpkix-jdk15on-1.58.jar and + bcprov-jdk15on-1.58.jar are pulled in via + container-dev and both contains the same set of + bouncycastle signature files in META-INF: + BC1024KE.DSA, BC1024KE.SF, BC2048KE.DSA, and + BC2048KE.SF. By merging any of these two with any + other JAR file like we're doing here, the signatures + are wrong. Worse, what we're doing is WRONG but not + yet fatal. + + The symptom of this happening is that the tester fails + to load the SystemTest class(!?), and subsequently + tries to run all test-like files in the fat test JAR. + + The solution is to exclude such files. This happens + automatically with maven-assembly-plugin. --> + <excludes>META-INF/*.SF,META-INF/*.DSA</excludes> + </configuration> + </execution> + <execution> + <!-- non-JAR-like dependencies --> + <id>non-jar-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <excludeTypes>jar,test-jar</excludeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>copy-resources</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>target/fat-test-classes</outputDirectory> + <resources> + <!-- application classes and resources, see 1. above --> + <resource> + <directory>target/classes</directory> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>fat-test-jar</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <classesDirectory>target/fat-test-classes</classesDirectory> + <classifier>fat-test</classifier> + </configuration> + </execution> + <execution> + <id>test-jar</id> + <phase>package</phase> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>attach-artifact</id> + <phase>package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <tasks> + <!-- copy fat test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-fat-test.jar" + todir="target/application-test/artifacts/" /> + + <!-- copy slim test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-tests.jar" + todir="target/application-test/artifacts/" /> + + <!-- zip application-test, see 5. above --> + <zip destfile="target/application-test.zip" + basedir="target/application-test/" /> + </tasks> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + + <profile> <!-- Alias vespaversion with a more descriptive vespa.compile.version --> + <id>set-vespa-compile-version</id> + <activation> + <property> + <name>vespa.compile.version</name> + </property> + </activation> + <properties> + <vespaversion>${vespa.compile.version}</vespaversion> + </properties> + </profile> + + <profile> <!-- Alias vespaVersion with a more descriptive vespa.runtime.version --> + <id>set-vespa-runtime-version</id> + <activation> + <property> + <name>vespa.runtime.version</name> + </property> + </activation> + <properties> + <vespaVersion>${vespa.runtime.version}</vespaVersion> + </properties> + </profile> + </profiles> + + <build> + <finalName>${project.artifactId}</finalName> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <groups>${test.categories}</groups> + <redirectTestOutputToFile>false</redirectTestOutputToFile> + <trimStackTrace>false</trimStackTrace> + <systemPropertyVariables> + <application>${application}</application> + <tenant>${tenant}</tenant> + <instance>${instance}</instance> + <environment>${environment}</environment> + <region>${region}</region> + <endpoint>${endpoint}</endpoint> + <apiKeyFile>${apiKeyFile}</apiKeyFile> + <apiCertificateFile>${apiCertificateFile}</apiCertificateFile> + <dataPlaneKeyFile>${dataPlaneKeyFile}</dataPlaneKeyFile> + <dataPlaneCertificateFile>${dataPlaneCertificateFile}</dataPlaneCertificateFile> + </systemPropertyVariables> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + </configuration> + </plugin> + </plugins> + </pluginManagement> + + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>3.0.0-M2</version> + <executions> + <execution> + <id>enforce-java</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireJavaVersion> + <version>[11, )</version> + </requireJavaVersion> + <requireMavenVersion> + <version>[3.5, )</version> + </requireMavenVersion> + </rules> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>${maven-compiler-plugin.version}</version> + <configuration> + <source>${target_jdk_version}</source> + <target>${target_jdk_version}</target> + <showWarnings>true</showWarnings> + <showDeprecation>true</showDeprecation> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-maven-plugin</artifactId> + <version>${vespaversion}</version> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-application-maven-plugin</artifactId> + <version>${vespaversion}</version> + <executions> + <execution> + <goals> + <goal>packageApplication</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${vespaversion}</version> + <extensions>true</extensions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/tenant-cd/pom.xml b/cloud-tenant-cd/pom.xml index 829b1de457b..cc0f0777008 100644 --- a/tenant-cd/pom.xml +++ b/cloud-tenant-cd/pom.xml @@ -1,14 +1,12 @@ <?xml version="1.0" encoding="UTF-8"?> -<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> - - <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-cd</artifactId> - <name>Hosted Vespa tenant CD</name> - <description>Test library for hosted Vespa applications.</description> + <artifactId>cloud-tenant-cd</artifactId> + <name>Vespa Cloud tenant CD implementation</name> + <description>Test library implementation for Vespa Cloud applications.</description> <url>https://github.com/vespa-engine</url> <packaging>jar</packaging> @@ -22,6 +20,11 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-cd-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>security-utils</artifactId> <version>${project.version}</version> </dependency> @@ -49,11 +52,5 @@ <artifactId>hosted-api</artifactId> <version>${project.version}</version> </dependency> - - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-engine</artifactId> - </dependency> </dependencies> - -</project> +</project>
\ No newline at end of file diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java index c479bab6e13..f77716064e5 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestRuntime.java +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/VespaTestRuntime.java @@ -1,8 +1,7 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; import ai.vespa.hosted.api.ControllerHttpClient; -import ai.vespa.hosted.api.EndpointAuthenticator; import ai.vespa.hosted.api.Properties; import ai.vespa.hosted.api.TestConfig; import ai.vespa.hosted.cd.http.HttpDeployment; @@ -14,46 +13,24 @@ import java.nio.file.Files; import java.nio.file.Paths; /** - * The place to obtain environment-dependent configuration for test of a Vespa deployment. - * - * @author jvenstad + * @author mortent */ -public class TestRuntime { - - private static TestRuntime theRuntime; - +public class VespaTestRuntime implements TestRuntime { private final TestConfig config; private final Deployment deploymentToTest; - private TestRuntime(TestConfig config, EndpointAuthenticator authenticator) { - this.config = config; - this.deploymentToTest = new HttpDeployment(config.deployments().get(config.zone()), authenticator); - } - - /** - * Returns the config and authenticator to use when running integration tests. - * - * If the system property {@code "vespa.test.config"} is set (to a file path), a file at that location - * is attempted read, and config parsed from it. - * Otherwise, config is fetched over HTTP from the hosted Vespa API, assuming the deployment indicated - * by the optional {@code "environment"} and {@code "region"} system properties exists. - * When environment is not specified, it defaults to {@link Environment#dev}, - * while region must be set unless the environment is {@link Environment#dev} or {@link Environment#perf}. - */ - public static synchronized TestRuntime get() { - if (theRuntime == null) { + public VespaTestRuntime() { String configPath = System.getProperty("vespa.test.config"); TestConfig config = configPath != null ? fromFile(configPath) : fromController(); - theRuntime = new TestRuntime(config, - new ai.vespa.hosted.auth.EndpointAuthenticator(config.system())); - } - return theRuntime; + this.config = config; + this.deploymentToTest = new HttpDeployment(config.deployments().get(config.zone()), new ai.vespa.hosted.auth.EndpointAuthenticator(config.system())); } - /** Returns a copy of this runtime, with the given endpoint authenticator. */ - public TestRuntime with(EndpointAuthenticator authenticator) { - return new TestRuntime(config, authenticator); - } +// In use ? +// /** Returns a copy of this runtime, with the given endpoint authenticator. */ +// public TestRuntime with(EndpointAuthenticator authenticator) { +// return new TestRuntime(config, authenticator); +// } /** Returns the full id of the application this is testing. */ public ApplicationId application() { return config.application(); } @@ -62,6 +39,7 @@ public class TestRuntime { public ZoneId zone() { return config.zone(); } /** Returns the deployment this is testing. */ + @Override public Deployment deploymentToTest() { return deploymentToTest; } private static TestConfig fromFile(String path) { @@ -78,8 +56,7 @@ public class TestRuntime { ApplicationId id = Properties.application(); Environment environment = Properties.environment().orElse(Environment.dev); ZoneId zone = Properties.region().map(region -> ZoneId.from(environment, region)) - .orElseGet(() -> controller.defaultZone(environment)); + .orElseGet(() -> controller.defaultZone(environment)); return controller.testConfig(id, zone); } - } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java index e17589c325d..80d5416ab34 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpDeployment.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.http; import ai.vespa.hosted.api.EndpointAuthenticator; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java index 74eb765dd0b..a803fc3e0e2 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java +++ b/cloud-tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java @@ -1,4 +1,4 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd.http; import ai.vespa.hosted.api.EndpointAuthenticator; diff --git a/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime b/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime new file mode 100644 index 00000000000..695fe363e4e --- /dev/null +++ b/cloud-tenant-cd/src/main/resources/META-INF/services/ai.vespa.hosted.cd.TestRuntime @@ -0,0 +1,2 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +ai.vespa.hosted.cd.VespaTestRuntime
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 0b42b3a481b..1d6e1a0893d 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -35,9 +35,6 @@ public class LogHandler extends ThreadedHttpRequestHandler { .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MAX); return new HttpResponse(200) { - { - headers().add("Content-Encoding", "gzip"); - } @Override public void render(OutputStream outputStream) { logReader.writeLogs(outputStream, from, to); diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 09330542bea..3cf849a6835 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -1,12 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; +import com.google.common.collect.Iterators; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.yolean.Exceptions; import java.io.BufferedReader; import java.io.BufferedWriter; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -23,15 +23,15 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; import java.util.List; import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; import static java.nio.charset.StandardCharsets.UTF_8; @@ -57,53 +57,105 @@ class LogReader { this.logFilePattern = logFilePattern; } - void writeLogs(OutputStream outputStream, Instant from, Instant to) { + void writeLogs(OutputStream out, Instant from, Instant to) { + double fromSeconds = from.getEpochSecond() + from.getNano() / 1e9; + double toSeconds = to.getEpochSecond() + to.getNano() / 1e9; + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out)); try { - List<List<Path>> logs = getMatchingFiles(from, to); - for (int i = 0; i < logs.size(); i++) { - for (Path log : logs.get(i)) { - boolean zipped = log.toString().endsWith(".gz"); - try (InputStream in = Files.newInputStream(log)) { - InputStream inProxy; - - // If the log needs filtering, possibly unzip (and rezip) it, and filter its lines on timestamp. - // When multiple log files exist for the same instant, their entries should ideally be sorted. This is not done here. - if (i == 0 || i == logs.size() - 1) { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(in) : in, UTF_8)); - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(zipped ? new GZIPOutputStream(buffer) : buffer, UTF_8))) { - for (String line; (line = reader.readLine()) != null; ) { - String[] parts = line.split("\t"); - if (parts.length != 7) - continue; - - Instant at = Instant.EPOCH.plus((long) (Double.parseDouble(parts[0]) * 1_000_000), ChronoUnit.MICROS); - if (at.isAfter(from) && !at.isAfter(to)) { - writer.write(line); - writer.newLine(); - } - } - } - inProxy = new ByteArrayInputStream(buffer.toByteArray()); - } - else - inProxy = in; - - if ( ! zipped) { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - try (OutputStream outProxy = new GZIPOutputStream(buffer)) { - inProxy.transferTo(outProxy); - } - inProxy = new ByteArrayInputStream(buffer.toByteArray()); - } - inProxy.transferTo(outputStream); + for (List<Path> logs : getMatchingFiles(from, to)) { + List<LogLineIterator> logLineIterators = new ArrayList<>(); + try { + // Logs in each sub-list contain entries covering the same time interval, so do a merge sort while reading + for (Path log : logs) + logLineIterators.add(new LogLineIterator(log, fromSeconds, toSeconds)); + + Iterator<LineWithTimestamp> lines = Iterators.mergeSorted(logLineIterators, + Comparator.comparingDouble(LineWithTimestamp::timestamp)); + while (lines.hasNext()) { + writer.write(lines.next().line()); + writer.newLine(); + } + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + finally { + for (LogLineIterator ll : logLineIterators) { + try { ll.close(); } catch (IOException ignored) { } } } } } - catch (IOException e) { - throw new UncheckedIOException(e); + finally { + Exceptions.uncheck(writer::flush); + } + } + + private static class LogLineIterator implements Iterator<LineWithTimestamp>, AutoCloseable { + + private final BufferedReader reader; + private final double from; + private final double to; + private LineWithTimestamp next; + + private LogLineIterator(Path log, double from, double to) throws IOException { + boolean zipped = log.toString().endsWith(".gz"); + InputStream in = Files.newInputStream(log); + this.reader = new BufferedReader(new InputStreamReader(zipped ? new GZIPInputStream(in) : in, UTF_8)); + this.from = from; + this.to = to; + this.next = readNext(); + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public LineWithTimestamp next() { + LineWithTimestamp current = next; + next = readNext(); + return current; + } + + @Override + public void close() throws IOException { + reader.close(); + } + + private LineWithTimestamp readNext() { + try { + for (String line; (line = reader.readLine()) != null; ) { + String[] parts = line.split("\t"); + if (parts.length != 7) + continue; + + double timestamp = Double.parseDouble(parts[0]); + if (timestamp > to) + return null; + + if (timestamp >= from) + return new LineWithTimestamp(line, timestamp); + } + return null; + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + } + + private static class LineWithTimestamp { + final String line; + final double timestamp; + LineWithTimestamp(String line, double timestamp) { + this.line = line; + this.timestamp = timestamp; } + String line() { return line; } + double timestamp() { return timestamp; } } /** Returns log files which may have relevant entries, grouped and sorted by {@link #extractTimestamp(Path)} — the first and last group must be filtered. */ diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index 01dcb885a97..ab0d0d54675 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -47,12 +47,12 @@ public class LogHandlerTest { } @Override - protected void writeLogs(OutputStream outputStream, Instant from, Instant to) { + protected void writeLogs(OutputStream out, Instant from, Instant to) { try { if (to.isAfter(Instant.ofEpochMilli(1000))) { - outputStream.write("newer log".getBytes()); + out.write("newer log".getBytes()); } else { - outputStream.write("older log".getBytes()); + out.write("older log".getBytes()); } } catch (Exception e) {} } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index 6b7be1d8cd3..3f7a78e13be 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -57,7 +57,7 @@ public class LogReaderTest { LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); logReader.writeLogs(baos, Instant.ofEpochMilli(150), Instant.ofEpochMilli(3601050)); - assertEquals(log100 + logv11 + log110, decompress(baos.toByteArray())); + assertEquals(log100 + logv11 + log110, baos.toString(UTF_8)); } @Test @@ -66,7 +66,7 @@ public class LogReaderTest { LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*-1.*")); logReader.writeLogs(baos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2))); - assertEquals(log101 + logv11, decompress(baos.toByteArray())); + assertEquals(log101 + logv11, baos.toString(UTF_8)); } @Test @@ -75,7 +75,7 @@ public class LogReaderTest { LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); logReader.writeLogs(zippedBaos, Instant.EPOCH, Instant.EPOCH.plus(Duration.ofDays(2))); - assertEquals(log100 + log101 + logv11 + log110 + log200 + logv, decompress(zippedBaos.toByteArray())); + assertEquals(log101 + log100 + logv11 + log110 + log200 + logv, zippedBaos.toString(UTF_8)); } private byte[] compress(String input) throws IOException { @@ -86,10 +86,4 @@ public class LogReaderTest { return baos.toByteArray(); } - private String decompress(byte[] input) throws IOException { - if (input.length == 0) return ""; - byte[] decompressed = new GZIPInputStream(new ByteArrayInputStream(input)).readAllBytes(); - return new String(decompressed); - } - } diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index be6a12526f5..1c4c9e9521b 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -446,7 +446,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.28.v20200408</jetty.version> + <jetty.version>9.4.30.v20200611</jetty.version> <org.lz4.version>1.7.1</org.lz4.version> <org.json.version>20090211</org.json.version> <slf4j.version>1.7.5</slf4j.version> diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index d9fcbaa3e2a..22073143ac0 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -176,7 +176,7 @@ DefaultTensorEngine::to_spec(const Value &value) const struct CallDenseTensorBuilder { template <typename CT> static Value::UP - call(const ValueType &type, const TensorSpec &spec) + invoke(const ValueType &type, const TensorSpec &spec) { TypedDenseTensorBuilder<CT> builder(type); for (const auto &cell: spec.cells()) { @@ -191,6 +191,8 @@ struct CallDenseTensorBuilder { } }; +using MyTypify = eval::TypifyCellType; + Value::UP DefaultTensorEngine::from_spec(const TensorSpec &spec) const { @@ -201,7 +203,7 @@ DefaultTensorEngine::from_spec(const TensorSpec &spec) const double value = spec.cells().empty() ? 0.0 : spec.cells().begin()->second.value; return std::make_unique<DoubleValue>(value); } else if (type.is_dense()) { - return dispatch_0<CallDenseTensorBuilder>(type.cell_type(), type, spec); + return typify_invoke<1,MyTypify,CallDenseTensorBuilder>(type.cell_type(), type, spec); } else if (type.is_sparse()) { DirectSparseTensorBuilder builder(type); SparseTensorAddressBuilder address_builder; @@ -449,7 +451,7 @@ const Value &concat_vectors(const Value &a, const Value &b, const vespalib::stri struct CallConcatVectors { template <typename OCT> - static const Value &call(const Value &a, const Value &b, const vespalib::string &dimension, size_t vector_size, Stash &stash) { + static const Value &invoke(const Value &a, const Value &b, const vespalib::string &dimension, size_t vector_size, Stash &stash) { return concat_vectors<OCT>(a, b, dimension, vector_size, stash); } }; @@ -461,7 +463,7 @@ DefaultTensorEngine::concat(const Value &a, const Value &b, const vespalib::stri size_t b_size = vector_size(b.type(), dimension); if ((a_size > 0) && (b_size > 0)) { CellType result_cell_type = ValueType::unify_cell_types(a.type(), b.type()); - return dispatch_0<CallConcatVectors>(result_cell_type, a, b, dimension, (a_size + b_size), stash); + return typify_invoke<1,MyTypify,CallConcatVectors>(result_cell_type, a, b, dimension, (a_size + b_size), stash); } return to_default(simple_engine().concat(to_simple(a, stash), to_simple(b, stash), dimension, stash), stash); } diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp index 493e4af3caf..4d6cfb1c9af 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp @@ -92,7 +92,7 @@ DenseBinaryFormat::serialize(nbostream &stream, const DenseTensorView &tensor) struct CallDecodeCells { template <typename CT> static std::unique_ptr<DenseTensorView> - call(nbostream &stream, size_t numCells, ValueType &&newType) { + invoke(nbostream &stream, size_t numCells, ValueType &&newType) { std::vector<CT> newCells; newCells.reserve(numCells); decodeCells<CT>(stream, numCells, newCells); @@ -106,7 +106,8 @@ DenseBinaryFormat::deserialize(nbostream &stream, CellType cell_type) std::vector<Dimension> dimensions; size_t numCells = decodeDimensions(stream, dimensions); ValueType newType = ValueType::tensor_type(std::move(dimensions), cell_type); - return dispatch_0<CallDecodeCells>(cell_type, stream, numCells, std::move(newType)); + using MyTypify = eval::TypifyCellType; + return typify_invoke<1,MyTypify,CallDecodeCells>(cell_type, stream, numCells, std::move(newType)); } template <typename T> diff --git a/hosted-tenant-base/OWNERS b/hosted-tenant-base/OWNERS new file mode 100644 index 00000000000..ff9741f2060 --- /dev/null +++ b/hosted-tenant-base/OWNERS @@ -0,0 +1,2 @@ +mortent +bjorncs
\ No newline at end of file diff --git a/hosted-tenant-base/README b/hosted-tenant-base/README new file mode 100644 index 00000000000..fed0672dcec --- /dev/null +++ b/hosted-tenant-base/README @@ -0,0 +1 @@ +Base pom for Vespa cloud application poms diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml new file mode 100644 index 00000000000..6fc65398f72 --- /dev/null +++ b/hosted-tenant-base/pom.xml @@ -0,0 +1,408 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <!-- <version>7-SNAPSHOT</version>--> + <!-- <url>https://github.com/vespa-engine</url>--> + <!-- <packaging>pom</packaging>--> + + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-tenant-base</artifactId> + <version>7-SNAPSHOT</version> + <name>Base pom for all tenant base poms</name> + <description>Parent POM for all Vespa base poms.</description> + <url>https://github.com/vespa-engine</url> + <packaging>pom</packaging> + + <licenses> + <license> + <name>The Apache License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + </license> + </licenses> + <developers> + <developer> + <name>Vespa</name> + <url>https://github.com/vespa-engine</url> + </developer> + </developers> + <scm> + <connection>scm:git:git@github.com:vespa-engine/vespa.git</connection> + <developerConnection>scm:git:git@github.com:vespa-engine/vespa.git</developerConnection> + <url>git@github.com:vespa-engine/vespa.git</url> + </scm> + + <properties> + <vespaversion>${project.version}</vespaversion> + <test-framework.version>${project.version}</test-framework.version> + <target_jdk_version>11</target_jdk_version> + <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version> + <maven-surefire-plugin.version>2.22.0</maven-surefire-plugin.version> + <junit.version>5.4.2</junit.version> + <endpoint>https://api.vespa-external.aws.oath.cloud:4443</endpoint> + <test.categories>!integration</test.categories> + </properties> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dependency-versions</artifactId> + <version>${vespaversion}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <version>${junit.version}</version> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <version>${junit.version}</version> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container</artifactId> + <version>${vespaversion}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${vespaversion}</version> + <scope>runtime</scope> + <exclusions> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-cd</artifactId> + <version>${test-framework.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>cloud-tenant-cd</artifactId> + <version>${test-framework.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + + <profiles> + <profile> + <!-- Build *-fat-test.jar file that includes all non-test classes and resources + that are part of the class path during test and and test.jar that includes + all test classes and resources, and put it inside a zip: + 1. application classes and resources + 2. test classes and resources + 3. classes and resources in all dependencies of both (1) and (2) + 4. copy the fat-test-jar and test-jar to application-test/artifacts directory + 5. zip application-test --> + <id>fat-test-application</id> + <build> + <plugins> + <plugin> + <!-- dependencies, see (3) above --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <!-- JAR-like dependencies --> + <id>unpack-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <configuration> + <includeTypes>jar,test-jar</includeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + <!-- WARNING(2018-06-27): bcpkix-jdk15on-1.58.jar and + bcprov-jdk15on-1.58.jar are pulled in via + container-dev and both contains the same set of + bouncycastle signature files in META-INF: + BC1024KE.DSA, BC1024KE.SF, BC2048KE.DSA, and + BC2048KE.SF. By merging any of these two with any + other JAR file like we're doing here, the signatures + are wrong. Worse, what we're doing is WRONG but not + yet fatal. + + The symptom of this happening is that the tester fails + to load the SystemTest class(!?), and subsequently + tries to run all test-like files in the fat test JAR. + + The solution is to exclude such files. This happens + automatically with maven-assembly-plugin. --> + <excludes>META-INF/*.SF,META-INF/*.DSA</excludes> + </configuration> + </execution> + <execution> + <!-- non-JAR-like dependencies --> + <id>non-jar-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <excludeTypes>jar,test-jar</excludeTypes> + <outputDirectory>target/fat-test-classes</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>copy-resources</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>target/fat-test-classes</outputDirectory> + <resources> + <!-- application classes and resources, see 1. above --> + <resource> + <directory>target/classes</directory> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.1.0</version> + <executions> + <execution> + <id>fat-test-jar</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <classesDirectory>target/fat-test-classes</classesDirectory> + <classifier>fat-test</classifier> + </configuration> + </execution> + <execution> + <id>test-jar</id> + <phase>package</phase> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>attach-artifact</id> + <phase>package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <tasks> + <!-- copy fat test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-fat-test.jar" + todir="target/application-test/artifacts/" /> + + <!-- copy slim test-jar to application-test artifacts directory, see 4. above --> + <copy file="target/${project.artifactId}-tests.jar" + todir="target/application-test/artifacts/" /> + + <!-- zip application-test, see 5. above --> + <zip destfile="target/application-test.zip" + basedir="target/application-test/" /> + </tasks> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + + <profile> <!-- Alias vespaversion with a more descriptive vespa.compile.version --> + <id>set-vespa-compile-version</id> + <activation> + <property> + <name>vespa.compile.version</name> + </property> + </activation> + <properties> + <vespaversion>${vespa.compile.version}</vespaversion> + </properties> + </profile> + + <profile> <!-- Alias vespaVersion with a more descriptive vespa.runtime.version --> + <id>set-vespa-runtime-version</id> + <activation> + <property> + <name>vespa.runtime.version</name> + </property> + </activation> + <properties> + <vespaVersion>${vespa.runtime.version}</vespaVersion> + </properties> + </profile> + </profiles> + + <build> + <finalName>${project.artifactId}</finalName> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <groups>${test.categories}</groups> + <redirectTestOutputToFile>false</redirectTestOutputToFile> + <trimStackTrace>false</trimStackTrace> + <systemPropertyVariables> + <application>${application}</application> + <tenant>${tenant}</tenant> + <instance>${instance}</instance> + <environment>${environment}</environment> + <region>${region}</region> + <endpoint>${endpoint}</endpoint> + <apiKeyFile>${apiKeyFile}</apiKeyFile> + <apiCertificateFile>${apiCertificateFile}</apiCertificateFile> + <dataPlaneKeyFile>${dataPlaneKeyFile}</dataPlaneKeyFile> + <dataPlaneCertificateFile>${dataPlaneCertificateFile}</dataPlaneCertificateFile> + </systemPropertyVariables> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + </configuration> + </plugin> + </plugins> + </pluginManagement> + + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>3.0.0-M2</version> + <executions> + <execution> + <id>enforce-java</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireJavaVersion> + <version>[11, )</version> + </requireJavaVersion> + <requireMavenVersion> + <version>[3.5, )</version> + </requireMavenVersion> + </rules> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>${maven-compiler-plugin.version}</version> + <configuration> + <source>${target_jdk_version}</source> + <target>${target_jdk_version}</target> + <showWarnings>true</showWarnings> + <showDeprecation>true</showDeprecation> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-maven-plugin</artifactId> + <version>${vespaversion}</version> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-application-maven-plugin</artifactId> + <version>${vespaversion}</version> + <executions> + <execution> + <goals> + <goal>packageApplication</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${vespaversion}</version> + <extensions>true</extensions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + </plugin> + </plugins> + </build> +</project>
\ No newline at end of file @@ -31,6 +31,8 @@ <module>bundle-plugin-test</module> <module>chain</module> <module>client</module> + <module>cloud-tenant-base</module> + <module>cloud-tenant-cd</module> <module>clustercontroller-apps</module> <module>clustercontroller-apputil</module> <module>clustercontroller-core</module> @@ -81,6 +83,7 @@ <module>flags</module> <module>fsa</module> <module>hosted-api</module> + <module>hosted-tenant-base</module> <module>hosted-zone-api</module> <module>http-utils</module> <module>indexinglanguage</module> @@ -125,7 +128,7 @@ <module>storage</module> <module>tenant-auth</module> <module>tenant-base</module> - <module>tenant-cd</module> + <module>tenant-cd-api</module> <module>testutil</module> <module>vdslib</module> <module>vespaclient-core</module> diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index 55342f91e93..d8b63909142 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -45,7 +45,7 @@ convert_cells<double,double>(std::unique_ptr<DenseTensorView> &, vespalib::eval: struct ConvertCellsSelector { template <typename LCT, typename RCT> - static auto get_fun() { return convert_cells<LCT, RCT>; } + static auto invoke() { return convert_cells<LCT, RCT>; } }; } // namespace <unnamed> @@ -67,7 +67,8 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f { auto lct = _query_tensor->cellsRef().type; auto rct = _attr_tensor.getTensorType().cell_type(); - auto fixup_fun = vespalib::tensor::select_2<ConvertCellsSelector>(lct, rct); + using MyTypify = vespalib::eval::TypifyCellType; + auto fixup_fun = vespalib::typify_invoke<2,MyTypify,ConvertCellsSelector>(lct, rct); fixup_fun(_query_tensor, _attr_tensor.getTensorType()); _fallback_dist_fun = search::tensor::make_distance_function(_attr_tensor.getConfig().distance_metric(), rct); _dist_fun = _fallback_dist_fun.get(); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 979eedec58a..f8db11ae9d8 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -20,7 +20,6 @@ using vespalib::tensor::SparseTensor; using vespalib::tensor::Tensor; using vespalib::tensor::TypedDenseTensorBuilder; using vespalib::tensor::WrappedSimpleTensor; -using vespalib::tensor::dispatch_0; using search::StateExplorerUtils; namespace search::tensor { @@ -34,7 +33,7 @@ constexpr size_t DEAD_SLACK = 0x10000u; struct CallMakeEmptyTensor { template <typename CT> - static Tensor::UP call(const ValueType &type) { + static Tensor::UP invoke(const ValueType &type) { TypedDenseTensorBuilder<CT> builder(type); return builder.build(); } @@ -46,7 +45,8 @@ createEmptyTensor(const ValueType &type) if (type.is_sparse()) { return std::make_unique<SparseTensor>(type, SparseTensor::Cells()); } else if (type.is_dense()) { - return dispatch_0<CallMakeEmptyTensor>(type.cell_type(), type); + using MyTypify = vespalib::eval::TypifyCellType; + return vespalib::typify_invoke<1,MyTypify,CallMakeEmptyTensor>(type.cell_type(), type); } else { return std::make_unique<WrappedSimpleTensor>(std::make_unique<SimpleTensor>(type, SimpleTensor::Cells())); } diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index a3fac22a93d..767119d2a02 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -93,23 +93,29 @@ <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-cd</artifactId> + <artifactId>tenant-cd-api</artifactId> <version>${test-framework.version}</version> <scope>test</scope> - <exclusions> - <exclusion> - <groupId>net.java.dev.jna</groupId> - <artifactId>jna</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </exclusion> - <exclusion> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </exclusion> - </exclusions> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>cloud-tenant-cd</artifactId> + <version>${test-framework.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> </dependency> </dependencies> diff --git a/tenant-cd/OWNERS b/tenant-cd-api/OWNERS index d0a102ecbf4..d0a102ecbf4 100644 --- a/tenant-cd/OWNERS +++ b/tenant-cd-api/OWNERS diff --git a/tenant-cd/README b/tenant-cd-api/README index a3803b81d53..a3803b81d53 100644 --- a/tenant-cd/README +++ b/tenant-cd-api/README diff --git a/tenant-cd-api/pom.xml b/tenant-cd-api/pom.xml new file mode 100644 index 00000000000..b9b4d136317 --- /dev/null +++ b/tenant-cd-api/pom.xml @@ -0,0 +1,29 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-cd-api</artifactId> + <name>Hosted Vespa tenant CD API</name> + <description>Test API library for hosted Vespa applications.</description> + <url>https://github.com/vespa-engine</url> + <packaging>jar</packaging> + + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent</relativePath> + </parent> + + <dependencies> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + </dependency> + </dependencies> + +</project> diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Deployment.java index 7d7b2f74981..7d7b2f74981 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Deployment.java diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Endpoint.java index bd6f30767f2..afc6aa1b519 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/Endpoint.java @@ -1,19 +1,12 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; -import ai.vespa.hosted.api.EndpointAuthenticator; - import java.net.URI; -import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; -import java.util.stream.Collectors; - -import static java.net.URLEncoder.encode; -import static java.nio.charset.StandardCharsets.UTF_8; /** * An endpoint in a Vespa application {@link Deployment}, which allows document retrieval. @@ -25,7 +18,7 @@ public interface Endpoint { /** Returns the URI of the endpoint, with scheme, host and port. */ URI uri(); - /** Sends the given request with required authentication. See {@link EndpointAuthenticator#authenticated} and {@link HttpClient#send}. */ + /** Sends the given request with required authentication. */ <T> HttpResponse<T> send(HttpRequest.Builder request, HttpResponse.BodyHandler<T> handler); /** Sends the given request with required authentication. */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java index f9dc15df32e..f9dc15df32e 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/IntegrationTest.java diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java index 53e7311fefc..b9054689b00 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java @@ -2,7 +2,6 @@ package ai.vespa.hosted.cd; import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; import java.lang.annotation.Retention; import java.lang.annotation.Target; diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingSetup.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java index bef3eabcef6..bef3eabcef6 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingSetup.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java index 59360b2753c..59360b2753c 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java index f01f2ca6c90..f01f2ca6c90 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/TestRuntime.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/TestRuntime.java new file mode 100644 index 00000000000..feea05b7064 --- /dev/null +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/TestRuntime.java @@ -0,0 +1,19 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.hosted.cd; + +import java.util.ServiceLoader; + +/** + * The place to obtain environment-dependent configuration for test of a Vespa deployment. + * + * @author jvenstad + * @author mortent + */ +public interface TestRuntime { + static TestRuntime get() { + ServiceLoader<TestRuntime> serviceLoader = ServiceLoader.load(TestRuntime.class); + return serviceLoader.findFirst().orElseThrow(() -> new RuntimeException("No TestRuntime implementation found")); + } + + Deployment deploymentToTest(); +} |