diff options
169 files changed, 1450 insertions, 617 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 7bf698b7a98..09d060f5d74 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,7 @@ add_subdirectory(configgen) add_subdirectory(configserver) add_subdirectory(configserver-flags) add_subdirectory(configutil) +add_subdirectory(container-apache-http-client-bundle) add_subdirectory(container-core) add_subdirectory(container-disc) add_subdirectory(container-jersey2) diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 855b3afafaf..3c5f96a1fec 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -28,6 +28,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>container-dev</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -79,16 +85,6 @@ <scope>provided</scope> </dependency> - <!-- COMPILE --> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - <!-- TEST --> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java index 00d194a37a2..bb88bcf75d9 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java @@ -35,9 +35,9 @@ public class Quota { } public static Quota fromSlime(Inspector inspector) { - var clusterSize = SlimeUtils.optionalLong(inspector.field("clusterSize")); + var clusterSize = SlimeUtils.optionalInteger(inspector.field("clusterSize")); var budget = budgetFromSlime(inspector.field("budget")); - return new Quota(clusterSize.map(Long::intValue), budget, true); + return new Quota(clusterSize.stream().boxed().findFirst(), budget, true); } public Quota withBudget(BigDecimal budget) { diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 36db55c206c..91e9e069e17 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -376,8 +376,8 @@ Tuning = element tuning { } TuningIoOptionsLight = string "normal" | string "directio" -TuningIoOptionsFull = string "normal" | string "directio" | string "mmap" | string "mlock" | string "populate" -TuningIoOptionsSearch = string "mmap" | string "mlock" | string "populate" +TuningIoOptionsFull = string "normal" | string "directio" | string "mmap" | string "populate" +TuningIoOptionsSearch = string "mmap" | string "populate" TuningCompression = element compression { element type { string "none" | string "lz4" | string "zstd" }? & diff --git a/config-model/src/test/derived/rankexpression/rankexpression.sd b/config-model/src/test/derived/rankexpression/rankexpression.sd index 20f9c7a9160..e615a1a7671 100644 --- a/config-model/src/test/derived/rankexpression/rankexpression.sd +++ b/config-model/src/test/derived/rankexpression/rankexpression.sd @@ -115,8 +115,8 @@ search rankexpression { expression { exp(0) + mysum(attribute(foo), - "attribute( bar )", - "attribute( \"baz\" )") + "attribute( bar )", + "attribute( \"baz\" )") } rerank-count: 101 } diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml index a02915c49a0..124d72f092d 100644 --- a/config-proxy/pom.xml +++ b/config-proxy/pom.xml @@ -73,6 +73,12 @@ <artifactId>slf4j-api</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/config/pom.xml b/config/pom.xml index 6e4e26ed0f1..8355587c10b 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -15,6 +15,12 @@ <dependencies> <!-- provided scope --> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>provided</scope> diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java index 182c30a0ece..26c5889c579 100644 --- a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java @@ -36,7 +36,7 @@ import java.util.concurrent.ThreadLocalRandom; import static com.yahoo.vespa.config.ConfigKey.createFull; /** - * A config client for generating load against a config server or config proxy. + * A client for generating load (config requests) against a config server or config proxy. * <p> * Log messages from a run will have a # first in the line, the end result will not. * @@ -44,13 +44,30 @@ import static com.yahoo.vespa.config.ConfigKey.createFull; */ public class LoadTester { - private static boolean debug = false; private final Transport transport = new Transport("rpc-client"); protected Supervisor supervisor = new Supervisor(transport); private List<ConfigKey<?>> configs = new ArrayList<>(); private Map<ConfigDefinitionKey, Tuple2<String, String[]>> defs = new HashMap<>(); private final CompressionType compressionType = JRTConfigRequestFactory.getCompressionType(); + private final String host; + private final int port; + private final int iterations; + private final int threads; + private final String configFile; + private final String defPath; + private final boolean debug; + + LoadTester(String host, int port, int iterations, int threads, String configFile, String defPath, boolean debug) { + this.host = host; + this.port = port; + this.iterations = iterations; + this.threads = threads; + this.configFile = configFile; + this.defPath = defPath; + this.debug = debug; + } + /** * @param args command-line arguments */ @@ -61,27 +78,29 @@ public class LoadTester { parser.addRequiredBinarySwitch("-p", "port"); parser.addRequiredBinarySwitch("-i", "iterations per thread"); parser.addRequiredBinarySwitch("-t", "threads"); - parser.addLegalBinarySwitch("-l", "configs file, on form name,configid. (To get list: vespa-configproxy-cmd -m cache | cut -d ',' -f1-2)"); + parser.addLegalBinarySwitch("-l", "config file, on form name,configid. (To get list: vespa-configproxy-cmd -m cache | cut -d ',' -f1-2)"); parser.addLegalBinarySwitch("-dd", "dir with def files, must be of form name.def"); parser.parse(); String host = parser.getBinarySwitches().get("-c"); int port = Integer.parseInt(parser.getBinarySwitches().get("-p")); int iterations = Integer.parseInt(parser.getBinarySwitches().get("-i")); int threads = Integer.parseInt(parser.getBinarySwitches().get("-t")); - String configsList = parser.getBinarySwitches().get("-l"); + String configFile = parser.getBinarySwitches().get("-l"); String defPath = parser.getBinarySwitches().get("-dd"); - debug = parser.getUnarySwitches().contains("-d"); - new LoadTester().runLoad(host, port, iterations, threads, configsList, defPath); + boolean debug = parser.getUnarySwitches().contains("-d"); + new LoadTester(host, port, iterations, threads, configFile, defPath, debug) + .runLoad(); } - private void runLoad(String host, int port, int iterations, int threads, - String configsList, String defPath) throws IOException, InterruptedException { - configs = readConfigs(configsList); + private void runLoad() throws IOException, InterruptedException { + configs = readConfigs(configFile); defs = readDefs(defPath); + validateConfigs(configs, defs); + List<LoadThread> threadList = new ArrayList<>(); - long startInNanos = System.nanoTime(); Metrics m = new Metrics(); + long startInNanos = System.nanoTime(); for (int i = 0; i < threads; i++) { LoadThread lt = new LoadThread(iterations, host, port); threadList.add(lt); @@ -92,7 +111,9 @@ public class LoadTester { lt.join(); m.merge(lt.metrics); } - printOutput(startInNanos, threads, iterations, m); + float durationInSeconds = (float) (System.nanoTime() - startInNanos) / 1_000_000_000f; + + printResults(durationInSeconds, threads, iterations, m); } private Map<ConfigDefinitionKey, Tuple2<String, String[]>> readDefs(String defPath) throws IOException { @@ -122,11 +143,10 @@ public class LoadTester { return ret; } - private void printOutput(long startInNanos, long threads, long iterations, Metrics metrics) { - float durSec = (float) (System.nanoTime() - startInNanos) / 1_000_000_000f; + private void printResults(float durationInSeconds, long threads, long iterations, Metrics metrics) { StringBuilder sb = new StringBuilder(); sb.append("#reqs/sec #avglatency #minlatency #maxlatency #failedrequests\n"); - sb.append(((float) (iterations * threads)) / durSec).append(","); + sb.append(((float) (iterations * threads)) / durationInSeconds).append(","); sb.append((metrics.latencyInMillis / threads / iterations)).append(","); sb.append((metrics.minLatency)).append(","); sb.append((metrics.maxLatency)).append(","); @@ -151,6 +171,16 @@ public class LoadTester { return ret; } + private void validateConfigs(List<ConfigKey<?>> configs, Map<ConfigDefinitionKey, Tuple2<String, String[]>> defs) { + for (ConfigKey<?> configKey : configs) { + ConfigDefinitionKey dKey = new ConfigDefinitionKey(configKey); + Tuple2<String, String[]> defContent = defs.get(dKey); + if (defContent == null) + throw new IllegalArgumentException("No matching config definition for " + configKey + + ", known config definitions: " + defs.keySet()); + } + } + private static class Metrics { long latencyInMillis = 0; @@ -189,34 +219,28 @@ public class LoadTester { private class LoadThread extends Thread { private final int iterations; - private final String host; - private final int port; + private final Spec spec; private final Metrics metrics = new Metrics(); LoadThread(int iterations, String host, int port) { this.iterations = iterations; - this.host = host; - this.port = port; + this.spec = new Spec(host, port); } @Override public void run() { - Spec spec = new Spec(host, port); Target target = connect(spec); - + int numberOfConfigs = configs.size(); for (int i = 0; i < iterations; i++) { - ConfigKey<?> reqKey = configs.get(ThreadLocalRandom.current().nextInt(configs.size())); - ConfigDefinitionKey dKey = new ConfigDefinitionKey(reqKey); - Tuple2<String, String[]> defContent = defs.get(dKey); - if (defContent == null && defs.size() > 0) { // Only complain if we actually did run with a def dir - System.out.println("# No def found for " + dKey + ", not sending in request."); - } - ConfigKey<?> configKey = createFull(reqKey.getName(), reqKey.getConfigId(), reqKey.getNamespace(), defContent.first); - JRTClientConfigRequest request = createRequest(configKey, defContent.second); - if (debug) System.out.println("# Requesting: " + reqKey); + ConfigKey<?> reqKey = configs.get(ThreadLocalRandom.current().nextInt(numberOfConfigs)); + JRTClientConfigRequest request = createRequest(reqKey); + if (debug) + System.out.println("# Requesting: " + reqKey); + long start = System.nanoTime(); target.invokeSync(request.getRequest(), 10.0); long durationInMillis = (System.nanoTime() - start) / 1_000_000; + if (request.isError()) { target = handleError(request, spec, target); } else { @@ -225,11 +249,15 @@ public class LoadTester { } } - private JRTClientConfigRequest createRequest(ConfigKey<?> reqKey, String[] defContent) { - if (defContent == null) defContent = new String[0]; + private JRTClientConfigRequest createRequest(ConfigKey<?> reqKey) { + ConfigDefinitionKey dKey = new ConfigDefinitionKey(reqKey); + Tuple2<String, String[]> defContent = defs.get(dKey); + ConfigKey<?> fullKey = createFull(reqKey.getName(), reqKey.getConfigId(), reqKey.getNamespace(), defContent.first); + final long serverTimeout = 1000; - return JRTClientConfigRequestV3.createWithParams(reqKey, DefContent.fromList(Arrays.asList(defContent)), - ConfigUtils.getCanonicalHostName(), "", 0, serverTimeout, Trace.createDummy(), + return JRTClientConfigRequestV3.createWithParams(fullKey, DefContent.fromList(List.of(defContent.second)), + ConfigUtils.getCanonicalHostName(), "", + 0, serverTimeout, Trace.createDummy(), compressionType, Optional.empty()); } diff --git a/configserver-client/pom.xml b/configserver-client/pom.xml index 0a29ba003f4..39005c9ccab 100644 --- a/configserver-client/pom.xml +++ b/configserver-client/pom.xml @@ -30,6 +30,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>security-utils</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/configserver/pom.xml b/configserver/pom.xml index 3b7fef085b1..a237d7e00ad 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -142,6 +142,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <!-- To get all necessary test deps. --> <groupId>com.yahoo.vespa</groupId> <artifactId>container-test</artifactId> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 41d050025bf..9778b1fc1f2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -175,7 +175,11 @@ public class SessionRepository { private void loadSessions(BooleanFlag loadLocalSessions) { ExecutorService executor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("load-sessions-")); - if (loadLocalSessions.value()) + loadSessions(loadLocalSessions.value(), executor); + } + + void loadSessions(boolean loadLocalSessions, ExecutorService executor) { + if (loadLocalSessions) loadLocalSessions(executor); loadRemoteSessions(executor); try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index 9f1bfa0b4e4..fa562c4813b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.zookeeper; import com.google.common.base.Joiner; @@ -138,11 +138,6 @@ public class ZKApplicationPackage implements ApplicationPackage { if (sd.endsWith(SD_NAME_SUFFIX)) schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd)))); } - // TODO: Remove when 7.414.19 is oldest version in use - for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) { - if (sd.endsWith(SD_NAME_SUFFIX)) - schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd)))); - } return schemas; } @@ -257,9 +252,7 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public Reader getRankingExpression(String name) { - Optional<Reader> reader = zkApplication.getOptionalDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name); - // TODO: Remove when 7.414.19 is oldest version in use - return reader.orElseGet(() -> zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, name)); + return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java index 31d29e7b78e..4790d8f4ae2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; +import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; @@ -37,6 +38,7 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import java.io.File; @@ -79,6 +81,9 @@ public class SessionRepositoryTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule + public ExpectedException expectedException = ExpectedException.none(); + public void setup() throws Exception { setup(new ModelFactoryRegistry(List.of(new VespaModelFactory(new NullConfigModelRegistry())))); } @@ -172,21 +177,25 @@ public class SessionRepositoryTest { assertStatusChange(sessionId, Session.Status.ACTIVATE); } - // If reading a session throws an exception it should be handled and not prevent other applications - // from loading. In this test we just show that we end up with one session in remote session - // repo even if it had bad data (by making getSessionIdForApplication() in FailingTenantApplications - // throw an exception). + // If reading a session throws an exception when bootstrapping SessionRepository it should fail, + // to make sure config server does not comes up and serves invalid/old config or, if this is hosted, + // serves empty config (takes down services on all nodes belonging to an application) @Test - public void testBadApplicationRepoOnActivate() throws Exception { + public void testInvalidSessionWhenBootstrappingSessionRepo() throws Exception { setup(); - long sessionId = 3L; - TenantName mytenant = TenantName.from("mytenant"); - curator.set(TenantRepository.getApplicationsPath(mytenant).append("mytenant:appX:default"), new byte[0]); // Invalid data - tenantRepository.addTenant(mytenant); - curator.create(TenantRepository.getSessionsPath(mytenant)); + + // Create a session with invalid data and set active session for application to this session + String sessionIdString = "3"; + Path sessionPath = TenantRepository.getSessionsPath(tenantName).append(sessionIdString); + curator.create(sessionPath); + curator.set(sessionPath.append("applicationId"), new byte[0]); // Invalid data + Path applicationsPath = TenantRepository.getApplicationsPath(tenantName); + curator.set(applicationsPath.append(applicationId.serializedForm()), Utf8.toBytes(sessionIdString)); + + expectedException.expectMessage("Could not load remote session " + sessionIdString); + expectedException.expect(RuntimeException.class); + sessionRepository.loadSessions(false, new InThreadExecutorService()); assertThat(sessionRepository.getRemoteSessionsFromZooKeeper().size(), is(0)); - createSession(sessionId, true); - assertThat(sessionRepository.getRemoteSessionsFromZooKeeper().size(), is(1)); } @Test(expected = InvalidApplicationException.class) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 6f7e0541cc7..464b3d1ab64 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -191,7 +191,6 @@ public class TenantRepositoryTest { public void testFailingBootstrap() { tenantRepository.close(); // stop using the one setup in Before method - // Should get exception if config is true expectedException.expect(RuntimeException.class); expectedException.expectMessage("Could not create all tenants when bootstrapping, failed to create: [default]"); new FailingDuringBootstrapTenantRepository(configserverConfig); @@ -213,7 +212,7 @@ public class TenantRepositoryTest { Metrics.createTestMetrics(), new StripedExecutor<>(new InThreadExecutorService()), new StripedExecutor<>(new InThreadExecutorService()), - new FileDistributionFactory(new ConfigserverConfig.Builder().build()), + new FileDistributionFactory(configserverConfig), new InMemoryFlagSource(), new InThreadExecutorService(), new MockSecretStore(), diff --git a/container-apache-http-client-bundle/CMakeLists.txt b/container-apache-http-client-bundle/CMakeLists.txt new file mode 100644 index 00000000000..8daac7a5030 --- /dev/null +++ b/container-apache-http-client-bundle/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(container-apache-http-client-bundle) diff --git a/container-apache-http-client-bundle/README.md b/container-apache-http-client-bundle/README.md new file mode 100644 index 00000000000..99606a95820 --- /dev/null +++ b/container-apache-http-client-bundle/README.md @@ -0,0 +1,3 @@ +# container-apache-http-client-bundle + +Apache http client 4.x/5.x packaged as bundle diff --git a/container-apache-http-client-bundle/pom.xml b/container-apache-http-client-bundle/pom.xml new file mode 100644 index 00000000000..5d7598d6f45 --- /dev/null +++ b/container-apache-http-client-bundle/pom.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <artifactId>container-apache-http-client-bundle</artifactId> + <packaging>container-plugin</packaging> + <version>7-SNAPSHOT</version> + + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <properties> + <maven.javadoc.skip>true</maven.javadoc.skip> <!-- Javadoc plugin fails because of no source code in module --> + </properties> + <dependencies> + <!-- provided --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <!-- Not directly used in this module, but needed to get Import-Packages for JDK packages it exports. --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <scope>provided</scope> + </dependency> + + <!-- compile --> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpmime</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents.client5</groupId> + <artifactId>httpclient5</artifactId> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + </plugins> + </build> + +</project> diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/async/methods/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/async/methods/package-info.java new file mode 100644 index 00000000000..050fc29c7dd --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/async/methods/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.async.methods; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/async/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/async/package-info.java new file mode 100644 index 00000000000..f1387bb1464 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/async/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.async; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/classic/methods/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/classic/methods/package-info.java new file mode 100644 index 00000000000..b014ce24aff --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/classic/methods/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.classic.methods; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/classic/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/classic/package-info.java new file mode 100644 index 00000000000..90b162d3eec --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/classic/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.classic; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/config/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/config/package-info.java new file mode 100644 index 00000000000..f1a450fbaf0 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/config/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/entity/mime/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/entity/mime/package-info.java new file mode 100644 index 00000000000..489d168e19b --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/entity/mime/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.entity.mime; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/entity/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/entity/package-info.java new file mode 100644 index 00000000000..f4542fa34dc --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/entity/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.entity; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/async/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/async/package-info.java new file mode 100644 index 00000000000..e9acf08ff88 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/async/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.impl.async; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/classic/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/classic/package-info.java new file mode 100644 index 00000000000..aeb608544ab --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/classic/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.impl.classic; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/io/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/io/package-info.java new file mode 100644 index 00000000000..1106f18631a --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/io/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.impl.io; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/nio/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/nio/package-info.java new file mode 100644 index 00000000000..64636ffec3d --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/nio/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.impl.nio; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/package-info.java new file mode 100644 index 00000000000..5f0ae9df63b --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/impl/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.impl; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/io/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/io/package-info.java new file mode 100644 index 00000000000..386f6399057 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/io/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.io; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/nio/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/nio/package-info.java new file mode 100644 index 00000000000..41b75048e97 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/nio/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.nio; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/package-info.java new file mode 100644 index 00000000000..ec4bab39b4d --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/protocol/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/protocol/package-info.java new file mode 100644 index 00000000000..f72c2742b24 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/protocol/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.protocol; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/routing/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/routing/package-info.java new file mode 100644 index 00000000000..412d824c38c --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/routing/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.routing; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/socket/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/socket/package-info.java new file mode 100644 index 00000000000..ed95115cd6d --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/socket/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.socket; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/ssl/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/ssl/package-info.java new file mode 100644 index 00000000000..c7184a484f7 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/client5/http/ssl/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.client5.http.ssl; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/concurrent/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/concurrent/package-info.java new file mode 100644 index 00000000000..a61bb459a68 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/concurrent/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.concurrent; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/config/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/config/package-info.java new file mode 100644 index 00000000000..f5a797131d2 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/config/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/io/entity/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/io/entity/package-info.java new file mode 100644 index 00000000000..576666f8c08 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/io/entity/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http.io.entity; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/io/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/io/package-info.java new file mode 100644 index 00000000000..c7b0debd608 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/io/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http.io; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/io/support/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/io/support/package-info.java new file mode 100644 index 00000000000..71cb8438727 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/io/support/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http.io.support; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/message/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/message/package-info.java new file mode 100644 index 00000000000..27cf37f846f --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/message/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http.message; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/package-info.java new file mode 100644 index 00000000000..1b949e0a38a --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http.nio; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/ssl/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/ssl/package-info.java new file mode 100644 index 00000000000..ad657066eb7 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/ssl/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http.nio.ssl; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/support/classic/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/support/classic/package-info.java new file mode 100644 index 00000000000..98836b631df --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/support/classic/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http.nio.support.classic; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/support/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/support/package-info.java new file mode 100644 index 00000000000..3e089b8ca23 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/nio/support/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http.nio.support; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/package-info.java new file mode 100644 index 00000000000..30fac3d2de9 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/protocol/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/protocol/package-info.java new file mode 100644 index 00000000000..c5b27fbb871 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http/protocol/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http.protocol; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http2/config/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http2/config/package-info.java new file mode 100644 index 00000000000..d5482b5f2a2 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http2/config/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http2.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http2/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http2/package-info.java new file mode 100644 index 00000000000..3c2d9389c74 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/http2/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.http2; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/net/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/net/package-info.java new file mode 100644 index 00000000000..0009845386c --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/net/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.net; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/reactor/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/reactor/package-info.java new file mode 100644 index 00000000000..e05e877f730 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/reactor/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.reactor; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/util/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/util/package-info.java new file mode 100644 index 00000000000..75e38c99e65 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/hc/core5/util/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.hc.core5.util; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/annotation/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/annotation/package-info.java new file mode 100644 index 00000000000..1599489de0f --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/annotation/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.annotation; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/auth/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/auth/package-info.java new file mode 100644 index 00000000000..2bb0a297c86 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/auth/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.auth; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/client/config/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/config/package-info.java new file mode 100644 index 00000000000..33b3ec7cae9 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/config/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.client.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/client/methods/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/methods/package-info.java new file mode 100644 index 00000000000..3b8ad361db5 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/methods/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.client.methods; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/client/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/package-info.java new file mode 100644 index 00000000000..6a62880648e --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.client; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/client/protocol/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/protocol/package-info.java new file mode 100644 index 00000000000..3d8ffbbcd6f --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/protocol/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.client.protocol; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/client/utils/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/utils/package-info.java new file mode 100644 index 00000000000..f7d6fac2b0b --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/utils/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.client.utils; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/config/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/config/package-info.java new file mode 100644 index 00000000000..7fba26f38a7 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/config/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/package-info.java new file mode 100644 index 00000000000..5e64cb7eab2 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.conn; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/routing/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/routing/package-info.java new file mode 100644 index 00000000000..204dc62666b --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/routing/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.conn.routing; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/socket/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/socket/package-info.java new file mode 100644 index 00000000000..7ac337e6ca6 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/socket/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.conn.socket; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/ssl/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/ssl/package-info.java new file mode 100644 index 00000000000..23d1644ba84 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/conn/ssl/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.conn.ssl; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/entity/mime/content/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/entity/mime/content/package-info.java new file mode 100644 index 00000000000..6e31d8e733a --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/entity/mime/content/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.entity.mime.content; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/entity/mime/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/entity/mime/package-info.java new file mode 100644 index 00000000000..9a41d01fc4d --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/entity/mime/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.entity.mime; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/entity/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/entity/package-info.java new file mode 100644 index 00000000000..e5d7bce4f01 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/entity/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.entity; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/impl/client/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/impl/client/package-info.java new file mode 100644 index 00000000000..5ac4dc40fac --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/impl/client/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.impl.client; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/impl/conn/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/impl/conn/package-info.java new file mode 100644 index 00000000000..b3a27272300 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/impl/conn/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.impl.conn; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/impl/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/impl/package-info.java new file mode 100644 index 00000000000..ebc2aebb981 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/impl/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.impl; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/message/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/message/package-info.java new file mode 100644 index 00000000000..5ede836011d --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/message/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.message; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/package-info.java new file mode 100644 index 00000000000..dbf7b50caa9 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/params/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/params/package-info.java new file mode 100644 index 00000000000..92123f26b4d --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/params/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.params; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/pool/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/pool/package-info.java new file mode 100644 index 00000000000..3c2509cd960 --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/pool/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.pool; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/protocol/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/protocol/package-info.java new file mode 100644 index 00000000000..1985b5b056a --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/protocol/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.protocol; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/util/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/util/package-info.java new file mode 100644 index 00000000000..90e4f8e1cff --- /dev/null +++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/util/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package org.apache.http.util; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/pom.xml b/container-core/pom.xml index 2b87d79daa4..c7fe2998530 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -140,10 +140,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>org.hdrhistogram</groupId> <artifactId>HdrHistogram</artifactId> </dependency> @@ -211,6 +207,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>defaults</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -262,11 +264,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpmime</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.assertj</groupId> <artifactId>assertj-core</artifactId> <scope>test</scope> @@ -302,17 +299,6 @@ <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.httpcomponents.client5</groupId> - <artifactId>httpclient5</artifactId> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - </exclusions> - </dependency> </dependencies> <build> <plugins> diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java index 71d0e539b5a..179abba42c8 100644 --- a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java +++ b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java @@ -258,7 +258,7 @@ public class ComponentGraph { if (component.isEmpty()) { Object instance; try { - log.log(Level.INFO, "Trying the fallback injector to create" + messageForNoGlobalComponent(clazz, node)); + log.log(Level.FINE, () -> "Trying the fallback injector to create" + messageForNoGlobalComponent(clazz, node)); instance = fallbackInjector.getInstance(key); } catch (ConfigurationException e) { throw removeStackTrace(new IllegalStateException( diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java index 2183098da2b..4c45319daf6 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java @@ -12,12 +12,11 @@ import com.yahoo.jdisc.http.server.jetty.testutils.ConnectorFactoryRegistryModul import com.yahoo.jdisc.test.ServerProviderConformanceTest; import org.apache.http.HttpResponse; import org.apache.http.HttpVersion; -import org.apache.http.ProtocolVersion; -import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; import org.hamcrest.Description; @@ -27,6 +26,7 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -34,6 +34,7 @@ import java.util.Collections; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; @@ -64,6 +65,8 @@ public class HttpServerConformanceTest extends ServerProviderConformanceTest { @SuppressWarnings("LoggerInitializedWithForeignClass") private static Logger httpRequestDispatchLogger = Logger.getLogger(HttpRequestDispatch.class.getName()); private static Level httpRequestDispatchLoggerOriginalLevel; + private static CloseableHttpClient httpClient; + private static ExecutorService executorService; /* * Reduce logging of every stack trace for {@link ServerProviderConformanceTest.ConformanceException} thrown. @@ -73,11 +76,16 @@ public class HttpServerConformanceTest extends ServerProviderConformanceTest { public static void reduceExcessiveLogging() { httpRequestDispatchLoggerOriginalLevel = httpRequestDispatchLogger.getLevel(); httpRequestDispatchLogger.setLevel(Level.SEVERE); + httpClient = HttpClientBuilder.create().build(); + executorService = Executors.newSingleThreadExecutor(); } @AfterClass - public static void restoreExcessiveLogging() { + public static void restoreExcessiveLogging() throws IOException, InterruptedException { httpRequestDispatchLogger.setLevel(httpRequestDispatchLoggerOriginalLevel); + httpClient.close(); + executorService.shutdownNow(); + executorService.awaitTermination(30, TimeUnit.SECONDS); } @AfterClass @@ -742,20 +750,12 @@ public class HttpServerConformanceTest extends ServerProviderConformanceTest { } } - private class TestRunner implements Adapter<JettyHttpServer, ClientProxy, Future<HttpResponse>> { + private class TestRunner implements Adapter<JettyHttpServer, Integer, Future<HttpResponse>> { private Matcher<ResponseGist> expectedResponse = null; - HttpVersion requestVersion; - private final ExecutorService executorService = Executors.newSingleThreadExecutor(); void execute() throws Throwable { - requestVersion = HttpVersion.HTTP_1_0; runTest(this); - - requestVersion = HttpVersion.HTTP_1_1; - runTest(this); - - executorService.shutdown(); } TestRunner expect(final Matcher<ResponseGist> matcher) { @@ -790,30 +790,27 @@ public class HttpServerConformanceTest extends ServerProviderConformanceTest { } @Override - public ClientProxy newClient(final JettyHttpServer server) throws Throwable { - return new ClientProxy(server.getListenPort(), requestVersion); + public Integer newClient(final JettyHttpServer server) throws Throwable { + return server.getListenPort(); } @Override public Future<HttpResponse> executeRequest( - final ClientProxy client, + final Integer listenPort, final boolean withRequestContent) throws Throwable { final HttpUriRequest request; - final URI requestUri = URI.create("http://localhost:" + client.listenPort + "/status.html"); + final URI requestUri = URI.create("http://localhost:" + listenPort + "/status.html"); if (!withRequestContent) { HttpGet httpGet = new HttpGet(requestUri); - httpGet.setProtocolVersion(client.requestVersion); + httpGet.setProtocolVersion(HttpVersion.HTTP_1_1); request = httpGet; } else { final HttpPost post = new HttpPost(requestUri); post.setEntity(new StringEntity(REQUEST_CONTENT, StandardCharsets.UTF_8)); - post.setProtocolVersion(client.requestVersion); + post.setProtocolVersion(HttpVersion.HTTP_1_1); request = post; } - log.fine(() -> "executorService:" - + " .isShutDown()=" + executorService.isShutdown() - + " .isTerminated()=" + executorService.isTerminated()); - return executorService.submit(() -> client.delegate.execute(request)); + return executorService.submit(() -> httpClient.execute(request)); } @Override @@ -831,17 +828,4 @@ public class HttpServerConformanceTest extends ServerProviderConformanceTest { assertThat(responseGist, expectedResponse); } } - - private static class ClientProxy { - - final HttpClient delegate; - final int listenPort; - final ProtocolVersion requestVersion; - - ClientProxy(final int listenPort, final HttpVersion requestVersion) { - this.delegate = HttpClientBuilder.create().build(); - this.requestVersion = requestVersion; - this.listenPort = listenPort; - } - } } diff --git a/container-disc/pom.xml b/container-disc/pom.xml index ae2e460094b..5446c9e1698 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -59,6 +59,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>container-core</artifactId> <version>${project.version}</version> <exclusions> @@ -178,6 +184,8 @@ defaults-jar-with-dependencies.jar, zkfacade-jar-with-dependencies.jar, zookeeper-server-jar-with-dependencies.jar, + <!-- Apache http client repackaged as bundle --> + container-apache-http-client-bundle-jar-with-dependencies.jar, <!-- Jetty --> alpn-api-${jetty-alpn.version}.jar, http2-server-${jetty.version}.jar, diff --git a/controller-api/pom.xml b/controller-api/pom.xml index 2f01f45edaa..02a7028b8ca 100644 --- a/controller-api/pom.xml +++ b/controller-api/pom.xml @@ -27,6 +27,13 @@ <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>serviceview</artifactId> <scope>provided</scope> <version>${project.version}</version> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index ed84a9b0a76..396be0adf92 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -77,7 +77,7 @@ public class ZmsClientMock implements ZmsClient { } @Override - public void addRoleMember(AthenzRole role, AthenzIdentity member) { + public void addRoleMember(AthenzRole role, AthenzIdentity member, Optional<String> reason) { if ( ! role.roleName().equals("tenancy.vespa.hosting.admin")) throw new IllegalArgumentException("Mock only supports adding tenant admins, not " + role.roleName()); getDomainOrThrow(role.domain(), true).tenantAdmin(member); diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 0c05f7d70bb..3673e3cb9f9 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -20,6 +20,13 @@ <!-- provided --> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>controller-api</artifactId> <version>${project.version}</version> @@ -128,16 +135,6 @@ </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - - <dependency> <groupId>com.auth0</groupId> <artifactId>java-jwt</artifactId> <exclusions> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 073451f6309..cb3e5ea97c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -215,7 +215,7 @@ public class AthenzFacade implements AccessControl { } public void addTenantAdmin(AthenzDomain tenantDomain, AthenzUser user) { - zmsClient.addRoleMember(new AthenzRole(tenantDomain, "tenancy." + service.getFullName() + ".admin"), user); + zmsClient.addRoleMember(new AthenzRole(tenantDomain, "tenancy." + service.getFullName() + ".admin"), user, Optional.empty()); } private void deleteApplication(AthenzDomain domain, ApplicationName application, OktaIdentityToken identityToken, OktaAccessToken accessToken) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java index e40d772a673..5f147d6e048 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java @@ -22,7 +22,7 @@ public class SystemRoutingPolicyMaintainer extends ControllerMaintainer { @Override protected double maintain() { - for (var zone : controller().zoneRegistry().zones().all().ids()) { + for (var zone : controller().zoneRegistry().zones().reachable().ids()) { for (var application : SystemApplication.values()) { if (!application.hasEndpoint()) continue; controller().routing().policies().refresh(application.id(), DeploymentSpec.empty, zone); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java index 7c2d990750c..0cf1aebeba9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.notification; import com.yahoo.collections.Pair; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; @@ -17,7 +16,6 @@ import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -36,13 +34,6 @@ public class NotificationsDb { public NotificationsDb(Controller controller) { this(controller.clock(), controller.curator()); - - Set<DeploymentId> allDeployments = controller.applications().asList().stream() - .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> instance.deployments().keySet().stream() - .map(zone -> new DeploymentId(instance.id(), zone))) - .collect(Collectors.toSet()); - removeNotificationsForRemovedInstances(allDeployments); } NotificationsDb(Clock clock, CuratorDb curatorDb) { @@ -50,26 +41,6 @@ public class NotificationsDb { this.curatorDb = curatorDb; } - // TODO (freva): Remove after 7.423 - void removeNotificationsForRemovedInstances(Set<DeploymentId> allDeployments) { - // Prior to 7.423, notifications created for instances that were later removed by being removed from - // deployment.xml were not cleared. This should only affect notifications with type 'deployment' - allDeployments.stream() - .map(deploymentId -> deploymentId.applicationId().tenant()) - .distinct() - .flatMap(tenant -> curatorDb.readNotifications(tenant).stream() - .filter(notification -> notification.type() == Type.deployment && notification.source().zoneId().isPresent()) - .map(Notification::source)) - .filter(source -> { - ApplicationId sourceApplication = ApplicationId.from(source.tenant(), - source.application().get(), - source.instance().get()); - DeploymentId sourceDeployment = new DeploymentId(sourceApplication, source.zoneId().get()); - return ! allDeployments.contains(sourceDeployment); - }) - .forEach(source -> removeNotification(source, Type.deployment)); - } - public List<Notification> listNotifications(NotificationSource source, boolean productionOnly) { return curatorDb.readNotifications(source.tenant()).stream() .filter(notification -> source.contains(notification.source()) && (!productionOnly || notification.source().isProduction())) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 06442779b9c..26fb4be04af 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -295,18 +295,18 @@ public class ApplicationSerializer { Inspector root = slime.get(); TenantAndApplicationId id = TenantAndApplicationId.fromSerialized(root.field(idField).asString()); - Instant createdAt = Serializers.instant(root.field(createdAtField)); + Instant createdAt = SlimeUtils.instant(root.field(createdAtField)); DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(root.field(deploymentSpecField).asString(), false); ValidationOverrides validationOverrides = ValidationOverrides.fromXml(root.field(validationOverridesField).asString()); - Optional<IssueId> deploymentIssueId = Serializers.optionalString(root.field(deploymentIssueField)).map(IssueId::from); - Optional<IssueId> ownershipIssueId = Serializers.optionalString(root.field(ownershipIssueIdField)).map(IssueId::from); - Optional<User> owner = Serializers.optionalString(root.field(ownerField)).map(User::from); - OptionalInt majorVersion = Serializers.optionalInteger(root.field(majorVersionField)); + Optional<IssueId> deploymentIssueId = SlimeUtils.optionalString(root.field(deploymentIssueField)).map(IssueId::from); + Optional<IssueId> ownershipIssueId = SlimeUtils.optionalString(root.field(ownershipIssueIdField)).map(IssueId::from); + Optional<User> owner = SlimeUtils.optionalString(root.field(ownerField)).map(User::from); + OptionalInt majorVersion = SlimeUtils.optionalInteger(root.field(majorVersionField)); ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), root.field(writeQualityField).asDouble()); Set<PublicKey> deployKeys = deployKeysFromSlime(root.field(pemDeployKeysField)); List<Instance> instances = instancesFromSlime(id, root.field(instancesField)); - OptionalLong projectId = Serializers.optionalLong(root.field(projectIdField)); + OptionalLong projectId = SlimeUtils.optionalLong(root.field(projectIdField)); Optional<ApplicationVersion> latestVersion = latestVersionFromSlime(root.field(latestVersionField)); return new Application(id, createdAt, deploymentSpec, validationOverrides, @@ -354,18 +354,18 @@ public class ApplicationSerializer { return new Deployment(zoneIdFromSlime(deploymentObject.field(zoneField)), applicationVersionFromSlime(deploymentObject.field(applicationPackageRevisionField)), Version.fromString(deploymentObject.field(versionField).asString()), - Serializers.instant(deploymentObject.field(deployTimeField)), + SlimeUtils.instant(deploymentObject.field(deployTimeField)), deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)), - DeploymentActivity.create(Serializers.optionalInstant(deploymentObject.field(lastQueriedField)), - Serializers.optionalInstant(deploymentObject.field(lastWrittenField)), - Serializers.optionalDouble(deploymentObject.field(lastQueriesPerSecondField)), - Serializers.optionalDouble(deploymentObject.field(lastWritesPerSecondField))), - QuotaUsage.create(Serializers.optionalDouble(deploymentObject.field(quotaUsageRateField))), - Serializers.optionalDouble(deploymentObject.field(deploymentCostField))); + DeploymentActivity.create(SlimeUtils.optionalInstant(deploymentObject.field(lastQueriedField)), + SlimeUtils.optionalInstant(deploymentObject.field(lastWrittenField)), + SlimeUtils.optionalDouble(deploymentObject.field(lastQueriesPerSecondField)), + SlimeUtils.optionalDouble(deploymentObject.field(lastWritesPerSecondField))), + QuotaUsage.create(SlimeUtils.optionalDouble(deploymentObject.field(quotaUsageRateField))), + SlimeUtils.optionalDouble(deploymentObject.field(deploymentCostField))); } private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) { - Optional<Instant> instant = Serializers.optionalInstant(object.field(deploymentMetricsUpdateTime)); + Optional<Instant> instant = SlimeUtils.optionalInstant(object.field(deploymentMetricsUpdateTime)); return new DeploymentMetrics(object.field(deploymentMetricsQPSField).asDouble(), object.field(deploymentMetricsWPSField).asDouble(), object.field(deploymentMetricsDocsField).asDouble(), @@ -388,7 +388,7 @@ public class ApplicationSerializer { object.traverse((ArrayTraverser) (idx, statusObject) -> statusMap.put(new RotationId(statusObject.field(rotationIdField).asString()), new RotationStatus.Targets( singleRotationStatusFromSlime(statusObject.field(statusField)), - Serializers.instant(statusObject.field(lastUpdatedField))))); + SlimeUtils.instant(statusObject.field(lastUpdatedField))))); return RotationStatus.from(statusMap); } @@ -411,16 +411,16 @@ public class ApplicationSerializer { private ApplicationVersion applicationVersionFromSlime(Inspector object) { if ( ! object.valid()) return ApplicationVersion.unknown; - OptionalLong applicationBuildNumber = Serializers.optionalLong(object.field(applicationBuildNumberField)); + OptionalLong applicationBuildNumber = SlimeUtils.optionalLong(object.field(applicationBuildNumberField)); if (applicationBuildNumber.isEmpty()) return ApplicationVersion.unknown; Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); - Optional<String> authorEmail = Serializers.optionalString(object.field(authorEmailField)); - Optional<Version> compileVersion = Serializers.optionalString(object.field(compileVersionField)).map(Version::fromString); - Optional<Instant> buildTime = Serializers.optionalInstant(object.field(buildTimeField)); - Optional<String> sourceUrl = Serializers.optionalString(object.field(sourceUrlField)); - Optional<String> commit = Serializers.optionalString(object.field(commitField)); + Optional<String> authorEmail = SlimeUtils.optionalString(object.field(authorEmailField)); + Optional<Version> compileVersion = SlimeUtils.optionalString(object.field(compileVersionField)).map(Version::fromString); + Optional<Instant> buildTime = SlimeUtils.optionalInstant(object.field(buildTimeField)); + Optional<String> sourceUrl = SlimeUtils.optionalString(object.field(sourceUrlField)); + Optional<String> commit = SlimeUtils.optionalString(object.field(commitField)); return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit); } @@ -437,7 +437,7 @@ public class ApplicationSerializer { object.field(jobStatusField).traverse((ArrayTraverser) (__, jobPauseObject) -> JobType.fromOptionalJobName(jobPauseObject.field(jobTypeField).asString()) .ifPresent(jobType -> jobPauses.put(jobType, - Serializers.instant(jobPauseObject.field(pausedUntilField))))); + SlimeUtils.instant(jobPauseObject.field(pausedUntilField))))); return jobPauses; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java index 7ea722bf5de..e5253462730 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java @@ -5,6 +5,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import java.util.ArrayList; @@ -51,11 +52,11 @@ public class AuditLogSerializer { Cursor root = slime.get(); root.field(entriesField).traverse((ArrayTraverser) (i, entryObject) -> { entries.add(new AuditLog.Entry( - Serializers.instant(entryObject.field(atField)), + SlimeUtils.instant(entryObject.field(atField)), entryObject.field(principalField).asString(), methodFrom(entryObject.field(methodField)), entryObject.field(resourceField).asString(), - Serializers.optionalString(entryObject.field(dataField)) + SlimeUtils.optionalString(entryObject.field(dataField)) )); }); return new AuditLog(entries); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java index 30fcc0e40c6..24a6ef72438 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerVersionSerializer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; /** @@ -36,7 +37,7 @@ public class ControllerVersionSerializer { var root = slime.get(); var version = Version.fromString(root.field(VERSION_FIELD).asString()); var commitSha = root.field(COMMIT_SHA_FIELD).asString(); - var commitDate = Serializers.instant(root.field(COMMIT_DATE_FIELD)); + var commitDate = SlimeUtils.instant(root.field(COMMIT_DATE_FIELD)); return new ControllerVersion(version, commitSha, commitDate); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java index 6416d077ce4..4e3ab293a02 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java @@ -92,7 +92,7 @@ class LogSerializer { private LogEntry fromSlime(Inspector entryObject) { return new LogEntry(entryObject.field(idField).asLong(), - Serializers.instant(entryObject.field(timestampField)), + SlimeUtils.instant(entryObject.field(timestampField)), typeOf(entryObject.field(typeField).asString()), entryObject.field(messageField).asString()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java index ff1dd4d95c6..dd431d94d94 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java @@ -6,6 +6,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; @@ -113,8 +114,8 @@ public class NameServiceQueueSerializer { private RemoveRecords removeRecordsFromSlime(Inspector object) { var type = Record.Type.valueOf(object.field(typeField).asString()); - var name = Serializers.optionalString(object.field(nameField)).map(RecordName::from); - var data = Serializers.optionalString(object.field(dataField)).map(RecordData::from); + var name = SlimeUtils.optionalString(object.field(nameField)).map(RecordName::from); + var data = SlimeUtils.optionalString(object.field(dataField)).map(RecordData::from); return new RemoveRecords(type, name, data); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java index f9f8de96591..2861e8922a5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; @@ -47,7 +48,7 @@ public class NodeVersionSerializer { var hostname = HostName.from(entry.field(hostnameField).asString()); var zone = ZoneId.from(entry.field(zoneField).asString()); var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString()); - var suspendedAt = Serializers.optionalInstant(entry.field(suspendedAtField)); + var suspendedAt = SlimeUtils.optionalInstant(entry.field(suspendedAtField)); nodeVersions.put(hostname, new NodeVersion(hostname, zone, version, wantedVersion, suspendedAt)); }); return new NodeVersions(nodeVersions.build()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java index 06263329091..ba1c5350580 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java @@ -74,18 +74,18 @@ public class NotificationsSerializer { private static Notification fromInspector(TenantName tenantName, Inspector inspector) { return new Notification( - Serializers.instant(inspector.field(atFieldName)), - typeFrom(inspector.field(typeField)), - levelFrom(inspector.field(levelField)), - new NotificationSource( - tenantName, - Serializers.optionalString(inspector.field(applicationField)).map(ApplicationName::from), - Serializers.optionalString(inspector.field(instanceField)).map(InstanceName::from), - Serializers.optionalString(inspector.field(zoneField)).map(ZoneId::from), - Serializers.optionalString(inspector.field(clusterIdField)).map(ClusterSpec.Id::from), - Serializers.optionalString(inspector.field(jobTypeField)).map(JobType::fromJobName), - Serializers.optionalLong(inspector.field(runNumberField))), - SlimeUtils.entriesStream(inspector.field(messagesField)).map(Inspector::asString).collect(Collectors.toUnmodifiableList())); + SlimeUtils.instant(inspector.field(atFieldName)), + typeFrom(inspector.field(typeField)), + levelFrom(inspector.field(levelField)), + new NotificationSource( + tenantName, + SlimeUtils.optionalString(inspector.field(applicationField)).map(ApplicationName::from), + SlimeUtils.optionalString(inspector.field(instanceField)).map(InstanceName::from), + SlimeUtils.optionalString(inspector.field(zoneField)).map(ZoneId::from), + SlimeUtils.optionalString(inspector.field(clusterIdField)).map(ClusterSpec.Id::from), + SlimeUtils.optionalString(inspector.field(jobTypeField)).map(JobType::fromJobName), + SlimeUtils.optionalLong(inspector.field(runNumberField))), + SlimeUtils.entriesStream(inspector.field(messagesField)).map(Inspector::asString).collect(Collectors.toUnmodifiableList())); } private static String asString(Notification.Type type) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java index 6942fc0549f..7c27533c144 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionTargetSerializer.java @@ -42,9 +42,7 @@ public class OsVersionTargetSerializer { Set<OsVersionTarget> osVersionTargets = new TreeSet<>(); array.traverse((ArrayTraverser) (i, inspector) -> { OsVersion osVersion = osVersionSerializer.fromSlime(inspector); - // TODO(mpolden): Require this field after 2021-05-01 - Duration upgradeBudget = Serializers.optionalDuration(inspector.field(upgradeBudgetField)) - .orElse(Duration.ZERO); + Duration upgradeBudget = Duration.ofMillis(inspector.field(upgradeBudgetField).asLong()); osVersionTargets.add(new OsVersionTarget(osVersion, upgradeBudget)); }); return Collections.unmodifiableSet(osVersionTargets); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 2697651f61b..d14cd780a8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -9,6 +9,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; @@ -80,7 +81,7 @@ public class RoutingPolicySerializer { ZoneId.from(inspect.field(zoneField).asString())); policies.put(id, new RoutingPolicy(id, HostName.from(inspect.field(canonicalNameField).asString()), - Serializers.optionalString(inspect.field(dnsZoneField)), + SlimeUtils.optionalString(inspect.field(dnsZoneField)), endpointIds, new Status(inspect.field(loadBalancerActiveField).asBool(), globalRoutingFromSlime(inspect.field(globalRoutingField))))); @@ -97,7 +98,7 @@ public class RoutingPolicySerializer { public GlobalRouting globalRoutingFromSlime(Inspector object) { var status = GlobalRouting.Status.valueOf(object.field(statusField).asString()); var agent = GlobalRouting.Agent.valueOf(object.field(agentField).asString()); - var changedAt = Serializers.optionalInstant(object.field(changedAtField)).orElse(Instant.EPOCH); + var changedAt = SlimeUtils.optionalInstant(object.field(changedAtField)).orElse(Instant.EPOCH); return new GlobalRouting(status, agent, changedAt); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 60d8afe0f5e..87527085237 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -9,6 +9,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -121,7 +122,7 @@ class RunSerializer { // For historical reasons are the step details stored in a separate JSON structure from the step statuses. Inspector stepDetailsField = detailsField.field(step); Inspector startTimeValue = stepDetailsField.field(startTimeField); - Optional<Instant> startTime = Serializers.optionalInstant(startTimeValue); + Optional<Instant> startTime = SlimeUtils.optionalInstant(startTimeValue); steps.put(typedStep, new StepInfo(typedStep, stepStatusOf(status.asString()), startTime)); }); @@ -130,12 +131,12 @@ class RunSerializer { runObject.field(numberField).asLong()), steps, versionsFromSlime(runObject.field(versionsField)), - Serializers.instant(runObject.field(startField)), - Serializers.optionalInstant(runObject.field(endField)), + SlimeUtils.instant(runObject.field(startField)), + SlimeUtils.optionalInstant(runObject.field(endField)), runStatusOf(runObject.field(statusField).asString()), runObject.field(lastTestRecordField).asLong(), Instant.EPOCH.plus(runObject.field(lastVespaLogTimestampField).asLong(), ChronoUnit.MICROS), - Serializers.optionalInstant(runObject.field(noNodesDownSinceField)), + SlimeUtils.optionalInstant(runObject.field(noNodesDownSinceField)), convergenceSummaryFrom(runObject.field(convergenceSummaryField)), Optional.of(runObject.field(testerCertificateField)) .filter(Inspector::valid) @@ -166,11 +167,11 @@ class RunSerializer { versionObject.field(branchField).asString(), versionObject.field(commitField).asString())) .filter(revision -> ! revision.commit().isBlank() && ! revision.repository().isBlank() && ! revision.branch().isBlank()); - Optional<String> authorEmail = Serializers.optionalString(versionObject.field(authorEmailField)); - Optional<Version> compileVersion = Serializers.optionalString(versionObject.field(compileVersionField)).map(Version::fromString); - Optional<Instant> buildTime = Serializers.optionalInstant(versionObject.field(buildTimeField)); - Optional<String> sourceUrl = Serializers.optionalString(versionObject.field(sourceUrlField)); - Optional<String> commit = Serializers.optionalString(versionObject.field(commitField)); + Optional<String> authorEmail = SlimeUtils.optionalString(versionObject.field(authorEmailField)); + Optional<Version> compileVersion = SlimeUtils.optionalString(versionObject.field(compileVersionField)).map(Version::fromString); + Optional<Instant> buildTime = SlimeUtils.optionalInstant(versionObject.field(buildTimeField)); + Optional<String> sourceUrl = SlimeUtils.optionalString(versionObject.field(sourceUrlField)); + Optional<String> commit = SlimeUtils.optionalString(versionObject.field(commitField)); return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, compileVersion, buildTime, sourceUrl, commit); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java deleted file mode 100644 index 7c8a09e244e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.slime.Inspector; -import com.yahoo.slime.SlimeUtils; - -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; -import java.util.OptionalDouble; -import java.util.OptionalInt; -import java.util.OptionalLong; - -/** - * Reusable serialization logic. - * - * @author mpolden - */ -public class Serializers { - - private Serializers() {} - - public static Instant instant(Inspector field) { - return Instant.ofEpochMilli(field.asLong()); - } - - public static OptionalLong optionalLong(Inspector field) { - return field.valid() ? OptionalLong.of(field.asLong()) : OptionalLong.empty(); - } - - public static OptionalInt optionalInteger(Inspector field) { - return field.valid() ? OptionalInt.of((int) field.asLong()) : OptionalInt.empty(); - } - - public static OptionalDouble optionalDouble(Inspector field) { - return field.valid() ? OptionalDouble.of(field.asDouble()) : OptionalDouble.empty(); - } - - public static Optional<String> optionalString(Inspector field) { - return SlimeUtils.optionalString(field); - } - - public static Optional<Instant> optionalInstant(Inspector field) { - return optionalLong(field).stream().mapToObj(Instant::ofEpochMilli).findFirst(); - } - - public static Optional<Duration> optionalDuration(Inspector field) { - return optionalLong(field).stream().mapToObj(Duration::ofMillis).findFirst(); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 8e97368624d..6b167f26314 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -151,14 +151,14 @@ public class TenantSerializer { Property property = new Property(tenantObject.field(propertyField).asString()); Optional<PropertyId> propertyId = SlimeUtils.optionalString(tenantObject.field(propertyIdField)).map(PropertyId::new); Optional<Contact> contact = contactFrom(tenantObject.field(contactField)); - Instant createdAt = Serializers.instant(tenantObject.field(createdAtField)); + Instant createdAt = SlimeUtils.instant(tenantObject.field(createdAtField)); LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo); } private CloudTenant cloudTenantFrom(Inspector tenantObject) { TenantName name = TenantName.from(tenantObject.field(nameField).asString()); - Instant createdAt = Serializers.instant(tenantObject.field(createdAtField)); + Instant createdAt = SlimeUtils.instant(tenantObject.field(createdAtField)); LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); Optional<Principal> creator = SlimeUtils.optionalString(tenantObject.field(creatorField)).map(SimplePrincipal::new); BiMap<PublicKey, Principal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); @@ -227,7 +227,7 @@ public class TenantSerializer { private LastLoginInfo lastLoginInfoFromSlime(Inspector lastLoginInfoObject) { Map<LastLoginInfo.UserLevel, Instant> lastLoginByUserLevel = new HashMap<>(); lastLoginInfoObject.traverse((String name, Inspector value) -> - lastLoginByUserLevel.put(userLevelOf(name), Serializers.instant(value))); + lastLoginByUserLevel.put(userLevelOf(name), SlimeUtils.instant(value))); return new LastLoginInfo(lastLoginByUserLevel); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 12d15aa7cdd..eccda7332e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -7,6 +7,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -106,7 +107,7 @@ public class VersionStatusSerializer { var version = Version.fromString(object.field(deploymentStatisticsField).field(versionField).asString()); return new VespaVersion(version, object.field(releaseCommitField).asString(), - Serializers.instant(object.field(committedAtField)), + SlimeUtils.instant(object.field(committedAtField)), object.field(isControllerVersionField).asBool(), object.field(isSystemVersionField).asBool(), object.field(isReleasedField).asBool(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 23754f6d57f..1510bb05a62 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2095,14 +2095,16 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { for (Application application : applications) { DeploymentStatus status = null; for (Instance instance : showOnlyProductionInstances(request) ? application.productionInstances().values() - : application.instances().values()) + : application.instances().values()) { + if (showOnlyActiveInstances(request) && instance.deployments().isEmpty()) + continue; if (recurseOverApplications(request)) { if (status == null) status = controller.jobController().deploymentStatus(application); toSlime(applicationArray.addObject(), instance, status, request); - } - else { + } else { toSlime(instance.id(), applicationArray.addObject(), request); } + } } tenantMetaDataToSlime(tenant, applications, object.setObject("metaData")); } @@ -2393,6 +2395,10 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return "true".equals(request.getProperty("production")); } + private static boolean showOnlyActiveInstances(HttpRequest request) { + return "true".equals(request.getProperty("activeInstances")); + } + private static String tenantType(Tenant tenant) { switch (tenant.type()) { case athenz: return "ATHENS"; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index 454a4f81524..786809fb7b1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -22,17 +22,14 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; +import static com.yahoo.vespa.hosted.controller.notification.Notification.Level; +import static com.yahoo.vespa.hosted.controller.notification.Notification.Type; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static com.yahoo.vespa.hosted.controller.notification.Notification.Level; -import static com.yahoo.vespa.hosted.controller.notification.Notification.Type; - /** * @author freva */ @@ -160,41 +157,6 @@ public class NotificationsDbTest { assertEquals(expected, curatorDb.readNotifications(tenant)); } - @Test - public void removes_invalid_deployment_notifications() { - curatorDb.deleteNotifications(tenant); // Remove notifications set in init() - - ZoneId z1 = ZoneId.from("prod", "us-west-1"); - ZoneId z2 = ZoneId.from("prod", "eu-south-2"); - DeploymentId d1 = new DeploymentId(ApplicationId.from("t1", "a1", "i1"), z1); - DeploymentId d2 = new DeploymentId(ApplicationId.from("t1", "a1", "i1"), z2); - DeploymentId d3 = new DeploymentId(ApplicationId.from("t1", "a1", "i2"), z1); - DeploymentId d4 = new DeploymentId(ApplicationId.from("t1", "a2", "i1"), z2); - DeploymentId d5 = new DeploymentId(ApplicationId.from("t2", "a1", "i1"), z2); - - List<Notification> notifications = Stream.of(d1, d2, d3, d4, d5) - .flatMap(deployment -> Stream.of(Type.deployment, Type.feedBlock) - .map(type -> new Notification(Instant.EPOCH, type, Level.warning, NotificationSource.from(deployment), List.of("msg")))) - .collect(Collectors.toUnmodifiableList()); - notifications.stream().collect(Collectors.groupingBy(notification -> notification.source().tenant(), Collectors.toList())) - .forEach(curatorDb::writeNotifications); - - // All except d3 plus a deployment that has no notifications - Set<DeploymentId> allDeployments = Set.of(d1, d2, d4, d5, new DeploymentId(ApplicationId.from("t3", "a1", "i1"), z1)); - notificationsDb.removeNotificationsForRemovedInstances(allDeployments); - - List<Notification> expectedNotifications = new ArrayList<>(notifications); - // Only the deployment notification for d3 should be cleared (the other types already correctly clear themselves) - expectedNotifications.remove(4); - - List<Notification> actualNotifications = curatorDb.listNotifications().stream() - .flatMap(tenant -> curatorDb.readNotifications(tenant).stream()) - .collect(Collectors.toUnmodifiableList()); - - assertEquals(expectedNotifications.stream().map(Notification::toString).collect(Collectors.joining("\n")), - actualNotifications.stream().map(Notification::toString).collect(Collectors.joining("\n"))); - } - @Before public void init() { curatorDb.writeNotifications(tenant, notifications); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 01d39c0ea1c..ed81ce36600 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -217,6 +217,11 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1", GET).userIdentity(USER_ID), new File("tenant-with-application.json")); + tester.assertResponse(request("/application/v4/tenant/tenant1", GET) + .userIdentity(USER_ID) + .properties(Map.of("activeInstances", "true")), + new File("tenant-without-applications.json")); + // GET tenant applications tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET).userIdentity(USER_ID), new File("application-list.json")); diff --git a/dist/vespa.spec b/dist/vespa.spec index f10d6cd3bc3..13ea9a733e1 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -769,6 +769,7 @@ fi %{_prefix}/lib/jars/config-model-api-jar-with-dependencies.jar %{_prefix}/lib/jars/config-model-jar-with-dependencies.jar %{_prefix}/lib/jars/config-provisioning-jar-with-dependencies.jar +%{_prefix}/lib/jars/container-apache-http-client-bundle-jar-with-dependencies.jar %{_prefix}/lib/jars/container-disc-jar-with-dependencies.jar %{_prefix}/lib/jars/container-jersey2-jar-with-dependencies.jar %{_prefix}/lib/jars/container-search-and-docproc-jar-with-dependencies.jar diff --git a/docker-api/pom.xml b/docker-api/pom.xml index 749eca97c53..7de2d502dc3 100644 --- a/docker-api/pom.xml +++ b/docker-api/pom.xml @@ -31,6 +31,12 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <!-- Compile --> <dependency> @@ -93,24 +99,6 @@ </exclusion> </exclusions> </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - <!-- We explicitly specify the version of httpcore to be used by - docker-java so the dependency is declared closer to the root of maven and - more likely be the version that is finally being used. --> - <version>4.4.1</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <!-- We explicitly specify the version of httpclient to be used by - docker-java so the dependency is declared closer to the root of maven and - more likely be the version that is finally being used. --> - <version>4.5</version> - <scope>compile</scope> - </dependency> <!-- Test --> <dependency> diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index 9ae33c1234f..8a596ad38d4 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -232,11 +232,24 @@ void generate_join_expr(const vespalib::string &expr, const Sequence &seq, TestB } } +void generate_join_expr(const vespalib::string &expr, const Sequence &seq_a, const Sequence &seq_b, TestBuilder &dst) { + for (const auto &layouts: join_layouts) { + GenSpec a = GenSpec::from_desc(layouts.first).seq(seq_a); + GenSpec b = GenSpec::from_desc(layouts.second).seq(seq_b); + generate(expr, a, b, dst); + } +} + void generate_op2_join(const vespalib::string &op2_expr, const Sequence &seq, TestBuilder &dst) { generate_join_expr(op2_expr, seq, dst); generate_join_expr(fmt("join(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst); } +void generate_op2_join(const vespalib::string &op2_expr, const Sequence &seq_a, const Sequence &seq_b, TestBuilder &dst) { + generate_join_expr(op2_expr, seq_a, seq_b, dst); + generate_join_expr(fmt("join(a,b,f(a,b)(%s))", op2_expr.c_str()), seq_a, seq_b, dst); +} + void generate_join(TestBuilder &dst) { generate_op2_join("a+b", Div16(N()), dst); generate_op2_join("a-b", Div16(N()), dst); @@ -259,6 +272,7 @@ void generate_join(TestBuilder &dst) { generate_op2_join("fmod(a,b)", Div16(N()), dst); generate_op2_join("min(a,b)", Div16(N()), dst); generate_op2_join("max(a,b)", Div16(N()), dst); + generate_op2_join("bit(a,b)", Seq({-128, -43, -1, 0, 85, 127}), Seq({0, 1, 2, 3, 4, 5, 6, 7}), dst); // inverted lambda generate_join_expr("join(a,b,f(a,b)(b-a))", Div16(N()), dst); // custom lambda @@ -276,11 +290,24 @@ void generate_merge_expr(const vespalib::string &expr, const Sequence &seq, Test } } +void generate_merge_expr(const vespalib::string &expr, const Sequence &seq_a, const Sequence &seq_b, TestBuilder &dst) { + for (const auto &layouts: merge_layouts) { + GenSpec a = GenSpec::from_desc(layouts.first).seq(seq_a); + GenSpec b = GenSpec::from_desc(layouts.second).seq(seq_b); + generate(expr, a, b, dst); + } +} + void generate_op2_merge(const vespalib::string &op2_expr, const Sequence &seq, TestBuilder &dst) { generate_merge_expr(op2_expr, seq, dst); generate_merge_expr(fmt("merge(a,b,f(a,b)(%s))", op2_expr.c_str()), seq, dst); } +void generate_op2_merge(const vespalib::string &op2_expr, const Sequence &seq_a, const Sequence &seq_b, TestBuilder &dst) { + generate_merge_expr(op2_expr, seq_a, seq_b, dst); + generate_merge_expr(fmt("merge(a,b,f(a,b)(%s))", op2_expr.c_str()), seq_a, seq_b, dst); +} + void generate_merge(TestBuilder &dst) { generate_op2_merge("a+b", Div16(N()), dst); generate_op2_merge("a-b", Div16(N()), dst); @@ -303,6 +330,7 @@ void generate_merge(TestBuilder &dst) { generate_op2_merge("fmod(a,b)", Div16(N()), dst); generate_op2_merge("min(a,b)", Div16(N()), dst); generate_op2_merge("max(a,b)", Div16(N()), dst); + generate_op2_merge("bit(a,b)", Seq({-128, -43, -1, 0, 85, 127}), Seq({0, 1, 2, 3, 4, 5, 6, 7}), dst); // inverted lambda generate_merge_expr("merge(a,b,f(a,b)(b-a))", Div16(N()), dst); // custom lambda diff --git a/eval/src/tests/eval/inline_operation/inline_operation_test.cpp b/eval/src/tests/eval/inline_operation/inline_operation_test.cpp index de5a3fbf395..ae5f503b680 100644 --- a/eval/src/tests/eval/inline_operation/inline_operation_test.cpp +++ b/eval/src/tests/eval/inline_operation/inline_operation_test.cpp @@ -115,6 +115,7 @@ TEST(InlineOperationTest, op2_lambdas_are_recognized) { EXPECT_EQ(as_op2("fmod(a,b)"), &Mod::f); EXPECT_EQ(as_op2("min(a,b)"), &Min::f); EXPECT_EQ(as_op2("max(a,b)"), &Max::f); + EXPECT_EQ(as_op2("bit(a,b)"), &Bit::f); } TEST(InlineOperationTest, op2_lambdas_are_recognized_with_different_parameter_names) { diff --git a/eval/src/tests/eval/node_tools/node_tools_test.cpp b/eval/src/tests/eval/node_tools/node_tools_test.cpp index 13185065f57..e8296c01d73 100644 --- a/eval/src/tests/eval/node_tools/node_tools_test.cpp +++ b/eval/src/tests/eval/node_tools/node_tools_test.cpp @@ -100,6 +100,7 @@ TEST("require that call node types can be copied") { TEST_DO(verify_copy("sigmoid(a)")); TEST_DO(verify_copy("elu(a)")); TEST_DO(verify_copy("erf(a)")); + TEST_DO(verify_copy("bit(a,b)")); } TEST("require that tensor node types can NOT be copied (yet)") { diff --git a/eval/src/tests/eval/node_types/node_types_test.cpp b/eval/src/tests/eval/node_types/node_types_test.cpp index 504f66ac717..b2373f0d8f5 100644 --- a/eval/src/tests/eval/node_types/node_types_test.cpp +++ b/eval/src/tests/eval/node_types/node_types_test.cpp @@ -218,6 +218,7 @@ TEST("require that various operations resolve appropriate type") { TEST_DO(verify_op1("sigmoid(%s)")); // Sigmoid TEST_DO(verify_op1("elu(%s)")); // Elu TEST_DO(verify_op1("erf(%s)")); // Erf + TEST_DO(verify_op2("bit(%s,%s)")); // Bit } TEST("require that map resolves correct type") { diff --git a/eval/src/vespa/eval/eval/aggr.h b/eval/src/vespa/eval/eval/aggr.h index 516ead0f0bf..fe70107af79 100644 --- a/eval/src/vespa/eval/eval/aggr.h +++ b/eval/src/vespa/eval/eval/aggr.h @@ -10,13 +10,10 @@ #include <algorithm> #include <cmath> -namespace vespalib { +namespace vespalib { class Stash; } -class Stash; +namespace vespalib::eval { -namespace eval { - -struct BinaryOperation; /** * Enumeration of all different aggregators that are allowed to be @@ -237,4 +234,3 @@ struct TypifyAggr { }; } // namespace vespalib::eval -} // namespace vespalib diff --git a/eval/src/vespa/eval/eval/call_nodes.cpp b/eval/src/vespa/eval/eval/call_nodes.cpp index 2fc25bdbc77..798583cf89a 100644 --- a/eval/src/vespa/eval/eval/call_nodes.cpp +++ b/eval/src/vespa/eval/eval/call_nodes.cpp @@ -43,6 +43,7 @@ CallRepo::CallRepo() : _map() { add(nodes::Sigmoid()); add(nodes::Elu()); add(nodes::Erf()); + add(nodes::Bit()); } } // namespace vespalib::eval::nodes diff --git a/eval/src/vespa/eval/eval/call_nodes.h b/eval/src/vespa/eval/eval/call_nodes.h index 2a7d4173e64..945aba69596 100644 --- a/eval/src/vespa/eval/eval/call_nodes.h +++ b/eval/src/vespa/eval/eval/call_nodes.h @@ -139,6 +139,7 @@ struct Relu : CallHelper<Relu> { Relu() : Helper("relu", 1) {} }; struct Sigmoid : CallHelper<Sigmoid> { Sigmoid() : Helper("sigmoid", 1) {} }; struct Elu : CallHelper<Elu> { Elu() : Helper("elu", 1) {} }; struct Erf : CallHelper<Erf> { Erf() : Helper("erf", 1) {} }; +struct Bit : CallHelper<Bit> { Bit() : Helper("bit", 2) {} }; //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/extract_bit.h b/eval/src/vespa/eval/eval/extract_bit.h new file mode 100644 index 00000000000..ecf56b33b02 --- /dev/null +++ b/eval/src/vespa/eval/eval/extract_bit.h @@ -0,0 +1,13 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib::eval { + +inline double extract_bit(double a, double b) { + int8_t value = (int8_t) a; + uint32_t n = (uint32_t) b; + return ((n < 8) && bool(value & (1 << n))) ? 1.0 : 0.0; +} + +} diff --git a/eval/src/vespa/eval/eval/key_gen.cpp b/eval/src/vespa/eval/eval/key_gen.cpp index a8fb205f124..a40a8887119 100644 --- a/eval/src/vespa/eval/eval/key_gen.cpp +++ b/eval/src/vespa/eval/eval/key_gen.cpp @@ -87,6 +87,7 @@ struct KeyGen : public NodeVisitor, public NodeTraverser { void visit(const Sigmoid &) override { add_byte(60); } void visit(const Elu &) override { add_byte(61); } void visit(const Erf &) override { add_byte(62); } + void visit(const Bit &) override { add_byte(63); } // traverse bool open(const Node &node) override { node.accept(*this); return true; } diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index 42911a56c14..2a9b7815aa8 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -4,6 +4,7 @@ #include "llvm_wrapper.h" #include <vespa/eval/eval/node_visitor.h> #include <vespa/eval/eval/node_traverser.h> +#include <vespa/eval/eval/extract_bit.h> #include <llvm/IR/Verifier.h> #include <llvm/Support/TargetSelect.h> #include <llvm/IR/IRBuilder.h> @@ -29,6 +30,7 @@ double vespalib_eval_approx(double a, double b) { return (vespalib::approx_equal double vespalib_eval_relu(double a) { return std::max(a, 0.0); } double vespalib_eval_sigmoid(double a) { return 1.0 / (1.0 + std::exp(-1.0 * a)); } double vespalib_eval_elu(double a) { return (a < 0) ? std::exp(a) - 1.0 : a; } +double vespalib_eval_bit(double a, double b) { return vespalib::eval::extract_bit(a, b); } using vespalib::eval::gbdt::Forest; using resolve_function = double (*)(void *ctx, size_t idx); @@ -646,6 +648,9 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { void visit(const Erf &) override { make_call_1("erf"); } + void visit(const Bit &) override { + make_call_2("vespalib_eval_bit"); + } }; FunctionBuilder::~FunctionBuilder() { } diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h index 040c0bdb73f..e04b477750d 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h @@ -19,6 +19,7 @@ extern "C" { double vespalib_eval_relu(double a); double vespalib_eval_sigmoid(double a); double vespalib_eval_elu(double a); + double vespalib_eval_bit(double a, double b); }; namespace vespalib::eval { diff --git a/eval/src/vespa/eval/eval/make_tensor_function.cpp b/eval/src/vespa/eval/eval/make_tensor_function.cpp index b65c3d5aaa7..498be2a738b 100644 --- a/eval/src/vespa/eval/eval/make_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/make_tensor_function.cpp @@ -357,6 +357,9 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void visit(const Erf &node) override { make_map(node, operation::Erf::f); } + void visit(const Bit &node) override { + make_join(node, operation::Bit::f); + } //------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/node_tools.cpp b/eval/src/vespa/eval/eval/node_tools.cpp index e7341bc1755..fa2d16a2271 100644 --- a/eval/src/vespa/eval/eval/node_tools.cpp +++ b/eval/src/vespa/eval/eval/node_tools.cpp @@ -182,6 +182,7 @@ struct CopyNode : NodeTraverser, NodeVisitor { void visit(const Sigmoid &node) override { copy_call(node); } void visit(const Elu &node) override { copy_call(node); } void visit(const Erf &node) override { copy_call(node); } + void visit(const Bit &node) override { copy_call(node); } // traverse nodes bool open(const Node &) override { return !error; } diff --git a/eval/src/vespa/eval/eval/node_types.cpp b/eval/src/vespa/eval/eval/node_types.cpp index 63da6d79c6f..8622fd734f1 100644 --- a/eval/src/vespa/eval/eval/node_types.cpp +++ b/eval/src/vespa/eval/eval/node_types.cpp @@ -278,6 +278,7 @@ struct TypeResolver : public NodeVisitor, public NodeTraverser { void visit(const Sigmoid &node) override { resolve_op1(node); } void visit(const Elu &node) override { resolve_op1(node); } void visit(const Erf &node) override { resolve_op1(node); } + void visit(const Bit &node) override { resolve_op2(node); } //------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/node_visitor.h b/eval/src/vespa/eval/eval/node_visitor.h index 172cd48fe2a..475bbf5405c 100644 --- a/eval/src/vespa/eval/eval/node_visitor.h +++ b/eval/src/vespa/eval/eval/node_visitor.h @@ -85,6 +85,7 @@ struct NodeVisitor { virtual void visit(const nodes::Sigmoid &) = 0; virtual void visit(const nodes::Elu &) = 0; virtual void visit(const nodes::Erf &) = 0; + virtual void visit(const nodes::Bit &) = 0; virtual ~NodeVisitor() {} }; @@ -154,6 +155,7 @@ struct EmptyNodeVisitor : NodeVisitor { void visit(const nodes::Sigmoid &) override {} void visit(const nodes::Elu &) override {} void visit(const nodes::Erf &) override {} + void visit(const nodes::Bit &) override {} }; } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/operation.cpp b/eval/src/vespa/eval/eval/operation.cpp index b97ac3f2261..a82a79e6bc4 100644 --- a/eval/src/vespa/eval/eval/operation.cpp +++ b/eval/src/vespa/eval/eval/operation.cpp @@ -3,6 +3,7 @@ #include "operation.h" #include "function.h" #include "key_gen.h" +#include "extract_bit.h" #include <vespa/vespalib/util/approx.h> #include <algorithm> @@ -50,6 +51,7 @@ double Relu::f(double a) { return std::max(a, 0.0); } double Sigmoid::f(double a) { return 1.0 / (1.0 + std::exp(-1.0 * a)); } double Elu::f(double a) { return (a < 0) ? std::exp(a) - 1 : a; } double Erf::f(double a) { return std::erf(a); } +double Bit::f(double a, double b) { return extract_bit(a, b); } //----------------------------------------------------------------------------- double Inv::f(double a) { return (1.0 / a); } double Square::f(double a) { return (a * a); } @@ -143,6 +145,7 @@ std::map<vespalib::string,op2_t> make_op2_map() { add_op2(map, "fmod(a,b)", Mod::f); add_op2(map, "min(a,b)", Min::f); add_op2(map, "max(a,b)", Max::f); + add_op2(map, "bit(a,b)", Bit::f); return map; } diff --git a/eval/src/vespa/eval/eval/operation.h b/eval/src/vespa/eval/eval/operation.h index 3170c868214..438b510b714 100644 --- a/eval/src/vespa/eval/eval/operation.h +++ b/eval/src/vespa/eval/eval/operation.h @@ -49,6 +49,7 @@ struct Relu { static double f(double a); }; struct Sigmoid { static double f(double a); }; struct Elu { static double f(double a); }; struct Erf { static double f(double a); }; +struct Bit { static double f(double a, double b); }; //----------------------------------------------------------------------------- struct Inv { static double f(double a); }; struct Square { static double f(double a); }; diff --git a/eval/src/vespa/eval/eval/test/eval_spec.cpp b/eval/src/vespa/eval/eval/test/eval_spec.cpp index 63a3a23d9ae..5d51a1d23b5 100644 --- a/eval/src/vespa/eval/eval/test/eval_spec.cpp +++ b/eval/src/vespa/eval/eval/test/eval_spec.cpp @@ -158,6 +158,17 @@ EvalSpec::add_function_call_cases() { add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "fmod(a,b)", [](double a, double b){ return std::fmod(a, b); }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "min(a,b)", [](double a, double b){ return std::min(a, b); }); add_rule({"a", -1.0, 1.0}, {"b", -1.0, 1.0}, "max(a,b)", [](double a, double b){ return std::max(a, b); }); + add_expression({"a", "b"}, "bit(a,b)") + .add_case({-128, 7}, 1.0).add_case({-128, 6}, 0.0).add_case({-128, 5}, 0.0).add_case({-128, 4}, 0.0) + .add_case({-128, 3}, 0.0).add_case({-128, 2}, 0.0).add_case({-128, 1}, 0.0).add_case({-128, 0}, 0.0) + .add_case({-43, 7}, 1.0).add_case({-43, 6}, 1.0).add_case({-43, 5}, 0.0).add_case({-43, 4}, 1.0) + .add_case({-43, 3}, 0.0).add_case({-43, 2}, 1.0).add_case({-43, 1}, 0.0).add_case({-43, 0}, 1.0) + .add_case({0, 7}, 0.0).add_case({0, 6}, 0.0).add_case({0, 5}, 0.0).add_case({0, 4}, 0.0) + .add_case({0, 3}, 0.0).add_case({0, 2}, 0.0).add_case({0, 1}, 0.0).add_case({0, 0}, 0.0) + .add_case({85, 7}, 0.0).add_case({85, 6}, 1.0).add_case({85, 5}, 0.0).add_case({85, 4}, 1.0) + .add_case({85, 3}, 0.0).add_case({85, 2}, 1.0).add_case({85, 1}, 0.0).add_case({85, 0}, 1.0) + .add_case({127, 7}, 0.0).add_case({127, 6}, 1.0).add_case({127, 5}, 1.0).add_case({127, 4}, 1.0) + .add_case({127, 3}, 1.0).add_case({127, 2}, 1.0).add_case({127, 1}, 1.0).add_case({127, 0}, 1.0); } void diff --git a/eval/src/vespa/eval/eval/test/reference_evaluation.cpp b/eval/src/vespa/eval/eval/test/reference_evaluation.cpp index 4824751bb14..58e4b91f6d9 100644 --- a/eval/src/vespa/eval/eval/test/reference_evaluation.cpp +++ b/eval/src/vespa/eval/eval/test/reference_evaluation.cpp @@ -335,6 +335,9 @@ struct EvalNode : public NodeVisitor { void visit(const Erf &node) override { eval_map(node.get_child(0), operation::Erf::f); } + void visit(const Bit &node) override { + eval_join(node.get_child(0), node.get_child(1), operation::Bit::f); + } }; TensorSpec eval_node(const Node &node, const std::vector<TensorSpec> ¶ms) { diff --git a/eval/src/vespa/eval/eval/visit_stuff.cpp b/eval/src/vespa/eval/eval/visit_stuff.cpp index 9306a720837..786562d823f 100644 --- a/eval/src/vespa/eval/eval/visit_stuff.cpp +++ b/eval/src/vespa/eval/eval/visit_stuff.cpp @@ -59,6 +59,7 @@ vespalib::string name_of(join_fun_t fun) { if (fun == operation::Ldexp::f) return "ldexp"; if (fun == operation::Min::f) return "min"; if (fun == operation::Max::f) return "max"; + if (fun == operation::Bit::f) return "bit"; return "[other join function]"; } diff --git a/fbench/src/fbench/client.cpp b/fbench/src/fbench/client.cpp index c1b444e9a6b..108d44e2b9b 100644 --- a/fbench/src/fbench/client.cpp +++ b/fbench/src/fbench/client.cpp @@ -5,38 +5,32 @@ #include <util/clientstatus.h> #include <httpclient/httpclient.h> #include <util/filereader.h> -#include <cassert> #include <cstring> #include <iostream> #include <vespa/vespalib/encoding/base64.h> using namespace vespalib; -Client::Client(vespalib::CryptoEngine::SP engine, ClientArguments *args) - : _args(args), - _status(new ClientStatus()), - _reqTimer(new Timer()), - _cycleTimer(new Timer()), - _masterTimer(new Timer()), - _http(new HTTPClient(std::move(engine), _args->_hostname, _args->_port, - _args->_keepAlive, _args->_headerBenchmarkdataCoverage, - _args->_extraHeaders, _args->_authority)), - _reader(new FileReader()), +Client::Client(vespalib::CryptoEngine::SP engine, std::unique_ptr<ClientArguments> args) + : _args(std::move(args)), + _status(std::make_unique<ClientStatus>()), + _reqTimer(std::make_unique<Timer>()), + _cycleTimer(std::make_unique<Timer>()), + _masterTimer(std::make_unique<Timer>()), + _http(std::make_unique<HTTPClient>(std::move(engine), _args->_hostname, _args->_port, _args->_keepAlive, + _args->_headerBenchmarkdataCoverage, _args->_extraHeaders, _args->_authority)), + _reader(std::make_unique<FileReader>()), _output(), - _linebufsize(args->_maxLineSize), - _linebuf(new char[_linebufsize]), + _linebufsize(_args->_maxLineSize), + _linebuf(std::make_unique<char[]>(_linebufsize)), _stop(false), _done(false), _thread() { - assert(args != NULL); _cycleTimer->SetMax(_args->_cycle); } -Client::~Client() -{ - delete [] _linebuf; -} +Client::~Client() = default; void Client::runMe(Client * me) { me->run(); @@ -173,15 +167,15 @@ Client::run() std::this_thread::sleep_for(std::chrono::milliseconds(_args->_delay)); // open query file - snprintf(inputFilename, 1024, _args->_filenamePattern, _args->_myNum); + snprintf(inputFilename, 1024, _args->_filenamePattern.c_str(), _args->_myNum); if (!_reader->Open(inputFilename)) { printf("Client %d: ERROR: could not open file '%s' [read mode]\n", _args->_myNum, inputFilename); _status->SetError("Could not open query file."); return; } - if (_args->_outputPattern != NULL) { - snprintf(outputFilename, 1024, _args->_outputPattern, _args->_myNum); + if ( ! _args->_outputPattern.empty()) { + snprintf(outputFilename, 1024, _args->_outputPattern.c_str(), _args->_myNum); _output = std::make_unique<std::ofstream>(outputFilename, std::ofstream::out | std::ofstream::binary); if (_output->fail()) { printf("Client %d: ERROR: could not open file '%s' [write mode]\n", @@ -208,7 +202,7 @@ Client::run() _cycleTimer->Start(); - linelen = urlSource.nextUrl(_linebuf, _linebufsize); + linelen = urlSource.nextUrl(_linebuf.get(), _linebufsize); if (linelen > 0) { ++urlNumber; } else { @@ -222,11 +216,11 @@ Client::run() if (linelen < _linebufsize) { if (_output) { _output->write("URL: ", strlen("URL: ")); - _output->write(_linebuf, linelen); + _output->write(_linebuf.get(), linelen); _output->write("\n\n", 2); } if (linelen + (int)_args->_queryStringToAppend.length() < _linebufsize) { - strcat(_linebuf, _args->_queryStringToAppend.c_str()); + strcat(_linebuf.get(), _args->_queryStringToAppend.c_str()); } int cLen = _args->_usePostMode ? urlSource.nextContent() : 0; @@ -239,7 +233,7 @@ Client::run() } _reqTimer->Start(); - auto fetch_status = _http->Fetch(_linebuf, _output.get(), _args->_usePostMode, content, cLen); + auto fetch_status = _http->Fetch(_linebuf.get(), _output.get(), _args->_usePostMode, content, cLen); _reqTimer->Stop(); _status->AddRequestStatus(fetch_status.RequestStatus()); if (fetch_status.Ok() && fetch_status.TotalHitCount() == 0) diff --git a/fbench/src/fbench/client.h b/fbench/src/fbench/client.h index 70d83f71971..3349a112fa2 100644 --- a/fbench/src/fbench/client.h +++ b/fbench/src/fbench/client.h @@ -21,23 +21,17 @@ struct ClientArguments int _myNum; /** - * The total number of clients controlled by the parent fbench - * application - **/ - int _totNum; - - /** * Pattern that combined with the client number will become the name * of the file containing the urls this client should request. **/ - const char *_filenamePattern; + std::string _filenamePattern; /** * Pattern that combined with the client number will become the name * of the file this client should dump url content to. If this * pattern is set to NULL no output file is generated. **/ - const char *_outputPattern; + std::string _outputPattern; /** * The server the client should fetch urls from. @@ -116,9 +110,9 @@ struct ClientArguments std::string _extraHeaders; std::string _authority; - ClientArguments(int myNum, int totNum, - const char *filenamePattern, - const char *outputPattern, + ClientArguments(int myNum, + const std::string & filenamePattern, + const std::string & outputPattern, const char *hostname, int port, long cycle, long delay, int ignoreCount, int byteLimit, @@ -129,7 +123,6 @@ struct ClientArguments const std::string & queryStringToAppend, const std::string & extraHeaders, const std::string &authority, bool postMode) : _myNum(myNum), - _totNum(totNum), _filenamePattern(filenamePattern), _outputPattern(outputPattern), _hostname(hostname), @@ -181,13 +174,11 @@ private: std::unique_ptr<FileReader> _reader; std::unique_ptr<std::ofstream> _output; int _linebufsize; - char *_linebuf; + std::unique_ptr<char[]> _linebuf; std::atomic<bool> _stop; std::atomic<bool> _done; std::thread _thread; - Client(const Client &); - Client &operator=(const Client &); static void runMe(Client * client); void run(); @@ -197,7 +188,9 @@ public: * The client arguments given to this method becomes the * responsibility of the client. **/ - Client(vespalib::CryptoEngine::SP engine, ClientArguments *args); + Client(vespalib::CryptoEngine::SP engine, std::unique_ptr<ClientArguments> args); + Client(const Client &) = delete; + Client &operator=(const Client &) = delete; /** * Delete objects owned by this client, including the client arguments. diff --git a/fbench/src/fbench/fbench.cpp b/fbench/src/fbench/fbench.cpp index 57efb8a47e0..1ba49e9897a 100644 --- a/fbench/src/fbench/fbench.cpp +++ b/fbench/src/fbench/fbench.cpp @@ -1,4 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "fbench.h" +#include "client.h" + #include <util/timer.h> #include <httpclient/httpclient.h> #include <util/filereader.h> @@ -9,8 +13,6 @@ #include <vespa/vespalib/net/tls/tls_crypto_engine.h> #include <vespa/vespalib/io/mapped_file_input.h> #include <vespa/vespalib/util/size_literals.h> -#include "client.h" -#include "fbench.h" #include <cstring> #include <cmath> #include <csignal> @@ -19,7 +21,8 @@ namespace { -std::string maybe_load(const std::string &file_name, bool &failed) { +std::string +maybe_load(const std::string &file_name, bool &failed) { std::string content; if (!file_name.empty()) { vespalib::MappedFileInput file(file_name); @@ -42,8 +45,8 @@ FBench::FBench() _clients(), _ignoreCount(0), _cycle(0), - _filenamePattern(NULL), - _outputPattern(NULL), + _filenamePattern(), + _outputPattern(), _byteLimit(0), _restartLimit(0), _maxLineSize(0), @@ -58,8 +61,6 @@ FBench::FBench() FBench::~FBench() { _clients.clear(); - free(_filenamePattern); - free(_outputPattern); } bool @@ -121,11 +122,13 @@ FBench::InitBenchmark(int numClients, int ignoreCount, int cycle, _ignoreCount = ignoreCount; _cycle = cycle; - free(_filenamePattern); - _filenamePattern = strdup(filenamePattern); - free(_outputPattern); - _outputPattern = (outputPattern == NULL) ? - NULL : strdup(outputPattern); + _filenamePattern = filenamePattern; + if (outputPattern != nullptr) { + _outputPattern = outputPattern; + } else { + _outputPattern.clear(); + } + _queryStringToAppend = queryStringToAppend; _extraHeaders = extraHeaders; _authority = authority; @@ -154,15 +157,12 @@ FBench::CreateClients() off_end = _queryfileOffset[i+1]; } client = std::make_unique<Client>(_crypto_engine, - new ClientArguments(i, _clients.size(), _filenamePattern, - _outputPattern, _hostnames[i % _hostnames.size()].c_str(), - _ports[i % _ports.size()], _cycle, - random() % spread, _ignoreCount, - _byteLimit, _restartLimit, _maxLineSize, - _keepAlive, _base64Decode, - _headerBenchmarkdataCoverage, - off_beg, off_end, - _singleQueryFile, _queryStringToAppend, _extraHeaders, _authority, _usePostMode)); + std::make_unique<ClientArguments>(i, _filenamePattern, _outputPattern, + _hostnames[i % _hostnames.size()].c_str(), + _ports[i % _ports.size()], _cycle,random() % spread, + _ignoreCount, _byteLimit, _restartLimit, _maxLineSize, _keepAlive, + _base64Decode, _headerBenchmarkdataCoverage, off_beg, off_end, + _singleQueryFile, _queryStringToAppend, _extraHeaders, _authority, _usePostMode)); ++i; } } @@ -278,6 +278,8 @@ FBench::PrintSummary() printf("utilization: %8.2f %%\n", (maxRate > 0) ? 100 * (actualRate / maxRate) : 0); printf("zero hit queries: %8ld\n", status._zeroHitQueries); + printf("zero hit percentage: %8.2f %%\n", + (status._requestCnt > 0) ? 100.0*(double(status._zeroHitQueries)/status._requestCnt) : 0.0); printf("http request status breakdown:\n"); for (const auto& entry : status._requestStatusDistribution) printf(" %8u : %8u \n", entry.first, entry.second); @@ -345,7 +347,7 @@ FBench::Main(int argc, char *argv[]) const int minLineSize = 1024; const char *queryFilePattern = "query%03d.txt"; - const char *outputFilePattern = NULL; + const char *outputFilePattern = nullptr; std::string queryStringToAppend; std::string extraHeaders; std::string ca_certs_file_name; // -T @@ -599,8 +601,8 @@ main(int argc, char** argv) sigemptyset(&act.sa_mask); act.sa_flags = 0; - sigaction(SIGINT, &act, NULL); - sigaction(SIGPIPE, &act, NULL); + sigaction(SIGINT, &act, nullptr); + sigaction(SIGPIPE, &act, nullptr); FBench myApp; return myApp.Main(argc, argv); diff --git a/fbench/src/fbench/fbench.h b/fbench/src/fbench/fbench.h index 362a463a4f1..e66fc28683d 100644 --- a/fbench/src/fbench/fbench.h +++ b/fbench/src/fbench/fbench.h @@ -1,6 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vector> +#include <string> +#include <memory> + +class Client; + +namespace vespalib { class CryptoEngine; } + /** * This is the application class of the fbench program. It controls * the operation of the test clients and collects overall results. @@ -10,29 +18,27 @@ class FBench { private: - vespalib::CryptoEngine::SP _crypto_engine; - std::vector<Client::UP> _clients; - int _numClients; - int _ignoreCount; - int _cycle; + std::shared_ptr<vespalib::CryptoEngine> _crypto_engine; + std::vector<std::unique_ptr<Client>> _clients; + int _ignoreCount; + int _cycle; std::vector<std::string> _hostnames; - std::vector<int> _ports; - char *_filenamePattern; - char *_outputPattern; - int _byteLimit; - int _restartLimit; - int _maxLineSize; - bool _keepAlive; - bool _base64Decode; - bool _usePostMode; - bool _headerBenchmarkdataCoverage; - int _seconds; - std::vector<uint64_t> _queryfileOffset; - int _numberOfQueries; - bool _singleQueryFile; - std::string _queryStringToAppend; - std::string _extraHeaders; - std::string _authority; + std::vector<int> _ports; + std::string _filenamePattern; + std::string _outputPattern; + int _byteLimit; + int _restartLimit; + int _maxLineSize; + bool _keepAlive; + bool _base64Decode; + bool _usePostMode; + bool _headerBenchmarkdataCoverage; + int _seconds; + std::vector<uint64_t> _queryfileOffset; + bool _singleQueryFile; + std::string _queryStringToAppend; + std::string _extraHeaders; + std::string _authority; bool init_crypto_engine(const std::string &ca_certs_file_name, const std::string &cert_chain_file_name, diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 8bfa85d8b16..e922d878dd7 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -20,6 +20,12 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> </dependency> @@ -58,10 +64,6 @@ <artifactId>airline</artifactId> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> </dependency> diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java index eb0976edc40..eab6cd39352 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/Downloads.java @@ -7,6 +7,7 @@ import java.io.File; import java.time.Instant; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -29,17 +30,13 @@ public class Downloads { public DownloadStatuses downloadStatuses() { return downloadStatuses; } void setDownloadStatus(FileReference fileReference, double completeness) { - Optional<Downloads.DownloadStatus> downloadStatus = downloadStatuses.get(fileReference); - if (downloadStatus.isPresent()) - downloadStatus.get().setProgress(completeness); - else - downloadStatuses.add(fileReference, completeness); + downloadStatuses.put(fileReference, completeness); } void completedDownloading(FileReference fileReference, File file) { Optional<FileReferenceDownload> download = get(fileReference); + setDownloadStatus(fileReference, 1.0); if (download.isPresent()) { - downloadStatuses().get(fileReference).ifPresent(Downloads.DownloadStatus::finished); downloads.remove(fileReference); download.get().future().complete(Optional.of(file)); } else { @@ -49,11 +46,11 @@ public class Downloads { void add(FileReferenceDownload fileReferenceDownload) { downloads.put(fileReferenceDownload.fileReference(), fileReferenceDownload); - downloadStatuses.add(fileReferenceDownload.fileReference()); + downloadStatuses.put(fileReferenceDownload.fileReference()); } void remove(FileReference fileReference) { - downloadStatuses.get(fileReference).ifPresent(d -> d.setProgress(0.0)); + downloadStatuses.get(fileReference).ifPresent(d -> new DownloadStatus(d.fileReference(), 0.0)); downloads.remove(fileReference); } @@ -79,16 +76,14 @@ public class Downloads { private static final int maxEntries = 100; - private final Map<FileReference, DownloadStatus> downloadStatus = new ConcurrentHashMap<>(); + private final Map<FileReference, DownloadStatus> downloadStatus = Collections.synchronizedMap(new HashMap<>()); - void add(FileReference fileReference) { - add(fileReference, 0.0); + void put(FileReference fileReference) { + put(fileReference, 0.0); } - void add(FileReference fileReference, double progress) { - DownloadStatus ds = new DownloadStatus(fileReference); - ds.setProgress(progress); - downloadStatus.put(fileReference, ds); + void put(FileReference fileReference, double progress) { + downloadStatus.put(fileReference, new DownloadStatus(fileReference, progress)); if (downloadStatus.size() > maxEntries) { Map.Entry<FileReference, DownloadStatus> oldest = Collections.min(downloadStatus.entrySet(), Comparator.comparing(e -> e.getValue().created)); @@ -104,16 +99,21 @@ public class Downloads { return Map.copyOf(downloadStatus); } + @Override + public String toString() { + return downloadStatus.entrySet().stream().map(entry -> entry.getKey().value() + "=>" + entry.getValue().progress).collect(Collectors.joining(", ")); + } + } static class DownloadStatus { private final FileReference fileReference; - private double progress; // between 0 and 1 + private final double progress; // between 0 and 1 private final Instant created; - DownloadStatus(FileReference fileReference) { + DownloadStatus(FileReference fileReference, double progress) { this.fileReference = fileReference; - this.progress = 0.0; + this.progress = progress; this.created = Instant.now(); } @@ -125,14 +125,6 @@ public class Downloads { return progress; } - public void setProgress(double progress) { - this.progress = progress; - } - - public void finished() { - setProgress(1.0); - } - public Instant created() { return created; } diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index c4848140b2d..6169f6fbe55 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -112,7 +112,7 @@ public class FileDownloaderTest { // Receives fileReference, should return and make it available to caller String filename = "abc.jar"; - receiveFile(fileDownloader, fileReference, filename, FileReferenceData.Type.file, "some other content"); + receiveFile(fileReference, filename, FileReferenceData.Type.file, "some other content"); Optional<File> downloadedFile = fileDownloader.getFile(fileReference); assertTrue(downloadedFile.isPresent()); @@ -121,6 +121,7 @@ public class FileDownloaderTest { assertEquals("some other content", IOUtils.readFile(downloadedFile.get())); // Verify download status when downloaded + System.out.println(downloads.downloadStatuses()); assertDownloadStatus(fileReference, 1.0); } @@ -146,7 +147,7 @@ public class FileDownloaderTest { File tarFile = CompressedFileReference.compress(tempPath.toFile(), Arrays.asList(fooFile, barFile), new File(tempPath.toFile(), filename)); byte[] tarredContent = IOUtils.readFileBytes(tarFile); - receiveFile(fileDownloader, fileReference, filename, FileReferenceData.Type.compressed, tarredContent); + receiveFile(fileReference, filename, FileReferenceData.Type.compressed, tarredContent); Optional<File> downloadedFile = fileDownloader.getFile(fileReference); assertTrue(downloadedFile.isPresent()); @@ -179,7 +180,7 @@ public class FileDownloaderTest { // Receives fileReference, should return and make it available to caller String filename = "abc.jar"; - receiveFile(fileDownloader, fileReference, filename, FileReferenceData.Type.file, "some other content"); + receiveFile(fileReference, filename, FileReferenceData.Type.file, "some other content"); Optional<File> downloadedFile = fileDownloader.getFile(fileReference); assertTrue(downloadedFile.isPresent()); File downloadedFileFullPath = new File(fileReferenceFullPath, filename); @@ -216,7 +217,7 @@ public class FileDownloaderTest { Future<Future<Optional<File>>> future2 = executor.submit(() -> fileDownloader.getFutureFile(fileReferenceDownload)); // Receive file, will complete downloading and futures - receiveFile(fileDownloader, fileReference, filename, FileReferenceData.Type.file, "some other content"); + receiveFile(fileReference, filename, FileReferenceData.Type.file, "some other content"); // Check that we got file correctly with first request Optional<File> downloadedFile = future1.get().get(); @@ -232,7 +233,7 @@ public class FileDownloaderTest { } @Test - public void setFilesToDownload() throws IOException { + public void setFilesToDownload() { Duration timeout = Duration.ofMillis(200); MockConnection connectionPool = new MockConnection(); connectionPool.setResponseHandler(new MockConnection.WaitResponseHandler(timeout.plus(Duration.ofMillis(1000)))); @@ -243,7 +244,7 @@ public class FileDownloaderTest { assertTrue(fileDownloader.isDownloading(xyzzy)); assertFalse(fileDownloader.getFile(xyzzy).isPresent()); // Receive files to simulate download - receiveFile(fileDownloader, xyzzy, "xyzzy.jar", FileReferenceData.Type.file, "content"); + receiveFile(xyzzy, "xyzzy.jar", FileReferenceData.Type.file, "content"); // Should not download, since file has already been downloaded fileDownloader.downloadIfNeeded(new FileReferenceDownload(xyzzy)); // and file should be available @@ -254,7 +255,7 @@ public class FileDownloaderTest { public void receiveFile() throws IOException { FileReference foobar = new FileReference("foobar"); String filename = "foo.jar"; - receiveFile(fileDownloader, foobar, filename, FileReferenceData.Type.file, "content"); + receiveFile(foobar, filename, FileReferenceData.Type.file, "content"); File downloadedFile = new File(fileReferenceFullPath(downloadDir, foobar), filename); assertEquals("content", IOUtils.readFile(downloadedFile)); } @@ -272,15 +273,17 @@ public class FileDownloaderTest { private void assertDownloadStatus(FileReference fileReference, double expectedDownloadStatus) { double downloadStatus = downloads.downloadStatus(fileReference); - assertEquals(expectedDownloadStatus, downloadStatus, 0.0001); + assertEquals("Download statuses: " + downloads.downloadStatuses().toString(), + expectedDownloadStatus, + downloadStatus, + 0.0001); } - private void receiveFile(FileDownloader fileDownloader, FileReference fileReference, String filename, - FileReferenceData.Type type, String content) { - receiveFile(fileDownloader, fileReference, filename, type, Utf8.toBytes(content)); + private void receiveFile(FileReference fileReference, String filename, FileReferenceData.Type type, String content) { + receiveFile(fileReference, filename, type, Utf8.toBytes(content)); } - private void receiveFile(FileDownloader fileDownloader, FileReference fileReference, String filename, + private void receiveFile(FileReference fileReference, String filename, FileReferenceData.Type type, byte[] content) { XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); FileReceiver.Session session = diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 4885f5c9ae5..5f784bcefa1 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -132,7 +132,7 @@ public class Flags { public static final UnboundBooleanFlag GROUP_SUSPENSION = defineFeatureFlag( "group-suspension", true, - List.of("hakon"), "2021-01-22", "2021-06-22", + List.of("hakon"), "2021-01-22", "2021-08-22", "Allow all content nodes in a hierarchical group to suspend at the same time", "Takes effect on the next suspension request to the Orchestrator.", APPLICATION_ID); diff --git a/http-utils/pom.xml b/http-utils/pom.xml index 1f85658430f..2a8ec1b9bb9 100644 --- a/http-utils/pom.xml +++ b/http-utils/pom.xml @@ -31,22 +31,21 @@ <artifactId>slf4j-api</artifactId> <scope>provided</scope> </dependency> - - <!-- compile scope --> + <!-- Apache client artifacts are provided by the jdisc container and are therefore scoped as such --> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> - <scope>compile</scope> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> - <scope>compile</scope> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.httpcomponents.client5</groupId> <artifactId>httpclient5</artifactId> - <scope>compile</scope> + <scope>provided</scope> <exclusions> <exclusion> <groupId>org.slf4j</groupId> diff --git a/jdisc-cloud-aws/pom.xml b/jdisc-cloud-aws/pom.xml index 9f4572736c8..0c89872aa46 100644 --- a/jdisc-cloud-aws/pom.xml +++ b/jdisc-cloud-aws/pom.xml @@ -24,8 +24,20 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-core</artifactId> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.amazonaws</groupId> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java b/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java index c9ce0e0b2e5..79fe8ea11d4 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java @@ -45,7 +45,7 @@ public abstract class ServerProviderConformanceTest { private static final Logger log = Logger.getLogger(ServerProviderConformanceTest.class.getName()); - private static final int NUM_RUNS_EACH_TEST = 10; + private static final int NUM_RUNS_EACH_TEST = 1; /** * <p>This interface declares the adapter between the general conformance test and an actual <code>ServerProvider</code> diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index a7579aeb2ea..19b545df616 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -60,6 +60,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>container-core</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/node-admin/pom.xml b/node-admin/pom.xml index 52873501744..5d3bdb6207c 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -21,6 +21,12 @@ <!-- Provided --> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>docker-api</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -68,16 +74,6 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <scope>compile</scope> - </dependency> - <dependency> <groupId>org.apache.velocity</groupId> <artifactId>velocity</artifactId> <scope>compile</scope> diff --git a/node-repository/pom.xml b/node-repository/pom.xml index d2deaf51afe..e7fdc560bc5 100644 --- a/node-repository/pom.xml +++ b/node-repository/pom.xml @@ -26,6 +26,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -75,14 +81,10 @@ <dependency> <groupId>org.questdb</groupId> <artifactId>questdb</artifactId> - <version>6.0.2</version> + <version>6.0.3</version> <scope>compile</scope> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>http-utils</artifactId> <version>${project.version}</version> diff --git a/parent/pom.xml b/parent/pom.xml index 2a763dcc6ac..4c100397e9d 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -605,11 +605,6 @@ </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> - <artifactId>fluent-hc</artifactId> - <version>4.3.6</version> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <version>${apache.httpclient.version}</version> </dependency> @@ -631,7 +626,7 @@ <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpmime</artifactId> - <version>4.3.6</version> + <version>${apache.httpclient.version}</version> </dependency> <dependency> <groupId>org.apache.maven</groupId> @@ -53,6 +53,7 @@ <module>configserver-flags</module> <module>config_test</module> <module>container</module> + <module>container-apache-http-client-bundle</module> <module>container-core</module> <module>container-dependencies-enforcer</module> <module>container-dependency-versions</module> diff --git a/searchcore/src/vespa/searchcore/config/onnx-models.def b/searchcore/src/vespa/searchcore/config/onnx-models.def index 8d1291fa61e..33ea90002c8 100644 --- a/searchcore/src/vespa/searchcore/config/onnx-models.def +++ b/searchcore/src/vespa/searchcore/config/onnx-models.def @@ -7,3 +7,4 @@ model[].input[].name string model[].input[].source string model[].output[].name string model[].output[].as string +model[].dry_run_on_setup bool default=false diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 3b94bb9984e..dd411d7f30f 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -175,7 +175,7 @@ index.cache.size long default=0 restart attribute.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart ## Multiple optional options for use with mmap -search.mmap.options[] enum {MLOCK, POPULATE, HUGETLB} restart +search.mmap.options[] enum {POPULATE, HUGETLB} restart ## Advise to give to os when mapping memory. search.mmap.advise enum {NORMAL, RANDOM, SEQUENTIAL} default=NORMAL restart @@ -277,7 +277,7 @@ summary.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO summary.read.io enum {NORMAL, DIRECTIO, MMAP } default=MMAP restart ## Multiple optional options for use with mmap -summary.read.mmap.options[] enum {MLOCK, POPULATE, HUGETLB} restart +summary.read.mmap.options[] enum {POPULATE, HUGETLB} restart ## Advise to give to os when mapping memory. summary.read.mmap.advise enum {NORMAL, RANDOM, SEQUENTIAL} default=NORMAL restart diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index ae3edc93f6d..d9d269f1e91 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -170,7 +170,7 @@ FlushableAttribute::FlushableAttribute(const AttributeVectorSP attr, if (config.basicType() == search::attribute::BasicType::Type::TENSOR && config.tensorType().is_dense() && config.hnsw_index_params().has_value()) { - _replay_operation_cost = 100.0; // replaying operations to hnsw index is 100 times more expensive than reading from tls + _replay_operation_cost = 400.0; // replaying operations to hnsw index is 400 times more expensive than reading from tls } } diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 5035a5f583f..65151dd6ff0 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1033,6 +1033,7 @@ "public static final int FMOD", "public static final int LDEXP", "public static final int POW", + "public static final int BIT", "public static final int MAP", "public static final int REDUCE", "public static final int JOIN", @@ -1387,7 +1388,8 @@ "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function ldexp", "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function max", "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function min", - "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function pow" + "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function pow", + "public static final enum com.yahoo.searchlib.rankingexpression.rule.Function bit" ] }, "com.yahoo.searchlib.rankingexpression.rule.FunctionNode": { diff --git a/searchlib/src/apps/docstore/benchmarkdatastore.cpp b/searchlib/src/apps/docstore/benchmarkdatastore.cpp index 20f06a087f5..5277c71cea2 100644 --- a/searchlib/src/apps/docstore/benchmarkdatastore.cpp +++ b/searchlib/src/apps/docstore/benchmarkdatastore.cpp @@ -29,7 +29,7 @@ class BenchmarkDataStoreApp : public FastOS_Application void BenchmarkDataStoreApp::usage() { - printf("Usage: %s <direcory> <numreads> <numthreads> <objects per read> <normal,directio,mmap,mlock>\n", _argv[0]); + printf("Usage: %s <direcory> <numreads> <numthreads> <objects per read> <normal,directio,mmap>\n", _argv[0]); fflush(stdout); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java index b109e6503e3..e41732f9d16 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java @@ -156,6 +156,7 @@ public class TensorValue extends Value { case pow: return value.pow(argument); case fmod: return value.fmod(argument); case ldexp: return value.ldexp(argument); + case bit: return value.bit(argument); default: throw new UnsupportedOperationException("Cannot combine two tensors using " + function); } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java index 99afb3b38d0..16aa947986d 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/Function.java @@ -45,7 +45,8 @@ public enum Function implements Serializable { ldexp(2) { public double evaluate(double x, double y) { return x*pow(2,(int)y); } }, max(2) { public double evaluate(double x, double y) { return max(x,y); } }, min(2) { public double evaluate(double x, double y) { return min(x,y); } }, - pow(2) { public double evaluate(double x, double y) { return pow(x,y); } }; + pow(2) { public double evaluate(double x, double y) { return pow(x,y); } }, + bit(2) { public double evaluate(double x, double y) { return ((int)y < 8 && (int)y >= 0 && ((int)x & (1 << (int)y)) != 0) ? 1.0 : 0.0; } }; private final int arity; diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj index 7506fe250fc..99eff010628 100755 --- a/searchlib/src/main/javacc/RankingExpressionParser.jj +++ b/searchlib/src/main/javacc/RankingExpressionParser.jj @@ -123,6 +123,7 @@ TOKEN : // MAX // MIN <POW: "pow"> | + <BIT: "bit"> | <MAP: "map"> | <REDUCE: "reduce"> | @@ -733,7 +734,8 @@ Function binaryFunctionName() : { } <LDEXP> { return Function.ldexp; } | <MAX> { return Function.max; } | <MIN> { return Function.min; } | - <POW> { return Function.pow; } + <POW> { return Function.pow; } | + <BIT> { return Function.bit; } } List<ExpressionNode> expressionList() : diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index d6302d7026e..4a3c4b248be 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -403,6 +403,24 @@ public class EvaluationTestCase { } @Test + public void testBitExtraction() { + EvaluationTester tester = new EvaluationTester(); + tester.assertEvaluates(1.0, "bit(-43,7)"); + tester.assertEvaluates(1.0, "bit(-43,6)"); + tester.assertEvaluates(0.0, "bit(-43,5)"); + tester.assertEvaluates(1.0, "bit(-43,4)"); + tester.assertEvaluates(0.0, "bit(-43,3)"); + tester.assertEvaluates(1.0, "bit(-43,2)"); + tester.assertEvaluates(0.0, "bit(-43,1)"); + tester.assertEvaluates(1.0, "bit(-43,0)"); + tester.assertEvaluates( + "tensor<int8>(x[40]):[1,1,0,1,0,1,0,1, 0,0,0,0,0,0,0,0, 0,1,0,1,0,1,0,1, 0,1,1,1,1,1,1,1, 1,0,0,0,0,0,0,0]", + "tensor<int8>(x[40])(bit(tensor0{y:x / 8}, 7 - x % 8))", + "tensor<int8>(y[5]):[-43,0,85,127,-128]" + ); + } + + @Test public void testCellTypeCasting() { EvaluationTester tester = new EvaluationTester(); diff --git a/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp b/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp index 7cba56d5d72..a99bf8fcfd0 100644 --- a/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp +++ b/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp @@ -26,7 +26,6 @@ TuneFileRandRead::setFromMmapConfig(const MMapConfig & mmapFlags) { for (size_t i(0), m(mmapFlags.options.size()); i < m; i++) { #ifdef __linux__ switch (mmapFlags.options[i]) { - case MMapConfig::Options::MLOCK: _mmapFlags |= MAP_LOCKED; break; case MMapConfig::Options::POPULATE: _mmapFlags |= MAP_POPULATE; break; case MMapConfig::Options::HUGETLB: _mmapFlags |= MAP_HUGETLB; break; } diff --git a/service-monitor/pom.xml b/service-monitor/pom.xml index 578fcc83006..df28737f352 100644 --- a/service-monitor/pom.xml +++ b/service-monitor/pom.xml @@ -16,14 +16,6 @@ <description>Service monitor component for hosted vespa.</description> <dependencies> - <!-- compile scope --> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <version>4.5</version> - <!-- This is necessary to get 4.4's HostnameVerifier API of SSLConnectionSocketFactory::new --> - <scope>compile</scope> - </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>http-utils</artifactId> @@ -34,6 +26,12 @@ <!-- provided scope --> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml index 653eb58d76d..c4dc849a460 100644 --- a/vespa-athenz/pom.xml +++ b/vespa-athenz/pom.xml @@ -25,6 +25,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-apache-http-client-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>security-utils</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -131,14 +137,6 @@ </exclusions> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-core</artifactId> <exclusions> @@ -154,6 +152,10 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-annotations</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -166,6 +168,13 @@ </exclusion> </exclusions> </dependency> + <dependency> + <!-- required by java-jwt --> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>1.15</version> + <scope>compile</scope> + </dependency> </dependencies> <build> diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index 5817eb0c8d2..7503b5a39ed 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -32,6 +32,8 @@ import java.net.URI; import java.time.Instant; import java.util.Collections; import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.OptionalInt; import java.util.Set; import java.util.function.Function; @@ -111,13 +113,17 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void addRoleMember(AthenzRole role, AthenzIdentity member) { + public void addRoleMember(AthenzRole role, AthenzIdentity member, Optional<String> reason) { URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s/member/%s", role.domain().getName(), role.roleName(), member.getFullName())); MembershipEntity membership = new MembershipEntity.RoleMembershipEntity(member.getFullName(), true, role.roleName(), null); - HttpUriRequest request = RequestBuilder.put(uri) - .setEntity(toJsonStringEntity(membership)) - .build(); - execute(request, response -> readEntity(response, Void.class)); + + + RequestBuilder requestBuilder = RequestBuilder.put(uri) + .setEntity(toJsonStringEntity(membership)); + if (reason.filter(s -> !s.isBlank()).isPresent()) { + requestBuilder.addHeader("Y-Audit-Ref", reason.get()); + } + execute(requestBuilder.build(), response -> readEntity(response, Void.class)); } @Override diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index 245078e3679..03afc9278cc 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.athenz.api.OktaIdentityToken; import java.time.Instant; import java.util.List; +import java.util.Optional; import java.util.Set; /** @@ -31,7 +32,7 @@ public interface ZmsClient extends AutoCloseable { void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, OktaIdentityToken identityToken, OktaAccessToken accessToken); - void addRoleMember(AthenzRole role, AthenzIdentity member); + void addRoleMember(AthenzRole role, AthenzIdentity member, Optional<String> reason); void deleteRoleMember(AthenzRole role, AthenzIdentity member); diff --git a/vespa-documentgen-plugin/README b/vespa-documentgen-plugin/README index 5afe4630355..3b016dccc9c 100644 --- a/vespa-documentgen-plugin/README +++ b/vespa-documentgen-plugin/README @@ -1 +1 @@ -Maven plugin which generates Vespa document classes from SD files. +Maven plugin which generates Vespa document classes from schema files. diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/Annotation.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/Annotation.java index 31046206392..465abae9504 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/Annotation.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/Annotation.java @@ -1,13 +1,14 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa; /** * Represents one configured provided annotation type * - * @author vegardh + * @author Vegard Balgaard Havdal */ public class Annotation { String type; String clazz; + } diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index b086e0d8a26..3b9babf1782 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa; import com.yahoo.collections.Pair; @@ -30,13 +30,11 @@ import org.apache.maven.project.MavenProject; import java.io.File; import java.io.FileWriter; -import java.io.FilenameFilter; import java.io.IOException; import java.io.Writer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -44,10 +42,10 @@ import java.util.List; import java.util.Map; import java.util.Set; - /** - * Goal which generates Vespa document classes from SD files. - * @author vegardh + * Generates Vespa document classes from schema files. + * + * @author Vegard Balgaard Havdal */ @Mojo(name = "document-gen", defaultPhase = LifecyclePhase.GENERATE_SOURCES) public class DocumentGenMojo extends AbstractMojo { @@ -60,12 +58,22 @@ public class DocumentGenMojo extends AbstractMojo { private MavenProject project; /** - * Directory containing the SD files + * Directory containing the searchdefinition files + * @deprecated use {@link #schemasDirectory} instead */ - @Parameter(defaultValue = ".", required = true) + // TODO: Remove in Vespa 8 + @Deprecated + @Parameter(defaultValue = ".", required = false) private File sdDirectory; /** + * Directory containing the schema files + */ + // TODO: Make this required and with defaultValue "." when sdDirectory is removed in Vespa 8 + @Parameter + private File schemasDirectory; + + /** * Java package for generated classes */ @Parameter(defaultValue = "com.yahoo.vespa.document", required = true) @@ -98,7 +106,7 @@ public class DocumentGenMojo extends AbstractMojo { private Map<String, String> structTypes; private Map<String, String> annotationTypes; - void execute(File sdDir, File outputDir, String packageName) { + void execute(File schemasDir, File outputDir, String packageName) { if ("".equals(packageName)) throw new IllegalArgumentException("You may not use empty package for generated types."); searches = new HashMap<>(); docTypes = new HashMap<>(); @@ -106,7 +114,7 @@ public class DocumentGenMojo extends AbstractMojo { annotationTypes = new HashMap<>(); outputDir.mkdirs(); - SearchBuilder builder = buildSearches(sdDir); + SearchBuilder builder = buildSearches(schemasDir); boolean annotationsExported=false; for (NewDocumentType docType : builder.getModel().getDocumentManager().getTypes()) { @@ -127,10 +135,7 @@ public class DocumentGenMojo extends AbstractMojo { } private SearchBuilder buildSearches(File sdDir) { - File[] sdFiles = sdDir.listFiles(new FilenameFilter() { - public boolean accept(File dir, String name) { - return name.endsWith(".sd"); - }}); + File[] sdFiles = sdDir.listFiles((dir, name) -> name.endsWith(".sd")); SearchBuilder builder = new SearchBuilder(true); for (File f : sdFiles) { try { @@ -980,7 +985,12 @@ public class DocumentGenMojo extends AbstractMojo { @Override public void execute() { - execute(this.sdDirectory, this.outputDirectory, this.packageName); + File dir = sdDirectory; + // Prefer schemasDirectory if set + if (this.schemasDirectory != null) + dir = this.schemasDirectory; + + execute(dir, this.outputDirectory, packageName); } Map<String, Search> getSearches() { @@ -990,4 +1000,5 @@ public class DocumentGenMojo extends AbstractMojo { private static String upperCaseFirstChar(String s) { return s.substring(0, 1).toUpperCase()+s.substring(1); } + } diff --git a/vespa-feed-client/abi-spec.json b/vespa-feed-client/abi-spec.json index ecac167cd8e..70cb4c3f09f 100644 --- a/vespa-feed-client/abi-spec.json +++ b/vespa-feed-client/abi-spec.json @@ -37,6 +37,21 @@ ], "fields": [] }, + "ai.vespa.feed.client.DynamicThrottler": { + "superClass": "ai.vespa.feed.client.StaticThrottler", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(ai.vespa.feed.client.FeedClientBuilder)", + "public void sent(long, java.util.concurrent.CompletableFuture)", + "public void success()", + "public void throttled(long)", + "public long targetInflight()" + ], + "fields": [] + }, "ai.vespa.feed.client.FeedClient$CircuitBreaker$State": { "superClass": "java.lang.Enum", "interfaces": [], @@ -102,6 +117,22 @@ ], "fields": [] }, + "ai.vespa.feed.client.FeedClient$Throttler": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void sent(long, java.util.concurrent.CompletableFuture)", + "public abstract void success()", + "public abstract void throttled(long)", + "public abstract long targetInflight()" + ], + "fields": [] + }, "ai.vespa.feed.client.FeedClient": { "superClass": "java.lang.Object", "interfaces": [ @@ -331,5 +362,25 @@ "public void <init>(ai.vespa.feed.client.DocumentId, java.lang.Throwable)" ], "fields": [] + }, + "ai.vespa.feed.client.StaticThrottler": { + "superClass": "java.lang.Object", + "interfaces": [ + "ai.vespa.feed.client.FeedClient$Throttler" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(ai.vespa.feed.client.FeedClientBuilder)", + "public void sent(long, java.util.concurrent.CompletableFuture)", + "public void success()", + "public void throttled(long)", + "public long targetInflight()" + ], + "fields": [ + "protected final long maxInflight", + "protected final long minInflight" + ] } }
\ No newline at end of file diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/DynamicThrottler.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/DynamicThrottler.java new file mode 100644 index 00000000000..6f4e4e752f0 --- /dev/null +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/DynamicThrottler.java @@ -0,0 +1,86 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.feed.client; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static java.lang.Math.log; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.lang.Math.pow; +import static java.lang.Math.random; + +/** + * Samples latency as a function of inflight requests, and regularly adjusts to the optimal value. + * + * @author jonmv + */ +public class DynamicThrottler extends StaticThrottler { + + private final AtomicLong targetInflight; + private long updateNanos = 0; + private final List<AtomicLong> latencies = new ArrayList<>(); + private final double weight = 0.9; // Higher weight favours higher (own) throughput, at the cost of (shared) latency. + + public DynamicThrottler(FeedClientBuilder builder) { + super(builder); + this.targetInflight = new AtomicLong(128L * builder.connectionsPerEndpoint * builder.endpoints.size()); + for (int i = 0; i < 128; i++) + latencies.add(new AtomicLong(-1)); + } + + @Override + public void sent(long inflight, CompletableFuture<HttpResponse> vessel) { + long startNanos = System.nanoTime(); + if (updateNanos == 0) updateNanos = System.nanoTime(); + boolean update = startNanos - updateNanos >= 1e8; // Ship ten updates per second. + if (update) updateNanos = startNanos; + + vessel.whenComplete((response, thrown) -> { + // Use buckets for latency measurements, with inflight along a log scale, + // and with minInflight and maxInflight at the ends. + int index = (int) (latencies.size() * log(max(1, (double) inflight / minInflight)) + / log(256)); // 4096 (server max streams per connection) / 16 (our min per connection) + long nowNanos = System.nanoTime(); + long latencyNanos = nowNanos - startNanos; + latencies.get(index).set(latencyNanos); + if ( ! update) + return; + + // Loop over latency measurements and pick the one which optimises throughput and latency. + double choice = -1; + double max = -1; + for (int i = latencies.size(); i-- > 0; ) { + double latency = latencies.get(i).get(); + if (latency < 0) continue; // Skip unknown values. + double target = minInflight * pow(256, (i + 0.5) / latencies.size()); + double objective = pow(target, weight) / latency; // Optimise throughput (weight), but also latency (1 - weight). + if (objective > max) { + max = objective; + choice = target; + } + } + long target = (long) ((random() * 0.25 + 0.90) * choice); // Random walk, skewed towards increase. + targetInflight.set(max(minInflight, min(maxInflight, target))); + }); + } + + @Override + public void success() { + super.success(); + } + + @Override + public void throttled(long inflight) { + super.throttled(inflight); + } + + @Override + public long targetInflight() { + return min(super.targetInflight(), targetInflight.get()); + } + +} diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java index 952edfb5464..f39b56ad50f 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java @@ -92,4 +92,22 @@ public interface FeedClient extends Closeable { } + + /** Determines the number of requests to have inflight at any point. */ + interface Throttler { + + /** A request was just sent with {@code vessel}, with {@code inflight} total in flight. */ + void sent(long inflight, CompletableFuture<HttpResponse> vessel); + + /** A successful response was obtained. */ + void success(); + + /** A throttle signal was obtained from the server. */ + void throttled(long inflight); + + /** The target inflight operations right now. */ + long targetInflight(); + + } + } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java index 8b5eb9efea7..0f685ec5b7f 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClientBuilder.java @@ -35,7 +35,7 @@ public class FeedClientBuilder { SSLContext sslContext; HostnameVerifier hostnameVerifier; int connectionsPerEndpoint = 4; - int maxStreamsPerConnection = 128; + int maxStreamsPerConnection = 4096; FeedClient.RetryStrategy retryStrategy = defaultRetryStrategy; FeedClient.CircuitBreaker circuitBreaker = new GracePeriodCircuitBreaker(Duration.ofSeconds(1), Duration.ofMinutes(10)); Path certificateFile; diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java index 98ff3a5d921..e9cd0baba5b 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java @@ -5,8 +5,10 @@ import ai.vespa.feed.client.FeedClient.CircuitBreaker; import ai.vespa.feed.client.FeedClient.RetryStrategy; import java.io.IOException; +import java.nio.channels.CancelledKeyException; import java.util.Map; import java.util.Queue; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; @@ -19,8 +21,6 @@ import java.util.logging.Logger; import static ai.vespa.feed.client.FeedClient.CircuitBreaker.State.CLOSED; import static ai.vespa.feed.client.FeedClient.CircuitBreaker.State.HALF_OPEN; import static ai.vespa.feed.client.FeedClient.CircuitBreaker.State.OPEN; -import static java.lang.Math.max; -import static java.lang.Math.min; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.logging.Level.FINE; import static java.util.logging.Level.WARNING; @@ -44,14 +44,11 @@ class HttpRequestStrategy implements RequestStrategy { private final Map<DocumentId, CompletableFuture<?>> inflightById = new ConcurrentHashMap<>(); private final RetryStrategy strategy; private final CircuitBreaker breaker; + final FeedClient.Throttler throttler; private final Queue<Runnable> queue = new ConcurrentLinkedQueue<>(); - private final long maxInflight; - private final long minInflight; - private final AtomicLong targetInflightX10; // 10x target, so we can increment one every tenth success. private final AtomicLong inflight = new AtomicLong(0); private final AtomicBoolean destroyed = new AtomicBoolean(false); private final AtomicLong delayedCount = new AtomicLong(0); - private final AtomicLong retries = new AtomicLong(0); private final ExecutorService resultExecutor = Executors.newSingleThreadExecutor(runnable -> { Thread thread = new Thread(runnable, "feed-client-result-executor"); thread.setDaemon(true); @@ -66,9 +63,7 @@ class HttpRequestStrategy implements RequestStrategy { this.cluster = builder.benchmark ? new BenchmarkingCluster(cluster) : cluster; this.strategy = builder.retryStrategy; this.breaker = builder.circuitBreaker; - this.maxInflight = builder.connectionsPerEndpoint * (long) builder.maxStreamsPerConnection; - this.minInflight = builder.connectionsPerEndpoint * (long) min(16, builder.maxStreamsPerConnection); - this.targetInflightX10 = new AtomicLong(10 * (long) (Math.sqrt(minInflight) * Math.sqrt(maxInflight))); + this.throttler = new DynamicThrottler(builder); Thread dispatcher = new Thread(this::dispatch, "feed-client-dispatcher"); dispatcher.setDaemon(true); @@ -100,9 +95,11 @@ class HttpRequestStrategy implements RequestStrategy { destroy(); } - private void offer(Runnable task) { + private void offer(HttpRequest request, CompletableFuture<HttpResponse> vessel) { delayedCount.incrementAndGet(); - queue.offer(task); + queue.offer(() -> { + cluster.dispatch(request, vessel); + }); } private boolean poll() { @@ -113,8 +110,9 @@ class HttpRequestStrategy implements RequestStrategy { return true; } + private boolean isInExcess() { - return inflight.get() - delayedCount.get() > targetInflight(); + return inflight.get() - delayedCount.get() > throttler.targetInflight(); } private boolean retry(HttpRequest request, int attempt) { @@ -137,29 +135,19 @@ class HttpRequestStrategy implements RequestStrategy { breaker.failure(); log.log(FINE, thrown, () -> "Failed attempt " + attempt + " at " + request); - if ( ! (thrown instanceof IOException)) - return false; - - return retry(request, attempt); - } - - private void incrementTargetInflight() { - targetInflightX10.incrementAndGet(); - } - - private void decreaseTargetInflight() { - targetInflightX10.set(max((inflight.get() - delayedCount.get()) * 9, minInflight * 10)); - } + if ( (thrown instanceof IOException) // General IO problems. + || (thrown instanceof CancellationException) // TLS session disconnect. + || (thrown instanceof CancelledKeyException)) // Selection cancelled. + return retry(request, attempt); - private long targetInflight() { - return min(targetInflightX10.get() / 10, maxInflight); + return false; } /** Retries throttled requests (429, 503), adjusting the target inflight count, and server errors (500, 502). */ private boolean retry(HttpRequest request, HttpResponse response, int attempt) { if (response.code() / 100 == 2) { breaker.success(); - incrementTargetInflight(); + throttler.success(); return false; } @@ -167,7 +155,7 @@ class HttpRequestStrategy implements RequestStrategy { ") on attempt " + attempt + " at " + request); if (response.code() == 429 || response.code() == 503) { // Throttling; reduce target inflight. - decreaseTargetInflight(); + throttler.throttled((inflight.get() - delayedCount.get())); return true; } @@ -180,7 +168,7 @@ class HttpRequestStrategy implements RequestStrategy { private void acquireSlot() { try { - while (inflight.get() >= targetInflight()) + while (inflight.get() >= throttler.targetInflight()) Thread.sleep(1); inflight.incrementAndGet(); @@ -216,27 +204,24 @@ class HttpRequestStrategy implements RequestStrategy { if (previous == null) { acquireSlot(); - offer(() -> cluster.dispatch(request, vessel)); + offer(request, vessel); + throttler.sent(inflight.get(), result); } else - previous.whenComplete((__, ___) -> offer(() -> cluster.dispatch(request, vessel))); + previous.whenComplete((__, ___) -> offer(request, vessel)); handleAttempt(vessel, request, result, 1); - result.whenComplete((__, ___) -> { + return result.handle((response, error) -> { if (inflightById.compute(documentId, (____, current) -> current == result ? null : current) == null) releaseSlot(); - }); - result.handle((response, error) -> { if (error != null) { - if (error instanceof FeedException) throw (FeedException)error; + if (error instanceof FeedException) throw (FeedException) error; throw new FeedException(documentId, error); } return response; }); - - return result; } /** Handles the result of one attempt at the given operation, retrying if necessary. */ @@ -245,10 +230,9 @@ class HttpRequestStrategy implements RequestStrategy { // Retry the operation if it failed with a transient error ... if (thrown != null ? retry(request, thrown, attempt) : retry(request, response, attempt)) { - retries.incrementAndGet(); CircuitBreaker.State state = breaker.state(); CompletableFuture<HttpResponse> retry = new CompletableFuture<>(); - offer(() -> cluster.dispatch(request, retry)); + offer(request, retry); handleAttempt(retry, request, result, attempt + (state == HALF_OPEN ? 0 : 1)); } // ... or accept the outcome and mark the operation as complete. @@ -262,11 +246,11 @@ class HttpRequestStrategy implements RequestStrategy { @Override public void destroy() { - if ( ! destroyed.getAndSet(true)) + if ( ! destroyed.getAndSet(true)) { inflightById.values().forEach(result -> result.cancel(true)); - - cluster.close(); - resultExecutor.shutdown(); + cluster.close(); + resultExecutor.shutdown(); + } } } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/StaticThrottler.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/StaticThrottler.java new file mode 100644 index 00000000000..4e0c4fe90f0 --- /dev/null +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/StaticThrottler.java @@ -0,0 +1,45 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.feed.client; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicLong; + +import static java.lang.Math.max; +import static java.lang.Math.min; + +/** + * Reduces max throughput whenever throttled; increases it slowly whenever successful responses are obtained. + * + * @author jonmv + */ +public class StaticThrottler implements FeedClient.Throttler { + + protected final long maxInflight; + protected final long minInflight; + private final AtomicLong targetX10; + + public StaticThrottler(FeedClientBuilder builder) { + this.maxInflight = builder.connectionsPerEndpoint * (long) builder.maxStreamsPerConnection; + this.minInflight = builder.connectionsPerEndpoint * (long) min(16, builder.maxStreamsPerConnection); + this.targetX10 = new AtomicLong(10 * maxInflight); // 10x the actual value to allow for smaller updates. + } + + @Override + public void sent(long inflight, CompletableFuture<HttpResponse> vessel) { } + + @Override + public void success() { + targetX10.incrementAndGet(); + } + + @Override + public void throttled(long inflight) { + targetX10.set(max(inflight * 5, minInflight * 10)); + } + + @Override + public long targetInflight() { + return min(maxInflight, targetX10.get() / 10); + } + +} diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java index d3005227184..21ab6889e6e 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/HttpRequestStrategyTest.java @@ -13,6 +13,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -36,17 +37,29 @@ class HttpRequestStrategyTest { HttpRequest request = new HttpRequest("PUT", "/", null, null); HttpResponse response = HttpResponse.of(200, "{}".getBytes(UTF_8)); ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); - Cluster cluster = new BenchmarkingCluster((__, vessel) -> executor.schedule(() -> vessel.complete(response), 100, TimeUnit.MILLISECONDS)); + Cluster cluster = new BenchmarkingCluster((__, vessel) -> executor.schedule(() -> vessel.complete(response), (int) (Math.random() * 2 * 10), TimeUnit.MILLISECONDS)); HttpRequestStrategy strategy = new HttpRequestStrategy(FeedClientBuilder.create(URI.create("https://dummy.com:123")) - .setConnectionsPerEndpoint(1 << 12) - .setMaxStreamPerConnection(1 << 4), + .setConnectionsPerEndpoint(1 << 10) + .setMaxStreamPerConnection(1 << 12), cluster); + CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> { + try { + while ( ! latch.await(1, TimeUnit.SECONDS)) { + System.err.println(cluster.stats().inflight()); + System.err.println(strategy.throttler.targetInflight()); + System.err.println(); + } + } + catch (InterruptedException ignored) { } + }).start(); long startNanos = System.nanoTime(); for (int i = 0; i < documents; i++) strategy.enqueue(DocumentId.of("ns", "type", Integer.toString(i)), request); strategy.await(); + latch.countDown(); executor.shutdown(); cluster.close(); OperationStats stats = cluster.stats(); @@ -84,7 +97,7 @@ class HttpRequestStrategyTest { HttpRequest request = new HttpRequest("POST", "/", null, null); // Runtime exception is not retried. - cluster.expect((__, vessel) -> vessel.completeExceptionally(new RuntimeException("boom"))); + cluster.expect((__, vessel) -> vessel.completeExceptionally(new FeedException("boom"))); ExecutionException expected = assertThrows(ExecutionException.class, () -> strategy.enqueue(id1, request).get()); assertEquals("boom", expected.getCause().getMessage()); @@ -94,7 +107,7 @@ class HttpRequestStrategyTest { cluster.expect((__, vessel) -> vessel.completeExceptionally(new IOException("retry me"))); expected = assertThrows(ExecutionException.class, () -> strategy.enqueue(id1, request).get()); - assertEquals("retry me", expected.getCause().getMessage()); + assertEquals("retry me", expected.getCause().getCause().getMessage()); assertEquals(3, strategy.stats().requests()); // Successful response is returned diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index 834c3d7c988..17443b11b6a 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -38,11 +38,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespa-http-client</artifactId> <version>${project.version}</version> diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 4ddf8b83cdc..c0ac0d0f3be 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1201,6 +1201,7 @@ "public com.yahoo.tensor.Tensor equal(com.yahoo.tensor.Tensor)", "public com.yahoo.tensor.Tensor notEqual(com.yahoo.tensor.Tensor)", "public com.yahoo.tensor.Tensor approxEqual(com.yahoo.tensor.Tensor)", + "public com.yahoo.tensor.Tensor bit(com.yahoo.tensor.Tensor)", "public com.yahoo.tensor.Tensor avg()", "public com.yahoo.tensor.Tensor avg(java.lang.String)", "public com.yahoo.tensor.Tensor avg(java.util.List)", diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java index 144a9a585f6..4a53d70ff38 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java @@ -4,8 +4,13 @@ package com.yahoo.slime; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; import java.util.Iterator; import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalInt; +import java.util.OptionalLong; import java.util.Spliterator; import java.util.Spliterators; import java.util.stream.Stream; @@ -57,7 +62,7 @@ public class SlimeUtils { } } - private static void copyArray(Inspector from, final Cursor to) { + private static void copyArray(Inspector from, Cursor to) { from.traverse((ArrayTraverser) (i, inspector) -> addValue(inspector, to)); } @@ -124,15 +129,32 @@ public class SlimeUtils { return slime; } + public static Instant instant(Inspector field) { + return Instant.ofEpochMilli(field.asLong()); + } + public static Optional<String> optionalString(Inspector inspector) { return Optional.of(inspector.asString()).filter(s -> !s.isEmpty()); } - public static Optional<Long> optionalLong(Inspector inspector) { - if (inspector.type() == Type.LONG) { - return Optional.of(inspector.asLong()); - } - return Optional.empty(); + public static OptionalLong optionalLong(Inspector field) { + return field.valid() ? OptionalLong.of(field.asLong()) : OptionalLong.empty(); + } + + public static OptionalInt optionalInteger(Inspector field) { + return field.valid() ? OptionalInt.of((int) field.asLong()) : OptionalInt.empty(); + } + + public static OptionalDouble optionalDouble(Inspector field) { + return field.valid() ? OptionalDouble.of(field.asDouble()) : OptionalDouble.empty(); + } + + public static Optional<Instant> optionalInstant(Inspector field) { + return optionalLong(field).stream().mapToObj(Instant::ofEpochMilli).findFirst(); + } + + public static Optional<Duration> optionalDuration(Inspector field) { + return optionalLong(field).stream().mapToObj(Duration::ofMillis).findFirst(); } public static Iterator<Inspector> entriesIterator(Inspector inspector) { @@ -146,8 +168,9 @@ public class SlimeUtils { /** Returns stream of entries for given inspector. If the inspector is not an array, empty stream is returned */ public static Stream<Inspector> entriesStream(Inspector inspector) { int characteristics = Spliterator.NONNULL | Spliterator.SIZED | Spliterator.ORDERED; - return StreamSupport.stream( - Spliterators.spliteratorUnknownSize(entriesIterator(inspector), characteristics), - false); + return StreamSupport.stream(Spliterators.spliteratorUnknownSize(entriesIterator(inspector), + characteristics), + false); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 3133752bc49..ab475e25387 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -240,6 +240,7 @@ public interface Tensor { default Tensor equal(Tensor argument) { return join(argument, (a, b) -> ( a == b ? 1.0 : 0.0)); } default Tensor notEqual(Tensor argument) { return join(argument, (a, b) -> ( a != b ? 1.0 : 0.0)); } default Tensor approxEqual(Tensor argument) { return join(argument, (a, b) -> ( approxEquals(a,b) ? 1.0 : 0.0)); } + default Tensor bit(Tensor argument) { return join(argument, (a,b) -> ((int)b < 8 && (int)b >= 0 && ((int)a & (1 << (int)b)) != 0) ? 1.0 : 0.0); } default Tensor avg() { return avg(Collections.emptyList()); } default Tensor avg(String dimension) { return avg(Collections.singletonList(dimension)); } |